1
0
mirror of https://github.com/certbot/certbot.git synced 2026-01-26 07:41:33 +03:00

[Windows|Linux] Launch integration tests on Pebble without Docker (#7157)

This PR is a part of the actions necessary to make Certbot-CI work on Windows, in order to execute the integration tests on this platform.

Following #7156, this PR changes how the integration tests are setup against Pebble to not need Docker anymore.

As a reminder, one can check #7156 and letsencrypt/pebble#240 to see the rationale about why using Docker is a problem to run the integration tests on Windows.

Basically, this PR executes directly Pebble using its executable, since it is build using Go, and Go produces self-contained executable that can run without any installation on Linux and on Windows. During the integration tests setup, Certbot-CI will get the Pebble (and Challtestsrv) executables for the defined target version on the GitHub releases. The binaries are persisted on the filesystem, so it is not needed to download them again on the second integration tests execution. Nonetheless, we are talking about 20MB of executables.

Since the setup needs to hold a state, I also took this occasion to refactor the acme_server, in order to use on object oriented approach and improve the readability/maintainability.

Once this PR and #7156 are merged, Docker will not be needed anymore for the main integration tests usecase, that is to use Pebble.

* Complete process

* Fix nginx cert path

* Check conditionnally docker

* Update gitignore, fix apacheconftest

* Full object

* Carriage return

* Move to official v2.1.0 of pebble

* Fix name

* Update acme_server.py

* Relaunch CI

* Update certbot-ci/certbot_integration_tests/utils/acme_server.py

Co-Authored-By: Brad Warren <bmw@users.noreply.github.com>

* Update certbot-ci/certbot_integration_tests/utils/acme_server.py

Co-Authored-By: Brad Warren <bmw@users.noreply.github.com>

* Update docstring

* Update documentation

* Configure a stdout to ACMEServer

* Map all process through defined stdout

* Remove unused variable

* Handle using signals

* Use failsafe entering context

* Remove failsafe rmtree, that is not needed anymore
This commit is contained in:
Adrien Ferrand
2019-07-10 23:29:57 +02:00
committed by Brad Warren
parent 43f58ca803
commit 2ac99fefe0
8 changed files with 213 additions and 169 deletions

2
.gitignore vendored
View File

@@ -47,3 +47,5 @@ tests/letstest/venv/
# certbot tests
.certbot_test_workspace
**/assets/pebble*
**/assets/challtestsrv*

View File

@@ -15,7 +15,7 @@ SCRIPT_DIRNAME = os.path.dirname(__file__)
def main(args=None):
if not args:
args = sys.argv[1:]
with acme_server.setup_acme_server('pebble', [], False) as acme_xdist:
with acme_server.ACMEServer('pebble', [], False) as acme_xdist:
environ = os.environ.copy()
environ['SERVER'] = acme_xdist['directory_url']
command = [os.path.join(SCRIPT_DIRNAME, 'apache-conf-test')]

View File

@@ -68,17 +68,18 @@ def _setup_primary_node(config):
:param config: Configuration of the pytest primary node
"""
# Check for runtime compatibility: some tools are required to be available in PATH
try:
subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT)
except (subprocess.CalledProcessError, OSError):
raise ValueError('Error: docker is required in PATH to launch the integration tests, '
'but is not installed or not available for current user.')
if 'boulder' in config.option.acme_server:
try:
subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT)
except (subprocess.CalledProcessError, OSError):
raise ValueError('Error: docker is required in PATH to launch the integration tests on'
'boulder, but is not installed or not available for current user.')
try:
subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT)
except (subprocess.CalledProcessError, OSError):
raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, '
'but is not installed or not available for current user.')
try:
subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT)
except (subprocess.CalledProcessError, OSError):
raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, '
'but is not installed or not available for current user.')
# Parameter numprocesses is added to option by pytest-xdist
workers = ['primary'] if not config.option.numprocesses\
@@ -86,7 +87,7 @@ def _setup_primary_node(config):
# By calling setup_acme_server we ensure that all necessary acme server instances will be
# fully started. This runtime is reflected by the acme_xdist returned.
acme_server = acme_lib.setup_acme_server(config.option.acme_server, workers)
acme_server = acme_lib.ACMEServer(config.option.acme_server, workers)
config.add_cleanup(acme_server.stop)
print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist))
acme_server.start()

View File

@@ -21,9 +21,9 @@ def construct_nginx_config(nginx_root, nginx_webroot, http_port, https_port, oth
:rtype: str
"""
key_path = key_path if key_path \
else pkg_resources.resource_filename('certbot_integration_tests', 'assets/nginx_key.pem')
else pkg_resources.resource_filename('certbot_integration_tests', 'assets/key.pem')
cert_path = cert_path if cert_path \
else pkg_resources.resource_filename('certbot_integration_tests', 'assets/nginx_cert.pem')
else pkg_resources.resource_filename('certbot_integration_tests', 'assets/cert.pem')
return '''\
# This error log will be written regardless of server scope error_log
# definitions, so we have to set this here in the main scope.

View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python
"""Module to setup an ACME CA server environment able to run multiple tests in parallel"""
from __future__ import print_function
import errno
import json
import tempfile
import time
@@ -11,184 +12,178 @@ import sys
from os.path import join
import requests
import yaml
from certbot_integration_tests.utils import misc, proxy
from certbot_integration_tests.utils import misc, proxy, pebble_artifacts
from certbot_integration_tests.utils.constants import *
class ACMEServer(object):
"""
Handler exposing methods to start and stop the ACME server, and get its configuration
(eg. challenges ports). ACMEServer is also a context manager, and so can be used to
ensure ACME server is started/stopped upon context enter/exit.
ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy
instance, to allow parallel execution of integration tests against the unique http-01 port
expected by the ACME CA server.
Typically all pytest integration tests will be executed in this context.
ACMEServer gives access the acme_xdist parameter, listing the ports and directory url to use
for each pytest node. It exposes also start and stop methods in order to start the stack, and
stop it with proper resources cleanup.
ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped
upon context enter/exit.
"""
def __init__(self, acme_xdist, start, server_cleanup):
self._proxy_process = None
self._server_cleanup = server_cleanup
self.acme_xdist = acme_xdist
self.start = start
def __init__(self, acme_server, nodes, http_proxy=True, stdout=False):
"""
Create an ACMEServer instance.
:param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
:param list nodes: list of node names that will be setup by pytest xdist
:param bool http_proxy: if False do not start the HTTP proxy
:param bool stdout: if True stream subprocesses stdout to standard stdout
"""
self._construct_acme_xdist(acme_server, nodes)
self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
self._proxy = http_proxy
self._workspace = tempfile.mkdtemp()
self._processes = []
self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
def start(self):
"""Start the test stack"""
try:
if self._proxy:
self._prepare_http_proxy()
if self._acme_type == 'pebble':
self._prepare_pebble_server()
if self._acme_type == 'boulder':
self._prepare_boulder_server()
except BaseException as e:
self.stop()
raise e
def stop(self):
if self._proxy_process:
self._proxy_process.terminate()
self._proxy_process.wait()
self._server_cleanup()
"""Stop the test stack, and clean its resources"""
print('=> Tear down the test infrastructure...')
try:
for process in self._processes:
try:
process.terminate()
except OSError as e:
# Process may be not started yet, so no PID and terminate fails.
# Then the process never started, and the situation is acceptable.
if e.errno != errno.ESRCH:
raise
for process in self._processes:
process.wait()
if os.path.exists(os.path.join(self._workspace, 'boulder')):
# Boulder docker generates build artifacts owned by root with 0o744 permissions.
# If we started the acme server from a normal user that has access to the Docker
# daemon, this user will not be able to delete these artifacts from the host.
# We need to do it through a docker.
process = self._launch_process(['docker', 'run', '--rm', '-v',
'{0}:/workspace'.format(self._workspace),
'alpine', 'rm', '-rf', '/workspace/boulder'])
process.wait()
finally:
shutil.rmtree(self._workspace)
if self._stdout != sys.stdout:
self._stdout.close()
print('=> Test infrastructure stopped and cleaned up.')
def __enter__(self):
self._proxy_process = self.start()
self.start()
return self.acme_xdist
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
def _construct_acme_xdist(self, acme_server, nodes):
"""Generate and return the acme_xdist dict"""
acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT}
def setup_acme_server(acme_server, nodes, proxy=True):
"""
This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel
execution of integration tests against the unique http-01 port expected by the ACME CA server.
Typically all pytest integration tests will be executed in this context.
An ACMEServer instance will be returned, giving access to the ports and directory url to use
for each pytest node, and its start and stop methods are appropriately configured to
respectively start the server, and stop it with proper resources cleanup.
:param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
:param str[] nodes: list of node names that will be setup by pytest xdist
:param bool proxy: set to False to not start the HTTP proxy
:return: a properly configured ACMEServer instance
:rtype: ACMEServer
"""
acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
acme_xdist = _construct_acme_xdist(acme_server, nodes)
workspace, server_cleanup = _construct_workspace(acme_type)
# Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble.
if acme_server == 'pebble':
acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL
else: # boulder
acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \
if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL
def start():
proxy_process = _prepare_http_proxy(acme_xdist) if proxy else None
_prepare_acme_server(workspace, acme_type, acme_xdist)
acme_xdist['http_port'] = {node: port for (node, port)
in zip(nodes, range(5200, 5200 + len(nodes)))}
acme_xdist['https_port'] = {node: port for (node, port)
in zip(nodes, range(5100, 5100 + len(nodes)))}
acme_xdist['other_port'] = {node: port for (node, port)
in zip(nodes, range(5300, 5300 + len(nodes)))}
return proxy_process
self.acme_xdist = acme_xdist
return ACMEServer(acme_xdist, start, server_cleanup)
def _prepare_pebble_server(self):
"""Configure and launch the Pebble server"""
print('=> Starting pebble instance deployment...')
pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(self._workspace)
# Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
# nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
environ = os.environ.copy()
environ['PEBBLE_VA_NOSLEEP'] = '1'
environ['PEBBLE_WFE_NONCEREJECT'] = '0'
def _construct_acme_xdist(acme_server, nodes):
"""Generate and return the acme_xdist dict"""
acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT}
self._launch_process(
[pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'],
env=environ)
# Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble.
if acme_server == 'pebble':
acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL
else: # boulder
acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \
if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL
acme_xdist['http_port'] = {node: port for (node, port)
in zip(nodes, range(5200, 5200 + len(nodes)))}
acme_xdist['https_port'] = {node: port for (node, port)
in zip(nodes, range(5100, 5100 + len(nodes)))}
acme_xdist['other_port'] = {node: port for (node, port)
in zip(nodes, range(5300, 5300 + len(nodes)))}
return acme_xdist
def _construct_workspace(acme_type):
"""Create a temporary workspace for integration tests stack"""
workspace = tempfile.mkdtemp()
def cleanup():
"""Cleanup function to call that will teardown relevant dockers and their configuration."""
print('=> Tear down the {0} instance...'.format(acme_type))
instance_path = join(workspace, acme_type)
try:
if os.path.isfile(join(instance_path, 'docker-compose.yml')):
_launch_command(['docker-compose', 'down'], cwd=instance_path)
except subprocess.CalledProcessError:
pass
print('=> Finished tear down of {0} instance.'.format(acme_type))
if acme_type == 'boulder' and os.path.exists(os.path.join(workspace, 'boulder')):
# Boulder docker generates build artifacts owned by root user with 0o744 permissions.
# If we started the acme server from a normal user that has access to the Docker
# daemon, this user will not be able to delete these artifacts from the host.
# We need to do it through a docker.
_launch_command(['docker', 'run', '--rm', '-v', '{0}:/workspace'.format(workspace),
'alpine', 'rm', '-rf', '/workspace/boulder'])
shutil.rmtree(workspace)
return workspace, cleanup
def _prepare_acme_server(workspace, acme_type, acme_xdist):
"""Configure and launch the ACME server, Boulder or Pebble"""
print('=> Starting {0} instance deployment...'.format(acme_type))
instance_path = join(workspace, acme_type)
try:
# Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production.
_launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type),
'--single-branch', '--depth=1', instance_path])
if acme_type == 'boulder':
# Allow Boulder to ignore usual limit rate policies, useful for tests.
os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
join(instance_path, 'test/rate-limit-policies.yml'))
if acme_type == 'pebble':
with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler:
config = yaml.load(file_handler.read())
# Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
# nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
config['services']['pebble'].setdefault('environment', [])\
.extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0'])
# Also disable strict mode for now, since Pebble v2.1.0 added specs in
# strict mode for which Certbot is not compliant for now.
# See https://github.com/certbot/certbot/pull/7175
# TODO: Add back -strict mode once Certbot is compliant with Pebble v2.1.0+
config['services']['pebble']['command'] = config['services']['pebble']['command']\
.replace('-strict', '')
with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler:
file_handler.write(yaml.dump(config))
# Launch the ACME CA server.
_launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
self._launch_process(
[challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""',
'-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""'])
# Wait for the ACME CA server to be up.
print('=> Waiting for {0} instance to respond...'.format(acme_type))
misc.check_until_timeout(acme_xdist['directory_url'])
print('=> Waiting for pebble instance to respond...')
misc.check_until_timeout(self.acme_xdist['directory_url'])
print('=> Finished pebble instance deployment.')
def _prepare_boulder_server(self):
"""Configure and launch the Boulder server"""
print('=> Starting boulder instance deployment...')
instance_path = join(self._workspace, 'boulder')
# Load Boulder from git, that includes a docker-compose.yml ready for production.
process = self._launch_process(['git', 'clone', 'https://github.com/letsencrypt/boulder',
'--single-branch', '--depth=1', instance_path])
process.wait()
# Allow Boulder to ignore usual limit rate policies, useful for tests.
os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
join(instance_path, 'test/rate-limit-policies.yml'))
# Launch the Boulder server
self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
# Wait for the ACME CA server to be up.
print('=> Waiting for boulder instance to respond...')
misc.check_until_timeout(self.acme_xdist['directory_url'])
# Configure challtestsrv to answer any A record request with ip of the docker host.
acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50'
response = requests.post('http://localhost:{0}/set-default-ipv4'
.format(acme_xdist['challtestsrv_port']),
json={'ip': '{0}.1'.format(acme_subnet)})
response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
json={'ip': '10.77.77.1'})
response.raise_for_status()
print('=> Finished {0} instance deployment.'.format(acme_type))
except BaseException:
print('Error while setting up {0} instance.'.format(acme_type))
raise
print('=> Finished boulder instance deployment.')
def _prepare_http_proxy(self):
"""Configure and launch an HTTP proxy"""
print('=> Configuring the HTTP proxy...')
mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port)
for node, port in self.acme_xdist['http_port'].items()}
command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)]
self._launch_process(command)
print('=> Finished configuring the HTTP proxy.')
def _prepare_http_proxy(acme_xdist):
"""Configure and launch an HTTP proxy"""
print('=> Configuring the HTTP proxy...')
mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port)
for node, port in acme_xdist['http_port'].items()}
command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)]
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
print('=> Finished configuring the HTTP proxy.')
return process
def _launch_command(command, cwd=os.getcwd()):
"""Launch silently an OS command, output will be displayed in case of failure"""
try:
subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True)
except subprocess.CalledProcessError as e:
sys.stderr.write(e.output)
raise
def _launch_process(self, command, cwd=os.getcwd(), env=None):
"""Launch silently an subprocess OS command"""
if not env:
env = os.environ
process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env)
self._processes.append(process)
return process
def main():
@@ -199,8 +194,7 @@ def main():
raise ValueError('Invalid server value {0}, should be one of {1}'
.format(server_type, possible_values))
acme_server = setup_acme_server(server_type, [], False)
process = None
acme_server = ACMEServer(server_type, [], http_proxy=False, stdout=True)
try:
with acme_server as acme_xdist:
@@ -208,15 +202,10 @@ def main():
.format(acme_xdist['directory_url']))
print('--> Press CTRL+C to stop the ACME server.')
docker_name = 'pebble_pebble_1' if 'pebble' in server_type else 'boulder_boulder_1'
process = subprocess.Popen(['docker', 'logs', '-f', docker_name])
while True:
time.sleep(3600)
except KeyboardInterrupt:
if process:
process.terminate()
process.wait()
pass
if __name__ == '__main__':

View File

@@ -0,0 +1,52 @@
import json
import platform
import os
import stat
import pkg_resources
import requests
PEBBLE_VERSION = 'v2.1.0'
ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')
def fetch(workspace):
suffix = '{0}-{1}{2}'.format(platform.system().lower(),
platform.machine().lower().replace('x86_64', 'amd64'),
'.exe' if platform.system() == 'Windows' else '')
pebble_path = _fetch_asset('pebble', suffix)
challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix)
pebble_config_path = _build_pebble_config(workspace)
return pebble_path, challtestsrv_path, pebble_config_path
def _fetch_asset(asset, suffix):
asset_path = os.path.join(ASSETS_PATH, '{0}_{1}_{2}'.format(asset, PEBBLE_VERSION, suffix))
if not os.path.exists(asset_path):
asset_url = ('https://github.com/letsencrypt/pebble/releases/download/{0}/{1}_{2}'
.format(PEBBLE_VERSION, asset, suffix))
response = requests.get(asset_url)
response.raise_for_status()
with open(asset_path, 'wb') as file_h:
file_h.write(response.content)
os.chmod(asset_path, os.stat(asset_path).st_mode | stat.S_IEXEC)
return asset_path
def _build_pebble_config(workspace):
config_path = os.path.join(workspace, 'pebble-config.json')
with open(config_path, 'w') as file_h:
file_h.write(json.dumps({
'pebble': {
'listenAddress': '0.0.0.0:14000',
'certificate': os.path.join(ASSETS_PATH, 'cert.pem'),
'privateKey': os.path.join(ASSETS_PATH, 'key.pem'),
'httpPort': 5002,
'tlsPort': 5001,
},
}))
return config_path