diff --git a/.gitignore b/.gitignore index 56ec23fbd..68762da6b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ tests/letstest/venv/ # certbot tests .certbot_test_workspace +**/assets/pebble* +**/assets/challtestsrv* diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py index 34f32f2d7..68bd6287d 100755 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py @@ -15,7 +15,7 @@ SCRIPT_DIRNAME = os.path.dirname(__file__) def main(args=None): if not args: args = sys.argv[1:] - with acme_server.setup_acme_server('pebble', [], False) as acme_xdist: + with acme_server.ACMEServer('pebble', [], False) as acme_xdist: environ = os.environ.copy() environ['SERVER'] = acme_xdist['directory_url'] command = [os.path.join(SCRIPT_DIRNAME, 'apache-conf-test')] diff --git a/certbot-ci/certbot_integration_tests/assets/nginx_cert.pem b/certbot-ci/certbot_integration_tests/assets/cert.pem similarity index 100% rename from certbot-ci/certbot_integration_tests/assets/nginx_cert.pem rename to certbot-ci/certbot_integration_tests/assets/cert.pem diff --git a/certbot-ci/certbot_integration_tests/assets/nginx_key.pem b/certbot-ci/certbot_integration_tests/assets/key.pem similarity index 100% rename from certbot-ci/certbot_integration_tests/assets/nginx_key.pem rename to certbot-ci/certbot_integration_tests/assets/key.pem diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index e84a866b9..d52e4fb58 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -68,17 +68,18 @@ def _setup_primary_node(config): :param config: Configuration of the pytest primary node """ # Check for runtime compatibility: some tools are required to be available in PATH - try: - subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT) - except (subprocess.CalledProcessError, OSError): - raise ValueError('Error: docker is required in PATH to launch the integration tests, ' - 'but is not installed or not available for current user.') + if 'boulder' in config.option.acme_server: + try: + subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker is required in PATH to launch the integration tests on' + 'boulder, but is not installed or not available for current user.') - try: - subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) - except (subprocess.CalledProcessError, OSError): - raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' - 'but is not installed or not available for current user.') + try: + subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') # Parameter numprocesses is added to option by pytest-xdist workers = ['primary'] if not config.option.numprocesses\ @@ -86,7 +87,7 @@ def _setup_primary_node(config): # By calling setup_acme_server we ensure that all necessary acme server instances will be # fully started. This runtime is reflected by the acme_xdist returned. - acme_server = acme_lib.setup_acme_server(config.option.acme_server, workers) + acme_server = acme_lib.ACMEServer(config.option.acme_server, workers) config.add_cleanup(acme_server.stop) print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist)) acme_server.start() diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py index a6305c3cc..18991ae62 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/nginx_config.py @@ -21,9 +21,9 @@ def construct_nginx_config(nginx_root, nginx_webroot, http_port, https_port, oth :rtype: str """ key_path = key_path if key_path \ - else pkg_resources.resource_filename('certbot_integration_tests', 'assets/nginx_key.pem') + else pkg_resources.resource_filename('certbot_integration_tests', 'assets/key.pem') cert_path = cert_path if cert_path \ - else pkg_resources.resource_filename('certbot_integration_tests', 'assets/nginx_cert.pem') + else pkg_resources.resource_filename('certbot_integration_tests', 'assets/cert.pem') return '''\ # This error log will be written regardless of server scope error_log # definitions, so we have to set this here in the main scope. diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index e9226e17c..8e3419f70 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """Module to setup an ACME CA server environment able to run multiple tests in parallel""" from __future__ import print_function +import errno import json import tempfile import time @@ -11,184 +12,178 @@ import sys from os.path import join import requests -import yaml -from certbot_integration_tests.utils import misc, proxy +from certbot_integration_tests.utils import misc, proxy, pebble_artifacts from certbot_integration_tests.utils.constants import * class ACMEServer(object): """ - Handler exposing methods to start and stop the ACME server, and get its configuration - (eg. challenges ports). ACMEServer is also a context manager, and so can be used to - ensure ACME server is started/stopped upon context enter/exit. + ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy + instance, to allow parallel execution of integration tests against the unique http-01 port + expected by the ACME CA server. + Typically all pytest integration tests will be executed in this context. + ACMEServer gives access the acme_xdist parameter, listing the ports and directory url to use + for each pytest node. It exposes also start and stop methods in order to start the stack, and + stop it with proper resources cleanup. + ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped + upon context enter/exit. """ - def __init__(self, acme_xdist, start, server_cleanup): - self._proxy_process = None - self._server_cleanup = server_cleanup - self.acme_xdist = acme_xdist - self.start = start + def __init__(self, acme_server, nodes, http_proxy=True, stdout=False): + """ + Create an ACMEServer instance. + :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) + :param list nodes: list of node names that will be setup by pytest xdist + :param bool http_proxy: if False do not start the HTTP proxy + :param bool stdout: if True stream subprocesses stdout to standard stdout + """ + self._construct_acme_xdist(acme_server, nodes) + + self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' + self._proxy = http_proxy + self._workspace = tempfile.mkdtemp() + self._processes = [] + self._stdout = sys.stdout if stdout else open(os.devnull, 'w') + + def start(self): + """Start the test stack""" + try: + if self._proxy: + self._prepare_http_proxy() + if self._acme_type == 'pebble': + self._prepare_pebble_server() + if self._acme_type == 'boulder': + self._prepare_boulder_server() + except BaseException as e: + self.stop() + raise e def stop(self): - if self._proxy_process: - self._proxy_process.terminate() - self._proxy_process.wait() - self._server_cleanup() + """Stop the test stack, and clean its resources""" + print('=> Tear down the test infrastructure...') + try: + for process in self._processes: + try: + process.terminate() + except OSError as e: + # Process may be not started yet, so no PID and terminate fails. + # Then the process never started, and the situation is acceptable. + if e.errno != errno.ESRCH: + raise + for process in self._processes: + process.wait() + + if os.path.exists(os.path.join(self._workspace, 'boulder')): + # Boulder docker generates build artifacts owned by root with 0o744 permissions. + # If we started the acme server from a normal user that has access to the Docker + # daemon, this user will not be able to delete these artifacts from the host. + # We need to do it through a docker. + process = self._launch_process(['docker', 'run', '--rm', '-v', + '{0}:/workspace'.format(self._workspace), + 'alpine', 'rm', '-rf', '/workspace/boulder']) + process.wait() + finally: + shutil.rmtree(self._workspace) + if self._stdout != sys.stdout: + self._stdout.close() + print('=> Test infrastructure stopped and cleaned up.') def __enter__(self): - self._proxy_process = self.start() + self.start() return self.acme_xdist def __exit__(self, exc_type, exc_val, exc_tb): self.stop() + def _construct_acme_xdist(self, acme_server, nodes): + """Generate and return the acme_xdist dict""" + acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} -def setup_acme_server(acme_server, nodes, proxy=True): - """ - This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel - execution of integration tests against the unique http-01 port expected by the ACME CA server. - Typically all pytest integration tests will be executed in this context. - An ACMEServer instance will be returned, giving access to the ports and directory url to use - for each pytest node, and its start and stop methods are appropriately configured to - respectively start the server, and stop it with proper resources cleanup. - :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) - :param str[] nodes: list of node names that will be setup by pytest xdist - :param bool proxy: set to False to not start the HTTP proxy - :return: a properly configured ACMEServer instance - :rtype: ACMEServer - """ - acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' - acme_xdist = _construct_acme_xdist(acme_server, nodes) - workspace, server_cleanup = _construct_workspace(acme_type) + # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. + if acme_server == 'pebble': + acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL + else: # boulder + acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \ + if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL - def start(): - proxy_process = _prepare_http_proxy(acme_xdist) if proxy else None - _prepare_acme_server(workspace, acme_type, acme_xdist) + acme_xdist['http_port'] = {node: port for (node, port) + in zip(nodes, range(5200, 5200 + len(nodes)))} + acme_xdist['https_port'] = {node: port for (node, port) + in zip(nodes, range(5100, 5100 + len(nodes)))} + acme_xdist['other_port'] = {node: port for (node, port) + in zip(nodes, range(5300, 5300 + len(nodes)))} - return proxy_process + self.acme_xdist = acme_xdist - return ACMEServer(acme_xdist, start, server_cleanup) + def _prepare_pebble_server(self): + """Configure and launch the Pebble server""" + print('=> Starting pebble instance deployment...') + pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(self._workspace) + # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid + # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. + environ = os.environ.copy() + environ['PEBBLE_VA_NOSLEEP'] = '1' + environ['PEBBLE_WFE_NONCEREJECT'] = '0' -def _construct_acme_xdist(acme_server, nodes): - """Generate and return the acme_xdist dict""" - acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} + self._launch_process( + [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'], + env=environ) - # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. - if acme_server == 'pebble': - acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL - else: # boulder - acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \ - if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL - - acme_xdist['http_port'] = {node: port for (node, port) - in zip(nodes, range(5200, 5200 + len(nodes)))} - acme_xdist['https_port'] = {node: port for (node, port) - in zip(nodes, range(5100, 5100 + len(nodes)))} - acme_xdist['other_port'] = {node: port for (node, port) - in zip(nodes, range(5300, 5300 + len(nodes)))} - - return acme_xdist - - -def _construct_workspace(acme_type): - """Create a temporary workspace for integration tests stack""" - workspace = tempfile.mkdtemp() - - def cleanup(): - """Cleanup function to call that will teardown relevant dockers and their configuration.""" - print('=> Tear down the {0} instance...'.format(acme_type)) - instance_path = join(workspace, acme_type) - try: - if os.path.isfile(join(instance_path, 'docker-compose.yml')): - _launch_command(['docker-compose', 'down'], cwd=instance_path) - except subprocess.CalledProcessError: - pass - print('=> Finished tear down of {0} instance.'.format(acme_type)) - - if acme_type == 'boulder' and os.path.exists(os.path.join(workspace, 'boulder')): - # Boulder docker generates build artifacts owned by root user with 0o744 permissions. - # If we started the acme server from a normal user that has access to the Docker - # daemon, this user will not be able to delete these artifacts from the host. - # We need to do it through a docker. - _launch_command(['docker', 'run', '--rm', '-v', '{0}:/workspace'.format(workspace), - 'alpine', 'rm', '-rf', '/workspace/boulder']) - - shutil.rmtree(workspace) - - return workspace, cleanup - - -def _prepare_acme_server(workspace, acme_type, acme_xdist): - """Configure and launch the ACME server, Boulder or Pebble""" - print('=> Starting {0} instance deployment...'.format(acme_type)) - instance_path = join(workspace, acme_type) - try: - # Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production. - _launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type), - '--single-branch', '--depth=1', instance_path]) - if acme_type == 'boulder': - # Allow Boulder to ignore usual limit rate policies, useful for tests. - os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), - join(instance_path, 'test/rate-limit-policies.yml')) - if acme_type == 'pebble': - with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler: - config = yaml.load(file_handler.read()) - - # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid - # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. - config['services']['pebble'].setdefault('environment', [])\ - .extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0']) - - # Also disable strict mode for now, since Pebble v2.1.0 added specs in - # strict mode for which Certbot is not compliant for now. - # See https://github.com/certbot/certbot/pull/7175 - # TODO: Add back -strict mode once Certbot is compliant with Pebble v2.1.0+ - config['services']['pebble']['command'] = config['services']['pebble']['command']\ - .replace('-strict', '') - - with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler: - file_handler.write(yaml.dump(config)) - - # Launch the ACME CA server. - _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + self._launch_process( + [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""', + '-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""']) # Wait for the ACME CA server to be up. - print('=> Waiting for {0} instance to respond...'.format(acme_type)) - misc.check_until_timeout(acme_xdist['directory_url']) + print('=> Waiting for pebble instance to respond...') + misc.check_until_timeout(self.acme_xdist['directory_url']) + + print('=> Finished pebble instance deployment.') + + def _prepare_boulder_server(self): + """Configure and launch the Boulder server""" + print('=> Starting boulder instance deployment...') + instance_path = join(self._workspace, 'boulder') + + # Load Boulder from git, that includes a docker-compose.yml ready for production. + process = self._launch_process(['git', 'clone', 'https://github.com/letsencrypt/boulder', + '--single-branch', '--depth=1', instance_path]) + process.wait() + + # Allow Boulder to ignore usual limit rate policies, useful for tests. + os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), + join(instance_path, 'test/rate-limit-policies.yml')) + + # Launch the Boulder server + self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) + + # Wait for the ACME CA server to be up. + print('=> Waiting for boulder instance to respond...') + misc.check_until_timeout(self.acme_xdist['directory_url']) # Configure challtestsrv to answer any A record request with ip of the docker host. - acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50' - response = requests.post('http://localhost:{0}/set-default-ipv4' - .format(acme_xdist['challtestsrv_port']), - json={'ip': '{0}.1'.format(acme_subnet)}) + response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), + json={'ip': '10.77.77.1'}) response.raise_for_status() - print('=> Finished {0} instance deployment.'.format(acme_type)) - except BaseException: - print('Error while setting up {0} instance.'.format(acme_type)) - raise + print('=> Finished boulder instance deployment.') + def _prepare_http_proxy(self): + """Configure and launch an HTTP proxy""" + print('=> Configuring the HTTP proxy...') + mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port) + for node, port in self.acme_xdist['http_port'].items()} + command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)] + self._launch_process(command) + print('=> Finished configuring the HTTP proxy.') -def _prepare_http_proxy(acme_xdist): - """Configure and launch an HTTP proxy""" - print('=> Configuring the HTTP proxy...') - mapping = {r'.+\.{0}\.wtf'.format(node): 'http://127.0.0.1:{0}'.format(port) - for node, port in acme_xdist['http_port'].items()} - command = [sys.executable, proxy.__file__, str(HTTP_01_PORT), json.dumps(mapping)] - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print('=> Finished configuring the HTTP proxy.') - - return process - - -def _launch_command(command, cwd=os.getcwd()): - """Launch silently an OS command, output will be displayed in case of failure""" - try: - subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True) - except subprocess.CalledProcessError as e: - sys.stderr.write(e.output) - raise + def _launch_process(self, command, cwd=os.getcwd(), env=None): + """Launch silently an subprocess OS command""" + if not env: + env = os.environ + process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) + self._processes.append(process) + return process def main(): @@ -199,8 +194,7 @@ def main(): raise ValueError('Invalid server value {0}, should be one of {1}' .format(server_type, possible_values)) - acme_server = setup_acme_server(server_type, [], False) - process = None + acme_server = ACMEServer(server_type, [], http_proxy=False, stdout=True) try: with acme_server as acme_xdist: @@ -208,15 +202,10 @@ def main(): .format(acme_xdist['directory_url'])) print('--> Press CTRL+C to stop the ACME server.') - docker_name = 'pebble_pebble_1' if 'pebble' in server_type else 'boulder_boulder_1' - process = subprocess.Popen(['docker', 'logs', '-f', docker_name]) - while True: time.sleep(3600) except KeyboardInterrupt: - if process: - process.terminate() - process.wait() + pass if __name__ == '__main__': diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py new file mode 100644 index 000000000..9f154bb0e --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -0,0 +1,52 @@ +import json +import platform +import os +import stat + +import pkg_resources +import requests + +PEBBLE_VERSION = 'v2.1.0' +ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') + + +def fetch(workspace): + suffix = '{0}-{1}{2}'.format(platform.system().lower(), + platform.machine().lower().replace('x86_64', 'amd64'), + '.exe' if platform.system() == 'Windows' else '') + + pebble_path = _fetch_asset('pebble', suffix) + challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix) + pebble_config_path = _build_pebble_config(workspace) + + return pebble_path, challtestsrv_path, pebble_config_path + + +def _fetch_asset(asset, suffix): + asset_path = os.path.join(ASSETS_PATH, '{0}_{1}_{2}'.format(asset, PEBBLE_VERSION, suffix)) + if not os.path.exists(asset_path): + asset_url = ('https://github.com/letsencrypt/pebble/releases/download/{0}/{1}_{2}' + .format(PEBBLE_VERSION, asset, suffix)) + response = requests.get(asset_url) + response.raise_for_status() + with open(asset_path, 'wb') as file_h: + file_h.write(response.content) + os.chmod(asset_path, os.stat(asset_path).st_mode | stat.S_IEXEC) + + return asset_path + + +def _build_pebble_config(workspace): + config_path = os.path.join(workspace, 'pebble-config.json') + with open(config_path, 'w') as file_h: + file_h.write(json.dumps({ + 'pebble': { + 'listenAddress': '0.0.0.0:14000', + 'certificate': os.path.join(ASSETS_PATH, 'cert.pem'), + 'privateKey': os.path.join(ASSETS_PATH, 'key.pem'), + 'httpPort': 5002, + 'tlsPort': 5001, + }, + })) + + return config_path