diff --git a/.gitignore b/.gitignore index 54545e883..56ec23fbd 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ tests/letstest/venv/ # docker files .docker + +# certbot tests +.certbot_test_workspace diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index c82793d3d..c4c02a25e 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -1,12 +1,10 @@ """Module to handle the context of integration tests.""" import os import shutil -import subprocess import sys import tempfile -from distutils.version import LooseVersion -from certbot_integration_tests.utils import misc +from certbot_integration_tests.utils import misc, certbot_call class IntegrationTestsContext(object): @@ -30,11 +28,6 @@ class IntegrationTestsContext(object): # is listening on challtestsrv_port. self.challtestsrv_port = acme_xdist['challtestsrv_port'] - # Certbot version does not depend on the test context. But getting its value requires - # calling certbot from a subprocess. Since it will be called a lot of times through - # _common_test_no_force_renew, we cache its value as a member of the fixture context. - self.certbot_version = misc.get_certbot_version() - self.workspace = tempfile.mkdtemp() self.config_dir = os.path.join(self.workspace, 'conf') self.hook_probe = tempfile.mkstemp(dir=self.workspace)[1] @@ -60,71 +53,18 @@ class IntegrationTestsContext(object): """Cleanup the integration test context.""" shutil.rmtree(self.workspace) - def _common_test_no_force_renew(self, args): - """ - Base command to execute certbot in a distributed integration test context, - not renewing certificates by default. - """ - new_environ = os.environ.copy() - new_environ['TMPDIR'] = self.workspace - - additional_args = [] - if self.certbot_version >= LooseVersion('0.30.0'): - additional_args.append('--no-random-sleep-on-renew') - - command = [ - 'certbot', - '--server', self.directory_url, - '--no-verify-ssl', - '--http-01-port', str(self.http_01_port), - '--https-port', str(self.tls_alpn_01_port), - '--manual-public-ip-logging-ok', - '--config-dir', self.config_dir, - '--work-dir', os.path.join(self.workspace, 'work'), - '--logs-dir', os.path.join(self.workspace, 'logs'), - '--non-interactive', - '--no-redirect', - '--agree-tos', - '--register-unsafely-without-email', - '--debug', - '-vv' - ] - - command.extend(args) - command.extend(additional_args) - - print('Invoke command:\n{0}'.format(subprocess.list2cmdline(command))) - return subprocess.check_output(command, universal_newlines=True, - cwd=self.workspace, env=new_environ) - - def _common_test(self, args): - """ - Base command to execute certbot in a distributed integration test context, - renewing certificates by default. - """ - command = ['--renew-by-default'] - command.extend(args) - return self._common_test_no_force_renew(command) - - def certbot_no_force_renew(self, args): + def certbot(self, args, force_renew=True): """ Execute certbot with given args, not renewing certificates by default. :param args: args to pass to certbot + :param force_renew: set to False to not renew by default :return: output of certbot execution """ command = ['--authenticator', 'standalone', '--installer', 'null'] command.extend(args) - return self._common_test_no_force_renew(command) - - def certbot(self, args): - """ - Execute certbot with given args, renewing certificates by default. - :param args: args to pass to certbot - :return: output of certbot execution - """ - command = ['--renew-by-default'] - command.extend(args) - return self.certbot_no_force_renew(command) + return certbot_call.certbot_test( + command, self.directory_url, self.http_01_port, self.tls_alpn_01_port, + self.config_dir, self.workspace, force_renew=force_renew) def get_domain(self, subdomain='le'): """ diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 5ce19dcb8..5428f1a09 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -229,8 +229,8 @@ def test_graceful_renew_it_is_not_time(context): assert_cert_count_for_lineage(context.config_dir, certname, 1) - context.certbot_no_force_renew([ - 'renew', '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)]) + context.certbot(['renew', '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)], + force_renew=False) assert_cert_count_for_lineage(context.config_dir, certname, 1) with pytest.raises(AssertionError): @@ -250,8 +250,8 @@ def test_graceful_renew_it_is_time(context): with open(join(context.config_dir, 'renewal', '{0}.conf'.format(certname)), 'w') as file: file.writelines(lines) - context.certbot_no_force_renew([ - 'renew', '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)]) + context.certbot(['renew', '--deploy-hook', 'echo deploy >> "{0}"'.format(context.hook_probe)], + force_renew=False) assert_cert_count_for_lineage(context.config_dir, certname, 2) assert_hook_execution(context.hook_probe, 'deploy') diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 892c16266..e84a866b9 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -86,7 +86,9 @@ def _setup_primary_node(config): # By calling setup_acme_server we ensure that all necessary acme server instances will be # fully started. This runtime is reflected by the acme_xdist returned. - acme_xdist = acme_lib.setup_acme_server(config.option.acme_server, workers) - print('ACME xdist config:\n{0}'.format(acme_xdist)) + acme_server = acme_lib.setup_acme_server(config.option.acme_server, workers) + config.add_cleanup(acme_server.stop) + print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist)) + acme_server.start() - return acme_xdist + return acme_server.acme_xdist diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py index 3da8a7dd9..61facc6af 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/context.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -2,7 +2,7 @@ import os import subprocess from certbot_integration_tests.certbot_tests import context as certbot_context -from certbot_integration_tests.utils import misc +from certbot_integration_tests.utils import misc, certbot_call from certbot_integration_tests.nginx_tests import nginx_config as config @@ -33,11 +33,14 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """ Main command to execute certbot using the nginx plugin. :param list args: list of arguments to pass to nginx + :param bool force_renew: set to False to not renew by default """ command = ['--authenticator', 'nginx', '--installer', 'nginx', '--nginx-server-root', self.nginx_root] command.extend(args) - return self._common_test(command) + return certbot_call.certbot_test( + command, self.directory_url, self.http_01_port, self.tls_alpn_01_port, + self.config_dir, self.workspace, force_renew=True) def _start_nginx(self, default_server): self.nginx_config = config.construct_nginx_config( diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 176bb220a..1a62ea8d7 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -30,6 +30,7 @@ def context(request): ('nginx6.{0}.wtf,nginx7.{0}.wtf', ['--preferred-challenges', 'http'], {'default_server': False}), ], indirect=['context']) def test_certificate_deployment(certname_pattern, params, context): + # type: (str, list, nginx_context.IntegrationTestsContext) -> None """ Test various scenarios to deploy a certificate to nginx using certbot. """ @@ -45,10 +46,7 @@ def test_certificate_deployment(certname_pattern, params, context): assert server_cert == certbot_cert - command = ['--authenticator', 'nginx', '--installer', 'nginx', - '--nginx-server-root', context.nginx_root, - 'rollback', '--checkpoints', '1'] - context._common_test_no_force_renew(command) + context.certbot_test_nginx(['rollback', '--checkpoints', '1']) with open(context.nginx_config_path, 'r') as file_h: current_nginx_config = file_h.read() diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py old mode 100644 new mode 100755 index 44010d899..ed17d1fd9 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -1,11 +1,11 @@ +#!/usr/bin/env python """Module to setup an ACME CA server environment able to run multiple tests in parallel""" from __future__ import print_function import tempfile -import atexit +import time import os import subprocess import shutil -import stat import sys from os.path import join @@ -14,33 +14,52 @@ import json import yaml from certbot_integration_tests.utils import misc - -# These ports are set implicitly in the docker-compose.yml files of Boulder/Pebble. -CHALLTESTSRV_PORT = 8055 -HTTP_01_PORT = 5002 +from certbot_integration_tests.utils.constants import * -def setup_acme_server(acme_server, nodes): +class ACMEServer(object): + """ + Handler exposing methods to start and stop the ACME server, and get its configuration + (eg. challenges ports). ACMEServer is also a context manager, and so can be used to + ensure ACME server is started/stopped upon context enter/exit. + """ + def __init__(self, acme_xdist, start, stop): + self.acme_xdist = acme_xdist + self.start = start + self.stop = stop + + def __enter__(self): + self.start() + return self.acme_xdist + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + + +def setup_acme_server(acme_server, nodes, proxy=True): """ This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel execution of integration tests against the unique http-01 port expected by the ACME CA server. - Instances are properly closed and cleaned when the Python process exits using atexit. Typically all pytest integration tests will be executed in this context. - This method returns an object describing ports and directory url to use for each pytest node - with the relevant pytest xdist node. + An ACMEServer instance will be returned, giving access to the ports and directory url to use + for each pytest node, and its start and stop methods are appropriately configured to + respectively start the server, and stop it with proper resources cleanup. :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) :param str[] nodes: list of node names that will be setup by pytest xdist - :return: a dict describing the challenge ports that have been setup for the nodes - :rtype: dict + :param bool proxy: set to False to not start the Traefik proxy + :return: a properly configured ACMEServer instance + :rtype: ACMEServer """ acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' acme_xdist = _construct_acme_xdist(acme_server, nodes) - workspace = _construct_workspace(acme_type) + workspace, stop = _construct_workspace(acme_type) - _prepare_traefik_proxy(workspace, acme_xdist) - _prepare_acme_server(workspace, acme_type, acme_xdist) + def start(): + if proxy: + _prepare_traefik_proxy(workspace, acme_xdist) + _prepare_acme_server(workspace, acme_type, acme_xdist) - return acme_xdist + return ACMEServer(acme_xdist, start, stop) def _construct_acme_xdist(acme_server, nodes): @@ -49,10 +68,10 @@ def _construct_acme_xdist(acme_server, nodes): # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. if acme_server == 'pebble': - acme_xdist['directory_url'] = 'https://localhost:14000/dir' + acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL else: # boulder - port = 4001 if acme_server == 'boulder-v2' else 4000 - acme_xdist['directory_url'] = 'http://localhost:{0}/directory'.format(port) + acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL \ + if acme_server == 'boulder-v2' else BOULDER_V1_DIRECTORY_URL acme_xdist['http_port'] = {node: port for (node, port) in zip(nodes, range(5200, 5200 + len(nodes)))} @@ -82,10 +101,7 @@ def _construct_workspace(acme_type): shutil.rmtree(workspace) - # Here with atexit we ensure that clean function is called no matter what. - atexit.register(cleanup) - - return workspace + return workspace, cleanup def _prepare_acme_server(workspace, acme_type, acme_xdist): @@ -136,7 +152,6 @@ def _prepare_traefik_proxy(workspace, acme_xdist): print('=> Starting traefik instance deployment...') instance_path = join(workspace, 'traefik') traefik_subnet = '10.33.33' - traefik_api_port = 8056 try: os.mkdir(instance_path) @@ -159,12 +174,12 @@ networks: config: - subnet: {traefik_subnet}.0/24 '''.format(traefik_subnet=traefik_subnet, - traefik_api_port=traefik_api_port, + traefik_api_port=TRAEFIK_API_PORT, http_01_port=HTTP_01_PORT)) _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) - misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port)) + misc.check_until_timeout('http://localhost:{0}/api'.format(TRAEFIK_API_PORT)) config = { 'backends': { node: { @@ -178,7 +193,7 @@ networks: } for node in acme_xdist['http_port'].keys() } } - response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port), + response = requests.put('http://localhost:{0}/api/providers/rest'.format(TRAEFIK_API_PORT), data=json.dumps(config)) response.raise_for_status() @@ -195,3 +210,35 @@ def _launch_command(command, cwd=os.getcwd()): except subprocess.CalledProcessError as e: sys.stderr.write(e.output) raise + + +def main(): + args = sys.argv[1:] + server_type = args[0] if args else 'pebble' + possible_values = ('pebble', 'boulder-v1', 'boulder-v2') + if server_type not in possible_values: + raise ValueError('Invalid server value {0}, should be one of {1}' + .format(server_type, possible_values)) + + acme_server = setup_acme_server(server_type, [], False) + process = None + + try: + with acme_server as acme_xdist: + print('--> Instance of {0} is running, directory URL is {0}' + .format(acme_xdist['directory_url'])) + print('--> Press CTRL+C to stop the ACME server.') + + docker_name = 'pebble_pebble_1' if 'pebble' in server_type else 'boulder_boulder_1' + process = subprocess.Popen(['docker', 'logs', '-f', docker_name]) + + while True: + time.sleep(3600) + except KeyboardInterrupt: + if process: + process.terminate() + process.wait() + + +if __name__ == '__main__': + main() diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py new file mode 100755 index 000000000..1bff94e75 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +"""Module to call certbot in test mode""" +from __future__ import absolute_import +from distutils.version import LooseVersion +import subprocess +import sys +import os + +from certbot_integration_tests.utils import misc +from certbot_integration_tests.utils.constants import * + + +def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port, + config_dir, workspace, force_renew=True): + """ + Invoke the certbot executable available in PATH in a test context for the given args. + The test context consists in running certbot in debug mode, with various flags suitable + for tests (eg. no ssl check, customizable ACME challenge ports and config directory ...). + This command captures stdout and returns it to the caller. + :param list certbot_args: the arguments to pass to the certbot executable + :param str directory_url: URL of the ACME directory server to use + :param int http_01_port: port for the HTTP-01 challenges + :param int tls_alpn_01_port: port for the TLS-ALPN-01 challenges + :param str config_dir: certbot configuration directory to use + :param str workspace: certbot current directory to use + :param bool force_renew: set False to not force renew existing certificates (default: True) + :return: stdout as string + :rtype: str + """ + command, env = _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_port, + config_dir, workspace, force_renew) + + return subprocess.check_output(command, universal_newlines=True, cwd=workspace, env=env) + + +def _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_port, + config_dir, workspace, force_renew): + new_environ = os.environ.copy() + new_environ['TMPDIR'] = workspace + + additional_args = [] + if misc.get_certbot_version() >= LooseVersion('0.30.0'): + additional_args.append('--no-random-sleep-on-renew') + + if force_renew: + additional_args.append('--renew-by-default') + + command = [ + 'certbot', + '--server', directory_url, + '--no-verify-ssl', + '--http-01-port', str(http_01_port), + '--https-port', str(tls_alpn_01_port), + '--manual-public-ip-logging-ok', + '--config-dir', config_dir, + '--work-dir', os.path.join(workspace, 'work'), + '--logs-dir', os.path.join(workspace, 'logs'), + '--non-interactive', + '--no-redirect', + '--agree-tos', + '--register-unsafely-without-email', + '--debug', + '-vv' + ] + + command.extend(certbot_args) + command.extend(additional_args) + + print('--> Invoke command:\n=====\n{0}\n====='.format(subprocess.list2cmdline(command))) + + return command, new_environ + + +def main(): + args = sys.argv[1:] + + # Default config is pebble + directory_url = os.environ.get('SERVER', PEBBLE_DIRECTORY_URL) + http_01_port = int(os.environ.get('HTTP_01_PORT', HTTP_01_PORT)) + tls_alpn_01_port = int(os.environ.get('TLS_ALPN_01_PORT', TLS_ALPN_01_PORT)) + + # Execution of certbot in a self-contained workspace + workspace = os.environ.get('WORKSPACE', os.path.join(os.getcwd(), '.certbot_test_workspace')) + if not os.path.exists(workspace): + print('--> Creating a workspace for certbot_test: {0}'.format(workspace)) + os.mkdir(workspace) + else: + print('--> Using an existing workspace for certbot_test: {0}'.format(workspace)) + config_dir = os.path.join(workspace, 'conf') + + # Invoke certbot in test mode, without capturing output so users see directly the outcome. + command, env = _prepare_args_env(args, directory_url, http_01_port, tls_alpn_01_port, + config_dir, workspace, True) + subprocess.check_call(command, universal_newlines=True, cwd=workspace, env=env) + + +if __name__ == '__main__': + main() diff --git a/certbot-ci/certbot_integration_tests/utils/constants.py b/certbot-ci/certbot_integration_tests/utils/constants.py new file mode 100644 index 000000000..5cf6fd776 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/constants.py @@ -0,0 +1,8 @@ +"""Some useful constants to use throughout certbot-ci integration tests""" +HTTP_01_PORT = 5002 +TLS_ALPN_01_PORT = 5001 +CHALLTESTSRV_PORT = 8055 +TRAEFIK_API_PORT = 8056 +BOULDER_V1_DIRECTORY_URL = 'http://localhost:4000/directory' +BOULDER_V2_DIRECTORY_URL = 'http://localhost:4001/directory' +PEBBLE_DIRECTORY_URL = 'https://localhost:14000/dir' diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index eecbe2887..2adf137b8 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -44,4 +44,11 @@ setup( packages=find_packages(), include_package_data=True, install_requires=install_requires, + + entry_points={ + 'console_scripts': [ + 'certbot_test=certbot_integration_tests.utils.certbot_call:main', + 'run_acme_server=certbot_integration_tests.utils.acme_server:main', + ], + } ) diff --git a/docs/contributing.rst b/docs/contributing.rst index eed7d1bce..dad886da9 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -17,6 +17,8 @@ its dependencies, Certbot needs to be run on a UNIX-like OS so if you're using Windows, you'll need to set up a (virtual) machine running an OS such as Linux and continue with these instructions on that UNIX-like OS. +.. _local copy: + Running a local copy of the client ---------------------------------- @@ -89,6 +91,17 @@ tests, and be compliant with the :ref:`coding style `. Testing ------- +You can test your code in several ways: + +- running the `automated unit`_ tests, +- running the `automated integration`_ tests +- running an *ad hoc* `manual integration`_ test + +.. _automated unit: + +Running automated unit tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + When you are working in a file ``foo.py``, there should also be a file ``foo_test.py`` either in the same directory as ``foo.py`` or in the ``tests`` subdirectory (if there isn't, make one). While you are working on your code and tests, run @@ -114,16 +127,16 @@ of output can make it hard to find specific failures when they happen. config if your user has sudo permissions, so it should not be run on a production Apache server. -.. _integration: +.. _automated integration: -Integration testing with the Pebble CA -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Running automated integration tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Generally it is sufficient to open a pull request and let Github and Travis run integration tests for you. However, you may want to run them locally before submitting your pull request. You need Docker and docker-compose installed and working. -The tox environment `integration` will setup Pebble, the Let's Encrypt ACME CA server +The tox environment `integration` will setup `Pebble`_, the Let's Encrypt ACME CA server for integration testing, then launch the Certbot integration tests. With a user allowed to access your local Docker daemon, run: @@ -135,6 +148,52 @@ With a user allowed to access your local Docker daemon, run: Tests will be run using pytest. A test report and a code coverage report will be displayed at the end of the integration tests execution. +.. _Pebble: https://github.com/letsencrypt/pebble + +.. _manual integration: + +Running manual integration tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also manually execute Certbot against a local instance of the `Pebble`_ ACME server. +This is useful to verify that the modifications done to the code makes Certbot behave as expected. + +To do so you need: + +- Docker installed, and a user with access to the Docker client, +- an available `local copy`_ of Certbot. + +The virtual environment set up with `python tools/venv.py` contains two commands +that can be used once the virtual environment is activated: + +.. code-block:: shell + + run_acme_server + +- Starts a local instance of Pebble and runs in the foreground printing its logs. +- Press CTRL+C to stop this instance. +- This instance is configured to validate challenges against certbot executed locally. + +.. code-block:: shell + + certbot_tests [ARGS...] + +- Execute certbot with the provided arguments and other arguments useful for testing purposes, + such as: verbose output, full tracebacks in case Certbot crashes, *etc.* +- Execution is preconfigured to interact with the Pebble CA started with ``run_acme_server``. +- Any arguments can be passed as they would be to Certbot (eg. ``certbot_test certonly -d test.example.com``). + +Here is a typical workflow to verify that Certbot successfully issued a certificate +using an HTTP-01 challenge on a machine with Python 3: + +.. code-block:: shell + + python tools/venv3.py + source venv3/bin/activate + run_acme_server & + certbot_test certonly --standalone -d test.example.com + # To stop Pebble, launch `fg` to get back the background job, then press CTRL+C + Code components and layout ==========================