From 7254ea5fb04de0699695abb0bcaf763c0dce0b70 Mon Sep 17 00:00:00 2001 From: Fan Jiang Date: Wed, 13 Jan 2016 14:05:22 -0500 Subject: [PATCH 1/5] enable config_test in configurator prepare --- .../letsencrypt_nginx/configurator.py | 31 ++++++--------- .../tests/configurator_test.py | 15 ++++--- .../letsencrypt_nginx/tests/util.py | 39 ++++++++++--------- letsencrypt/tests/cli_test.py | 3 +- 4 files changed, 42 insertions(+), 46 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 4a5a3ddcd..efa7e08b4 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -5,7 +5,6 @@ import re import shutil import socket import subprocess -import sys import time import OpenSSL @@ -106,11 +105,18 @@ class NginxConfigurator(common.Plugin): # This is called in determine_authenticator and determine_installer def prepare(self): - """Prepare the authenticator/installer.""" + """Prepare the authenticator/installer. + + :raises .errors.NoInstallationError: If Nginx ctl cannot be found + :raises .errors.MisconfigurationError: If Nginx is misconfigured + """ # Verify Nginx is installed if not le_util.exe_exists(self.conf('ctl')): raise errors.NoInstallationError + # Make sure configuration is valid + self.config_test() + self.parser = parser.NginxParser( self.conf('server-root'), self.mod_ssl_conf) @@ -409,26 +415,13 @@ class NginxConfigurator(common.Plugin): def config_test(self): # pylint: disable=no-self-use """Check the configuration of Nginx for errors. - :returns: Success - :rtype: bool + :raises .errors.MisconfigurationError: If config_test fails """ try: - proc = subprocess.Popen( - [self.conf('ctl'), "-c", self.nginx_conf, "-t"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - except (OSError, ValueError): - logger.fatal("Unable to run nginx config test") - sys.exit(1) - - if proc.returncode != 0: - # Enter recovery routine... - logger.error("Config test failed\n%s\n%s", stdout, stderr) - return False - - return True + le_util.run_script([self.conf('ctl'), "-c", self.nginx_conf, "-t"]) + except errors.SubprocessError as err: + raise errors.MisconfigurationError(str(err)) def _verify_setup(self): """Verify the setup to ensure safe operating environment. diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index f9af5183a..4fce33079 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -54,6 +54,7 @@ class NginxConfiguratorTest(util.NginxTest): mock_exe_exists.return_value = True self.config.version = None + self.config.config_test = mock.Mock() self.config.prepare() self.assertEquals((1, 6, 2), self.config.version) @@ -361,12 +362,14 @@ class NginxConfiguratorTest(util.NginxTest): mock_popen.side_effect = OSError("Can't find program") self.assertRaises(errors.MisconfigurationError, self.config.restart) - @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") - def test_config_test(self, mock_popen): - mocked = mock_popen() - mocked.communicate.return_value = ('', '') - mocked.returncode = 0 - self.assertTrue(self.config.config_test()) + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test(self, _): + self.config.config_test() + + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test_bad_process(self, mock_run_script): + mock_run_script.side_effect = errors.SubprocessError + self.assertRaises(errors.MisconfigurationError, self.config.config_test) def test_get_snakeoil_paths(self): # pylint: disable=protected-access diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py index 3d70f7ac7..7a16e3738 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py @@ -49,25 +49,26 @@ def get_nginx_configurator( backups = os.path.join(work_dir, "backups") - with mock.patch("letsencrypt_nginx.configurator.le_util." - "exe_exists") as mock_exe_exists: - mock_exe_exists.return_value = True - - config = configurator.NginxConfigurator( - config=mock.MagicMock( - nginx_server_root=config_path, - le_vhost_ext="-le-ssl.conf", - config_dir=config_dir, - work_dir=work_dir, - backup_dir=backups, - temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), - in_progress_dir=os.path.join(backups, "IN_PROGRESS"), - server="https://acme-server.org:443/new", - tls_sni_01_port=5001, - ), - name="nginx", - version=version) - config.prepare() + with mock.patch("letsencrypt_nginx.configurator.NginxConfigurator." + "config_test"): + with mock.patch("letsencrypt_nginx.configurator.le_util." + "exe_exists") as mock_exe_exists: + mock_exe_exists.return_value = True + config = configurator.NginxConfigurator( + config=mock.MagicMock( + nginx_server_root=config_path, + le_vhost_ext="-le-ssl.conf", + config_dir=config_dir, + work_dir=work_dir, + backup_dir=backups, + temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), + in_progress_dir=os.path.join(backups, "IN_PROGRESS"), + server="https://acme-server.org:443/new", + tls_sni_01_port=5001, + ), + name="nginx", + version=version) + config.prepare() # Provide general config utility. nsconfig = configuration.NamespaceConfig(config.config) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 39c09dede..16ef5c093 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -202,8 +202,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods # (we can only do that if letsencrypt-nginx is actually present) ret, _, _, _ = self._call(args) self.assertTrue("The nginx plugin is not working" in ret) - self.assertTrue("Could not find configuration root" in ret) - self.assertTrue("NoInstallationError" in ret) + self.assertTrue("MisconfigurationError" in ret) args = ["certonly", "--webroot"] ret, _, _, _ = self._call(args) From e87de72662e3ca0f78fc3ef8ddf550f457aa50bc Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 12:13:51 -0800 Subject: [PATCH 2/5] Revert "Fix "global" max_attempt bug (#1719)" --- acme/acme/client.py | 30 ++++++++++-------------------- acme/acme/client_test.py | 5 +---- acme/acme/errors.py | 17 +++++++++-------- acme/acme/errors_test.py | 7 ++++--- 4 files changed, 24 insertions(+), 35 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 478536ecc..49c6bcb21 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,5 +1,4 @@ """ACME client API.""" -import collections import datetime import heapq import logging @@ -335,9 +334,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param authzrs: `list` of `.AuthorizationResource` :param int mintime: Minimum time before next attempt, used if ``Retry-After`` is not present in the response. - :param int max_attempts: Maximum number of attempts (per - authorization) before `PollError` with non-empty ``waiting`` - is raised. + :param int max_attempts: Maximum number of attempts before + `PollError` with non-empty ``waiting`` is raised. :returns: ``(cert, updated_authzrs)`` `tuple` where ``cert`` is the issued certificate (`.messages.CertificateResource`), @@ -351,11 +349,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes was marked by the CA as invalid """ - # pylint: disable=too-many-locals - assert max_attempts > 0 - attempts = collections.defaultdict(int) - exhausted = set() - # priority queue with datetime (based on Retry-After) as key, # and original Authorization Resource as value waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] @@ -363,7 +356,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) - while waiting: + while waiting and max_attempts: + max_attempts -= 1 # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) now = datetime.datetime.now() @@ -377,20 +371,16 @@ class Client(object): # pylint: disable=too-many-instance-attributes updated_authzr, response = self.poll(updated[authzr]) updated[authzr] = updated_authzr - attempts[authzr] += 1 # pylint: disable=no-member if updated_authzr.body.status not in ( messages.STATUS_VALID, messages.STATUS_INVALID): - if attempts[authzr] < max_attempts: - # push back to the priority queue, with updated retry_after - heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) - else: - exhausted.add(authzr) + # push back to the priority queue, with updated retry_after + heapq.heappush(waiting, (self.retry_after( + response, default=mintime), authzr)) - if exhausted or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): - raise errors.PollError(exhausted, updated) + if not max_attempts or any(authzr.body.status == messages.STATUS_INVALID + for authzr in six.itervalues(updated)): + raise errors.PollError(waiting, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) return self.request_issuance(csr, updated_authzrs), updated_authzrs diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 9abc69c7c..449bd695e 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -319,10 +319,7 @@ class ClientTest(unittest.TestCase): ) cert, updated_authzrs = self.client.poll_and_request_issuance( - csr, authzrs, mintime=mintime, - # make sure that max_attempts is per-authorization, rather - # than global - max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) + csr, authzrs, mintime=mintime) self.assertTrue(cert[0] is csr) self.assertTrue(cert[1] is updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 77d47c522..0385667c7 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -56,25 +56,26 @@ class MissingNonce(NonceError): class PollError(ClientError): """Generic error when polling for authorization fails. - This might be caused by either timeout (`exhausted` will be non-empty) + This might be caused by either timeout (`waiting` will be non-empty) or by some authorization being invalid. - :ivar exhausted: Set of `.AuthorizationResource` that didn't finish - within max allowed attempts. + :ivar waiting: Priority queue with `datetime.datatime` (based on + ``Retry-After``) as key, and original `.AuthorizationResource` + as value. :ivar updated: Mapping from original `.AuthorizationResource` to the most recently updated one """ - def __init__(self, exhausted, updated): - self.exhausted = exhausted + def __init__(self, waiting, updated): + self.waiting = waiting self.updated = updated super(PollError, self).__init__() @property def timeout(self): """Was the error caused by timeout?""" - return bool(self.exhausted) + return bool(self.waiting) def __repr__(self): - return '{0}(exhausted={1!r}, updated={2!r})'.format( - self.__class__.__name__, self.exhausted, self.updated) + return '{0}(waiting={1!r}, updated={2!r})'.format( + self.__class__.__name__, self.waiting, self.updated) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 966be8f1e..45b269a0b 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -1,4 +1,5 @@ """Tests for acme.errors.""" +import datetime import unittest import mock @@ -35,9 +36,9 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - exhausted=set([mock.sentinel.AR]), + waiting=[(datetime.datetime(2015, 11, 29), mock.sentinel.AR)], updated={}) - self.invalid = PollError(exhausted=set(), updated={ + self.invalid = PollError(waiting=[], updated={ mock.sentinel.AR: mock.sentinel.AR2}) def test_timeout(self): @@ -45,7 +46,7 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(exhausted=set([]), updated={sentinel.AR: ' + self.assertEqual('PollError(waiting=[], updated={sentinel.AR: ' 'sentinel.AR2})', repr(self.invalid)) From c05fa8934c3f3a56e94ab5d7db18c2aaa78b9e95 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 12:34:19 -0800 Subject: [PATCH 3/5] Revert "Cache Python packages" --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09e580c3c..d9b4cb5ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: python -cache: - directories: - - $HOME/.cache/pip - services: - rabbitmq - mariadb From 5535c0675b9f0602b26c23690ad6e2d246501008 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 12:46:10 -0800 Subject: [PATCH 4/5] Revert "Revert "Fix "global" max_attempt bug (#1719)"" --- acme/acme/client.py | 30 ++++++++++++++++++++---------- acme/acme/client_test.py | 5 ++++- acme/acme/errors.py | 17 ++++++++--------- acme/acme/errors_test.py | 7 +++---- 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 49c6bcb21..478536ecc 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,4 +1,5 @@ """ACME client API.""" +import collections import datetime import heapq import logging @@ -334,8 +335,9 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param authzrs: `list` of `.AuthorizationResource` :param int mintime: Minimum time before next attempt, used if ``Retry-After`` is not present in the response. - :param int max_attempts: Maximum number of attempts before - `PollError` with non-empty ``waiting`` is raised. + :param int max_attempts: Maximum number of attempts (per + authorization) before `PollError` with non-empty ``waiting`` + is raised. :returns: ``(cert, updated_authzrs)`` `tuple` where ``cert`` is the issued certificate (`.messages.CertificateResource`), @@ -349,6 +351,11 @@ class Client(object): # pylint: disable=too-many-instance-attributes was marked by the CA as invalid """ + # pylint: disable=too-many-locals + assert max_attempts > 0 + attempts = collections.defaultdict(int) + exhausted = set() + # priority queue with datetime (based on Retry-After) as key, # and original Authorization Resource as value waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] @@ -356,8 +363,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) - while waiting and max_attempts: - max_attempts -= 1 + while waiting: # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) now = datetime.datetime.now() @@ -371,16 +377,20 @@ class Client(object): # pylint: disable=too-many-instance-attributes updated_authzr, response = self.poll(updated[authzr]) updated[authzr] = updated_authzr + attempts[authzr] += 1 # pylint: disable=no-member if updated_authzr.body.status not in ( messages.STATUS_VALID, messages.STATUS_INVALID): - # push back to the priority queue, with updated retry_after - heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) + if attempts[authzr] < max_attempts: + # push back to the priority queue, with updated retry_after + heapq.heappush(waiting, (self.retry_after( + response, default=mintime), authzr)) + else: + exhausted.add(authzr) - if not max_attempts or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): - raise errors.PollError(waiting, updated) + if exhausted or any(authzr.body.status == messages.STATUS_INVALID + for authzr in six.itervalues(updated)): + raise errors.PollError(exhausted, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) return self.request_issuance(csr, updated_authzrs), updated_authzrs diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 449bd695e..9abc69c7c 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -319,7 +319,10 @@ class ClientTest(unittest.TestCase): ) cert, updated_authzrs = self.client.poll_and_request_issuance( - csr, authzrs, mintime=mintime) + csr, authzrs, mintime=mintime, + # make sure that max_attempts is per-authorization, rather + # than global + max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) self.assertTrue(cert[0] is csr) self.assertTrue(cert[1] is updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 0385667c7..77d47c522 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -56,26 +56,25 @@ class MissingNonce(NonceError): class PollError(ClientError): """Generic error when polling for authorization fails. - This might be caused by either timeout (`waiting` will be non-empty) + This might be caused by either timeout (`exhausted` will be non-empty) or by some authorization being invalid. - :ivar waiting: Priority queue with `datetime.datatime` (based on - ``Retry-After``) as key, and original `.AuthorizationResource` - as value. + :ivar exhausted: Set of `.AuthorizationResource` that didn't finish + within max allowed attempts. :ivar updated: Mapping from original `.AuthorizationResource` to the most recently updated one """ - def __init__(self, waiting, updated): - self.waiting = waiting + def __init__(self, exhausted, updated): + self.exhausted = exhausted self.updated = updated super(PollError, self).__init__() @property def timeout(self): """Was the error caused by timeout?""" - return bool(self.waiting) + return bool(self.exhausted) def __repr__(self): - return '{0}(waiting={1!r}, updated={2!r})'.format( - self.__class__.__name__, self.waiting, self.updated) + return '{0}(exhausted={1!r}, updated={2!r})'.format( + self.__class__.__name__, self.exhausted, self.updated) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 45b269a0b..966be8f1e 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -1,5 +1,4 @@ """Tests for acme.errors.""" -import datetime import unittest import mock @@ -36,9 +35,9 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - waiting=[(datetime.datetime(2015, 11, 29), mock.sentinel.AR)], + exhausted=set([mock.sentinel.AR]), updated={}) - self.invalid = PollError(waiting=[], updated={ + self.invalid = PollError(exhausted=set(), updated={ mock.sentinel.AR: mock.sentinel.AR2}) def test_timeout(self): @@ -46,7 +45,7 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(waiting=[], updated={sentinel.AR: ' + self.assertEqual('PollError(exhausted=set([]), updated={sentinel.AR: ' 'sentinel.AR2})', repr(self.invalid)) From 3a90b4c7c5e2e8656fd111dbf28d8d29344a32b3 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 18 Jan 2016 21:39:25 +0000 Subject: [PATCH 5/5] acme: fix empty set repr py3 compat --- acme/acme/errors_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 966be8f1e..1e5f3d479 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -45,8 +45,8 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(exhausted=set([]), updated={sentinel.AR: ' - 'sentinel.AR2})', repr(self.invalid)) + self.assertEqual('PollError(exhausted=%s, updated={sentinel.AR: ' + 'sentinel.AR2})' % repr(set()), repr(self.invalid)) if __name__ == "__main__":