From 74ce332b5a4a0747cad56afc4cbe72fa519ce31f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 27 Jun 2015 19:38:00 +0000 Subject: [PATCH 1/7] Manual SimpleHTTP integration tests. --- letsencrypt/auth_handler.py | 7 +- letsencrypt/plugins/manual.py | 77 ++++++++++++++++--- letsencrypt/plugins/manual_test.py | 3 +- .../plugins/standalone/authenticator.py | 4 + tests/boulder-integration.sh | 2 + tests/integration/_common.sh | 3 +- 6 files changed, 81 insertions(+), 15 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 2929166c2..3d8275901 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -156,7 +156,7 @@ class AuthHandler(object): active_achalls = [] for achall, resp in itertools.izip(achalls, resps): # Don't send challenges for None and False authenticator responses - if resp: + if resp is not None and resp: self.network.answer_challenge(achall.challb, resp) # TODO: answer_challenge returns challr, with URI, # that can be used in _find_updated_challr @@ -166,6 +166,11 @@ class AuthHandler(object): chall_update[achall.domain].append(achall) else: chall_update[achall.domain] = [achall] + else: # resp is None or not resp + # XXX: make sure that achalls corresponding to None or + # False returned from Authenticator are removed from + # the queue and thus avoid infinite loop + active_achalls.append(achall) return active_achalls diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 700759194..771833d6f 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -1,6 +1,12 @@ """Manual plugin.""" import os +import logging +import shutil +import signal +import subprocess import sys +import tempfile +import time import zope.component import zope.interface @@ -8,10 +14,14 @@ import zope.interface from acme import challenges from acme import jose +from letsencrypt import errors from letsencrypt import interfaces from letsencrypt.plugins import common +logger = logging.getLogger(__name__) + + class ManualAuthenticator(common.Plugin): """Manual Authenticator. @@ -43,8 +53,8 @@ command on the target server (as root): # anything recursively under the cwd HTTP_TEMPLATE = """\ -mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH} -cd /tmp/letsencrypt/public_html +mkdir -p {root}/public_html/{response.URI_ROOT_PATH} +cd {root}/public_html echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path} # run only once per server: python -c "import BaseHTTPServer, SimpleHTTPServer; \\ @@ -55,8 +65,8 @@ s.serve_forever()" """ # https://www.piware.de/2011/01/creating-an-https-server-in-python/ HTTPS_TEMPLATE = """\ -mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH} -cd /tmp/letsencrypt/public_html +mkdir -p {root}/public_html/{response.URI_ROOT_PATH} +cd {root}/public_html echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path} # run only once per server: openssl req -new -newkey rsa:4096 -subj "/" -days 1 -nodes -x509 -keyout ../key.pem -out ../cert.pem @@ -77,6 +87,14 @@ s.serve_forever()" """ super(ManualAuthenticator, self).__init__(*args, **kwargs) self.template = (self.HTTP_TEMPLATE if self.config.no_simple_http_tls else self.HTTPS_TEMPLATE) + self._root = (tempfile.mkdtemp() if self.conf("test-mode") + else "/tmp/letsencrypt") + + @classmethod + def add_parser_arguments(cls, add): + add("test-mode", action="store_true", + help="Test mode. Executes the manual command in subprocess. " + "Requires openssl to be installed unless --no-simple-http-tls.") def prepare(self): # pylint: disable=missing-docstring,no-self-use pass # pragma: no cover @@ -110,17 +128,44 @@ binary for temporary key/certificate generation.""".replace("\n", "") tls=(not self.config.no_simple_http_tls)) assert response.good_path # is encoded os.urandom(18) good? - self._notify_and_wait(self.MESSAGE_TEMPLATE.format( - achall=achall, response=response, uri=response.uri(achall.domain), - ct=response.CONTENT_TYPE, command=self.template.format( - achall=achall, response=response, ct=response.CONTENT_TYPE, - port=(response.port if self.config.simple_http_port is None - else self.config.simple_http_port)))) + command = self.template.format( + root=self._root, achall=achall, response=response, + ct=response.CONTENT_TYPE, port=( + response.port if self.config.simple_http_port is None + else self.config.simple_http_port)) + if self.conf("test-mode"): + logger.debug("Test mode. Executing the manual command: %s", command) + try: + # pylint: disable=attribute-defined-outside-init + self._httpd = subprocess.Popen( + command, + # don't care about setting stdout and stderr, + # we're in test mode anyway + shell=True, + # "preexec_fn" is UNIX specific, but so is "command" + preexec_fn=os.setsid) + except OSError as error: # ValueError should not happen! + logging.debug( + "Couldn't execute manual command", error, exc_info=True) + return False + logger.debug("Manual command running as PID %s.", self._httpd.pid) + # give it some time to bootstrap, before we try to verify + # (cert generation in case of simpleHttpS might take time) + time.sleep(4) # XXX + if self._httpd.poll(): + raise errors.Error("Couldn't execute manual command") + else: + self._notify_and_wait(self.MESSAGE_TEMPLATE.format( + achall=achall, response=response, + uri=response.uri(achall.domain), ct=response.CONTENT_TYPE, + command=command)) if response.simple_verify( achall.challb, achall.domain, self.config.simple_http_port): return response else: + if self.conf("test-mode") and self._httpd.poll(): + return False return None def _notify_and_wait(self, message): # pylint: disable=no-self-use @@ -130,5 +175,13 @@ binary for temporary key/certificate generation.""".replace("\n", "") sys.stdout.write(message) raw_input("Press ENTER to continue") - def cleanup(self, achalls): # pylint: disable=missing-docstring,no-self-use - pass # pragma: no cover + def cleanup(self, achalls): + # pylint: disable=missing-docstring,no-self-use,unused-argument + if self.conf("test-mode"): + if self._httpd.poll() is None: + logger.debug("Terminating manual command process") + os.killpg(self._httpd.pid, signal.SIGTERM) + else: + logger.debug("Manual command process already terminated " + "with %s code", self._httpd.returncode) + shutil.rmtree(self._root) diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index a533bcc75..1360ebf44 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -15,7 +15,8 @@ class ManualAuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.manual import ManualAuthenticator self.config = mock.MagicMock( - no_simple_http_tls=True, simple_http_port=4430) + no_simple_http_tls=True, simple_http_port=4430, + manual_test_mode=False) self.auth = ManualAuthenticator(config=self.config, name="manual") self.achalls = [achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP, domain="foo.com", key=None)] diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index 971f90266..d55d70aa0 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -197,6 +197,10 @@ class StandaloneAuthenticator(common.Plugin): """ signal.signal(signal.SIGINT, self.subproc_signal_handler) self.sock = socket.socket() + # SO_REUSEADDR flag tells the kernel to reuse a local socket + # in TIME_WAIT state, without waiting for its natural timeout + # to expire. + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: self.sock.bind(("0.0.0.0", port)) except socket.error, error: diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 06a1d8aa9..975d030da 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -23,6 +23,8 @@ common() { common --domains le1.wtf auth common --domains le2.wtf run +common -a manual -d le.wtf auth +common -a manual -d le.wtf --no-simple-http-tls auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 0f26d3815..de16a939a 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -10,11 +10,12 @@ store_flags="$store_flags --logs-dir $root/logs" export root store_flags letsencrypt_test () { - # first three flags required, rest is handy defaults letsencrypt \ --server "${SERVER:-http://localhost:4000/acme/new-reg}" \ --no-verify-ssl \ --dvsni-port 5001 \ + --simple-http-port 5001 \ + --manual-test-mode \ $store_flags \ --text \ --agree-eula \ From fe3c3be675f175fbb9dfed9b97f1d556253d0014 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 18 Jul 2015 19:25:30 +0000 Subject: [PATCH 2/7] Travis: run Boulder in AMQP mode (fixes #620). --- .travis.yml | 5 ++++- tests/boulder-start.sh | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7cccd20c9..52ad49506 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: python +services: + - rabbitmq + # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS before_install: - travis_retry sudo ./bootstrap/ubuntu.sh @@ -20,7 +23,7 @@ env: - TOXENV=cover install: "travis_retry pip install tox coveralls" -before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh' +before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh amqp' script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || (source .tox/$TOXENV/bin/activate && ./tests/boulder-integration.sh))' after_success: '[ "$TOXENV" == "cover" ] && coveralls' diff --git a/tests/boulder-start.sh b/tests/boulder-start.sh index 49139ff3c..20f64bcce 100755 --- a/tests/boulder-start.sh +++ b/tests/boulder-start.sh @@ -11,5 +11,10 @@ export GOPATH="${GOPATH:-/tmp/go}" go get -d github.com/letsencrypt/boulder/cmd/boulder cd $GOPATH/src/github.com/letsencrypt/boulder make -j4 # Travis has 2 cores per build instance. -./start.sh & -# Hopefully start.sh bootstraps before integration test is started... +if [ "$1" = "amqp" ]; +then + ./start.py & +else + ./start.sh & +fi +# Hopefully start.py/start.sh bootstraps before integration test is started... From 9e2682a025f32dabe8fe7256141785a698d0c257 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 19 Jul 2015 11:02:17 +0000 Subject: [PATCH 3/7] 100% coverage for manual test mode and related code. --- letsencrypt/auth_handler.py | 12 ++++---- letsencrypt/plugins/manual.py | 11 ++++--- letsencrypt/plugins/manual_test.py | 48 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index e1a964f77..5969dc36f 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -153,22 +153,22 @@ class AuthHandler(object): """ active_achalls = [] for achall, resp in itertools.izip(achalls, resps): + # XXX: make sure that all achalls, including those + # corresponding to None or False returned from + # Authenticator are removed from the queue and thus avoid + # infinite loop + active_achalls.append(achall) + # Don't send challenges for None and False authenticator responses if resp is not None and resp: self.acme.answer_challenge(achall.challb, resp) # TODO: answer_challenge returns challr, with URI, # that can be used in _find_updated_challr # comparisons... - active_achalls.append(achall) if achall.domain in chall_update: chall_update[achall.domain].append(achall) else: chall_update[achall.domain] = [achall] - else: # resp is None or not resp - # XXX: make sure that achalls corresponding to None or - # False returned from Authenticator are removed from - # the queue and thus avoid infinite loop - active_achalls.append(achall) return active_achalls diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 771833d6f..b5ddd2140 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -89,6 +89,7 @@ s.serve_forever()" """ else self.HTTPS_TEMPLATE) self._root = (tempfile.mkdtemp() if self.conf("test-mode") else "/tmp/letsencrypt") + self._httpd = None @classmethod def add_parser_arguments(cls, add): @@ -136,7 +137,6 @@ binary for temporary key/certificate generation.""".replace("\n", "") if self.conf("test-mode"): logger.debug("Test mode. Executing the manual command: %s", command) try: - # pylint: disable=attribute-defined-outside-init self._httpd = subprocess.Popen( command, # don't care about setting stdout and stderr, @@ -146,13 +146,13 @@ binary for temporary key/certificate generation.""".replace("\n", "") preexec_fn=os.setsid) except OSError as error: # ValueError should not happen! logging.debug( - "Couldn't execute manual command", error, exc_info=True) + "Couldn't execute manual command: %s", error, exc_info=True) return False logger.debug("Manual command running as PID %s.", self._httpd.pid) # give it some time to bootstrap, before we try to verify # (cert generation in case of simpleHttpS might take time) time.sleep(4) # XXX - if self._httpd.poll(): + if self._httpd.poll() is not None: raise errors.Error("Couldn't execute manual command") else: self._notify_and_wait(self.MESSAGE_TEMPLATE.format( @@ -164,7 +164,8 @@ binary for temporary key/certificate generation.""".replace("\n", "") achall.challb, achall.domain, self.config.simple_http_port): return response else: - if self.conf("test-mode") and self._httpd.poll(): + if self.conf("test-mode") and self._httpd.poll() is not None: + # simply verify cause command failure... return False return None @@ -178,6 +179,8 @@ binary for temporary key/certificate generation.""".replace("\n", "") def cleanup(self, achalls): # pylint: disable=missing-docstring,no-self-use,unused-argument if self.conf("test-mode"): + assert self._httpd is not None, ( + "cleanup() must be called after perform()") if self._httpd.poll() is None: logger.debug("Terminating manual command process") os.killpg(self._httpd.pid, signal.SIGTERM) diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index e57864b9d..fe95a00f0 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -1,4 +1,5 @@ """Tests for letsencrypt.plugins.manual.""" +import signal import unittest import mock @@ -6,6 +7,8 @@ import mock from acme import challenges from letsencrypt import achallenges +from letsencrypt import errors + from letsencrypt.tests import acme_util @@ -21,6 +24,12 @@ class ManualAuthenticatorTest(unittest.TestCase): self.achalls = [achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP, domain="foo.com", key=None)] + config_test_mode = mock.MagicMock( + no_simple_http_tls=True, simple_http_port=4430, + manual_test_mode=True) + self.auth_test_mode = ManualAuthenticator( + config=config_test_mode, name="manual") + def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), str)) @@ -52,6 +61,45 @@ class ManualAuthenticatorTest(unittest.TestCase): mock_verify.return_value = False self.assertEqual([None], self.auth.perform(self.achalls)) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_command_oserror(self, mock_popen): + mock_popen.side_effect = OSError + self.assertEqual([False], self.auth_test_mode.perform(self.achalls)) + + @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_command_run_failure( + self, mock_popen, unused_mock_sleep): + mock_popen.poll.return_value = 10 + mock_popen.return_value.pid = 1234 + self.assertRaises( + errors.Error, self.auth_test_mode.perform, self.achalls) + + @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) + @mock.patch("acme.challenges.SimpleHTTPResponse.simple_verify", + autospec=True) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_mode(self, mock_popen, mock_verify, mock_sleep): + mock_popen.return_value.poll.side_effect = [None, 10] + mock_popen.return_value.pid = 1234 + mock_verify.return_value = False + self.assertEqual([False], self.auth_test_mode.perform(self.achalls)) + self.assertEqual(1, mock_sleep.call_count) + + def test_cleanup_test_mode_already_terminated(self): + # pylint: disable=protected-access + self.auth_test_mode._httpd = httpd = mock.Mock() + httpd.poll.return_value = 0 + self.auth_test_mode.cleanup(self.achalls) + + @mock.patch("letsencrypt.plugins.manual.os.killpg", autospec=True) + def test_cleanup_test_mode_kills_still_running(self, mock_killpg): + # pylint: disable=protected-access + self.auth_test_mode._httpd = httpd = mock.Mock(pid=1234) + httpd.poll.return_value = None + self.auth_test_mode.cleanup(self.achalls) + mock_killpg.assert_called_once_with(1234, signal.SIGTERM) + if __name__ == "__main__": unittest.main() # pragma: no cover From 82147f1f5e3373ed9e623cbee618339ba8a3f2a6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 19 Jul 2015 11:22:57 +0000 Subject: [PATCH 4/7] Travis: add le.wtf to /etc/hosts. --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7cccd20c9..587ac6ccc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,11 @@ env: - TOXENV=lint - TOXENV=cover +# make sure simplehttp simple verification works (custom /etc/hosts) +addons: + hosts: + - le.wtf + install: "travis_retry pip install tox coveralls" before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh' script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || (source .tox/$TOXENV/bin/activate && ./tests/boulder-integration.sh))' From 7908ea0b868f159aace798c67f6e5987dfef1e78 Mon Sep 17 00:00:00 2001 From: Peter Mosmans Date: Mon, 20 Jul 2015 10:17:58 +0200 Subject: [PATCH 5/7] Fixed typo Changed config-changes in the short help (wrong) to config_changes (right) --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0797f23b4..6692d9c99 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -57,7 +57,7 @@ USAGE = SHORT_USAGE + """Major SUBCOMMANDS are: install Install a previously obtained cert in a server revoke Revoke a previously obtained certificate rollback Rollback server configuration changes made during install - config-changes Show changes made to server config during installation + config_changes Show changes made to server config during installation Choice of server for authentication/installation: From 8e0b271ccd52a7abfb016e64b799a6e7ba46c78c Mon Sep 17 00:00:00 2001 From: Bigfish Date: Thu, 23 Jul 2015 15:47:11 +0800 Subject: [PATCH 6/7] remove sudo before brew (OS X) brew will refuse sudo ref: https://github.com/Homebrew/homebrew/issues/9953 --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index fa0f7b8a9..d22f22076 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -83,7 +83,7 @@ Mac OSX .. code-block:: shell - sudo ./bootstrap/mac.sh + ./bootstrap/mac.sh Fedora From ab097d128b502eaa44a8d77d2ff9d5a59dcb5126 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 24 Jul 2015 06:41:56 +0000 Subject: [PATCH 7/7] Fix logging -> logger typo. --- letsencrypt/plugins/manual.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index b5ddd2140..83f2c0f70 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -145,7 +145,7 @@ binary for temporary key/certificate generation.""".replace("\n", "") # "preexec_fn" is UNIX specific, but so is "command" preexec_fn=os.setsid) except OSError as error: # ValueError should not happen! - logging.debug( + logger.debug( "Couldn't execute manual command: %s", error, exc_info=True) return False logger.debug("Manual command running as PID %s.", self._httpd.pid)