diff --git a/.travis.yml b/.travis.yml index 7cccd20c9..9076c52f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: python +services: + - rabbitmq + # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS before_install: - travis_retry sudo ./bootstrap/ubuntu.sh @@ -19,8 +22,13 @@ env: - TOXENV=lint - TOXENV=cover +# make sure simplehttp simple verification works (custom /etc/hosts) +addons: + hosts: + - le.wtf + install: "travis_retry pip install tox coveralls" -before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh' +before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh amqp' script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || (source .tox/$TOXENV/bin/activate && ./tests/boulder-integration.sh))' after_success: '[ "$TOXENV" == "cover" ] && coveralls' diff --git a/docs/using.rst b/docs/using.rst index fa0f7b8a9..d22f22076 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -83,7 +83,7 @@ Mac OSX .. code-block:: shell - sudo ./bootstrap/mac.sh + ./bootstrap/mac.sh Fedora diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 0cf1da7e4..709e8b7b8 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -153,13 +153,18 @@ class AuthHandler(object): """ active_achalls = [] for achall, resp in itertools.izip(achalls, resps): + # XXX: make sure that all achalls, including those + # corresponding to None or False returned from + # Authenticator are removed from the queue and thus avoid + # infinite loop + active_achalls.append(achall) + # Don't send challenges for None and False authenticator responses - if resp: + if resp is not None and resp: self.acme.answer_challenge(achall.challb, resp) # TODO: answer_challenge returns challr, with URI, # that can be used in _find_updated_challr # comparisons... - active_achalls.append(achall) if achall.domain in chall_update: chall_update[achall.domain].append(achall) else: diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0797f23b4..6692d9c99 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -57,7 +57,7 @@ USAGE = SHORT_USAGE + """Major SUBCOMMANDS are: install Install a previously obtained cert in a server revoke Revoke a previously obtained certificate rollback Rollback server configuration changes made during install - config-changes Show changes made to server config during installation + config_changes Show changes made to server config during installation Choice of server for authentication/installation: diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 700759194..83f2c0f70 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -1,6 +1,12 @@ """Manual plugin.""" import os +import logging +import shutil +import signal +import subprocess import sys +import tempfile +import time import zope.component import zope.interface @@ -8,10 +14,14 @@ import zope.interface from acme import challenges from acme import jose +from letsencrypt import errors from letsencrypt import interfaces from letsencrypt.plugins import common +logger = logging.getLogger(__name__) + + class ManualAuthenticator(common.Plugin): """Manual Authenticator. @@ -43,8 +53,8 @@ command on the target server (as root): # anything recursively under the cwd HTTP_TEMPLATE = """\ -mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH} -cd /tmp/letsencrypt/public_html +mkdir -p {root}/public_html/{response.URI_ROOT_PATH} +cd {root}/public_html echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path} # run only once per server: python -c "import BaseHTTPServer, SimpleHTTPServer; \\ @@ -55,8 +65,8 @@ s.serve_forever()" """ # https://www.piware.de/2011/01/creating-an-https-server-in-python/ HTTPS_TEMPLATE = """\ -mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH} -cd /tmp/letsencrypt/public_html +mkdir -p {root}/public_html/{response.URI_ROOT_PATH} +cd {root}/public_html echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path} # run only once per server: openssl req -new -newkey rsa:4096 -subj "/" -days 1 -nodes -x509 -keyout ../key.pem -out ../cert.pem @@ -77,6 +87,15 @@ s.serve_forever()" """ super(ManualAuthenticator, self).__init__(*args, **kwargs) self.template = (self.HTTP_TEMPLATE if self.config.no_simple_http_tls else self.HTTPS_TEMPLATE) + self._root = (tempfile.mkdtemp() if self.conf("test-mode") + else "/tmp/letsencrypt") + self._httpd = None + + @classmethod + def add_parser_arguments(cls, add): + add("test-mode", action="store_true", + help="Test mode. Executes the manual command in subprocess. " + "Requires openssl to be installed unless --no-simple-http-tls.") def prepare(self): # pylint: disable=missing-docstring,no-self-use pass # pragma: no cover @@ -110,17 +129,44 @@ binary for temporary key/certificate generation.""".replace("\n", "") tls=(not self.config.no_simple_http_tls)) assert response.good_path # is encoded os.urandom(18) good? - self._notify_and_wait(self.MESSAGE_TEMPLATE.format( - achall=achall, response=response, uri=response.uri(achall.domain), - ct=response.CONTENT_TYPE, command=self.template.format( - achall=achall, response=response, ct=response.CONTENT_TYPE, - port=(response.port if self.config.simple_http_port is None - else self.config.simple_http_port)))) + command = self.template.format( + root=self._root, achall=achall, response=response, + ct=response.CONTENT_TYPE, port=( + response.port if self.config.simple_http_port is None + else self.config.simple_http_port)) + if self.conf("test-mode"): + logger.debug("Test mode. Executing the manual command: %s", command) + try: + self._httpd = subprocess.Popen( + command, + # don't care about setting stdout and stderr, + # we're in test mode anyway + shell=True, + # "preexec_fn" is UNIX specific, but so is "command" + preexec_fn=os.setsid) + except OSError as error: # ValueError should not happen! + logger.debug( + "Couldn't execute manual command: %s", error, exc_info=True) + return False + logger.debug("Manual command running as PID %s.", self._httpd.pid) + # give it some time to bootstrap, before we try to verify + # (cert generation in case of simpleHttpS might take time) + time.sleep(4) # XXX + if self._httpd.poll() is not None: + raise errors.Error("Couldn't execute manual command") + else: + self._notify_and_wait(self.MESSAGE_TEMPLATE.format( + achall=achall, response=response, + uri=response.uri(achall.domain), ct=response.CONTENT_TYPE, + command=command)) if response.simple_verify( achall.challb, achall.domain, self.config.simple_http_port): return response else: + if self.conf("test-mode") and self._httpd.poll() is not None: + # simply verify cause command failure... + return False return None def _notify_and_wait(self, message): # pylint: disable=no-self-use @@ -130,5 +176,15 @@ binary for temporary key/certificate generation.""".replace("\n", "") sys.stdout.write(message) raw_input("Press ENTER to continue") - def cleanup(self, achalls): # pylint: disable=missing-docstring,no-self-use - pass # pragma: no cover + def cleanup(self, achalls): + # pylint: disable=missing-docstring,no-self-use,unused-argument + if self.conf("test-mode"): + assert self._httpd is not None, ( + "cleanup() must be called after perform()") + if self._httpd.poll() is None: + logger.debug("Terminating manual command process") + os.killpg(self._httpd.pid, signal.SIGTERM) + else: + logger.debug("Manual command process already terminated " + "with %s code", self._httpd.returncode) + shutil.rmtree(self._root) diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index 9daba668a..fe95a00f0 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -1,4 +1,5 @@ """Tests for letsencrypt.plugins.manual.""" +import signal import unittest import mock @@ -6,6 +7,8 @@ import mock from acme import challenges from letsencrypt import achallenges +from letsencrypt import errors + from letsencrypt.tests import acme_util @@ -15,11 +18,18 @@ class ManualAuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.manual import ManualAuthenticator self.config = mock.MagicMock( - no_simple_http_tls=True, simple_http_port=4430) + no_simple_http_tls=True, simple_http_port=4430, + manual_test_mode=False) self.auth = ManualAuthenticator(config=self.config, name="manual") self.achalls = [achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP, domain="foo.com", key=None)] + config_test_mode = mock.MagicMock( + no_simple_http_tls=True, simple_http_port=4430, + manual_test_mode=True) + self.auth_test_mode = ManualAuthenticator( + config=config_test_mode, name="manual") + def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), str)) @@ -51,6 +61,45 @@ class ManualAuthenticatorTest(unittest.TestCase): mock_verify.return_value = False self.assertEqual([None], self.auth.perform(self.achalls)) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_command_oserror(self, mock_popen): + mock_popen.side_effect = OSError + self.assertEqual([False], self.auth_test_mode.perform(self.achalls)) + + @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_command_run_failure( + self, mock_popen, unused_mock_sleep): + mock_popen.poll.return_value = 10 + mock_popen.return_value.pid = 1234 + self.assertRaises( + errors.Error, self.auth_test_mode.perform, self.achalls) + + @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) + @mock.patch("acme.challenges.SimpleHTTPResponse.simple_verify", + autospec=True) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_mode(self, mock_popen, mock_verify, mock_sleep): + mock_popen.return_value.poll.side_effect = [None, 10] + mock_popen.return_value.pid = 1234 + mock_verify.return_value = False + self.assertEqual([False], self.auth_test_mode.perform(self.achalls)) + self.assertEqual(1, mock_sleep.call_count) + + def test_cleanup_test_mode_already_terminated(self): + # pylint: disable=protected-access + self.auth_test_mode._httpd = httpd = mock.Mock() + httpd.poll.return_value = 0 + self.auth_test_mode.cleanup(self.achalls) + + @mock.patch("letsencrypt.plugins.manual.os.killpg", autospec=True) + def test_cleanup_test_mode_kills_still_running(self, mock_killpg): + # pylint: disable=protected-access + self.auth_test_mode._httpd = httpd = mock.Mock(pid=1234) + httpd.poll.return_value = None + self.auth_test_mode.cleanup(self.achalls) + mock_killpg.assert_called_once_with(1234, signal.SIGTERM) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index f95a1a49e..968063781 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -201,6 +201,10 @@ class StandaloneAuthenticator(common.Plugin): """ signal.signal(signal.SIGINT, self.subproc_signal_handler) self.sock = socket.socket() + # SO_REUSEADDR flag tells the kernel to reuse a local socket + # in TIME_WAIT state, without waiting for its natural timeout + # to expire. + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: self.sock.bind(("0.0.0.0", port)) except socket.error, error: diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 99829e661..23bfcf3ca 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -23,6 +23,8 @@ common() { common --domains le1.wtf auth common --domains le2.wtf run +common -a manual -d le.wtf auth +common -a manual -d le.wtf --no-simple-http-tls auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf diff --git a/tests/boulder-start.sh b/tests/boulder-start.sh index 49139ff3c..20f64bcce 100755 --- a/tests/boulder-start.sh +++ b/tests/boulder-start.sh @@ -11,5 +11,10 @@ export GOPATH="${GOPATH:-/tmp/go}" go get -d github.com/letsencrypt/boulder/cmd/boulder cd $GOPATH/src/github.com/letsencrypt/boulder make -j4 # Travis has 2 cores per build instance. -./start.sh & -# Hopefully start.sh bootstraps before integration test is started... +if [ "$1" = "amqp" ]; +then + ./start.py & +else + ./start.sh & +fi +# Hopefully start.py/start.sh bootstraps before integration test is started... diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 60e7d60ac..be4e75098 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -10,11 +10,12 @@ store_flags="$store_flags --logs-dir $root/logs" export root store_flags letsencrypt_test () { - # first three flags required, rest is handy defaults letsencrypt \ --server "${SERVER:-http://localhost:4000/acme/new-reg}" \ --no-verify-ssl \ --dvsni-port 5001 \ + --simple-http-port 5001 \ + --manual-test-mode \ $store_flags \ --text \ --agree-eula \