diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py
index d13f35f99..e16fc152f 100644
--- a/letsencrypt/plugins/manual.py
+++ b/letsencrypt/plugins/manual.py
@@ -4,6 +4,7 @@ import logging
 import pipes
 import shutil
 import signal
+import socket
 import subprocess
 import sys
 import tempfile
@@ -122,6 +123,20 @@ binary for temporary key/certificate generation.""".replace("\n", "")
             responses.append(self._perform_single(achall))
         return responses
 
+    @classmethod
+    def _test_mode_busy_wait(cls, port):
+        while True:
+            time.sleep(1)
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            try:
+                sock.connect(("localhost", port))
+            except socket.error:  # pragma: no cover
+                pass
+            else:
+                break
+            finally:
+                sock.close()
+
     def _perform_single(self, achall):
         # same path for each challenge response would be easier for
         # users, but will not work if multiple domains point at the
@@ -129,13 +144,13 @@ binary for temporary key/certificate generation.""".replace("\n", "")
         response, validation = achall.gen_response_and_validation(
             tls=(not self.config.no_simple_http_tls))
 
+        port = (response.port if self.config.simple_http_port is None
+                else int(self.config.simple_http_port))
         command = self.template.format(
             root=self._root, achall=achall, response=response,
             validation=pipes.quote(validation.json_dumps()),
             encoded_token=achall.chall.encode("token"),
-            ct=response.CONTENT_TYPE, port=(
-                response.port if self.config.simple_http_port is None
-                else self.config.simple_http_port))
+            ct=response.CONTENT_TYPE, port=port)
         if self.conf("test-mode"):
             logger.debug("Test mode. Executing the manual command: %s", command)
             try:
@@ -153,7 +168,7 @@ binary for temporary key/certificate generation.""".replace("\n", "")
             logger.debug("Manual command running as PID %s.", self._httpd.pid)
             # give it some time to bootstrap, before we try to verify
             # (cert generation in case of simpleHttpS might take time)
-            time.sleep(4)  # XXX
+            self._test_mode_busy_wait(port)
             if self._httpd.poll() is not None:
                 raise errors.Error("Couldn't execute manual command")
         else:
diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py
index caf7fb3c4..c1ca9f70e 100644
--- a/letsencrypt/plugins/manual_test.py
+++ b/letsencrypt/plugins/manual_test.py
@@ -71,25 +71,29 @@ class ManualAuthenticatorTest(unittest.TestCase):
         mock_popen.side_effect = OSError
         self.assertEqual([False], self.auth_test_mode.perform(self.achalls))
 
+    @mock.patch("letsencrypt.plugins.manual.socket.socket", autospec=True)
     @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True)
     @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True)
     def test_perform_test_command_run_failure(
-            self, mock_popen, unused_mock_sleep):
+            self, mock_popen, unused_mock_sleep, unused_mock_socket):
         mock_popen.poll.return_value = 10
         mock_popen.return_value.pid = 1234
         self.assertRaises(
             errors.Error, self.auth_test_mode.perform, self.achalls)
 
+    @mock.patch("letsencrypt.plugins.manual.socket.socket", autospec=True)
     @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True)
     @mock.patch("acme.challenges.SimpleHTTPResponse.simple_verify",
                 autospec=True)
     @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True)
-    def test_perform_test_mode(self, mock_popen, mock_verify, mock_sleep):
+    def test_perform_test_mode(self, mock_popen, mock_verify, mock_sleep,
+                               mock_socket):
         mock_popen.return_value.poll.side_effect = [None, 10]
         mock_popen.return_value.pid = 1234
         mock_verify.return_value = False
         self.assertEqual([False], self.auth_test_mode.perform(self.achalls))
         self.assertEqual(1, mock_sleep.call_count)
+        self.assertEqual(1, mock_socket.call_count)
 
     def test_cleanup_test_mode_already_terminated(self):
         # pylint: disable=protected-access