diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py
index ebc3bc0da..4b3b79e36 100644
--- a/letsencrypt/constants.py
+++ b/letsencrypt/constants.py
@@ -84,3 +84,7 @@ IConfig.work_dir)."""
 NETSTAT = "/bin/netstat"
 """Location of netstat binary for checking whether a listener is already
 running on the specified port (Linux-specific)."""
+
+BOULDER_TEST_MODE_CHALLENGE_PORT = 5001
+"""Port that Boulder will connect on for validations in test mode."""
+
diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py
index db4b629d2..1eb565289 100644
--- a/letsencrypt/crypto_util.py
+++ b/letsencrypt/crypto_util.py
@@ -72,7 +72,7 @@ def init_save_csr(privkey, names, cert_dir, csrname="csr-letsencrypt.pem"):
     csr_pem, csr_der = make_csr(privkey.pem, names)
 
     # Save CSR
-    le_util.make_or_verify_dir(cert_dir, 0o755)
+    le_util.make_or_verify_dir(cert_dir, 0o755, os.geteuid())
     csr_f, csr_filename = le_util.unique_file(
         os.path.join(cert_dir, csrname), 0o644)
     csr_f.write(csr_pem)
diff --git a/letsencrypt/le_util.py b/letsencrypt/le_util.py
index 9a3dd77a5..ec0be7514 100644
--- a/letsencrypt/le_util.py
+++ b/letsencrypt/le_util.py
@@ -70,8 +70,10 @@ def unique_file(path, mode=0o777):
         try:
             file_d = os.open(fname, os.O_CREAT | os.O_EXCL | os.O_RDWR, mode)
             return os.fdopen(file_d, "w"), fname
-        except OSError:
-            pass
+        except OSError as exception:
+            # "File exists," is okay, try a different name.
+            if exception.errno != errno.EEXIST:
+                raise
         count += 1
 
 
diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py
index 7d5c3ea6e..3947c1e3e 100644
--- a/letsencrypt/plugins/standalone/authenticator.py
+++ b/letsencrypt/plugins/standalone/authenticator.py
@@ -15,6 +15,7 @@ import zope.interface
 from acme import challenges
 
 from letsencrypt import achallenges
+from letsencrypt import constants
 from letsencrypt import interfaces
 
 from letsencrypt.plugins import common
@@ -378,7 +379,10 @@ class StandaloneAuthenticator(common.Plugin):
                 results_if_failure.append(False)
         if not self.tasks:
             raise ValueError("nothing for .perform() to do")
-        if self.already_listening(challenges.DVSNI.PORT):
+        port = challenges.DVSNI.PORT
+        if self.config and self.config.test_mode:
+            port = constants.BOULDER_TEST_MODE_CHALLENGE_PORT
+        if self.already_listening(port):
             # If we know a process is already listening on this port,
             # tell the user, and don't even attempt to bind it.  (This
             # test is Linux-specific and won't indicate that the port
@@ -386,7 +390,7 @@ class StandaloneAuthenticator(common.Plugin):
             return results_if_failure
         # Try to do the authentication; note that this creates
         # the listener subprocess via os.fork()
-        if self.start_listener(challenges.DVSNI.PORT, key):
+        if self.start_listener(port, key):
             return results_if_success
         else:
             # TODO: This should probably raise a DVAuthError exception