diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
index cb659786e..a83cbd826 100755
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -15,19 +15,56 @@ if ! command -v git ; then
         exit 1
     fi
 fi
-BRANCH=`git rev-parse --abbrev-ref HEAD`
 # 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's
 # the first version that pins package versions, properly supports
 # --no-self-upgrade, and works with newer versions of pip.
-git checkout -f v0.5.0
+git checkout -f v0.5.0 letsencrypt-auto
 if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then
     echo initial installation appeared to fail
     exit 1
 fi
 
-git checkout -f "$BRANCH"
-EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2)
-if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then
+# Now that python and openssl have been installed, we can set up a fake server
+# to provide a new version of letsencrypt-auto. First, we start the server and
+# directory to be served.
+MY_TEMP_DIR=$(mktemp -d)
+PORT_FILE="$MY_TEMP_DIR/port"
+SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py)
+cd "$MY_TEMP_DIR"
+"$SERVER_PATH" 0 > $PORT_FILE &
+SERVER_PID=$!
+trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT
+cd ~-
+
+# Then, we set up the files to be served.
+FAKE_VERSION_NUM="99.99.99"
+echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json"
+LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM"
+NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto"
+mkdir "$LE_AUTO_SOURCE_DIR"
+cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto"
+SIGNING_KEY="letsencrypt-auto-source/tests/signing.key"
+openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH"
+
+# Next, we wait for the server to start and get the port number.
+sleep 5s
+SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE")
+
+# Finally, we set the necessary certbot-auto environment variables.
+export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/"
+export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json"
+export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg
+tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G
+hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT
+uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl
+LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47
+Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68
+iQIDAQAB
+-----END PUBLIC KEY-----
+"
+
+if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
     echo upgrade appeared to fail
     exit 1
 fi
diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py
new file mode 100755
index 000000000..26bf231b7
--- /dev/null
+++ b/tools/simple_http_server.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""A version of Python 2.x's SimpleHTTPServer that flushes its output."""
+from BaseHTTPServer import HTTPServer
+from SimpleHTTPServer import SimpleHTTPRequestHandler
+import sys
+
+def serve_forever(port=0):
+    """Spins up an HTTP server on all interfaces and the given port.
+
+    A message is printed to stdout specifying the address and port being used
+    by the server.
+
+    :param int port: port number to use.
+
+    """
+    server = HTTPServer(('', port), SimpleHTTPRequestHandler)
+    print 'Serving HTTP on {0} port {1} ...'.format(*server.server_address)
+    sys.stdout.flush()
+    server.serve_forever()
+
+
+if __name__ == '__main__':
+    kwargs = {}
+    if len(sys.argv) > 1:
+        kwargs['port'] = int(sys.argv[1])
+    serve_forever(**kwargs)