From 4b06eeae6437b91b8b2e91c0f0490460d87a8ba2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 31 May 2019 18:08:52 -0700 Subject: [PATCH] Update Fedora AMI (#7102) Fixes #6955. This updates the Fedora version used in our test farm tests to Fedora 30. The AMI ID comes from https://alt.fedoraproject.org/cloud/ where it is listed as their standard HVM AMI for the region we use us-east-1 (US East (N. Virginia)). Unfortunately, there were a lot of small changes required for this. The big reason for this is on Fedora, there isn't a Python 2 executable installed. In fact, there's not even an executable named python. It's just python3. Rather than installing another Python in each test, I wrote a script that the test scripts can share to figure out the different paths and names that should be used in their script. (This isn't used in test_sdists.sh because the logic is a little different.) Other changes here worth flagging are: I changed the name of the variable RUN_PYTHON3_TESTS in test_leauto_upgrades.sh to RUN_RHEL6_TESTS. The tests that are run when this variable is set test the upgrade from Python 2 to Python 3 on RHEL 6. I think this new name is much better now that we also have Fedora running Python 3. I made tools/simple_http_server.py work on Python 3. You can see tests passing with these changes at https://travis-ci.com/certbot/certbot/builds/113821476. I also ran test_tests.sh and they passed. * Update to Fedora 30 in test farm tests. Fedora 28 is likely to reach its EOL soon. * Add set_python_envvars.sh. * Fix test_apache2.sh on python3 only distros. * Fix test_leauto_upgrades.sh on python3 systems. * Fix certonly_standalone tests with python3 only * Fix test_sdists.sh on python3 only distros. * Make simple_http_server.py work on Python 3. * add comments --- tests/letstest/apache2_targets.yaml | 4 ++-- tests/letstest/scripts/set_python_envvars.sh | 17 +++++++++++++++++ tests/letstest/scripts/test_apache2.sh | 11 ++++++++--- tests/letstest/scripts/test_leauto_upgrades.sh | 17 +++++++++++------ ...test_letsencrypt_auto_certonly_standalone.sh | 8 +++++++- tests/letstest/scripts/test_sdists.sh | 6 +++++- tests/letstest/targets.yaml | 4 ++-- tools/simple_http_server.py | 9 ++++++--- 8 files changed, 58 insertions(+), 18 deletions(-) create mode 100755 tests/letstest/scripts/set_python_envvars.sh diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 4da6abb15..44faf8027 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -40,8 +40,8 @@ targets: user: admin #----------------------------------------------------------------------------- # Fedora - - ami: ami-5c69df23 - name: fedora28 + - ami: ami-00bbc6858140f19ed + name: fedora30 type: centos virt: hvm user: fedora diff --git a/tests/letstest/scripts/set_python_envvars.sh b/tests/letstest/scripts/set_python_envvars.sh new file mode 100755 index 000000000..668444209 --- /dev/null +++ b/tests/letstest/scripts/set_python_envvars.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# This is a simple script that can be sourced to set Python environment +# variables for use in Certbot's letstest test farm tests. + +# Some distros like Fedora may only have an executable named python3 installed. +if command -v python; then + PYTHON_NAME="python" + VENV_SCRIPT="tools/venv.py" + VENV_PATH="venv" +else + # We could check for "python2" here, however, the addition of "python3" + # only systems is what necessitated this change so checking for "python2" + # isn't necessary. + PYTHON_NAME="python3" + VENV_PATH="venv3" + VENV_SCRIPT="tools/venv3.py" +fi diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index c52578003..007ab720e 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -45,8 +45,13 @@ if [ $? -ne 0 ] ; then exit 1 fi -python tools/venv.py -e acme[dev] -e .[dev,docs] -e certbot-apache -sudo venv/bin/certbot -v --debug --text --agree-dev-preview --agree-tos \ +# This script sets the environment variables PYTHON_NAME, VENV_PATH, and +# VENV_SCRIPT based on the version of Python available on the system. For +# instance, Fedora uses Python 3 and Python 2 is not installed. +. tests/letstest/scripts/set_python_envvars.sh + +"$VENV_SCRIPT" -e acme[dev] -e .[dev,docs] -e certbot-apache +sudo "$VENV_PATH/bin/certbot" -v --debug --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL if [ $? -ne 0 ] ; then @@ -55,7 +60,7 @@ fi if [ "$OS_TYPE" = "ubuntu" ] ; then export SERVER="$BOULDER_URL" - venv/bin/tox -e apacheconftest + "$VENV_PATH/bin/tox" -e apacheconftest else echo Not running hackish apache tests on $OS_TYPE fi diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 8cc1748ed..49606b49c 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -21,7 +21,7 @@ if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | se # 0.20.0 is the latest version of letsencrypt-auto that doesn't install # Python 3 on RHEL 6. INITIAL_VERSION="0.20.0" - RUN_PYTHON3_TESTS=1 + RUN_RHEL6_TESTS=1 else # 0.33.x is the oldest version of letsencrypt-auto that works on Fedora 29+. INITIAL_VERSION="0.33.1" @@ -32,14 +32,19 @@ if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | exit 1 fi +# This script sets the environment variables PYTHON_NAME, VENV_PATH, and +# VENV_SCRIPT based on the version of Python available on the system. For +# instance, Fedora uses Python 3 and Python 2 is not installed. +. tests/letstest/scripts/set_python_envvars.sh + # Now that python and openssl have been installed, we can set up a fake server # to provide a new version of letsencrypt-auto. First, we start the server and # directory to be served. MY_TEMP_DIR=$(mktemp -d) PORT_FILE="$MY_TEMP_DIR/port" -SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py) +SERVER_PATH=$("$PYTHON_NAME" tools/readlink.py tools/simple_http_server.py) cd "$MY_TEMP_DIR" -"$SERVER_PATH" 0 > $PORT_FILE & +"$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE & SERVER_PID=$! trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT cd ~- @@ -72,7 +77,7 @@ iQIDAQAB -----END PUBLIC KEY----- " -if [ "$RUN_PYTHON3_TESTS" = 1 ]; then +if [ "$RUN_RHEL6_TESTS" = 1 ]; then if command -v python3; then echo "Didn't expect Python 3 to be installed!" exit 1 @@ -110,7 +115,7 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then exit 1 fi -if [ "$RUN_PYTHON3_TESTS" = 1 ]; then +if [ "$RUN_RHEL6_TESTS" = 1 ]; then if ! command -v python3; then echo "Python3 wasn't properly installed" exit 1 @@ -120,7 +125,7 @@ if [ "$RUN_PYTHON3_TESTS" = 1 ]; then exit 1 fi - if [ "$(tools/readlink.py $OLD_VENV_PATH)" != "/opt/eff.org/certbot/venv" ]; then + if [ "$("$PYTHON_NAME" tools/readlink.py $OLD_VENV_PATH)" != "/opt/eff.org/certbot/venv" ]; then echo symlink from old venv path not properly created! exit 1 fi diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 0973bbc03..eb63b9ca7 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -18,6 +18,11 @@ export PATH="$LE_AUTO_DIR:$PATH" letsencrypt-auto --os-packages-only --debug --version +# This script sets the environment variables PYTHON_NAME, VENV_PATH, and +# VENV_SCRIPT based on the version of Python available on the system. For +# instance, Fedora uses Python 3 and Python 2 is not installed. +. tests/letstest/scripts/set_python_envvars.sh + # Create a venv-like layout at the old virtual environment path to test that a # symlink is properly created when letsencrypt-auto runs. HOME=${HOME:-~root} @@ -32,7 +37,8 @@ letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then +LINK_PATH=$("$PYTHON_NAME" tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt) +if [ "$LINK_PATH" != "/opt/eff.org/certbot/venv" ]; then echo symlink from old venv path not properly created! exit 1 fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index f407d5d2c..e48e95848 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -6,16 +6,20 @@ cd letsencrypt PLUGINS="certbot-apache certbot-nginx" PYTHON_MAJOR_VERSION=$(/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1) TEMP_DIR=$(mktemp -d) -VERSION=$(letsencrypt-auto-source/version.py) if [ "$PYTHON_MAJOR_VERSION" = "3" ]; then + # Some distros like Fedora may only have an executable named python3 installed. + PYTHON_NAME="python3" VENV_PATH="venv3" VENV_SCRIPT="tools/venv3.py" else + PYTHON_NAME="python" VENV_SCRIPT="tools/venv.py" VENV_PATH="venv" fi +VERSION=$("$PYTHON_NAME" letsencrypt-auto-source/version.py) + # setup venv "$VENV_SCRIPT" --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . "$VENV_PATH/bin/activate" diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 340fe6bf8..1ca605b5d 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -49,8 +49,8 @@ targets: type: centos virt: hvm user: ec2-user - - ami: ami-5c69df23 - name: fedora28 + - ami: ami-00bbc6858140f19ed + name: fedora30 type: centos virt: hvm user: fedora diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py index 14ac9a3d3..233aa6bd3 100755 --- a/tools/simple_http_server.py +++ b/tools/simple_http_server.py @@ -1,8 +1,11 @@ #!/usr/bin/env python -"""A version of Python 2.x's SimpleHTTPServer that flushes its output.""" -from BaseHTTPServer import HTTPServer -from SimpleHTTPServer import SimpleHTTPRequestHandler +"""A version of Python's SimpleHTTPServer that flushes its output.""" import sys +try: + from http.server import HTTPServer, SimpleHTTPRequestHandler +except ImportError: + from BaseHTTPServer import HTTPServer + from SimpleHTTPServer import SimpleHTTPRequestHandler def serve_forever(port=0): """Spins up an HTTP server on all interfaces and the given port.