Switch oldest tests to Python 3 (#8590)

Fixes https://github.com/certbot/certbot/issues/8580.

With this PR, it should now be possible to run the oldest tests natively on Linux, at least when using an older version of Python 3, which hasn't been possible in a long time. Unfortunately, this isn't possible on macOS which I opened https://github.com/certbot/certbot/issues/8589 to track.

You can see the full test suite running with these changes at https://dev.azure.com/certbot/certbot/_build/results?buildId=3283&view=results.

I took the version numbers for the packages I updated by searching for the oldest version of the dependency I think we should try and support based on the updated comments at the top of `oldest_constraints.txt`. While kind of annoying, I think it'd be a good idea for the reviewer to double check that I didn't make a mistake with the versions I used here.

To find these versions, I used https://packages.ubuntu.com, https://packages.debian.org, and a CentOS 7 Docker image with EPEL 7 installed. For the latter, not all packages are available in Python 3 yet (which is something Certbot's EPEL package maintainers are working on) and in that case I didn't worry about the system because I think they can/will package the newest version available. If they end up hitting any issues here when trying to package Certbot on Python 3, we can always work with them to fix it.

* remove py27 from oldest name

* update min cryptography version

* remove run_oldest_tests.sh

* upgrade setuptools and pyopenssl

* update cffi, pyparsing, and idna

* expand oldest_constraints comments

* clarify oldest comment

* update min configobj version

* update min parsedatetime version

* quote tox env name

* use Python 3.6 in the oldest tests

* use Python 3.6 for oldest integration tests

* properly pin asn1crypto

* update min six version

* set basepython for a nicer error message

* remove outdated python 2 oldest constraints

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -22,15 +22,19 @@ jobs:
           TOXENV: py37
           CERTBOT_NO_PIN: 1
         linux-boulder-v1-integration-certbot-oldest:
+          PYTHON_VERSION: 3.6
           TOXENV: integration-certbot-oldest
           ACME_SERVER: boulder-v1
         linux-boulder-v2-integration-certbot-oldest:
+          PYTHON_VERSION: 3.6
           TOXENV: integration-certbot-oldest
           ACME_SERVER: boulder-v2
         linux-boulder-v1-integration-nginx-oldest:
+          PYTHON_VERSION: 3.6
           TOXENV: integration-nginx-oldest
           ACME_SERVER: boulder-v1
         linux-boulder-v2-integration-nginx-oldest:
+          PYTHON_VERSION: 3.6
           TOXENV: integration-nginx-oldest
           ACME_SERVER: boulder-v2
         linux-boulder-v1-py27-integration:

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -26,10 +26,12 @@ jobs:
           TOXENV: integration-certbot
         linux-oldest-tests-1:
           IMAGE_NAME: ubuntu-18.04
-          TOXENV: py27-{acme,apache,apache-v2,certbot}-oldest
+          PYTHON_VERSION: 3.6
+          TOXENV: '{acme,apache,apache-v2,certbot}-oldest'
         linux-oldest-tests-2:
           IMAGE_NAME: ubuntu-18.04
-          TOXENV: py27-{dns,nginx}-oldest
+          PYTHON_VERSION: 3.6
+          TOXENV: '{dns,nginx}-oldest'
         linux-py27:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 2.7

.azure-pipelines/templates/steps/tox-steps.yml

@@ -45,11 +45,7 @@ steps:
       export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`"
       [ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}"
       env
-      if [[ "${TOXENV}" == *"oldest"* ]]; then
-        tools/run_oldest_tests.sh
-      else
-        python -m tox
-      fi
+      python -m tox
     env:
       AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
       AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)

acme/setup.py

@@ -9,21 +9,18 @@ version = '1.12.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
-    # load_pem_private/public_key (>=0.6)
-    # rsa_recover_prime_factors (>=0.8)
-    'cryptography>=1.2.3',
+    'cryptography>=2.1.4',
     # formerly known as acme.jose:
     # 1.1.0+ is required to avoid the warnings described at
     # https://github.com/certbot/josepy/issues/13.
     'josepy>=1.1.0',
-    # Connection.set_tlsext_host_name (>=0.13) + matching Xenial requirements (>=0.15.1)
-    'PyOpenSSL>=0.15.1',
+    'PyOpenSSL>=17.3.0',
     'pyrfc3339',
     'pytz',
     'requests[security]>=2.6.0',  # security extras added in 2.4.1
     'requests-toolbelt>=0.3.0',
-    'setuptools',
-    'six>=1.9.0',  # needed for python_2_unicode_compatible
+    'setuptools>=39.0.1',
+    'six>=1.11.0',
 ]
 
 setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2'))

certbot-apache/setup.py

@@ -13,7 +13,7 @@ install_requires = [
     'acme>=0.29.0',
     'certbot>=1.6.0',
     'python-augeas',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.component',
     'zope.interface',
 ]

certbot-dns-cloudflare/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'cloudflare>=1.5.1',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-cloudxns/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-digitalocean/setup.py

@@ -12,8 +12,8 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'python-digitalocean>=1.11',
-    'setuptools',
-    'six',
+    'setuptools>=39.0.1',
+    'six>=1.11.0',
     'zope.interface',
 ]
 

certbot-dns-dnsimple/setup.py

@@ -11,7 +11,7 @@ version = '1.12.0.dev0'
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-dnsmadeeasy/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-gehirn/setup.py

@@ -11,7 +11,7 @@ version = '1.12.0.dev0'
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
     'dns-lexicon>=2.1.22',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-google/setup.py

@@ -13,7 +13,7 @@ version = '1.12.0.dev0'
 install_requires = [
     'google-api-python-client>=1.5.5',
     'oauth2client>=4.0',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
     # already a dependency of google-api-python-client, but added for consistency
     'httplib2'

certbot-dns-linode/setup.py

@@ -11,7 +11,7 @@ version = '1.12.0.dev0'
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
     'dns-lexicon>=2.2.3',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-luadns/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-nsone/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-ovh/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'dns-lexicon>=2.7.14',  # Correct proxy use on OVH provider
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-rfc2136/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'dnspython',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-route53/setup.py

@@ -12,7 +12,7 @@ version = '1.12.0.dev0'
 # acme/certbot version.
 install_requires = [
     'boto3',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-dns-sakuracloud/setup.py

@@ -11,7 +11,7 @@ version = '1.12.0.dev0'
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
     'dns-lexicon>=2.1.23',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot-nginx/setup.py

@@ -12,9 +12,9 @@ version = '1.12.0.dev0'
 install_requires = [
     'acme>=1.4.0',
     'certbot>=1.6.0',
-    'PyOpenSSL',
-    'pyparsing>=1.5.5',  # Python3 support
-    'setuptools',
+    'PyOpenSSL>=17.3.0',
+    'pyparsing>=2.2.0',
+    'setuptools>=39.0.1',
     'zope.interface',
 ]
 

certbot/setup.py

@@ -40,16 +40,16 @@ install_requires = [
     # saying so here causes a runtime error against our temporary fork of 0.9.3
     # in which we added 2.6 support (see #2243), so we relax the requirement.
     'ConfigArgParse>=0.9.3',
-    'configobj',
-    'cryptography>=1.2.3',  # load_pem_x509_certificate
+    'configobj>=5.0.6',
+    'cryptography>=2.1.4',
     'distro>=1.0.1',
     # 1.1.0+ is required to avoid the warnings described at
     # https://github.com/certbot/josepy/issues/13.
     'josepy>=1.1.0',
-    'parsedatetime>=1.3',  # Calendar.parseDT
+    'parsedatetime>=2.4',
     'pyrfc3339',
     'pytz',
-    'setuptools',
+    'setuptools>=39.0.1',
     'zope.component',
     'zope.interface',
 ]

tools/oldest_constraints.txt

@@ -1,75 +1,79 @@
-# This file contains the oldest versions of our dependencies we say we require
-# in our packages or versions we need to support to maintain compatibility with
-# the versions included in the various Linux distros where we are packaged.
+# This file contains the oldest versions of our dependencies we're trying to
+# support. Usually these version numbers are taken from the packages of our
+# dependencies available in popular LTS Linux distros. Keeping compatibility
+# with those versions makes it much easier for OS maintainers to update their
+# Certbot packages.
+#
+# When updating these dependencies, we should try to only update them to the
+# oldest version of the package that is found in a non-EOL'd version of
+# CentOS, Debian, or Ubuntu that has Certbot packages in their OS repositories
+# using a version of Python we support. If the distro is EOL'd or using a
+# version of Python we don't support, it can be ignored.
 
 # CentOS/RHEL 7 EPEL constraints
-cffi==1.6.0
+# Some of these constraints may be stricter than necessary because they
+# initially referred to the Python 2 packages in CentOS/RHEL 7 with EPEL.
+cffi==1.9.1
 chardet==2.2.1
-configobj==4.7.2
 ipaddress==1.0.16
 mock==1.0.1
 ndg-httpsclient==0.3.2
 ply==3.4
+pyOpenSSL==17.3.0
 pyasn1==0.1.9
 pycparser==2.14
 pyRFC3339==1.0
 python-augeas==0.5.0
 oauth2client==4.0.0
-six==1.9.0
-# setuptools 0.9.8 is the actual version packaged, but some other dependencies
-# in this file require setuptools>=1.0 and there are no relevant changes for us
-# between these versions.
-setuptools==1.0.0
 urllib3==1.10.2
 zope.component==4.1.0
 zope.event==4.0.3
 zope.interface==4.0.5
 
 # Debian Jessie Backports constraints
-# Debian Jessie has reached end of life. However:
-# When it becomes necessary to upgrade any of these dependencies, you should only update them to the oldest version of the package found
-# in a non-EOL'd version of CentOS, Debian, or Ubuntu that has Certbot packages in their OS repositories.
+# Debian Jessie has reached end of life so these dependencies can probably be
+# updated as needed or desired.
 colorama==0.3.2
 enum34==1.0.3
 html5lib==0.999
-idna==2.0
 pbr==1.8.0
 pytz==2012rc0
 
 # Debian Buster constraints
 google-api-python-client==1.5.5
+pyparsing==2.2.0
 
 # Our setup.py constraints
 apacheconfig==0.3.2
 cloudflare==1.5.1
-cryptography==1.2.3
-parsedatetime==1.3
-pyparsing==1.5.5
 python-digitalocean==1.11
 requests[security]==2.6.0
 
 # Ubuntu Xenial constraints
+# Ubuntu Xenial only has versions of Python which we do not support available
+# so these dependencies can probably be updated as needed or desired.
 ConfigArgParse==0.10.0
-pyOpenSSL==0.15.1
 funcsigs==0.4
 zope.hookable==4.0.4
 
 # Ubuntu Bionic constraints.
+cryptography==2.1.4
 distro==1.0.1
 # Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins
 # using their local-oldest-requirements.txt
 dns-lexicon==2.2.1
 httplib2==0.9.2
+idna==2.6
+setuptools==39.0.1
+six==1.11.0
+
+# Ubuntu Focal constraints
+asn1crypto==0.24.0
+configobj==5.0.6
+parsedatetime==2.4
 
 # Plugin constraints
 # These aren't necessarily the oldest versions we need to support
 # Tracking at https://github.com/certbot/certbot/issues/6473
 boto3==1.4.7
 botocore==1.7.41
-
-# Old certbot[dev] constraints
-# Old versions of certbot[dev] required ipdb and our normally pinned version of
-# ipython which ipdb depends on doesn't support Python 2 so we pin an older
-# version here to keep tests working while we have Python 2 support.
-ipython==5.8.0
-prompt-toolkit==1.0.18

tools/run_oldest_tests.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-pushd "${DIR}/../"
-
-function cleanup() {
-  rm -f "${DOCKERFILE}"
-  popd
-}
-
-trap cleanup EXIT
-
-DOCKERFILE=$(mktemp /tmp/Dockerfile.XXXXXX)
-
-cat << "EOF" >> "${DOCKERFILE}"
-FROM ubuntu:16.04
-COPY letsencrypt-auto-source/pieces/dependency-requirements.txt /tmp/letsencrypt-auto-source/pieces/
-COPY tools/ /tmp/tools/
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
-        python-dev python-pip python-setuptools \
-        gcc libaugeas0 libssl-dev libffi-dev \
-        git ca-certificates nginx-light openssl curl \
- && curl -fsSL https://get.docker.com | bash /dev/stdin \
- && python /tmp/tools/pipstrap.py \
- && python /tmp/tools/pip_install.py tox \
- && rm -rf /var/lib/apt/lists/*
-EOF
-
-docker build -f "${DOCKERFILE}" -t oldest-worker .
-docker run --rm --network=host -w "${PWD}" \
-  -v /var/run/docker.sock:/var/run/docker.sock \
-  -v "${PWD}:${PWD}" -v /tmp:/tmp \
-  -e TOXENV -e ACME_SERVER -e PYTEST_ADDOPTS \
-  oldest-worker python -m tox

tox.ini

@@ -77,49 +77,65 @@ setenv =
     PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto}
     PYTHONHASHSEED = 0
 
-[testenv:py27-oldest]
+[testenv:oldest]
+# Setting basepython allows the tests to fail fast if that version of Python
+# isn't available instead of potentially trying to use a newer version of
+# Python which is unlikely to work.
+basepython = python3.6
 commands =
     {[testenv]commands}
 setenv =
     {[testenv]setenv}
     CERTBOT_OLDEST=1
 
-[testenv:py27-acme-oldest]
+[testenv:acme-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]install_and_test} acme[dev]
 setenv =
-    {[testenv:py27-oldest]setenv}
+    {[testenv:oldest]setenv}
 
-[testenv:py27-apache-oldest]
+[testenv:apache-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]install_and_test} certbot-apache
 setenv =
-    {[testenv:py27-oldest]setenv}
+    {[testenv:oldest]setenv}
 
-[testenv:py27-apache-v2-oldest]
+[testenv:apache-v2-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]install_and_test} certbot-apache[dev]
 setenv =
-    {[testenv:py27-oldest]setenv}
+    {[testenv:oldest]setenv}
 
-[testenv:py27-certbot-oldest]
+[testenv:certbot-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]install_and_test} certbot[dev]
 setenv =
-    {[testenv:py27-oldest]setenv}
+    {[testenv:oldest]setenv}
 
-[testenv:py27-dns-oldest]
+[testenv:dns-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]install_and_test} {[base]dns_packages}
 setenv =
-    {[testenv:py27-oldest]setenv}
+    {[testenv:oldest]setenv}
 
-[testenv:py27-nginx-oldest]
+[testenv:nginx-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]install_and_test} certbot-nginx
     python tests/lock_test.py
 setenv =
-    {[testenv:py27-oldest]setenv}
+    {[testenv:oldest]setenv}
 
 [testenv:lint]
 basepython = python3
@@ -238,22 +254,26 @@ commands =
 passenv = DOCKER_*
 
 [testenv:integration-certbot-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]pip_install} certbot
     {[base]pip_install} certbot-ci
     pytest certbot-ci/certbot_integration_tests/certbot_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
-setenv = {[testenv:py27-oldest]setenv}
+setenv = {[testenv:oldest]setenv}
 
 [testenv:integration-nginx-oldest]
+basepython =
+    {[testenv:oldest]basepython}
 commands =
     {[base]pip_install} certbot-nginx
     {[base]pip_install} certbot-ci
     pytest certbot-ci/certbot_integration_tests/nginx_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
-setenv = {[testenv:py27-oldest]setenv}
+setenv = {[testenv:oldest]setenv}
 
 [testenv:test-farm-tests-base]
 changedir = tests/letstest