Add Python 3.9 support and tests (#8460)

Fixes https://github.com/certbot/certbot/issues/8134.

* Test on Python 3.9.

* Mention Python 3.9 support in changelog.

* s/\( *'Pro.*3\.\)8\(',\)/\18\2\n\19\2/

* undo changes to tox.ini

* Move more tests to Python 3.9

* Update PyYAML and packages which pinned it back

* Upgrade typed-ast

* Use <= to "pin" dnspython

* Fix lint by telling pylint it cannot be trusted

* Disable mypy on RFC plugin

* add comment about <= support

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -4,7 +4,7 @@ jobs:
       - name: IMAGE_NAME
         value: ubuntu-18.04
       - name: PYTHON_VERSION
-        value: 3.8
+        value: 3.9
       - group: certbot-common
     strategy:
       matrix:
@@ -14,6 +14,9 @@ jobs:
         linux-py37:
           PYTHON_VERSION: 3.7
           TOXENV: py37
+        linux-py38:
+          PYTHON_VERSION: 3.8
+          TOXENV: py38
         linux-py37-nopin:
           PYTHON_VERSION: 3.7
           TOXENV: py37
@@ -62,6 +65,14 @@ jobs:
           PYTHON_VERSION: 3.8
           TOXENV: integration
           ACME_SERVER: boulder-v2
+        linux-boulder-v1-py39-integration:
+          PYTHON_VERSION: 3.9
+          TOXENV: integration
+          ACME_SERVER: boulder-v1
+        linux-boulder-v2-py39-integration:
+          PYTHON_VERSION: 3.9
+          TOXENV: integration
+          ACME_SERVER: boulder-v2
         nginx-compat:
           TOXENV: nginx_compat
         linux-integration-rfc2136:

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -1,17 +1,17 @@
 jobs:
   - job: test
     variables:
-      PYTHON_VERSION: 3.8
+      PYTHON_VERSION: 3.9
     strategy:
       matrix:
         macos-py27:
           IMAGE_NAME: macOS-10.15
           PYTHON_VERSION: 2.7
           TOXENV: py27
-        macos-py38:
+        macos-py39:
           IMAGE_NAME: macOS-10.15
-          PYTHON_VERSION: 3.8
+          PYTHON_VERSION: 3.9
-          TOXENV: py38
+          TOXENV: py39
         windows-py36:
           IMAGE_NAME: vs2017-win2016
           PYTHON_VERSION: 3.6
@@ -38,10 +38,10 @@ jobs:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.6
           TOXENV: py36
-        linux-py38-cover:
+        linux-py39-cover:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 3.8
+          PYTHON_VERSION: 3.9
-          TOXENV: py38-cover
+          TOXENV: py39-cover
         linux-py37-lint:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.7

acme/setup.py

@@ -66,6 +66,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-apache/setup.py

@@ -53,6 +53,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-ci/setup.py

@@ -52,6 +52,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-compatibility-test/setup.py

@@ -50,6 +50,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-dns-cloudflare/setup.py

@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-cloudxns/setup.py

@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-digitalocean/setup.py

@@ -64,6 +64,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsimple/setup.py

@@ -74,6 +74,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsmadeeasy/setup.py

@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-gehirn/setup.py

@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-google/setup.py

@@ -66,6 +66,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-linode/setup.py

@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-luadns/setup.py

@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-nsone/setup.py

@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-ovh/setup.py

@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py

@@ -1,3 +1,13 @@
+# type: ignore
+# pylint: disable=no-member
+# Many attributes of dnspython are now dynamically defined which causes both
+# mypy and pylint to error about accessing attributes they think do not exist.
+# This is the case even in up-to-date versions of mypy and pylint which as of
+# writing this are 0.790 and 2.6.0 respectively. This problem may be fixed in
+# dnspython 2.1.0. See https://github.com/rthalley/dnspython/issues/598. For
+# now, let's disable these checks. This is done at the very top of the file
+# like this because "type: ignore" must be the first line in the file to be
+# respected by mypy.
 """DNS Authenticator using RFC 2136 Dynamic Updates."""
 import logging
 

certbot-dns-rfc2136/setup.py

@@ -63,6 +63,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-route53/setup.py

@@ -58,6 +58,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-sakuracloud/setup.py

@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-nginx/setup.py

@@ -49,6 +49,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot/CHANGELOG.md

@@ -13,6 +13,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
   `secp256r1`, `secp284r1` and `secp521r1` are accepted values.
 * The command `certbot certficates` lists the which type of the private key that was used
   for the private key.
+* Support for Python 3.9 was added to Certbot and all of its components.
 
 ### Changed
 

certbot/setup.py

@@ -131,6 +131,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

tools/dev_constraints.txt

@@ -26,9 +26,15 @@ coverage==4.5.4
 decorator==4.4.1
 deprecated==1.2.10
 dns-lexicon==3.3.17
-dnspython==1.15.0
+# There is no version of dnspython that works on both Python 2 and Python 3.9.
-docker==3.7.2
+# To work around this, we make use of the fact that subject to other
-docker-compose==1.25.0
+# constraints, pip will install the newest version of a package while ignoring
+# versions that don't support the version of Python being used. The result of
+# this is dnspython 2.0.0 is installed in Python 3 while dnspython 1.16.0 is
+# installed in Python 2.
+dnspython<=2.0.0
+docker==4.3.1
+docker-compose==1.26.2
 docker-pycreds==0.4.0
 dockerpty==0.4.1
 docopt==0.6.2
@@ -94,8 +100,9 @@ pytest-sugar==0.9.2
 pytest-rerunfailures==4.2
 python-dateutil==2.8.1
 python-digitalocean==1.11
+python-dotenv==0.14.0
 pywin32==227
-PyYAML==3.13
+PyYAML==5.3.1
 repoze.sphinx.autointerface==0.8
 requests-file==1.4.2
 requests-oauthlib==1.3.0
@@ -116,7 +123,7 @@ tox==3.14.0
 tqdm==4.19.4
 traitlets==4.3.3
 twine==1.11.0
-typed-ast==1.4.0
+typed-ast==1.4.1
 typing==3.6.4
 uritemplate==3.0.0
 virtualenv==16.6.2

tools/merge_requirements.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 """Merges multiple Python requirements files into one file.
 
-Requirements files specified later take precedence over earlier ones. Only
+Requirements files specified later take precedence over earlier ones.
-simple SomeProject==1.2.3 format is currently supported.
+Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are
+currently supported.
 
 """
 from __future__ import print_function
@@ -16,17 +17,28 @@ def process_entries(entries):
 
     :param list entries: List of entries
 
-    :returns: mapping from a project to its pinned version
+    :returns: mapping from a project to its version specifier
     :rtype: dict
     """
     data = {}
     for e in entries:
         e = e.strip()
         if e and not e.startswith('#') and not e.startswith('-e'):
-            project, version = e.split('==')
+            # Support for <= was added as part of
-            if not version:
+            # https://github.com/certbot/certbot/pull/8460 because we weren't
+            # able to pin a package to an exact version. Normally, this
+            # functionality shouldn't be needed so we could remove it in the
+            # future. If you do so, make sure to update other places in this
+            # file related to this behavior such as this file's docstring.
+            for comparison in ('==', '<=',):
+                parts = e.split(comparison)
+                if len(parts) == 2:
+                    project_name = parts[0]
+                    version = parts[1]
+                    data[project_name] = comparison + version
+                    break
+            else:
                 raise ValueError("Unexpected syntax '{0}'".format(e))
-            data[project] = version
     return data
 
 def read_file(file_path):
@@ -44,10 +56,11 @@ def read_file(file_path):
 def output_requirements(requirements):
     """Prepare print requirements to stdout.
 
-    :param dict requirements: mapping from a project to its pinned version
+    :param dict requirements: mapping from a project to its version
+        specifier
 
     """
-    return '\n'.join('{0}=={1}'.format(key, value)
+    return '\n'.join('{0}{1}'.format(key, value)
                      for key, value in sorted(requirements.items()))