From 9a4e95e25a542a6a295e2a2800000970afed41e6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 19 Nov 2020 12:48:36 -0800 Subject: [PATCH] Add Python 3.9 support and tests (#8460) Fixes https://github.com/certbot/certbot/issues/8134. * Test on Python 3.9. * Mention Python 3.9 support in changelog. * s/\( *'Pro.*3\.\)8\(',\)/\18\2\n\19\2/ * undo changes to tox.ini * Move more tests to Python 3.9 * Update PyYAML and packages which pinned it back * Upgrade typed-ast * Use <= to "pin" dnspython * Fix lint by telling pylint it cannot be trusted * Disable mypy on RFC plugin * add comment about <= support --- .../templates/jobs/extended-tests-jobs.yml | 13 ++++++++- .../templates/jobs/standard-tests-jobs.yml | 14 ++++----- acme/setup.py | 1 + certbot-apache/setup.py | 1 + certbot-ci/setup.py | 1 + certbot-compatibility-test/setup.py | 1 + certbot-dns-cloudflare/setup.py | 1 + certbot-dns-cloudxns/setup.py | 1 + certbot-dns-digitalocean/setup.py | 1 + certbot-dns-dnsimple/setup.py | 1 + certbot-dns-dnsmadeeasy/setup.py | 1 + certbot-dns-gehirn/setup.py | 1 + certbot-dns-google/setup.py | 1 + certbot-dns-linode/setup.py | 1 + certbot-dns-luadns/setup.py | 1 + certbot-dns-nsone/setup.py | 1 + certbot-dns-ovh/setup.py | 1 + .../_internal/dns_rfc2136.py | 10 +++++++ certbot-dns-rfc2136/setup.py | 1 + certbot-dns-route53/setup.py | 1 + certbot-dns-sakuracloud/setup.py | 1 + certbot-nginx/setup.py | 1 + certbot/CHANGELOG.md | 1 + certbot/setup.py | 1 + tools/dev_constraints.txt | 17 +++++++---- tools/merge_requirements.py | 29 ++++++++++++++----- 26 files changed, 83 insertions(+), 21 deletions(-) diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index 3197501e1..f835078c8 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -4,7 +4,7 @@ jobs: - name: IMAGE_NAME value: ubuntu-18.04 - name: PYTHON_VERSION - value: 3.8 + value: 3.9 - group: certbot-common strategy: matrix: @@ -14,6 +14,9 @@ jobs: linux-py37: PYTHON_VERSION: 3.7 TOXENV: py37 + linux-py38: + PYTHON_VERSION: 3.8 + TOXENV: py38 linux-py37-nopin: PYTHON_VERSION: 3.7 TOXENV: py37 @@ -62,6 +65,14 @@ jobs: PYTHON_VERSION: 3.8 TOXENV: integration ACME_SERVER: boulder-v2 + linux-boulder-v1-py39-integration: + PYTHON_VERSION: 3.9 + TOXENV: integration + ACME_SERVER: boulder-v1 + linux-boulder-v2-py39-integration: + PYTHON_VERSION: 3.9 + TOXENV: integration + ACME_SERVER: boulder-v2 nginx-compat: TOXENV: nginx_compat linux-integration-rfc2136: diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 7a1f620bb..897ab40bb 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -1,17 +1,17 @@ jobs: - job: test variables: - PYTHON_VERSION: 3.8 + PYTHON_VERSION: 3.9 strategy: matrix: macos-py27: IMAGE_NAME: macOS-10.15 PYTHON_VERSION: 2.7 TOXENV: py27 - macos-py38: + macos-py39: IMAGE_NAME: macOS-10.15 - PYTHON_VERSION: 3.8 - TOXENV: py38 + PYTHON_VERSION: 3.9 + TOXENV: py39 windows-py36: IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.6 @@ -38,10 +38,10 @@ jobs: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.6 TOXENV: py36 - linux-py38-cover: + linux-py39-cover: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.8 - TOXENV: py38-cover + PYTHON_VERSION: 3.9 + TOXENV: py39-cover linux-py37-lint: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.7 diff --git a/acme/setup.py b/acme/setup.py index 96adc2963..55d1ae17d 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -66,6 +66,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index cd1a24f93..c6fadb929 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -53,6 +53,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 971db2b8e..ce29fe45d 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -52,6 +52,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 598454ae8..66e1c9f03 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -50,6 +50,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 96db7351a..34938aca1 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -63,6 +63,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 09225a42e..81f6afa12 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -63,6 +63,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 4da161146..f6c5936ae 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -64,6 +64,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index cd1246e07..e0d95eea1 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -74,6 +74,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c077acbef..0ff3c71ab 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -63,6 +63,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index ba58509f7..e57796e1a 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -62,6 +62,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 78a78cbc3..c52215f33 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -66,6 +66,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 4be13cf56..e7cdcabfb 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -62,6 +62,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 805158771..d38c9fcca 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -63,6 +63,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 100a48551..deaa60790 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -63,6 +63,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 585b038a3..349903900 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -63,6 +63,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index 57e9506f2..a3a943660 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -1,3 +1,13 @@ +# type: ignore +# pylint: disable=no-member +# Many attributes of dnspython are now dynamically defined which causes both +# mypy and pylint to error about accessing attributes they think do not exist. +# This is the case even in up-to-date versions of mypy and pylint which as of +# writing this are 0.790 and 2.6.0 respectively. This problem may be fixed in +# dnspython 2.1.0. See https://github.com/rthalley/dnspython/issues/598. For +# now, let's disable these checks. This is done at the very top of the file +# like this because "type: ignore" must be the first line in the file to be +# respected by mypy. """DNS Authenticator using RFC 2136 Dynamic Updates.""" import logging diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c841aefc1..c2c97a41f 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -63,6 +63,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2ad0de045..7d28d2703 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -58,6 +58,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b1c6ee6a3..22f5fa031 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -62,6 +62,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index a4faf470e..9dfa7bbb3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -49,6 +49,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index b7d57a62d..a7f48ea3b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -13,6 +13,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). `secp256r1`, `secp284r1` and `secp521r1` are accepted values. * The command `certbot certficates` lists the which type of the private key that was used for the private key. +* Support for Python 3.9 was added to Certbot and all of its components. ### Changed diff --git a/certbot/setup.py b/certbot/setup.py index 742900402..ea87d2301 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -131,6 +131,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 1d8001727..967596ded 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -26,9 +26,15 @@ coverage==4.5.4 decorator==4.4.1 deprecated==1.2.10 dns-lexicon==3.3.17 -dnspython==1.15.0 -docker==3.7.2 -docker-compose==1.25.0 +# There is no version of dnspython that works on both Python 2 and Python 3.9. +# To work around this, we make use of the fact that subject to other +# constraints, pip will install the newest version of a package while ignoring +# versions that don't support the version of Python being used. The result of +# this is dnspython 2.0.0 is installed in Python 3 while dnspython 1.16.0 is +# installed in Python 2. +dnspython<=2.0.0 +docker==4.3.1 +docker-compose==1.26.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -94,8 +100,9 @@ pytest-sugar==0.9.2 pytest-rerunfailures==4.2 python-dateutil==2.8.1 python-digitalocean==1.11 +python-dotenv==0.14.0 pywin32==227 -PyYAML==3.13 +PyYAML==5.3.1 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 requests-oauthlib==1.3.0 @@ -116,7 +123,7 @@ tox==3.14.0 tqdm==4.19.4 traitlets==4.3.3 twine==1.11.0 -typed-ast==1.4.0 +typed-ast==1.4.1 typing==3.6.4 uritemplate==3.0.0 virtualenv==16.6.2 diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 0d41d12c4..bbcb38051 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -1,8 +1,9 @@ #!/usr/bin/env python """Merges multiple Python requirements files into one file. -Requirements files specified later take precedence over earlier ones. Only -simple SomeProject==1.2.3 format is currently supported. +Requirements files specified later take precedence over earlier ones. +Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are +currently supported. """ from __future__ import print_function @@ -16,17 +17,28 @@ def process_entries(entries): :param list entries: List of entries - :returns: mapping from a project to its pinned version + :returns: mapping from a project to its version specifier :rtype: dict """ data = {} for e in entries: e = e.strip() if e and not e.startswith('#') and not e.startswith('-e'): - project, version = e.split('==') - if not version: + # Support for <= was added as part of + # https://github.com/certbot/certbot/pull/8460 because we weren't + # able to pin a package to an exact version. Normally, this + # functionality shouldn't be needed so we could remove it in the + # future. If you do so, make sure to update other places in this + # file related to this behavior such as this file's docstring. + for comparison in ('==', '<=',): + parts = e.split(comparison) + if len(parts) == 2: + project_name = parts[0] + version = parts[1] + data[project_name] = comparison + version + break + else: raise ValueError("Unexpected syntax '{0}'".format(e)) - data[project] = version return data def read_file(file_path): @@ -44,10 +56,11 @@ def read_file(file_path): def output_requirements(requirements): """Prepare print requirements to stdout. - :param dict requirements: mapping from a project to its pinned version + :param dict requirements: mapping from a project to its version + specifier """ - return '\n'.join('{0}=={1}'.format(key, value) + return '\n'.join('{0}{1}'.format(key, value) for key, value in sorted(requirements.items()))