From 717afebcff44c9abbb1338efc7831b0d3b3246f2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 9 Oct 2019 14:39:49 -0700 Subject: [PATCH 01/10] Upload coverage for integration tests (#7433) * Upload coverage for integration tests. * Use in not containsValue. --- .azure-pipelines/templates/tests-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/templates/tests-suite.yml b/.azure-pipelines/templates/tests-suite.yml index bb54c8eee..119f755a6 100644 --- a/.azure-pipelines/templates/tests-suite.yml +++ b/.azure-pipelines/templates/tests-suite.yml @@ -32,7 +32,7 @@ jobs: curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash" chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash" ./codecov-bash -F windows || echo "Codecov did not collect coverage reports" - condition: eq(variables['TOXENV'], 'py37-cover') + condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot') env: CODECOV_TOKEN: $(codecov_token) displayName: Publish coverage From 118cb3c9b1e62599b57082b71ab7313b2cdc1632 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Thu, 10 Oct 2019 09:09:25 +1100 Subject: [PATCH 02/10] cli: allow --dry-run to be combined with --server (#7436) The value of --server will now be respected, except when it is the default value, in which case it will be changed to the staging server, preserving Certbot's existing behavior. --- CHANGELOG.md | 2 ++ certbot/cli.py | 19 +++++++++++++------ certbot/tests/cli_test.py | 24 +++++++++++++++++------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 102eaf4bb..fa8ca2379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * Removed `--fast` flag from the test farm tests +* `--server` may now be combined with `--dry-run`. Certbot will, as before, use the + staging server instead of the live server when `--dry-run` is used. ### Fixed diff --git a/certbot/cli.py b/certbot/cli.py index d22a9a524..6715dfd9c 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -649,13 +649,20 @@ class HelpfulArgumentParser(object): def set_test_server(self, parsed_args): """We have --staging/--dry-run; perform sanity check and set config.server""" - if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): - conflicts = ["--staging"] if parsed_args.staging else [] - conflicts += ["--dry-run"] if parsed_args.dry_run else [] - raise errors.Error("--server value conflicts with {0}".format( - " and ".join(conflicts))) + # Flag combinations should produce these results: + # | --staging | --dry-run | + # ------------------------------------------------------------ + # | --server acme-v02 | Use staging | Use staging | + # | --server acme-staging-v02 | Use staging | Use staging | + # | --server | Conflict error | Use | - parsed_args.server = constants.STAGING_URI + default_servers = (flag_default("server"), constants.STAGING_URI) + + if parsed_args.staging and parsed_args.server not in default_servers: + raise errors.Error("--server value conflicts with --staging") + + if parsed_args.server in default_servers: + parsed_args.server = constants.STAGING_URI if parsed_args.dry_run: if self.verb not in ["certonly", "renew"]: diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 87b074a81..166559040 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -333,16 +333,26 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods self._assert_dry_run_flag_worked(self.parse(short_args + ['auth']), True) self._assert_dry_run_flag_worked(self.parse(short_args + ['renew']), True) + self._assert_dry_run_flag_worked(self.parse(short_args + ['certonly']), True) + short_args += ['certonly'] - self._assert_dry_run_flag_worked(self.parse(short_args), True) - short_args += '--server example.com'.split() - conflicts = ['--dry-run'] - self._check_server_conflict_message(short_args, '--dry-run') + # `--dry-run --server example.com` should emit example.com + self.assertEqual(self.parse(short_args + ['--server', 'example.com']).server, + 'example.com') - short_args += ['--staging'] - conflicts += ['--staging'] - self._check_server_conflict_message(short_args, conflicts) + # `--dry-run --server STAGING_URI` should emit STAGING_URI + self.assertEqual(self.parse(short_args + ['--server', constants.STAGING_URI]).server, + constants.STAGING_URI) + + # `--dry-run --server LIVE` should emit STAGING_URI + self.assertEqual(self.parse(short_args + ['--server', cli.flag_default("server")]).server, + constants.STAGING_URI) + + # `--dry-run --server example.com --staging` should emit an error + conflicts = ['--staging'] + self._check_server_conflict_message(short_args + ['--server', 'example.com', '--staging'], + conflicts) def test_option_was_set(self): key_size_option = 'rsa_key_size' From 032178bea05f2c17a734d64ed1f874e0f3eb9228 Mon Sep 17 00:00:00 2001 From: Victor Shih Date: Fri, 18 Oct 2019 13:36:45 -0700 Subject: [PATCH 03/10] Clarify possible existence of /etc/letsencrypt/cli.ini (#7449) --- docs/using.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 458ab9a01..83d824058 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -917,8 +917,9 @@ Certbot accepts a global configuration file that applies its options to all invo of Certbot. Certificate specific configuration choices should be set in the ``.conf`` files that can be found in ``/etc/letsencrypt/renewal``. -By default no cli.ini file is created, after creating one -it is possible to specify the location of this configuration file with +By default no cli.ini file is created (though it may exist already if you installed Certbot +via a package manager, for instance). +After creating one it is possible to specify the location of this configuration file with ``certbot --config cli.ini`` (or shorter ``-c cli.ini``). An example configuration file is shown below: From 37b3c22dee5576f4d799ae58e3e1d338ab8b9c7f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 Oct 2019 23:06:37 -0700 Subject: [PATCH 04/10] Run nightly on Azure even if no commits landed. (#7455) --- .azure-pipelines/advanced.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/advanced.yml b/.azure-pipelines/advanced.yml index a072a8a85..afcf16db4 100644 --- a/.azure-pipelines/advanced.yml +++ b/.azure-pipelines/advanced.yml @@ -12,6 +12,7 @@ schedules: branches: include: - master + always: true jobs: - template: templates/tests-suite.yml From f8e097a06176edfd69cd7306b6c83e3946b47f60 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 Oct 2019 23:09:08 -0700 Subject: [PATCH 05/10] Remove warning about rename. (#7453) --- certbot/cli.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 6715dfd9c..93cdc7408 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -163,24 +163,6 @@ def report_config_interaction(modified, modifiers): VAR_MODIFIERS.setdefault(var, set()).update(modifiers) -def possible_deprecation_warning(config): - "A deprecation warning for users with the old, not-self-upgrading letsencrypt-auto." - if cli_command != LEAUTO: - return - if config.no_self_upgrade: - # users setting --no-self-upgrade might be hanging on a client version like 0.3.0 - # or 0.5.0 which is the new script, but doesn't set CERTBOT_AUTO; they don't - # need warnings - return - if "CERTBOT_AUTO" not in os.environ: - logger.warning("You are running with an old copy of letsencrypt-auto" - " that does not receive updates, and is less reliable than more" - " recent versions. The letsencrypt client has also been renamed" - " to Certbot. We recommend upgrading to the latest certbot-auto" - " script, or using native OS packages.") - logger.debug("Deprecation warning circumstances: %s / %s", sys.argv[0], os.environ) - - class _Default(object): """A class to use as a default to detect if a value is set by a user""" @@ -642,8 +624,6 @@ class HelpfulArgumentParser(object): raise errors.Error( "Parameters --hsts and --auto-hsts cannot be used simultaneously.") - possible_deprecation_warning(parsed_args) - return parsed_args def set_test_server(self, parsed_args): From 44cc8d7a3ca72ba5c19b2939d7bd226764507f85 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 21 Oct 2019 13:54:17 -0700 Subject: [PATCH 06/10] Require newer versions of oauth2client (#7458) Over the weekend, nightly tests on Windows failed for certbot-dns-google: https://dev.azure.com/certbot/web/build.aspx?pcguid=74ef9c03-9faf-405b-9d03-9acf8c43e8d6&builduri=vstfs%3a%2f%2f%2fBuild%2fBuild%2f72 The error occurred inside `oauth2client`'s locking code and the failure seems spurious as it did not reproduce this morning: https://dev.azure.com/certbot/certbot/_build/results?buildId=73 I could not find a relevant changelog entry in `oauth2client` saying they've fixed the problem, but the problematic code no longer exists in `oauth2client>=4.0`. This PR updates our minimum dependency required in an attempt to avoid spurious failures for us in the future. The only downside I am aware of is it'll make it harder for certbot-dns-google to be packaged in Debian Old Stable or Ubuntu 16.04, but I don't expect either of those things to happen anytime soon. * bump oauth2client dep * Update dev_constraints.txt. * Add changelog entry for packagers. --- CHANGELOG.md | 2 ++ certbot-dns-google/setup.py | 6 ++---- tools/dev_constraints.txt | 14 +++++++++++--- tools/oldest_constraints.txt | 6 ++++-- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa8ca2379..ac1263e85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Removed `--fast` flag from the test farm tests * `--server` may now be combined with `--dry-run`. Certbot will, as before, use the staging server instead of the live server when `--dry-run` is used. +* Updated certbot-dns-google to depend on newer versions of + google-api-python-client and oauth2client. ### Fixed diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 9fd159c41..6bf12ddbf 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -9,11 +9,9 @@ version = '0.40.0.dev0' install_requires = [ 'acme>=0.29.0', 'certbot>=0.39.0', - # 1.5 is the first version that supports oauth2client>=2.0 - 'google-api-python-client>=1.5', + 'google-api-python-client>=1.5.5', 'mock', - # for oauth2client.service_account.ServiceAccountCredentials - 'oauth2client>=2.0', + 'oauth2client>=4.0', 'setuptools', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 419b65d6c..d97f23a71 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -4,6 +4,7 @@ # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 apipkg==1.4 +appnope==0.1.0 asn1crypto==0.22.0 astroid==1.6.5 attrs==17.3.0 @@ -15,6 +16,7 @@ botocore==1.12.36 cloudflare==1.5.1 codecov==2.0.15 configparser==3.7.4 +contextlib2==0.6.0.post1 coverage==4.5.4 decorator==4.1.2 dns-lexicon==3.2.1 @@ -23,9 +25,11 @@ docutils==0.12 execnet==1.5.0 future==0.16.0 futures==3.1.1 -google-api-python-client==1.5 +filelock==3.0.12 +google-api-python-client==1.5.5 httplib2==0.10.3 imagesize==0.7.1 +importlib-metadata==0.23 ipdb==0.10.2 ipython==5.5.0 ipython-genutils==0.2.0 @@ -38,9 +42,11 @@ logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 mccabe==0.6.1 +more-itertools==5.0.0 mypy==0.600 ndg-httpsclient==0.3.2 -oauth2client==2.0.0 +oauth2client==4.0.0 +packaging==19.2 pathlib2==2.3.0 pexpect==4.7.0 pickleshare==0.7.4 @@ -79,13 +85,15 @@ Sphinx==1.7.5 sphinx-rtd-theme==0.2.4 sphinxcontrib-websupport==1.0.1 tldextract==2.2.0 +toml==0.10.0 tox==3.14.0 tqdm==4.19.4 traitlets==4.3.2 twine==1.11.0 typed-ast==1.1.0 typing==3.6.4 -uritemplate==0.6 +uritemplate==3.0.0 virtualenv==16.6.2 wcwidth==0.1.7 wrapt==1.11.1 +zipp==0.6.0 diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 73465639f..c5a5c5aa0 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -16,6 +16,7 @@ pyOpenSSL==0.13.1 pyparsing==1.5.6 pyRFC3339==1.0 python-augeas==0.5.0 +oauth2client==4.0.0 six==1.9.0 # setuptools 0.9.8 is the actual version packaged, but some other dependencies # in this file require setuptools>=1.0 and there are no relevant changes for us @@ -35,11 +36,12 @@ idna==2.0 pbr==1.8.0 pytz==2012rc0 +# Debian Buster constraints +google-api-python-client==1.5.5 + # Our setup.py constraints cloudflare==1.5.1 cryptography==1.2.3 -google-api-python-client==1.5 -oauth2client==2.0 parsedatetime==1.3 pyparsing==1.5.5 python-digitalocean==1.11 From db46326e9597f2166483f5da459069fe8afd509c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 21 Oct 2019 14:42:51 -0700 Subject: [PATCH 07/10] Run at 4:00AM UTC not 0:04AM UTC. (#7460) Fixes [cron syntax](https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=azure-devops&tabs=yaml#supported-cron-syntax) to get the behavior I had in mind in https://github.com/certbot/certbot/pull/7377#discussion_r331295897. --- .azure-pipelines/advanced.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/advanced.yml b/.azure-pipelines/advanced.yml index afcf16db4..9832c0684 100644 --- a/.azure-pipelines/advanced.yml +++ b/.azure-pipelines/advanced.yml @@ -7,7 +7,7 @@ pr: - '*.x' # This pipeline is also nightly run on master schedules: - - cron: "4 0 * * *" + - cron: "0 4 * * *" displayName: Nightly build branches: include: From 3132c32c262e4d81b366a5df9aa72b4c6037bef4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Oct 2019 01:50:18 -0700 Subject: [PATCH 08/10] Update pluggy pinning. (#7459) --- tools/dev_constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index d97f23a71..6854be466 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,7 +51,7 @@ pathlib2==2.3.0 pexpect==4.7.0 pickleshare==0.7.4 pkginfo==1.4.2 -pluggy==0.5.2 +pluggy==0.13.0 prompt-toolkit==1.0.15 ptyprocess==0.6.0 py==1.8.0 From 60673e8a81d1a0695921a30df808bfeed18fb884 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Oct 2019 03:48:01 -0700 Subject: [PATCH 09/10] Remove AppVeyor. (#7440) --- appveyor.yml | 47 ----------------------------------------------- tox.cover.py | 6 +++--- 2 files changed, 3 insertions(+), 50 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 53f29a5e6..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,47 +0,0 @@ -image: Visual Studio 2015 - -environment: - matrix: - - TOXENV: py35 - - TOXENV: py37-cover - - TOXENV: integration-certbot - -branches: - only: - # apache-parser-v2 is a temporary branch for doing work related to - # rewriting the parser in the Apache plugin. - - apache-parser-v2 - - master - - /^\d+\.\d+\.x$/ # Version branches like X.X.X - - /^test-.*$/ - -init: - # Since master can receive only commits from PR that have already been tested, following - # condition avoid to launch all jobs except the coverage one for commits pushed to master. - - ps: | - if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' ` - -And -Not ($Env:TOXENV -Like '*-cover')) - { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } - -install: - # Use Python 3.7 by default - - SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH% - # Using 4 processes is proven to be the most efficient integration tests config for AppVeyor - - IF %TOXENV%==integration-certbot SET PYTEST_ADDOPTS=--numprocesses=4 - # Check env - - python --version - # Upgrade pip to avoid warnings - - python -m pip install --upgrade pip - # Ready to install tox and coverage - # tools/pip_install.py is used to pin packages to a known working version. - - python tools\\pip_install.py tox codecov - -build: off - -test_script: - - set TOX_TESTENV_PASSENV=APPVEYOR - # Test env is set by TOXENV env variable - - tox - -on_success: - - if exist .coverage codecov -F windows diff --git a/tox.cover.py b/tox.cover.py index c313419ed..6981bbb41 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -56,9 +56,9 @@ def cover(package): def main(): description = """ -This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order -to generate separate stats for each package. It should be removed once those -packages are moved to a separate repo. +This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in +order to generate separate stats for each package. It should be removed once +those packages are moved to a separate repo. Option -e makes sure we fail fast and don't submit to codecov.""" parser = argparse.ArgumentParser(description=description) From 0f31d9b7ac1bb01a3ff6ac396fe315cb657eef6f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 24 Oct 2019 05:46:55 -0700 Subject: [PATCH 10/10] Remove skip_unless cruft (#7410) * Remove skip_unless cruft. * remove unused import --- acme/acme/test_util.py | 21 --------------------- certbot/tests/util.py | 20 -------------------- 2 files changed, 41 deletions(-) diff --git a/acme/acme/test_util.py b/acme/acme/test_util.py index f04829deb..6d9cbc8dc 100644 --- a/acme/acme/test_util.py +++ b/acme/acme/test_util.py @@ -4,7 +4,6 @@ """ import os -import unittest import pkg_resources from cryptography.hazmat.backends import default_backend @@ -73,23 +72,3 @@ def load_pyopenssl_private_key(*names): loader = _guess_loader( names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1) return crypto.load_privatekey(loader, load_vector(*names)) - - -def skip_unless(condition, reason): # pragma: no cover - """Skip tests unless a condition holds. - - This implements the basic functionality of unittest.skipUnless - which is only available on Python 2.7+. - - :param bool condition: If ``False``, the test will be skipped - :param str reason: the reason for skipping the test - - :rtype: callable - :returns: decorator that hides tests unless condition is ``True`` - - """ - if hasattr(unittest, "skipUnless"): - return unittest.skipUnless(condition, reason) - elif condition: - return lambda cls: cls - return lambda cls: None diff --git a/certbot/tests/util.py b/certbot/tests/util.py index c46623e0a..dfd752511 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -94,26 +94,6 @@ def load_pyopenssl_private_key(*names): return OpenSSL.crypto.load_privatekey(loader, load_vector(*names)) -def skip_unless(condition, reason): # pragma: no cover - """Skip tests unless a condition holds. - - This implements the basic functionality of unittest.skipUnless - which is only available on Python 2.7+. - - :param bool condition: If ``False``, the test will be skipped - :param str reason: the reason for skipping the test - - :rtype: callable - :returns: decorator that hides tests unless condition is ``True`` - - """ - if hasattr(unittest, "skipUnless"): - return unittest.skipUnless(condition, reason) - elif condition: - return lambda cls: cls - return lambda cls: None - - def make_lineage(config_dir, testfile): """Creates a lineage defined by testfile.