Merge branch 'master' into apache-parser-v2

.azure-pipelines/advanced.yml

@@ -7,11 +7,12 @@ pr:
   - '*.x'
 # This pipeline is also nightly run on master
 schedules:
-  - cron: "4 0 * * *"
+  - cron: "0 4 * * *"
     displayName: Nightly build
     branches:
       include:
       - master
+    always: true
 
 jobs:
   - template: templates/tests-suite.yml

.azure-pipelines/templates/tests-suite.yml

@@ -32,7 +32,7 @@ jobs:
         curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash"
         chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash"
         ./codecov-bash -F windows  || echo "Codecov did not collect coverage reports"
-      condition: eq(variables['TOXENV'], 'py37-cover')
+      condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot')
       env:
         CODECOV_TOKEN: $(codecov_token)
       displayName: Publish coverage

CHANGELOG.md

@@ -11,6 +11,10 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 ### Changed
 
 * Removed `--fast` flag from the test farm tests
+* `--server` may now be combined with `--dry-run`. Certbot will, as before, use the
+  staging server instead of the live server when `--dry-run` is used.
+* Updated certbot-dns-google to depend on newer versions of
+  google-api-python-client and oauth2client.
 
 ### Fixed
 

acme/acme/test_util.py

@@ -4,7 +4,6 @@
 
 """
 import os
-import unittest
 import pkg_resources
 
 from cryptography.hazmat.backends import default_backend
@@ -73,23 +72,3 @@ def load_pyopenssl_private_key(*names):
     loader = _guess_loader(
         names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
     return crypto.load_privatekey(loader, load_vector(*names))
-
-
-def skip_unless(condition, reason):  # pragma: no cover
-    """Skip tests unless a condition holds.
-
-    This implements the basic functionality of unittest.skipUnless
-    which is only available on Python 2.7+.
-
-    :param bool condition: If ``False``, the test will be skipped
-    :param str reason: the reason for skipping the test
-
-    :rtype: callable
-    :returns: decorator that hides tests unless condition is ``True``
-
-    """
-    if hasattr(unittest, "skipUnless"):
-        return unittest.skipUnless(condition, reason)
-    elif condition:
-        return lambda cls: cls
-    return lambda cls: None

appveyor.yml

@@ -1,47 +0,0 @@
-image: Visual Studio 2015
-
-environment:
-  matrix:
-    - TOXENV: py35
-    - TOXENV: py37-cover
-    - TOXENV: integration-certbot
-
-branches:
-  only:
-    # apache-parser-v2 is a temporary branch for doing work related to
-    # rewriting the parser in the Apache plugin.
-    - apache-parser-v2
-    - master
-    - /^\d+\.\d+\.x$/ # Version branches like X.X.X
-    - /^test-.*$/
-
-init:
-  # Since master can receive only commits from PR that have already been tested, following
-  # condition avoid to launch all jobs except the coverage one for commits pushed to master.
-  - ps: |
-      if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' `
-          -And -Not ($Env:TOXENV -Like '*-cover'))
-      { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild }
-
-install:
-  # Use Python 3.7 by default
-  - SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%
-  # Using 4 processes is proven to be the most efficient integration tests config for AppVeyor
-  - IF %TOXENV%==integration-certbot SET PYTEST_ADDOPTS=--numprocesses=4
-  # Check env
-  - python --version
-  # Upgrade pip to avoid warnings
-  - python -m pip install --upgrade pip
-  # Ready to install tox and coverage
-  # tools/pip_install.py is used to pin packages to a known working version.
-  - python tools\\pip_install.py tox codecov
-
-build: off
-
-test_script:
-  - set TOX_TESTENV_PASSENV=APPVEYOR
-  # Test env is set by TOXENV env variable
-  - tox
-
-on_success:
-  - if exist .coverage codecov -F windows

certbot-dns-google/setup.py

@@ -9,11 +9,9 @@ version = '0.40.0.dev0'
 install_requires = [
     'acme>=0.29.0',
     'certbot>=0.39.0',
-    # 1.5 is the first version that supports oauth2client>=2.0
-    'google-api-python-client>=1.5',
+    'google-api-python-client>=1.5.5',
     'mock',
-    # for oauth2client.service_account.ServiceAccountCredentials
-    'oauth2client>=2.0',
+    'oauth2client>=4.0',
     'setuptools',
     'zope.interface',
     # already a dependency of google-api-python-client, but added for consistency

certbot/cli.py

@@ -163,24 +163,6 @@ def report_config_interaction(modified, modifiers):
         VAR_MODIFIERS.setdefault(var, set()).update(modifiers)
 
 
-def possible_deprecation_warning(config):
-    "A deprecation warning for users with the old, not-self-upgrading letsencrypt-auto."
-    if cli_command != LEAUTO:
-        return
-    if config.no_self_upgrade:
-        # users setting --no-self-upgrade might be hanging on a client version like 0.3.0
-        # or 0.5.0 which is the new script, but doesn't set CERTBOT_AUTO; they don't
-        # need warnings
-        return
-    if "CERTBOT_AUTO" not in os.environ:
-        logger.warning("You are running with an old copy of letsencrypt-auto"
-            " that does not receive updates, and is less reliable than more"
-            " recent versions. The letsencrypt client has also been renamed"
-            " to Certbot. We recommend upgrading to the latest certbot-auto"
-            " script, or using native OS packages.")
-        logger.debug("Deprecation warning circumstances: %s / %s", sys.argv[0], os.environ)
-
-
 class _Default(object):
     """A class to use as a default to detect if a value is set by a user"""
 
@@ -642,20 +624,25 @@ class HelpfulArgumentParser(object):
             raise errors.Error(
                 "Parameters --hsts and --auto-hsts cannot be used simultaneously.")
 
-        possible_deprecation_warning(parsed_args)
-
         return parsed_args
 
     def set_test_server(self, parsed_args):
         """We have --staging/--dry-run; perform sanity check and set config.server"""
 
-        if parsed_args.server not in (flag_default("server"), constants.STAGING_URI):
-            conflicts = ["--staging"] if parsed_args.staging else []
-            conflicts += ["--dry-run"] if parsed_args.dry_run else []
-            raise errors.Error("--server value conflicts with {0}".format(
-                " and ".join(conflicts)))
+        # Flag combinations should produce these results:
+        #                             | --staging      | --dry-run   |
+        # ------------------------------------------------------------
+        # | --server acme-v02         | Use staging    | Use staging |
+        # | --server acme-staging-v02 | Use staging    | Use staging |
+        # | --server <other>          | Conflict error | Use <other> |
 
-        parsed_args.server = constants.STAGING_URI
+        default_servers = (flag_default("server"), constants.STAGING_URI)
+
+        if parsed_args.staging and parsed_args.server not in default_servers:
+            raise errors.Error("--server value conflicts with --staging")
+
+        if parsed_args.server in default_servers:
+            parsed_args.server = constants.STAGING_URI
 
         if parsed_args.dry_run:
             if self.verb not in ["certonly", "renew"]:

certbot/tests/cli_test.py

@@ -333,16 +333,26 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
 
         self._assert_dry_run_flag_worked(self.parse(short_args + ['auth']), True)
         self._assert_dry_run_flag_worked(self.parse(short_args + ['renew']), True)
+        self._assert_dry_run_flag_worked(self.parse(short_args + ['certonly']), True)
+
         short_args += ['certonly']
-        self._assert_dry_run_flag_worked(self.parse(short_args), True)
 
-        short_args += '--server example.com'.split()
-        conflicts = ['--dry-run']
-        self._check_server_conflict_message(short_args, '--dry-run')
+        # `--dry-run --server example.com` should emit example.com
+        self.assertEqual(self.parse(short_args + ['--server', 'example.com']).server,
+                         'example.com')
 
-        short_args += ['--staging']
-        conflicts += ['--staging']
-        self._check_server_conflict_message(short_args, conflicts)
+        # `--dry-run --server STAGING_URI` should emit STAGING_URI
+        self.assertEqual(self.parse(short_args + ['--server', constants.STAGING_URI]).server,
+                         constants.STAGING_URI)
+
+        # `--dry-run --server LIVE` should emit STAGING_URI
+        self.assertEqual(self.parse(short_args + ['--server', cli.flag_default("server")]).server,
+                         constants.STAGING_URI)
+
+        # `--dry-run --server example.com --staging` should emit an error
+        conflicts = ['--staging']
+        self._check_server_conflict_message(short_args + ['--server', 'example.com', '--staging'],
+                                            conflicts)
 
     def test_option_was_set(self):
         key_size_option = 'rsa_key_size'

certbot/tests/util.py

@@ -94,26 +94,6 @@ def load_pyopenssl_private_key(*names):
     return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))
 
 
-def skip_unless(condition, reason):  # pragma: no cover
-    """Skip tests unless a condition holds.
-
-    This implements the basic functionality of unittest.skipUnless
-    which is only available on Python 2.7+.
-
-    :param bool condition: If ``False``, the test will be skipped
-    :param str reason: the reason for skipping the test
-
-    :rtype: callable
-    :returns: decorator that hides tests unless condition is ``True``
-
-    """
-    if hasattr(unittest, "skipUnless"):
-        return unittest.skipUnless(condition, reason)
-    elif condition:
-        return lambda cls: cls
-    return lambda cls: None
-
-
 def make_lineage(config_dir, testfile):
     """Creates a lineage defined by testfile.
 

docs/using.rst

@@ -917,8 +917,9 @@ Certbot accepts a global configuration file that applies its options to all invo
 of Certbot. Certificate specific configuration choices should be set in the ``.conf``
 files that can be found in ``/etc/letsencrypt/renewal``.
 
-By default no cli.ini file is created, after creating one
-it is possible to specify the location of this configuration file with
+By default no cli.ini file is created (though it may exist already if you installed Certbot
+via a package manager, for instance).
+After creating one it is possible to specify the location of this configuration file with
 ``certbot --config cli.ini`` (or shorter ``-c cli.ini``). An
 example configuration file is shown below:
 

tools/dev_constraints.txt

@@ -4,6 +4,7 @@
 # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).
 alabaster==0.7.10
 apipkg==1.4
+appnope==0.1.0
 asn1crypto==0.22.0
 astroid==1.6.5
 attrs==17.3.0
@@ -15,6 +16,7 @@ botocore==1.12.36
 cloudflare==1.5.1
 codecov==2.0.15
 configparser==3.7.4
+contextlib2==0.6.0.post1
 coverage==4.5.4
 decorator==4.1.2
 dns-lexicon==3.2.1
@@ -23,9 +25,11 @@ docutils==0.12
 execnet==1.5.0
 future==0.16.0
 futures==3.1.1
-google-api-python-client==1.5
+filelock==3.0.12
+google-api-python-client==1.5.5
 httplib2==0.10.3
 imagesize==0.7.1
+importlib-metadata==0.23
 ipdb==0.10.2
 ipython==5.5.0
 ipython-genutils==0.2.0
@@ -38,14 +42,16 @@ logger==1.4
 logilab-common==1.4.1
 MarkupSafe==1.0
 mccabe==0.6.1
+more-itertools==5.0.0
 mypy==0.600
 ndg-httpsclient==0.3.2
-oauth2client==2.0.0
+oauth2client==4.0.0
+packaging==19.2
 pathlib2==2.3.0
 pexpect==4.7.0
 pickleshare==0.7.4
 pkginfo==1.4.2
-pluggy==0.5.2
+pluggy==0.13.0
 prompt-toolkit==1.0.15
 ptyprocess==0.6.0
 py==1.8.0
@@ -79,13 +85,15 @@ Sphinx==1.7.5
 sphinx-rtd-theme==0.2.4
 sphinxcontrib-websupport==1.0.1
 tldextract==2.2.0
+toml==0.10.0
 tox==3.14.0
 tqdm==4.19.4
 traitlets==4.3.2
 twine==1.11.0
 typed-ast==1.1.0
 typing==3.6.4
-uritemplate==0.6
+uritemplate==3.0.0
 virtualenv==16.6.2
 wcwidth==0.1.7
 wrapt==1.11.1
+zipp==0.6.0

tools/oldest_constraints.txt

@@ -16,6 +16,7 @@ pyOpenSSL==0.13.1
 pyparsing==1.5.6
 pyRFC3339==1.0
 python-augeas==0.5.0
+oauth2client==4.0.0
 six==1.9.0
 # setuptools 0.9.8 is the actual version packaged, but some other dependencies
 # in this file require setuptools>=1.0 and there are no relevant changes for us
@@ -35,11 +36,12 @@ idna==2.0
 pbr==1.8.0
 pytz==2012rc0
 
+# Debian Buster constraints
+google-api-python-client==1.5.5
+
 # Our setup.py constraints
 cloudflare==1.5.1
 cryptography==1.2.3
-google-api-python-client==1.5
-oauth2client==2.0
 parsedatetime==1.3
 pyparsing==1.5.5
 python-digitalocean==1.11

tox.cover.py

@@ -56,9 +56,9 @@ def cover(package):
 
 def main():
     description = """
-This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order
-to generate separate stats for each package. It should be removed once those
-packages are moved to a separate repo.
+This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in
+order to generate separate stats for each package. It should be removed once
+those packages are moved to a separate repo.
 
 Option -e makes sure we fail fast and don't submit to codecov."""
     parser = argparse.ArgumentParser(description=description)