Finish dropping Python 2.6 and 3.3 support

* Undo letsencrypt-auto changes

* Remove ordereddict import

* Add Python 3.4 tests to replace 3.3

* Add python_requires

* update pipstrap

.travis.yml

@@ -29,6 +29,10 @@ matrix:
       env: TOXENV=py27-oldest
       sudo: required
       services: docker
+    - python: "3.4"
+      env: TOXENV=py34
+      sudo: required
+      services: docker
     - python: "3.6"
       env: TOXENV=py36
       sudo: required

acme/setup.py

@@ -45,6 +45,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',

certbot-apache/setup.py

@@ -32,6 +32,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-compatibility-test/setup.py

@@ -34,6 +34,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',

certbot-dns-cloudflare/setup.py

@@ -31,6 +31,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-cloudxns/setup.py

@@ -31,6 +31,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-digitalocean/setup.py

@@ -32,6 +32,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-dnsimple/setup.py

@@ -31,6 +31,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-dnsmadeeasy/setup.py

@@ -31,6 +31,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-google/setup.py

@@ -36,6 +36,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-luadns/setup.py

@@ -31,6 +31,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-nsone/setup.py

@@ -31,6 +31,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-rfc2136/setup.py

@@ -31,6 +31,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-dns-route53/setup.py

@@ -24,6 +24,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot-nginx/setup.py

@@ -32,6 +32,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',

certbot/plugins/disco.py

@@ -5,6 +5,8 @@ import logging
 import pkg_resources
 import six
 
+from collections import OrderedDict
+
 import zope.interface
 import zope.interface.verify
 
@@ -12,12 +14,6 @@ from certbot import constants
 from certbot import errors
 from certbot import interfaces
 
-try:
-    from collections import OrderedDict
-except ImportError:  # pragma: no cover
-    # OrderedDict was added in Python 2.7
-    from ordereddict import OrderedDict  # pylint: disable=import-error
-
 
 logger = logging.getLogger(__name__)
 

certbot/util.py

@@ -16,18 +16,14 @@ import stat
 import subprocess
 import sys
 
+from collections import OrderedDict
+
 import configargparse
 
 from certbot import constants
 from certbot import errors
 from certbot import lock
 
-try:
-    from collections import OrderedDict
-except ImportError:  # pragma: no cover
-    # OrderedDict was added in Python 2.7
-    from ordereddict import OrderedDict  # pylint: disable=import-error
-
 
 logger = logging.getLogger(__name__)
 

letsencrypt-auto-source/letsencrypt-auto

@@ -781,11 +781,20 @@ elif [ -f /etc/redhat-release ]; then
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
     Bootstrap() {
       BootstrapMessage "RedHat-based OSes"
       BootstrapRpmCommon
     }
     BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
@@ -956,11 +965,19 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
+    else
       if [ "$VERBOSE" = 1 ]; then
         "$LE_PYTHON" -m venv "$VENV_PATH"
       else
         "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
       fi
+    fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
       echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH"
@@ -1220,6 +1237,7 @@ anything goes wrong, it will exit with a non-zero status code.
 from __future__ import print_function
 from distutils.version import StrictVersion
 from hashlib import sha256
+from os import environ
 from os.path import join
 from pipes import quote
 from shutil import rmtree
@@ -1253,33 +1271,32 @@ except ImportError:
     from urllib.parse import urlparse  # 3.4
 
 
-__version__ = 1, 3, 0
+__version__ = 1, 5, 0
 PIP_VERSION = '9.0.1'
+DEFAULT_INDEX_BASE = 'https://pypi.python.org'
 
 
 # wheel has a conditional dependency on argparse:
 maybe_argparse = (
-    [('https://pypi.python.org/packages/18/dd/'
+    [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
-      'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
       'argparse-1.4.0.tar.gz',
       '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
     if version_info < (2, 7, 0) else [])
 
 
-PACKAGES = maybe_argparse + [
+# Pip has no dependencies, as it vendors everything:
-    # Pip has no dependencies, as it vendors everything:
+PIP_PACKAGE = [
-    ('https://pypi.python.org/packages/11/b6/'
+    ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
-     'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
+     'pip-{0}.tar.gz'.format(PIP_VERSION),
-     'pip-{0}.tar.gz'
+     '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')]
-     .format(PIP_VERSION),
+
-     '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'),
+
+OTHER_PACKAGES = maybe_argparse + [
     # This version of setuptools has only optional dependencies:
-    ('https://pypi.python.org/packages/69/65/'
+    ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/'
-     '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/'
+     'setuptools-29.0.1.tar.gz',
-     'setuptools-20.2.2.tar.gz',
+     'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'),
-     '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'),
+    ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
-    ('https://pypi.python.org/packages/c9/1d/'
-     'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
      'wheel-0.29.0.tar.gz',
      '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648')
 ]
@@ -1300,8 +1317,9 @@ def hashed_download(url, temp, digest):
     # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert
     # authenticity has only privacy (not arbitrary code execution)
     # implications, since we're checking hashes.
-    def opener():
+    def opener(using_https=True):
         opener = build_opener(HTTPSHandler())
+        if using_https:
             # Strip out HTTPHandler to prevent MITM spoof:
             for handler in opener.handlers:
                 if isinstance(handler, HTTPHandler):
@@ -1315,8 +1333,9 @@ def hashed_download(url, temp, digest):
                 break
             yield chunk
 
-    response = opener().open(url)
+    parsed_url = urlparse(url)
-    path = join(temp, urlparse(url).path.split('/')[-1])
+    response = opener(using_https=parsed_url.scheme == 'https').open(url)
+    path = join(temp, parsed_url.path.split('/')[-1])
     actual_hash = sha256()
     with open(path, 'wb') as file:
         for chunk in read_chunks(response, 4096):
@@ -1329,6 +1348,24 @@ def hashed_download(url, temp, digest):
     return path
 
 
+def get_index_base():
+    """Return the URL to the dir containing the "packages" folder.
+
+    Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the
+    end if it's there; that is likely to give us the right dir.
+
+    """
+    env_var = environ.get('PIP_INDEX_URL', '').rstrip('/')
+    if env_var:
+        SIMPLE = '/simple'
+        if env_var.endswith(SIMPLE):
+            return env_var[:-len(SIMPLE)]
+        else:
+            return env_var
+    else:
+        return DEFAULT_INDEX_BASE
+
+
 def main():
     pip_version = StrictVersion(check_output(['pip', '--version'])
                                 .decode('utf-8').split()[1])
@@ -1336,14 +1373,21 @@ def main():
     if pip_version >= min_pip_version:
         return 0
     has_pip_cache = pip_version >= StrictVersion('6.0')
-
+    index_base = get_index_base()
     temp = mkdtemp(prefix='pipstrap-')
     try:
-        downloads = [hashed_download(url, temp, digest)
+        # We download and install pip first, then the rest, to avoid the bug
-                     for url, digest in PACKAGES]
+        # https://github.com/certbot/certbot/issues/4938.
+        pip_downloads, other_downloads = [
+            [hashed_download(index_base + '/packages/' + path,
+                             temp,
+                             digest)
+             for path, digest in packages]
+            for packages in (PIP_PACKAGE, OTHER_PACKAGES)]
+        for downloads in (pip_downloads, other_downloads):
             check_output('pip install --no-index --no-deps -U ' +
-                     # Disable cache since we're not using it and it otherwise
+                         # Disable cache since we're not using it and it
-                     # sometimes throws permission warnings:
+                         # otherwise sometimes throws permission warnings:
                          ('--no-cache-dir ' if has_pip_cache else '') +
                          ' '.join(quote(d) for d in downloads),
                          shell=True)

letsencrypt-auto-source/letsencrypt-auto.template

@@ -320,11 +320,20 @@ elif [ -f /etc/redhat-release ]; then
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
     Bootstrap() {
       BootstrapMessage "RedHat-based OSes"
       BootstrapRpmCommon
     }
     BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
   LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
@@ -495,11 +504,19 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
+    else
       if [ "$VERBOSE" = 1 ]; then
         "$LE_PYTHON" -m venv "$VENV_PATH"
       else
         "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
       fi
+    fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
       echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH"

letsencrypt-auto-source/pieces/pipstrap.py

@@ -23,6 +23,7 @@ anything goes wrong, it will exit with a non-zero status code.
 from __future__ import print_function
 from distutils.version import StrictVersion
 from hashlib import sha256
+from os import environ
 from os.path import join
 from pipes import quote
 from shutil import rmtree
@@ -56,33 +57,32 @@ except ImportError:
     from urllib.parse import urlparse  # 3.4
 
 
-__version__ = 1, 3, 0
+__version__ = 1, 5, 0
 PIP_VERSION = '9.0.1'
+DEFAULT_INDEX_BASE = 'https://pypi.python.org'
 
 
 # wheel has a conditional dependency on argparse:
 maybe_argparse = (
-    [('https://pypi.python.org/packages/18/dd/'
+    [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
-      'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
       'argparse-1.4.0.tar.gz',
       '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
     if version_info < (2, 7, 0) else [])
 
 
-PACKAGES = maybe_argparse + [
+# Pip has no dependencies, as it vendors everything:
-    # Pip has no dependencies, as it vendors everything:
+PIP_PACKAGE = [
-    ('https://pypi.python.org/packages/11/b6/'
+    ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
-     'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
+     'pip-{0}.tar.gz'.format(PIP_VERSION),
-     'pip-{0}.tar.gz'
+     '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')]
-     .format(PIP_VERSION),
+
-     '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'),
+
+OTHER_PACKAGES = maybe_argparse + [
     # This version of setuptools has only optional dependencies:
-    ('https://pypi.python.org/packages/69/65/'
+    ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/'
-     '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/'
+     'setuptools-29.0.1.tar.gz',
-     'setuptools-20.2.2.tar.gz',
+     'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'),
-     '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'),
+    ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
-    ('https://pypi.python.org/packages/c9/1d/'
-     'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
      'wheel-0.29.0.tar.gz',
      '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648')
 ]
@@ -103,8 +103,9 @@ def hashed_download(url, temp, digest):
     # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert
     # authenticity has only privacy (not arbitrary code execution)
     # implications, since we're checking hashes.
-    def opener():
+    def opener(using_https=True):
         opener = build_opener(HTTPSHandler())
+        if using_https:
             # Strip out HTTPHandler to prevent MITM spoof:
             for handler in opener.handlers:
                 if isinstance(handler, HTTPHandler):
@@ -118,8 +119,9 @@ def hashed_download(url, temp, digest):
                 break
             yield chunk
 
-    response = opener().open(url)
+    parsed_url = urlparse(url)
-    path = join(temp, urlparse(url).path.split('/')[-1])
+    response = opener(using_https=parsed_url.scheme == 'https').open(url)
+    path = join(temp, parsed_url.path.split('/')[-1])
     actual_hash = sha256()
     with open(path, 'wb') as file:
         for chunk in read_chunks(response, 4096):
@@ -132,6 +134,24 @@ def hashed_download(url, temp, digest):
     return path
 
 
+def get_index_base():
+    """Return the URL to the dir containing the "packages" folder.
+
+    Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the
+    end if it's there; that is likely to give us the right dir.
+
+    """
+    env_var = environ.get('PIP_INDEX_URL', '').rstrip('/')
+    if env_var:
+        SIMPLE = '/simple'
+        if env_var.endswith(SIMPLE):
+            return env_var[:-len(SIMPLE)]
+        else:
+            return env_var
+    else:
+        return DEFAULT_INDEX_BASE
+
+
 def main():
     pip_version = StrictVersion(check_output(['pip', '--version'])
                                 .decode('utf-8').split()[1])
@@ -139,14 +159,21 @@ def main():
     if pip_version >= min_pip_version:
         return 0
     has_pip_cache = pip_version >= StrictVersion('6.0')
-
+    index_base = get_index_base()
     temp = mkdtemp(prefix='pipstrap-')
     try:
-        downloads = [hashed_download(url, temp, digest)
+        # We download and install pip first, then the rest, to avoid the bug
-                     for url, digest in PACKAGES]
+        # https://github.com/certbot/certbot/issues/4938.
+        pip_downloads, other_downloads = [
+            [hashed_download(index_base + '/packages/' + path,
+                             temp,
+                             digest)
+             for path, digest in packages]
+            for packages in (PIP_PACKAGE, OTHER_PACKAGES)]
+        for downloads in (pip_downloads, other_downloads):
             check_output('pip install --no-index --no-deps -U ' +
-                     # Disable cache since we're not using it and it otherwise
+                         # Disable cache since we're not using it and it
-                     # sometimes throws permission warnings:
+                         # otherwise sometimes throws permission warnings:
                          ('--no-cache-dir ' if has_pip_cache else '') +
                          ' '.join(quote(d) for d in downloads),
                          shell=True)

letshelp-certbot/setup.py

@@ -24,6 +24,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: System Administrators',

setup.py

@@ -82,6 +82,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Console',