Pin dependencies in oldest tests (#5316)

* Add tools/merge_requirements.py

* Revert "Fix oldest tests by pinning Google DNS deps (#5000)"

This reverts commit f68fba2be2.

* Add tools/oldest_constraints.txt

* Remove oldest constraints from tox.ini

* Rename dev constraints file

* Update tools/pip_install.sh

* Update install_and_test.sh

* Fix pip_install.sh

* Don't cat when you can cp

* Add ng-httpsclient to dev constraints for oldest tests

* Bump tested setuptools version

* Update dev_constraints comment

* Better document oldest dependencies

* test against oldest versions we say we require

* Update dev constraints

* Properly handle empty lines

* Update constraints gen in pip_install

* Remove duplicated zope.component

* Reduce pyasn1-modules dependency

* Remove blank line

* pin back google-api-python-client

* pin back uritemplate

* pin josepy for oldest tests

* Undo changes to install_and_test.sh

* Update install_and_test.sh description

* use split instead of partition

tools/dev_constraints.txt

@@ -1,16 +1,15 @@
 # Specifies Python package versions for packages not specified in
-# letsencrypt-auto's requirements file. We should avoid listing packages in
-# both places because if both files are used as constraints for the same pip
-# invocation, some constraints may be ignored due to pip's lack of dependency
-# resolution.
+# letsencrypt-auto's requirements file.
 alabaster==0.7.10
 apipkg==1.4
+asn1crypto==0.22.0
 astroid==1.3.5
+attrs==17.3.0
 Babel==2.5.1
 backports.shutil-get-terminal-size==1.0.0
 boto3==1.4.7
 botocore==1.7.41
-cloudflare==1.8.1
+cloudflare==1.5.1
 coverage==4.4.2
 decorator==4.1.2
 dns-lexicon==2.1.14
@@ -19,7 +18,7 @@ docutils==0.14
 execnet==1.5.0
 future==0.16.0
 futures==3.1.1
-google-api-python-client==1.6.4
+google-api-python-client==1.5
 httplib2==0.10.3
 imagesize==0.7.1
 ipdb==0.10.3
@@ -27,20 +26,22 @@ ipython==5.5.0
 ipython-genutils==0.2.0
 Jinja2==2.9.6
 jmespath==0.9.3
+josepy==1.0.1
+logger==1.4
 logilab-common==1.4.1
 MarkupSafe==1.0
-oauth2client==4.1.2
+ndg-httpsclient==0.3.2
+oauth2client==2.0.0
 pathlib2==2.3.0
 pexpect==4.2.1
 pickleshare==0.7.4
-pkg-resources==0.0.0
 pkginfo==1.4.1
 pluggy==0.5.2
 prompt-toolkit==1.0.15
 ptyprocess==0.5.2
 py==1.4.34
-pyasn1==0.3.7
-pyasn1-modules==0.1.5
+pyasn1==0.1.9
+pyasn1-modules==0.0.10
 Pygments==2.2.0
 pylint==1.4.2
 pytest==3.2.5
@@ -48,7 +49,7 @@ pytest-cov==2.5.1
 pytest-forked==0.2
 pytest-xdist==1.20.1
 python-dateutil==2.6.1
-python-digitalocean==1.12
+python-digitalocean==1.11
 PyYAML==3.12
 repoze.sphinx.autointerface==0.8
 requests-file==1.4.2
@@ -65,6 +66,6 @@ tox==2.9.1
 tqdm==4.19.4
 traitlets==4.3.2
 twine==1.9.1
-uritemplate==3.0.0
+uritemplate==0.6
 virtualenv==15.1.0
 wcwidth==0.1.7

tools/install_and_test.sh

@@ -2,8 +2,9 @@
 # pip installs the requested packages in editable mode and runs unit tests on
 # them. Each package is installed and tested in the order they are provided
 # before the script moves on to the next package. If CERTBOT_NO_PIN is set not
-# set to 1, packages are installed using certbot-auto's requirements file as
-# constraints.
+# set to 1, packages are installed using pinned versions of all of our
+# dependencies. See pip_install.sh for more information on the versions pinned
+# to.
 
 if [ "$CERTBOT_NO_PIN" = 1 ]; then
   pip_install="pip install -q -e"

tools/merge_requirements.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""Merges multiple Python requirements files into one file.
+
+Requirements files specified later take precedence over earlier ones. Only
+simple SomeProject==1.2.3 format is currently supported.
+
+"""
+
+from __future__ import print_function
+
+import sys
+
+
+def read_file(file_path):
+    """Reads in a Python requirements file.
+
+    :param str file_path: path to requirements file
+
+    :returns: mapping from a project to its pinned version
+    :rtype: dict
+
+    """
+    d = {}
+    with open(file_path) as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith('#'):
+                project, version = line.split('==')
+                if not version:
+                    raise ValueError("Unexpected syntax '{0}'".format(line))
+                d[project] = version
+    return d
+
+
+def print_requirements(requirements):
+    """Prints requirements to stdout.
+
+    :param dict requirements: mapping from a project to its pinned version
+
+    """
+    print('\n'.join('{0}=={1}'.format(k, v)
+          for k, v in sorted(requirements.items())))
+
+
+def merge_requirements_files(*files):
+    """Merges multiple requirements files together and prints the result.
+
+    Requirement files specified later in the list take precedence over earlier
+    files.
+
+    :param tuple files: paths to requirements files
+
+    """
+    d = {}
+    for f in files:
+        d.update(read_file(f))
+    print_requirements(d)
+
+
+if __name__ == '__main__':
+    merge_requirements_files(*sys.argv[1:])

tools/oldest_constraints.txt

@@ -0,0 +1,51 @@
+# This file contains the oldest versions of our dependencies we say we require
+# in our packages or versions we need to support to maintain compatibility with
+# the versions included in the various Linux distros where we are packaged.
+
+# CentOS/RHEL 7 EPEL constraints
+cffi==1.6.0
+chardet==2.2.1
+configobj==4.7.2
+ipaddress==1.0.16
+mock==1.0.1
+ndg-httpsclient==0.3.2
+ply==3.4
+pyasn1==0.1.9
+pycparser==2.14
+pyOpenSSL==0.13.1
+pyparsing==1.5.6
+pyRFC3339==1.0
+python-augeas==0.5.0
+six==1.9.0
+# setuptools 0.9.8 is the actual version packaged, but some other dependencies
+# in this file require setuptools>=1.0 and there are no relevant changes for us
+# between these versions.
+setuptools==1.0.0
+urllib3==1.10.2
+zope.component==4.1.0
+zope.event==4.0.3
+zope.interface==4.0.5
+
+# Debian Jessie Backports constraints
+PyICU==1.8
+colorama==0.3.2
+enum34==1.0.3
+html5lib==0.999
+idna==2.0
+pbr==1.8.0
+pytz==2012rc0
+
+# Our setup.py constraints
+cloudflare==1.5.1
+cryptography==1.2.0
+google-api-python-client==1.5
+oauth2client==2.0
+parsedatetime==1.3
+pyparsing==1.5.5
+python-digitalocean==1.11
+requests[security]==2.4.1
+
+# Ubuntu Xenial constraints
+ConfigArgParse==0.10.0
+funcsigs==0.4
+zope.hookable==4.0.4

tools/pip_install.sh

@@ -1,17 +1,26 @@
-#!/bin/sh -e
-# pip installs packages using pinned package versions
+#!/bin/bash -e
+# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
+# to 1, a combination of tools/oldest_constraints.txt and
+# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's
+# requirements file and tools/dev_constraints.txt is used. The other file
+# always takes precedence over tools/dev_constraints.txt.
 
 # get the root of the Certbot repo
-my_path=$("$(dirname $0)/readlink.py" $0)
-repo_root=$(dirname $(dirname $my_path))
-requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
-certbot_auto_constraints=$(mktemp)
-trap "rm -f $certbot_auto_constraints" EXIT
-# extract pinned requirements without hashes
-sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints
-dev_constraints="$(dirname $my_path)/pip_constraints.txt"
+tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0))
+dev_constraints="$tools_dir/dev_constraints.txt"
+merge_reqs="$tools_dir/merge_requirements.py"
+test_constraints=$(mktemp)
+trap "rm -f $test_constraints" EXIT
+
+if [ "$CERTBOT_OLDEST" = 1 ]; then
+    cp "$tools_dir/oldest_constraints.txt" "$test_constraints"
+else
+    repo_root=$(dirname "$tools_dir")
+    certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
+    sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints"
+fi
 
 set -x
 
 # install the requested packages using the pinned requirements as constraints
-pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@"
+pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@"

tox.ini

@@ -11,9 +11,8 @@ envlist = modification,py{26,33,34,35,36},cover,lint
 pip_install = {toxinidir}/tools/pip_install_editable.sh
 # pip installs the requested packages in editable mode and runs unit tests on
 # them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
-# set to 1, packages are installed using certbot-auto's requirements file as
-# constraints.
+# before the script moves on to the next package. All dependencies are pinned
+# to a specific version for increased stability for developers.
 install_and_test = {toxinidir}/tools/install_and_test.sh
 py26_packages =
     acme[dev] \
@@ -82,36 +81,7 @@ commands =
     {[testenv]commands}
 setenv =
     {[testenv]setenv}
-    CERTBOT_NO_PIN=1
-deps =
-    PyOpenSSL==0.13
-    cffi==1.5.2
-    configargparse==0.10.0
-    configargparse==0.10.0
-    configobj==4.7.2
-    cryptography==1.2.3
-    enum34==0.9.23
-    google-api-python-client==1.5
-    idna==2.0
-    ipaddress==1.0.16
-    mock==1.0.1
-    ndg-httpsclient==0.3.2
-    oauth2client==2.0
-    parsedatetime==1.4
-    pyasn1-modules==0.0.5
-    pyasn1==0.1.9
-    pyparsing==1.5.6
-    pyrfc3339==1.0
-    pytest==3.2.5
-    python-augeas==0.4.1
-    pytz==2012c
-    requests[security]==2.6.0
-    setuptools==0.9.8
-    six==1.9.0
-    urllib3==1.10
-    zope.component==4.0.2
-    zope.event==4.0.1
-    zope.interface==4.0.5
+    CERTBOT_OLDEST=1
 
 [testenv:py27_install]
 basepython = python2.7