Pin dependency versions when using tools/venv.sh (#4629)

* Revert "Pin python-augeas version to avoid error with 1.0.0 (#4422)"

This reverts commit 1c51ae2588.

* make dependency-requirements

* separate certbot and dependency requirements

* fix build.py

* update hashin comment

* simplify release pinning

* separate letsencrypt dependency

* pin hashes in venv

* error out when bad things happen

* use pinned dependencies in tox

* Revert "pin hashes in venv"

This reverts commit 1cd38a9e50.

* use pip_install.sh in venv_common

* quote pip install args

* bump mock version

certbot-apache/setup.py

@@ -11,7 +11,7 @@ install_requires = [
     'acme=={0}'.format(version),
     'certbot=={0}'.format(version),
     'mock',
-    'python-augeas<=0.5.0',
+    'python-augeas',
     # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
     # will tolerate; see #2599:
     'setuptools>=1.0',

letsencrypt-auto-source/build.py

@@ -21,14 +21,17 @@ def build(version=None, requirements=None):
     :arg version: The version to attach to the script. Default: the version of
         the certbot package
     :arg requirements: The contents of the requirements file to embed. Default:
-        contents of letsencrypt-auto-requirements.txt
+        contents of dependency-requirements.txt, letsencrypt-requirements.txt,
+        and certbot-requirements.txt
 
     """
     special_replacements = {
         'LE_AUTO_VERSION': version or certbot_version(DIR)
     }
     if requirements:
-        special_replacements['letsencrypt-auto-requirements.txt'] = requirements
+        special_replacements['dependency-requirements.txt'] = ''
+        special_replacements['letsencrypt-requirements.txt'] = ''
+        special_replacements['certbot-requirements.txt'] = requirements
 
     def replacer(match):
         token = match.group(1)

letsencrypt-auto-source/letsencrypt-auto

@@ -694,7 +694,7 @@ if [ "$1" = "--le-auto-phase2" ]; then
 
 # Hashin example:
 # pip install hashin
-# hashin -r letsencrypt-auto-requirements.txt cryptography==1.5.2
+# hashin -r dependency-requirements.txt cryptography==1.5.2
 # sets the new certbot-auto pinned version of cryptography to 1.5.2
 
 argparse==1.4.0 \
@@ -754,9 +754,9 @@ cryptography==1.5.3 \
 enum34==1.1.2 \
     --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
     --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
-funcsigs==0.4 \
+funcsigs==1.0.2 \
-    --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \
+    --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
-    --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033
+    --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
 idna==2.0 \
     --hash=sha256:9b2fc50bd3c4ba306b9651b69411ef22026d4d8335b93afc2214cef1246ce707 \
     --hash=sha256:16199aad938b290f5be1057c0e1efc6546229391c23cea61ca940c115f7d3d3b
@@ -851,15 +851,21 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-mock==1.0.1 \
+mock==2.0.0 \
-    --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \
+    --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
-    --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc
+    --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
+
+# Contains the requirements for the letsencrypt package.
+#
+# Since the letsencrypt package depends on certbot and using pip with hashes
+# requires that all installed packages have hashes listed, this allows
+# dependency-requirements.txt to be used without requiring a hash for a
+# (potentially unreleased) Certbot package.
+
 letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-# THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE.
-
 acme==0.14.0 \
     --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \
     --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543

letsencrypt-auto-source/letsencrypt-auto.template

@@ -317,7 +317,9 @@ if [ "$1" = "--le-auto-phase2" ]; then
     # There is no $ interpolation due to quotes on starting heredoc delimiter.
     # -------------------------------------------------------------------------
     cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt"
-{{ letsencrypt-auto-requirements.txt }}
+{{ dependency-requirements.txt }}
+{{ letsencrypt-requirements.txt }}
+{{ certbot-requirements.txt }}
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
     cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py"

letsencrypt-auto-source/pieces/certbot-requirements.txt

@@ -0,0 +1,12 @@
+acme==0.14.0 \
+    --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \
+    --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543
+certbot==0.14.0 \
+    --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \
+    --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741
+certbot-apache==0.14.0 \
+    --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \
+    --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f
+certbot-nginx==0.14.0 \
+    --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \
+    --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db

letsencrypt-auto-source/pieces/dependency-requirements.txt

@@ -5,7 +5,7 @@
 
 # Hashin example:
 # pip install hashin
-# hashin -r letsencrypt-auto-requirements.txt cryptography==1.5.2
+# hashin -r dependency-requirements.txt cryptography==1.5.2
 # sets the new certbot-auto pinned version of cryptography to 1.5.2
 
 argparse==1.4.0 \
@@ -65,9 +65,9 @@ cryptography==1.5.3 \
 enum34==1.1.2 \
     --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
     --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
-funcsigs==0.4 \
+funcsigs==1.0.2 \
-    --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \
+    --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
-    --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033
+    --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
 idna==2.0 \
     --hash=sha256:9b2fc50bd3c4ba306b9651b69411ef22026d4d8335b93afc2214cef1246ce707 \
     --hash=sha256:16199aad938b290f5be1057c0e1efc6546229391c23cea61ca940c115f7d3d3b
@@ -162,24 +162,6 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-mock==1.0.1 \
+mock==2.0.0 \
-    --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \
+    --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
-    --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc
+    --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
-letsencrypt==0.7.0 \
-    --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
-    --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
-
-# THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE.
-
-acme==0.14.0 \
-    --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \
-    --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543
-certbot==0.14.0 \
-    --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \
-    --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741
-certbot-apache==0.14.0 \
-    --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \
-    --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f
-certbot-nginx==0.14.0 \
-    --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \
-    --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db

letsencrypt-auto-source/pieces/letsencrypt-requirements.txt

@@ -0,0 +1,10 @@
+# Contains the requirements for the letsencrypt package.
+#
+# Since the letsencrypt package depends on certbot and using pip with hashes
+# requires that all installed packages have hashes listed, this allows
+# dependency-requirements.txt to be used without requiring a hash for a
+# (potentially unreleased) Certbot package.
+
+letsencrypt==0.7.0 \
+    --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
+    --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9

tools/_venv_common.sh

@@ -19,7 +19,7 @@ virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS
 # invocations use latest
 pip install -U pip
 pip install -U setuptools
-pip install "$@"
+./tools/pip_install.sh "$@"
 
 set +x
 echo "Please run the following command to activate developer environment:"

tools/pip_install.sh

@@ -0,0 +1,13 @@
+#!/bin/sh -e
+# pip installs packages using Certbot's requirements file as constraints
+
+# get the root of the Certbot repo
+repo_root=$(git rev-parse --show-toplevel)
+requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
+constraints=$(mktemp)
+trap "rm -f $constraints" EXIT
+# extract pinned requirements without hashes
+sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $constraints
+
+# install the requested packages using the pinned requirements as constraints
+pip install --constraint $constraints "$@"

tools/release.sh

@@ -170,19 +170,14 @@ cd ~-
 for pkg in acme certbot certbot-apache certbot-nginx ; do
     echo $pkg==$version \\
     pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print "   ", input.replace("\n--hash", " \\\n    --hash"),'
-done > /tmp/hashes.$$
+done > letsencrypt-auto-source/pieces/certbot-requirements.txt
 deactivate
 
-if ! wc -l /tmp/hashes.$$ | grep -qE "^\s*12 " ; then
+if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*12 " ; then
     echo Unexpected pip hash output
     exit 1
 fi
 
-# perform hideous surgery on requirements.txt...
-head -n -12 letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt > /tmp/req.$$
-cat /tmp/hashes.$$ >> /tmp/req.$$
-cp /tmp/req.$$ letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt
-
 # ensure we have the latest built version of leauto
 letsencrypt-auto-source/build.py
 

tox.ini

@@ -10,20 +10,22 @@ envlist = modification,py{26,33,34,35,36},cover,lint
 # loops, especially on Travis
 
 [base]
+# wraps pip install to use pinned versions of dependencies
+pip_install = {toxinidir}/tools/pip_install.sh
 # packages installed separately to ensure that downstream deps problems
 # are detected, c.f. #1002
 core_commands =
-    pip install -e acme[dev]
+    {[base]pip_install} -e acme[dev]
     nosetests -v acme --processes=-1
-    pip install -e .[dev]
+    {[base]pip_install} -e .[dev]
     nosetests -v certbot --processes=-1 --process-timeout=100
 core_install_args = -e acme[dev] -e .[dev]
 core_paths = acme/acme certbot
 
 plugin_commands =
-    pip install -e certbot-apache
+    {[base]pip_install} -e certbot-apache
     nosetests -v certbot_apache --processes=-1 --process-timeout=80
-    pip install -e certbot-nginx
+    {[base]pip_install} -e certbot-nginx
     nosetests -v certbot_nginx --processes=-1
 plugin_install_args = -e certbot-apache -e certbot-nginx
 plugin_paths = certbot-apache/certbot_apache certbot-nginx/certbot_nginx
@@ -38,7 +40,7 @@ compatibility_install_args = -e certbot-compatibility-test
 compatibility_paths = certbot-compatibility-test/certbot_compatibility_test
 
 other_commands =
-    pip install -e letshelp-certbot
+    {[base]pip_install} -e letshelp-certbot
     nosetests -v letshelp_certbot --processes=-1
     python tests/lock_test.py
 other_install_args = -e letshelp-certbot
@@ -69,12 +71,12 @@ deps =
 [testenv:py27_install]
 basepython = python2.7
 commands =
-    pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args}
+    {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args}
 
 [testenv:cover]
 basepython = python2.7
 commands =
-    pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args}
+    {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args}
     ./tox.cover.sh
 
 [testenv:lint]
@@ -84,25 +86,25 @@ basepython = python2.7
 # duplicate code checking; if one of the commands fails, others will
 # continue, but tox return code will reflect previous error
 commands =
-    pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args}
+    {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args}
     pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths}
 
 [testenv:mypy]
 basepython = python3.4
 commands =
-    pip install mypy
+    {[base]pip_install} mypy
-    pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args}
+    {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args}
     mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths}
 
 [testenv:apacheconftest]
 #basepython = python2.7
 commands =
-    pip install {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args}
+    {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args}
     {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules
 
 [testenv:nginxroundtrip]
 commands =
-    pip install {[base]core_install_args} -e certbot-nginx
+    {[base]pip_install} {[base]core_install_args} -e certbot-nginx
     python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata
 
 # This is a duplication of the command line in testenv:le_auto to