Write a replacement for pipstrap (#8401)

* Add a new, simplified version of pipstrap.

* Use tools/pipstrap.py

* Uncomment code

* Refactor pip_install.py and provide hashes.

* Fix test_sdists.sh.

* Make code work on Python 2.

* Call strip_hashes.py using Python 3.

* Pin the oldest version of httplib2 used in distros

* Strip enum34 dependency.

* Remove pip pinnings from dev_constraints.txt

* Correct pipstrap docstring.

* Don't set working_dir twice.

* Add comments

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -98,13 +98,9 @@ jobs:
           artifact: windows-installer
           path: $(Build.SourcesDirectory)/bin
         displayName: Retrieve Windows installer
-      # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency
-      # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install
-      # a recent version of pip, but we also to disable the isolated feature as described in
-      # https://github.com/certbot/certbot/issues/8256
       - script: |
           python -m venv venv
-          venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1
+          venv\Scripts\python tools\pipstrap.py
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         env:
           PIP_NO_BUILD_ISOLATION: no
@@ -174,7 +170,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends nginx-light snapd
           python3 -m venv venv
-          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+          venv/bin/python tools/pipstrap.py
           venv/bin/python tools/pip_install.py -U tox
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
@@ -212,7 +208,7 @@ jobs:
       - script: |
           set -e
           python3 -m venv venv
-          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+          venv/bin/python tools/pipstrap.py
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |

.azure-pipelines/templates/steps/tox-steps.yml

@@ -32,7 +32,7 @@ steps:
     # problems with its lack of real dependency resolution.
   - bash: |
       set -e
-      python letsencrypt-auto-source/pieces/pipstrap.py
+      python tools/pipstrap.py
       python tools/pip_install.py -I tox virtualenv
     displayName: Install runtime dependencies
   - task: DownloadSecureFile@1

tests/letstest/scripts/test_sdists.sh

@@ -15,7 +15,16 @@ if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | se
 fi
 
 # setup venv
-CERTBOT_PIP_NO_BINARY=:all: tools/venv3.py --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt
+# We strip the hashes because the venv creation script includes unhashed
+# constraints in the commands given to pip and the mix of hashed and unhashed
+# packages makes pip error out.
+python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > requirements.txt
+# We also strip out the requirement for enum34 because it cannot be installed
+# in newer versions of Python 3, tools/strip_hashes.py removes the environment
+# marker that'd normally prevent it from being installed, and this package is
+# not needed for any OS tested here.
+sed -i '/enum34/d' requirements.txt
+CERTBOT_PIP_NO_BINARY=:all: tools/venv3.py --requirement requirements.txt
 . "$VENV_PATH/bin/activate"
 # pytest is needed to run tests on some of our packages so we install a pinned version here.
 tools/pip_install.py pytest

tools/_venv_common.py

@@ -200,7 +200,7 @@ def install_packages(venv_name, pip_args):
     """
     # Using the python executable from venv, we ensure to execute following commands in this venv.
     py_venv = get_venv_python_path(venv_name)
-    subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')])
+    subprocess_with_print([py_venv, os.path.abspath('tools/pipstrap.py')])
     # We only use this value during pip install because:
     # 1) We're really only adding it for installing cryptography, which happens here, and
     # 2) There are issues with calling it along with VIRTUALENV_NO_DOWNLOAD, which applies at the

tools/docker/core/Dockerfile

@@ -14,16 +14,14 @@ WORKDIR /opt/certbot
 
 # Copy certbot code
 COPY CHANGELOG.md README.rst src/
-COPY letsencrypt-auto-source/pieces/dependency-requirements.txt .
+# We keep the relative path to the requirements file the same because, as of
-COPY letsencrypt-auto-source/pieces/pipstrap.py .
+# writing this, tools/pip_install.py is used in the Dockerfile for Certbot
+# plugins and this script expects to find the requirements file there.
+COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/
 COPY tools tools
 COPY acme src/acme
 COPY certbot src/certbot
 
-# Generate constraints file to pin dependency versions
-RUN cat dependency-requirements.txt | tools/strip_hashes.py > unhashed_requirements.txt \
- && cat tools/dev_constraints.txt unhashed_requirements.txt | tools/merge_requirements.py > docker_constraints.txt
-
 # Install certbot runtime dependencies
 RUN apk add --no-cache --virtual .certbot-deps \
         libffi \
@@ -33,15 +31,20 @@ RUN apk add --no-cache --virtual .certbot-deps \
         binutils
 
 # Install certbot from sources
+#
+# We don't use tools/pip_install.py below so the hashes in
+# dependency-requirements.txt can be used when installing packages for extra
+# security.
 RUN apk add --no-cache --virtual .build-deps \
         gcc \
         linux-headers \
         openssl-dev \
         musl-dev \
         libffi-dev \
-    && python pipstrap.py \
+    && python tools/pipstrap.py \
-    && pip install -r dependency-requirements.txt \
+    && pip install --no-build-isolation \
-    && pip install --no-cache-dir --no-deps \
+        -r letsencrypt-auto-source/pieces/dependency-requirements.txt \
+    && pip install --no-build-isolation --no-cache-dir --no-deps \
         --editable src/acme \
         --editable src/certbot \
 && apk del .build-deps

tools/docker/plugin/Dockerfile

@@ -11,4 +11,4 @@ COPY qemu-${QEMU_ARCH}-static /usr/bin/
 COPY . /opt/certbot/src/plugin
 
 # Install the DNS plugin
-RUN pip install --constraint /opt/certbot/docker_constraints.txt --no-cache-dir --editable /opt/certbot/src/plugin
+RUN tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin

tools/oldest_constraints.txt

@@ -60,6 +60,7 @@ distro==1.0.1
 # Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins
 # using their local-oldest-requirements.txt
 dns-lexicon==2.2.1
+httplib2==0.9.2
 
 # Plugin constraints
 # These aren't necessarily the oldest versions we need to support

tools/pip_install.py

@@ -11,6 +11,7 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import contextlib
 import os
 import re
 import shutil
@@ -23,6 +24,17 @@ import readlink
 import strip_hashes
 
 
+# Once this code doesn't need to support Python 2, we can simply use
+# tempfile.TemporaryDirectory.
+@contextlib.contextmanager
+def temporary_directory():
+    dirpath = tempfile.mkdtemp()
+    try:
+        yield dirpath
+    finally:
+        shutil.rmtree(dirpath)
+
+
 def find_tools_path():
     return os.path.dirname(readlink.main(__file__))
 
@@ -75,22 +87,18 @@ def call_with_print(command):
     subprocess.check_call(command, shell=True)
 
 
-def pip_install_with_print(args_str):
+def pip_install_with_print(args_str, disable_build_isolation=True):
-    command = '"{0}" -m pip install --disable-pip-version-check {1}'.format(sys.executable,
+    command = ['"', sys.executable, '" -m pip install --disable-pip-version-check ']
-                                                                            args_str)
+    if disable_build_isolation:
-    call_with_print(command)
+        command.append('--no-build-isolation ')
+    command.append(args_str)
+    call_with_print(''.join(command))
 
 
 def main(args):
     tools_path = find_tools_path()
-    working_dir = tempfile.mkdtemp()
 
-    if os.environ.get('TRAVIS'):
+    with temporary_directory() as working_dir:
-        # When this script is executed on Travis, the following print will make the log
-        # be folded until the end command is printed (see finally section).
-        print('travis_fold:start:install_certbot_deps')
-
-    try:
         test_constraints = os.path.join(working_dir, 'test_constraints.txt')
         all_constraints = os.path.join(working_dir, 'all_constraints.txt')
 
@@ -119,10 +127,6 @@ def main(args):
 
             pip_install_with_print('--constraint "{0}" {1}'.format(
                 all_constraints, ' '.join(args)))
-    finally:
-        if os.environ.get('TRAVIS'):
-            print('travis_fold:end:install_certbot_deps')
-        shutil.rmtree(working_dir)
 
 
 if __name__ == '__main__':

tools/pipstrap.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""Uses pip to upgrade Python packaging tools to pinned versions."""
+from __future__ import absolute_import
+
+import os
+import shutil
+import tempfile
+
+import pip_install
+
+
+# We include the hashes of the packages here for extra verification of
+# the packages downloaded from PyPI. This is especially valuable in our
+# builds of Certbot that we ship to our users such as our Docker images.
+#
+# An older version of setuptools is currently used here in order to keep
+# compatibility with Python 2 since newer versions of setuptools have dropped
+# support for it.
+REQUIREMENTS = r"""
+pip==20.2.4 \
+    --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \
+    --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1
+setuptools==44.1.1 \
+    --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \
+    --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b
+wheel==0.35.1 \
+    --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \
+    --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f
+"""
+
+
+def main():
+    with pip_install.temporary_directory() as tempdir:
+        requirements_filepath = os.path.join(tempdir, 'reqs.txt')
+        with open(requirements_filepath, 'w') as f:
+            f.write(REQUIREMENTS)
+        pip_install_args = '--requirement ' + requirements_filepath
+        # We don't disable build isolation because we may have an older
+        # version of pip that doesn't support the flag disabling it. We
+        # expect these packages to already have usable wheels available
+        # anyway so no building should be required.
+        pip_install.pip_install_with_print(pip_install_args,
+                                           disable_build_isolation=False)
+
+
+if __name__ == '__main__':
+    main()

tools/run_oldest_tests.sh

@@ -16,19 +16,20 @@ DOCKERFILE=$(mktemp /tmp/Dockerfile.XXXXXX)
 
 cat << "EOF" >> "${DOCKERFILE}"
 FROM ubuntu:16.04
-COPY pipstrap.py /tmp/pipstrap.py
+COPY letsencrypt-auto-source/pieces/dependency-requirements.txt /tmp/letsencrypt-auto-source/pieces/
+COPY tools/ /tmp/tools/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
         python-dev python-pip python-setuptools \
         gcc libaugeas0 libssl-dev libffi-dev \
         git ca-certificates nginx-light openssl curl \
  && curl -fsSL https://get.docker.com | bash /dev/stdin \
- && python /tmp/pipstrap.py \
+ && python /tmp/tools/pipstrap.py \
- && python -m pip install tox \
+ && python /tmp/tools/pip_install.py tox \
  && rm -rf /var/lib/apt/lists/*
 EOF
 
-docker build -f "${DOCKERFILE}" -t oldest-worker ./letsencrypt-auto-source/pieces
+docker build -f "${DOCKERFILE}" -t oldest-worker .
 docker run --rm --network=host -w "${PWD}" \
   -v /var/run/docker.sock:/var/run/docker.sock \
   -v "${PWD}:${PWD}" -v /tmp:/tmp \

tox.ini

@@ -62,7 +62,7 @@ source_paths =
 [testenv]
 passenv =
     CERTBOT_NO_PIN
-commands_pre = python {toxinidir}/letsencrypt-auto-source/pieces/pipstrap.py
+commands_pre = python {toxinidir}/tools/pipstrap.py
 commands =
     !cover: {[base]install_and_test} {[base]all_packages}
     !cover: python tests/lock_test.py

windows-installer/construct.py

@@ -54,7 +54,7 @@ def _compile_wheels(repo_path, build_path, venv_python):
 def _prepare_build_tools(venv_path, venv_python, repo_path):
     print('Prepare build tools')
     subprocess.check_call([sys.executable, '-m', 'venv', venv_path])
-    subprocess.check_call([venv_python, os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'pipstrap.py')])
+    subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pipstrap.py')])
     subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pip_install.py'), 'pynsist'])
     subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION])