Always "pipstrap" when running pip_install.py (#9658)

Fixes https://github.com/certbot/certbot/issues/7921.

In all cases when we run `pip_install.py`, we first run `pipstrap.py`. This PR combines these two steps for convenience and to make always doing that less error prone. This will also help me with some of the `tox.ini` refactoring I'm planning to do.

I ran the full test suite on everything and tested the release script changes locally.

This change shouldn't have any effect on cryptography's setup because they install `certbot[test]` which depends on pip, setuptools, and wheel.

* always pipstrap

* use pip_install.py during releases

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -62,7 +62,6 @@ jobs:
           addToPath: true
       - script: |
           python -m venv venv
-          venv\Scripts\python tools\pipstrap.py
           venv\Scripts\python tools\pip_install.py -e windows-installer
         displayName: Prepare Windows installer build environment
       - script: |
@@ -99,7 +98,6 @@ jobs:
         displayName: Retrieve Windows installer
       - script: |
           python -m venv venv
-          venv\Scripts\python tools\pipstrap.py
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         env:
           PIP_NO_BUILD_ISOLATION: no
@@ -171,7 +169,6 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends nginx-light snapd
           python3 -m venv venv
-          venv/bin/python tools/pipstrap.py
           venv/bin/python tools/pip_install.py -U tox
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
@@ -209,7 +206,6 @@ jobs:
       - script: |
           set -e
           python3 -m venv venv
-          venv/bin/python tools/pipstrap.py
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |

.azure-pipelines/templates/steps/tox-steps.yml

@@ -30,7 +30,6 @@ steps:
       addToPath: true
   - bash: |
       set -e
-      python3 tools/pipstrap.py
       python3 tools/pip_install.py tox
     displayName: Install runtime dependencies
   - task: DownloadSecureFile@1

tools/_release.sh

@@ -76,14 +76,8 @@ git tag --delete "$tag" || true
 tmpvenv=$(mktemp -d)
 python3 -m venv "$tmpvenv"
 . $tmpvenv/bin/activate
-# update setuptools/pip just like in other places in the repo
+# update packaging tools to their pinned versions
-pip install -U setuptools
+tools/pip_install.py virtualenv
-pip install -U pip  # latest pip => no --pre for dev releases
-pip install -U wheel  # setup.py bdist_wheel
-
-# newer versions of virtualenv inherit setuptools/pip/wheel versions
-# from current env when creating a child env
-pip install -U virtualenv
 
 root_without_le="$version.$$"
 root="$RELEASE_DIR/le.$root_without_le"

tools/docker/core/Dockerfile

@@ -40,7 +40,6 @@ RUN apk add --no-cache --virtual .build-deps \
         python3-dev \
         cargo \
         git \
-    && python tools/pipstrap.py \
     && python tools/pip_install.py --no-cache-dir \
             --editable src/acme \
             --editable src/certbot \

tools/pip_install.py

@@ -1,7 +1,14 @@
 #!/usr/bin/env python
 # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
 # to 1, tools/oldest_constraints.txt is used, otherwise, tools/requirements.txt
-# is used.
+# is used. Before installing the requested packages, core Python packaging
+# tools like pip, setuptools, and wheel are updated to pinned versions to
+# increase stability of the install.
+#
+# cryptography is currently using this script in their CI at
+# https://github.com/pyca/cryptography/blob/14d45c2259b01f1459eeab8bb7d85ce4cfb0841b/.github/downstream.d/certbot.sh#L8-L9.
+# We should try to remember to keep their repo updated if we make any changes
+# to this script which may break things for them.
 
 from __future__ import absolute_import
 from __future__ import print_function
@@ -9,43 +16,52 @@ from __future__ import print_function
 import os
 import subprocess
 import sys
-import tempfile
 
 
 def find_tools_path():
     return os.path.dirname(os.path.realpath(__file__))
 
 
-def call_with_print(command, env=None):
+def call_with_print(command, env):
-    if not env:
+    assert env is not None
-        env = os.environ
     print(command)
     subprocess.check_call(command, shell=True, env=env)
 
 
-def pip_install_with_print(args_str, env=None):
+def pip_install_with_print(args_str, env):
-    if not env:
-        env = os.environ
     command = ['"', sys.executable, '" -m pip install --disable-pip-version-check ', args_str]
     call_with_print(''.join(command), env=env)
 
 
-def main(args):
+def pip_constrained_environ():
     tools_path = find_tools_path()
 
-    with tempfile.TemporaryDirectory() as working_dir:
+    repo_path = os.path.dirname(tools_path)
-        repo_path = os.path.dirname(tools_path)
+    if os.environ.get('CERTBOT_OLDEST') == '1':
-        if os.environ.get('CERTBOT_OLDEST') == '1':
+        constraints_path = os.path.normpath(os.path.join(
-            constraints_path = os.path.normpath(os.path.join(
+            repo_path, 'tools', 'oldest_constraints.txt'))
-                repo_path, 'tools', 'oldest_constraints.txt'))
+    else:
-        else:
+        constraints_path = os.path.normpath(os.path.join(
-            constraints_path = os.path.normpath(os.path.join(
+            repo_path, 'tools', 'requirements.txt'))
-                repo_path, 'tools', 'requirements.txt'))
 
-        env = os.environ.copy()
+    env = os.environ.copy()
-        env["PIP_CONSTRAINT"] = constraints_path
+    # We set constraints for pip using an environment variable so that they
+    # are also used when installing build dependencies. See
+    # https://github.com/certbot/certbot/pull/8443 for more info.
+    env["PIP_CONSTRAINT"] = constraints_path
+    return env
 
-        pip_install_with_print(' '.join(args), env=env)
+
+def pipstrap(env=None):
+    if env is None:
+        env = pip_constrained_environ()
+    pip_install_with_print('pip setuptools wheel', env=env)
+
+
+def main(args):
+    env = pip_constrained_environ()
+    pipstrap(env)
+    pip_install_with_print(' '.join(args), env=env)
 
 
 if __name__ == '__main__':

tools/pip_install_editable.py

@@ -1,10 +1,5 @@
 #!/usr/bin/env python
 # pip installs packages in editable mode using pip_install.py
-#
-# cryptography is currently using this script in their CI at
-# https://github.com/pyca/cryptography/blob/a02fdd60d98273ca34427235c4ca96687a12b239/.travis/downstream.d/certbot.sh#L8-L9.
-# We should try to remember to keep their repo updated if we make any changes
-# to this script which may break things for them.
 import sys
 
 import pip_install

tools/pipstrap.py

@@ -4,7 +4,7 @@ import pip_install
 
 
 def main():
-    pip_install.main('pip setuptools wheel'.split())
+    pip_install.pipstrap()
 
 
 if __name__ == '__main__':

tools/venv.py

@@ -197,7 +197,6 @@ def install_packages(venv_name, pip_args):
     """
     # Using the python executable from venv, we ensure to execute following commands in this venv.
     py_venv = get_venv_python_path(venv_name)
-    subprocess_with_print([py_venv, os.path.abspath('tools/pipstrap.py')])
     command = [py_venv, os.path.abspath('tools/pip_install.py')]
     command.extend(pip_args)
     subprocess_with_print(command)

tox.ini

@@ -28,7 +28,6 @@ source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-c
 platform =
     win: win32
     posix: ^(?!.*win32).*$
-commands_pre = python {toxinidir}/tools/pipstrap.py
 commands =
     win: {[base]install_and_test} {[base]win_all_packages}
     !win: {[base]install_and_test} {[base]all_packages}