From 79297ef5cbb39e7d66cfa21039ea4b962e5619a5 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 25 Sep 2020 02:12:12 +0200 Subject: [PATCH] Invoke pipstrap in tox and during the CI (#8316) Partial fix for #8256 This PR makes tox calls pipstrap before any commands is executed, and Azure Pipelines calls pipstrap when appropriate (when an actual call to pip is done). * Invoke pipstrap in tox and during the CI * Set default value for PYTHON_VERSION and always set python interpreter * Set Python for snaps_build also * Fix the build for Windows installer * Add a warning comment for pinned versions in pipstrap * Rebuild letsencrypt-auto * Same version than the installer build * Let's update to latest pip for installer tests --- .../templates/jobs/extended-tests-jobs.yml | 2 ++ .../templates/jobs/packaging-jobs.yml | 18 ++++++++++++++++-- .../templates/jobs/standard-tests-jobs.yml | 2 ++ .azure-pipelines/templates/steps/tox-steps.yml | 2 +- letsencrypt-auto-source/letsencrypt-auto | 5 +++++ letsencrypt-auto-source/pieces/pipstrap.py | 5 +++++ tox.ini | 1 + 7 files changed, 32 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index 0e1a98861..67fa34880 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -3,6 +3,8 @@ jobs: variables: - name: IMAGE_NAME value: ubuntu-18.04 + - name: PYTHON_VERSION + value: 3.8 - group: certbot-common strategy: matrix: diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index b0c7998cb..2d659aef5 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -78,9 +78,16 @@ jobs: artifact: windows-installer path: $(Build.SourcesDirectory)/bin displayName: Retrieve Windows installer + # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency + # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install + # a recent version of pip, but we also to disable the isolated feature as described in + # https://github.com/certbot/certbot/issues/8256 - script: | py -3 -m venv venv + venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1 venv\Scripts\python tools\pip_install.py -e certbot-ci + env: + PIP_NO_BUILD_ISOLATION: no displayName: Prepare Certbot-CI - script: | set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH% @@ -135,10 +142,16 @@ jobs: pool: vmImage: ubuntu-18.04 steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.8 + addToPath: true - script: | sudo apt-get update sudo apt-get install -y --no-install-recommends nginx-light snapd - python tools/pip_install.py -U tox + python3 -m venv venv + venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py + venv/bin/python tools/pip_install.py -U tox displayName: Install dependencies - task: DownloadPipelineArtifact@2 inputs: @@ -149,7 +162,7 @@ jobs: sudo snap install --dangerous --classic snap/certbot_*_amd64.snap displayName: Install Certbot snap - script: | - python -m tox -e integration-external,apacheconftest-external-with-pebble + venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble displayName: Run tox - job: snap_dns_run dependsOn: snaps_build @@ -171,6 +184,7 @@ jobs: displayName: Retrieve Certbot snaps - script: | python3 -m venv venv + venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py venv/bin/python tools/pip_install.py -e certbot-ci displayName: Prepare Certbot-CI - script: | diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 3bb73b67e..d5b3a0a16 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -1,5 +1,7 @@ jobs: - job: test + variables: + PYTHON_VERSION: 3.8 strategy: matrix: macos-py27: diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml index 828552e43..7f6c3150f 100644 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ b/.azure-pipelines/templates/steps/tox-steps.yml @@ -21,7 +21,6 @@ steps: inputs: versionSpec: $(PYTHON_VERSION) addToPath: true - condition: ne(variables['PYTHON_VERSION'], '') # tools/pip_install.py is used to pin packages to a known working version # except in tests where the environment variable CERTBOT_NO_PIN is set. # virtualenv is listed here explicitly to make sure it is upgraded when @@ -30,6 +29,7 @@ steps: # set, pip updates dependencies it thinks are already satisfied to avoid some # problems with its lack of real dependency resolution. - bash: | + python letsencrypt-auto-source/pieces/pipstrap.py python tools/pip_install.py -I tox virtualenv displayName: Install runtime dependencies - task: DownloadSecureFile@1 diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 7168ee2d1..5c5434323 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1615,6 +1615,11 @@ maybe_argparse = ( if sys.version_info < (2, 7, 0) else []) +# Be careful when updating the pinned versions here, in particular for pip. +# Indeed starting from 10.0, pip will build dependencies in isolation if the +# related projects are compliant with PEP 517. This is not something we want +# as of now, so the isolation build will need to be disabled wherever +# pipstrap is used (see https://github.com/certbot/certbot/issues/8256). PACKAGES = maybe_argparse + [ # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 346e23938..7610c2686 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -67,6 +67,11 @@ maybe_argparse = ( if sys.version_info < (2, 7, 0) else []) +# Be careful when updating the pinned versions here, in particular for pip. +# Indeed starting from 10.0, pip will build dependencies in isolation if the +# related projects are compliant with PEP 517. This is not something we want +# as of now, so the isolation build will need to be disabled wherever +# pipstrap is used (see https://github.com/certbot/certbot/issues/8256). PACKAGES = maybe_argparse + [ # Pip has no dependencies, as it vendors everything: ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' diff --git a/tox.ini b/tox.ini index 932f9e947..befdeba93 100644 --- a/tox.ini +++ b/tox.ini @@ -62,6 +62,7 @@ source_paths = [testenv] passenv = CERTBOT_NO_PIN +commands_pre = python {toxinidir}/letsencrypt-auto-source/pieces/pipstrap.py commands = !cover: {[base]install_and_test} {[base]all_packages} !cover: python tests/lock_test.py