chore: add hack/update-requirements-build.sh (#2384)

.github/workflows/CI-nightly.yaml

@@ -24,11 +24,11 @@ jobs:
       matrix:
         platform: ['amd64','ppc64le','s390x']
     steps:
-    - name: install ibmcli and setup ibm login 
+    - name: install ibmcli and setup ibm login
       if: ${{ matrix.platform  == 's390x' }}
       run: |
         curl -fsSL https://clis.cloud.ibm.com/install/linux | sh
-        ibmcloud login -q --apikey ${{ secrets.IBMCLOUD_API_KEY }} -r eu-gb 
+        ibmcloud login -q --apikey ${{ secrets.IBMCLOUD_API_KEY }} -r eu-gb
         ibmcloud plugin install vpc-infrastructure
 
     - name: Creation of ZVSI
@@ -40,9 +40,9 @@ jobs:
         #Reserving a floating ip to the ZVSI
         ibmcloud is floating-ip-reserve $ZVSI_FP_NAME --zone $ZVSI_ZONE_NAME --resource-group-id $ZVSI_RG_ID --in $ZVSI_INSTANCE_NAME
         #Bouding the Floating ip to the ZVSI
-        ibmcloud is floating-ip-update $ZVSI_FP_NAME --nic primary --in $ZVSI_INSTANCE_NAME          
+        ibmcloud is floating-ip-update $ZVSI_FP_NAME --nic primary --in $ZVSI_INSTANCE_NAME
         sleep 60
-            
+
         #Saving the Floating IP to login ZVSI
         ZVSI_HOST=$(ibmcloud is floating-ip $ZVSI_FP_NAME | awk '/Address/{print $2}')
         echo $ZVSI_HOST
@@ -95,7 +95,7 @@ jobs:
         GH_REPOSITORY: ${{ github.server_url }}/${{ github.repository }}
         GH_REF: ${{ github.ref }}
         GRPC_PYTHON_BUILD_SYSTEM_OPENSSL: 1
-      with: 
+      with:
         host: ${{ matrix.platform  == 's390x' && steps.ZVSI.outputs.IP || secrets.BUILDER_PPC64LE_SSH_HOST_UBUNTU}}
         username: ${{ matrix.platform  == 's390x' && secrets.ZVSI_SSH_USER || secrets.BUILDER_PPC64LE_SSH_USER }}
         key: ${{ matrix.platform  == 's390x' && secrets.ZVSI_PR_KEY || secrets.BUILDER_PPC64LE_SSH_KEY}}
@@ -111,9 +111,7 @@ jobs:
           cd build && git checkout ${GH_REF}
           cat requirements-dev.txt | grep tox | xargs python3 -m pip -qq install
 
-          if [[ 'ppc64le' == $(uname -m) ]]; then
-            sed -i "s/PyYAML==5.4.1/PyYAML==6.0.1/" requirements.txt
-          else
+          if [[ 's390x' == $(uname -m) ]]; then
             GRPC_LATEST=$(grep "grpcio" requirements.txt |cut -d "=" -f 3)
             export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 && python3 -m pip install -qq grpcio==${GRPC_LATEST}
           fi
@@ -125,7 +123,7 @@ jobs:
           tox -e py39-e2e
 
           #registry
-          tox -e py39-registry    
+          tox -e py39-registry
 
     - name: Cleanup ZVSI
       if: ${{ steps.ZVSI.conclusion == 'success' && always() }}

.gitignore

@@ -29,6 +29,7 @@ test/dockerclients/.*
 .mypy_cache
 tmp/*
 /.build-image-quay-stamp
+/requirements-build.in
 
 # files generated by local dev,
 # do not need to check in and can be deleted

Dockerfile

@@ -90,8 +90,6 @@ RUN ARCH=$(uname -m) ; echo $ARCH; \
 
 RUN set -ex\
 	; python3 -m pip install --no-cache-dir --progress-bar off --user $(grep -e '^pip=' -e '^wheel=' -e '^setuptools=' ./requirements.txt) \
-	; python3 -m pip install --no-cache-dir --progress-bar off --user Cython==3.0.0a9 \
-	; python3 -m pip install --no-cache-dir --progress-bar off --user --no-build-isolation PyYAML==5.4.1 \
 	; python3 -m pip install --no-cache-dir --progress-bar off --user --requirement requirements.txt \
 	;
 RUN set -ex\

hack/update-requirements-build.sh

@@ -0,0 +1,12 @@
+#!/bin/sh -eux
+curl -fsSLO https://raw.githubusercontent.com/containerbuildsystem/cachito/master/bin/pip_find_builddeps.py
+python3 pip_find_builddeps.py requirements.txt -o requirements-build.in
+sed -i.bak '/^Cython>=3.0.2/d' requirements-build.in
+rm requirements-build.in.bak
+pip-compile -q requirements-build.in -o requirements-build.txt --allow-unsafe
+sed -i.bak '1i\
+# This file is autogenerated by hack/update-requirements-build.sh.
+/^cython==/a\
+cython==3.0.2
+' requirements-build.txt
+rm requirements-build.txt.bak

hack/verify-requirements.sh

@@ -2,15 +2,19 @@
 TMP=$(mktemp -d "${TMPDIR:-/tmp/}quay-verify-requirements.XXXXXXXX")
 TMP_PACKAGESDIR="$TMP/packages"
 TMP_REQUIREMENTS="$TMP/requirements.txt"
+TMP_CONSTRAINTS="$TMP/constraints.txt"
+TMP_REQUIREMENTS_FULL="$TMP/requirements-full.txt"
 TMP_PIP_FREEZE="$TMP/pip-freeze.txt"
 trap 'rc=$?; rm -rf "$TMP"; exit $rc' EXIT
 
-sed -n 's/ *#.*//; s/\[[^]]*\]//g; s/^pip==.*/pip/; /./p' ./requirements.txt | sort >"$TMP_REQUIREMENTS"
+sed '/^# Indirect dependencies/,$d' ./requirements.txt | sort >"$TMP_REQUIREMENTS"
+sed -n '/^# Indirect dependencies/,$p' ./requirements.txt | sort >"$TMP_CONSTRAINTS"
+sed -n 's/ *#.*//; s/\[[^]]*\]//g; /./p' ./requirements.txt | sort >"$TMP_REQUIREMENTS_FULL"
 
-pip install --target="$TMP_PACKAGESDIR" -r ./requirements.txt
-pip freeze --path="$TMP_PACKAGESDIR" --all | sed 's/^pip==.*/pip/' | sort >"$TMP_PIP_FREEZE"
+pip install --target="$TMP_PACKAGESDIR" -r "$TMP_REQUIREMENTS" -c "$TMP_CONSTRAINTS"
+pip freeze --path="$TMP_PACKAGESDIR" --all | sort >"$TMP_PIP_FREEZE"
 
-if ! diff -u "$TMP_REQUIREMENTS" "$TMP_PIP_FREEZE"; then
-  echo >&2 "requirements.txt doesn't have all dependencies pinned. Please check output above to see what should be added."
+if ! diff -u "$TMP_REQUIREMENTS_FULL" "$TMP_PIP_FREEZE"; then
+  echo >&2 "requirements.txt doesn't have all dependencies pinned correctly. Please check output above to see what should be added or removed."
   exit 1
 fi

requirements-build.txt

@@ -1,2 +1,83 @@
-Cython==0.29.36
-Cython==3.0.2
+# This file is autogenerated by hack/update-requirements-build.sh.
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --allow-unsafe --output-file=requirements-build.txt requirements-build.in
+#
+cffi==1.16.0
+    # via -r requirements-build.in
+cython==0.29.36
+cython==3.0.2
+    # via -r requirements-build.in
+editables==0.5
+    # via
+    #   -r requirements-build.in
+    #   hatchling
+flit-core==3.9.0
+    # via -r requirements-build.in
+greenlet==3.0.0
+    # via -r requirements-build.in
+hatch-vcs==0.3.0
+    # via -r requirements-build.in
+hatchling==1.18.0
+    # via
+    #   -r requirements-build.in
+    #   hatch-vcs
+packaging==23.2
+    # via
+    #   -r requirements-build.in
+    #   hatchling
+    #   setuptools-scm
+pathspec==0.11.2
+    # via
+    #   -r requirements-build.in
+    #   hatchling
+pbr==5.11.1
+    # via -r requirements-build.in
+pluggy==1.3.0
+    # via
+    #   -r requirements-build.in
+    #   hatchling
+pycparser==2.21
+    # via
+    #   -r requirements-build.in
+    #   cffi
+pyyaml==6.0.1
+    # via -r requirements-build.in
+semantic-version==2.10.0
+    # via
+    #   -r requirements-build.in
+    #   setuptools-rust
+setuptools-rust==1.7.0
+    # via -r requirements-build.in
+setuptools-scm[toml]==8.0.4
+    # via
+    #   -r requirements-build.in
+    #   hatch-vcs
+six==1.16.0
+    # via -r requirements-build.in
+tomli==2.0.1
+    # via
+    #   -r requirements-build.in
+    #   hatchling
+    #   setuptools-rust
+    #   setuptools-scm
+trove-classifiers==2023.9.19
+    # via
+    #   -r requirements-build.in
+    #   hatchling
+typing-extensions==4.8.0
+    # via
+    #   -r requirements-build.in
+    #   setuptools-rust
+    #   setuptools-scm
+wheel==0.41.2
+    # via -r requirements-build.in
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==68.2.2
+    # via
+    #   -r requirements-build.in
+    #   setuptools-rust
+    #   setuptools-scm

requirements.txt

@@ -16,7 +16,6 @@ boto3==1.28.61
 botocore==1.31.61
 cachetools==4.0.0
 certifi==2023.7.22
-cffi==1.14.3
 chardet==3.0.4
 charset-normalizer==2.0.12
 click==8.1.7
@@ -33,15 +32,12 @@ Flask-Login==0.6.2
 Flask-Mail==0.9.1
 Flask-Principal==0.4.0
 Flask-RESTful==0.3.10
-furl==2.1.0
 geoip2==3.0.0
 gevent==23.9.1
 greenlet==2.0.2
 grpcio==1.58.0
 gunicorn==21.2.0
 hashids==1.3.1
-html5lib==1.1
-idna==3.4
 importlib-metadata==6.7.0
 iso8601==0.1.12
 isodate==0.6.1
@@ -64,12 +60,6 @@ netifaces==0.11.0
 oauthlib==3.2.2
 orderedmultidict==1.0.1
 os-service-types==1.7.0
-oslo.config==7.0.0
-oslo.i18n==3.25.1
-oslo.serialization==2.29.2
-oslo.utils==6.2.1
-packaging==23.1
-pbr==5.11.1
 peewee==3.16.3
 Pillow==10.0.1
 ply==3.11
@@ -80,7 +70,6 @@ psycopg2-binary==2.9.9
 pyasn1==0.4.8
 pyasn1-modules==0.3.0
 py-bitbucket @ git+https://github.com/quay/py-bitbucket.git@3ba167408d92eb4ba2cae78c0047b8f1e8821c67
-pycparser==2.21
 PyGithub==2.1.1
 PyJWT==2.8.0
 pymemcache==3.0.0
@@ -97,7 +86,7 @@ python-magic==0.4.27
 python-redis-lock==3.7.0
 python-swiftclient==4.4.0
 pytz==2019.3
-PyYAML==5.4.1
+PyYAML==6.0.1
 raven==6.10.0
 recaptcha2==0.1
 redis==3.5.3
@@ -130,33 +119,38 @@ webencodings==0.5.1
 WebOb==1.8.6
 websocket-client==0.57.0
 Werkzeug==2.3.6
-wrapt==1.13.3
 xhtml2pdf==0.2.6
-zipp==3.17.0
-zope.event==5.0
-zope.interface==5.4.0
+
+# Leverage wheels for faster builds
+setuptools==68.2.2
+wheel==0.38.4
 
 # Indirect dependencies
 PyNaCl==1.5.0
 PyPDF3==1.0.6
 arabic-reshaper==3.0.0
+cffi==1.16.0
 cssselect2==0.7.0
 filelock==3.12.4
+furl==2.1.0
+html5lib==1.1
+idna==3.4
 lxml==4.9.3
+oslo.config==7.0.0
+oslo.i18n==3.25.1
+oslo.serialization==2.29.2
+oslo.utils==6.2.1
+packaging==23.2
+pbr==5.11.1
+pycparser==2.21
 python-bidi==0.4.2
 reportlab==3.6.13
 svglib==1.5.1
 tinycss2==1.2.1
 tqdm==4.66.1
+typing_extensions==4.8.0
 tzdata==2023.3
-
-# Build dependencies
-flit_core==3.7.1
-toml==0.10.2
-tomli==1.2.3
-pip==23.2.1
-setuptools==68.1.2
-setuptools-rust==1.5.2
-setuptools-scm[toml]==8.0.4
-typing_extensions==4.7.1
-wheel==0.38.4
+wrapt==1.13.3
+zipp==3.17.0
+zope.event==5.0
+zope.interface==5.4.0