Certbot snap multiarch build (#8016)

This PR proposes a way to build the certbot snap for several architectures, using a QEMU-base emulation approach, and several optimizations to keep the build time of each snap below 20 minutes. Most of the reasoning for the approach proposed here is described in the original PR: https://github.com/basak/certbot-snap-build/pull/27 On top of it, I added a docker pull to a pre-compiled snapcraft docker, instead of compiling it during the Travis pipeline, in order to save 5 to 7 minutes more on each snap build. The snap images are compiled and stored here: https://hub.docker.com/repository/docker/adferrand/snapcraft. Depending on the time the PR will be reviewed, we can: * continue to use `adferrand/snapcraft` * move its logic to certbot scope and use something like `certbot/snapcraft` * wait for https://github.com/snapcore/snapcraft/pull/3144 to be merged, and use `snapcore/snapcraft`. * Backport https://github.com/basak/certbot-snap-build/pull/27 into Certbot project * Fix build deps * Integrate proactively #8012 to fix builds on non-amd64 archs * Configure jobs on Travis * Focus on snap builds. Disable temporarily some jobs. Disable deploy actions by security. * Specify TARGET_ARCH during snap build * Do not do anything if TOXENV is not set * Various optimizations * Use recent version of ubuntu for get correct features on snap out of the box * Add up to date wheels * Organizing scripts * Set dest dir * Get back original configuration for Travis * Add comments * Update common_libs.sh * Use adferrand/snapcraft * Test build * Stable snapcraft * Update build_and_install.sh * Move back snap builds to the cron/release pipeline * Update snap/local/compile_native_wheels.sh Co-authored-by: Brad Warren <bmw@users.noreply.github.com> * Update snap/local/compile_native_wheels.sh Co-authored-by: Brad Warren <bmw@users.noreply.github.com> * Update snap/local/compile_native_wheels.sh Co-authored-by: Brad Warren <bmw@users.noreply.github.com> * Update snap/local/build_and_install.sh Co-authored-by: Brad Warren <bmw@users.noreply.github.com> * Enable i386 builds, various optimizations * Update dependencies * Configure a simple http server to serve the pre compiled wheels * Fix wheels compilation * Relax permissions Co-authored-by: Brad Warren <bmw@users.noreply.github.com>
2026-01-26 07:41:33 +03:00 · 2020-06-10 23:33:02 +02:00
parent 8d776fb7ac
commit 9ac476e87b
12 changed files with 204 additions and 62 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -58,3 +58,5 @@ parts
 prime
 stage
 *.snap
+snap-constraints.txt
+qemu-*
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,7 @@ before_script:
  # Use Travis retry feature for farm tests since they are flaky
  - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi'
  - export TOX_TESTENV_PASSENV=TRAVIS
-  - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh; fi'
+  - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh $TARGET_ARCH; fi'

 # Only build pushes to the master branch, PRs, and branches beginning with
 # `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or
@@ -39,6 +39,45 @@ not-on-master: &not-on-master
 extended-test-suite: &extended-test-suite
  if: type = cron OR (type = push AND branch != master)

+# Common configuration for all snap tasks
+snap-config: &snap-config
+  dist: bionic
+  addons:
+    apt:
+      packages:
+      - nginx-light
+  git:
+    # By default, Travis clones the repo to a depth of 50 commits which can
+    # break the ability to use `git describe` to set the version of the
+    # snap. This setting removes the --depth flag from git commands solving
+    # this problem. See
+    # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth
+    # for more info.
+    depth: false
+  deploy:
+    # This section relies on credentials stored in a SNAP_TOKEN environment
+    # variable in Travis. See
+    # https://docs.travis-ci.com/user/deployment/snaps/ for more info.
+    # This credential has a maximum lifetime of 1 year and the current
+    # credential will expire on 4/22/2021. The value of SNAP_TOKEN will
+    # need to be updated to use a new credential before then to prevent
+    # automated deploys from breaking. Remembering to do this is also
+    # tracked by https://github.com/certbot/certbot/issues/7931.
+    'on':
+      # Deploy on release tags or nightly runs from any branch. We only try
+      # to deploy from the certbot/certbot repo to prevent errors if forks
+      # of this repo try to run tests.
+      all_branches: true
+      condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron
+      repo: certbot/certbot
+    provider: snap
+    snap: certbot_*.snap
+    channel: edge
+    # skip_cleanup is needed to prevent Travis from deleting the snaps we
+    # just built and tested. See
+    # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup.
+    skip_cleanup: true
+
 matrix:
  include:
    # Main test suite
@@ -231,49 +270,17 @@ matrix:
            - libaugeas0
      <<: *extended-test-suite
    - stage: "Snap"
-      sudo: required
-      env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble
-      addons:
-        apt:
-          packages:
-          - nginx-light
-        snaps:
-          - name: snapcraft
-            channel: stable
-            confinement: classic
-          - name: lxd
-            channel: stable
-      git:
-        # By default, Travis clones the repo to a depth of 50 commits which can
-        # break the ability to use `git describe` to set the version of the
-        # snap. This setting removes the --depth flag from git commands solving
-        # this problem. See
-        # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth
-        # for more info.
-        depth: false
-      deploy:
-        # This section relies on credentials stored in a SNAP_TOKEN environment
-        # variable in Travis. See
-        # https://docs.travis-ci.com/user/deployment/snaps/ for more info.
-        # This credential has a maximum lifetime of 1 year and the current
-        # credential will expire on 4/22/2021. The value of SNAP_TOKEN will
-        # need to be updated to use a new credential before then to prevent
-        # automated deploys from breaking. Remembering to do this is also
-        # tracked by https://github.com/certbot/certbot/issues/7931.
-        'on':
-          # Deploy on release tags or nightly runs from any branch. We only try
-          # to deploy from the certbot/certbot repo to prevent errors if forks
-          # of this repo try to run tests.
-          all_branches: true
-          condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron
-          repo: certbot/certbot
-        provider: snap
-        snap: certbot_*.snap
-        channel: edge
-        # skip_cleanup is needed to prevent Travis from deleting the snaps we
-        # just built and tested. See
-        # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup.
-        skip_cleanup: true
+      env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble TARGET_ARCH=amd64
+      <<: *snap-config
+      <<: *extended-test-suite
+    - env: SNAP=true TARGET_ARCH=i386
+      <<: *snap-config
+      <<: *extended-test-suite
+    - env: SNAP=true TARGET_ARCH=arm64
+      <<: *snap-config
+      <<: *extended-test-suite
+    - env: SNAP=true TARGET_ARCH=armhf
+      <<: *snap-config
      <<: *extended-test-suite

 # container-based infrastructure
@@ -304,7 +311,7 @@ install: 'tools/pip_install.py -I tox virtualenv'
 # script command. It is set only to `travis_retry` during farm tests, in
 # order to trigger the Travis retry feature, and compensate the inherent
 # flakiness of these specific tests.
-script: '$TRAVIS_RETRY tox'
+script: 'if [[ ! -z "$TOXENV" ]]; then $TRAVIS_RETRY tox; fi'

 notifications:
  email: false
--- a/snap/local/build_and_install.sh
+++ b/snap/local/build_and_install.sh
@@ -1,20 +1,53 @@
 #!/bin/bash
+# Cross-compile the Certbot snap from local sources for the specified architecture,
+# and install it if this architecture is also the the current machine one.
+# This script is designed for CI tests purpose.
+# Usage: build_and_install.sh [amd64,arm64,armhf]
 set -ex

-if [[ -z "$TRAVIS" ]]; then
+if [[ -z "${TRAVIS}" ]]; then
    echo "This script makes global changes to the system it is run on so should only be run in CI."
    exit 1
 fi

-# Add the current user to the lxd group so they can run `snapcraft --use-lxd`
-# without sudo since running the command without sudo is required by newer
-# versions of snapcraft.
-sudo usermod -aG lxd "$USER"
-sudo /snap/bin/lxd.migrate -yes
-sudo /snap/bin/lxd waitready
-sudo /snap/bin/lxd init --auto
-tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > constraints.txt
-# Run snapcraft with the lxd group since it has not been added to the current
-# shell.
-sg lxd -c 'snapcraft --use-lxd'
-sudo snap install --dangerous --classic *.snap
+SNAP_ARCH=$1
+
+if [[ -z "${SNAP_ARCH}" ]]; then
+    echo "You need to specify the target architecture"
+    exit 1
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")"
+
+# shellcheck source=common.sh
+source "${DIR}/common.sh"
+
+RegisterQemuHandlers
+ResolveArch "${SNAP_ARCH}"
+
+tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > snap-constraints.txt
+
+pushd "${DIR}/packages"
+"${CERTBOT_DIR}/tools/simple_http_server.py" 8080 >/dev/null 2>&1 &
+HTTP_SERVER_PID="$!"
+popd
+
+function cleanup() {
+    kill "${HTTP_SERVER_PID}"
+}
+
+trap cleanup EXIT
+
+docker run \
+  --rm \
+  --net=host \
+  -v "${CERTBOT_DIR}:/certbot" \
+  -w "/certbot" \
+  -e "PIP_EXTRA_INDEX_URL=http://localhost:8080" \
+  "adferrand/snapcraft:${DOCKER_ARCH}-stable" \
+  snapcraft
+
+if [[ "$(arch)" == "${QEMU_ARCH}" ]]; then
+    sudo snap install --dangerous --classic *.snap
+fi
--- a/snap/local/common.sh
+++ b/snap/local/common.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# Common bash functions useful for cross-compiling Certbot snaps.
+
+# Resolve the Snap architecture to Docker architecture (DOCKER_ARCH variable)
+# and QEMU architecture (QEMU_ARCH variable).
+# Usage: ResolveArch [amd64|i386|arm64|armhf]
+ResolveArch() {
+    local SNAP_ARCH=$1
+
+    case "${SNAP_ARCH}" in
+        "amd64")
+            DOCKER_ARCH="amd64"
+            QEMU_ARCH="x86_64"
+            ;;
+        "i386")
+            DOCKER_ARCH="i386"
+            QEMU_ARCH="i386"
+            ;;
+        "arm64")
+            DOCKER_ARCH="arm64v8"
+            QEMU_ARCH="aarch64"
+            ;;
+        "armhf")
+            DOCKER_ARCH="arm32v7"
+            QEMU_ARCH="arm"
+            ;;
+        "*")
+            echo "Not supported build architecture '$1'." >&2
+            exit 1
+    esac
+}
+
+# Downloads QEMU static binary file for architecture
+# Usage: DownloadQemuStatic [x86_64|aarch64|arm] DEST_DIR
+DownloadQemuStatic() {
+    local QEMU_ARCH=$1
+    local DEST_DIR=$2
+    local QEMU_DOWNLOAD_URL
+    local QEMU_LATEST_TAG
+
+    if [ ! -f "${DIR}/qemu-${QEMU_ARCH}-static" ]; then
+        QEMU_DOWNLOAD_URL="https://github.com/multiarch/qemu-user-static/releases/download"
+        QEMU_LATEST_TAG=$(curl -s https://api.github.com/repos/multiarch/qemu-user-static/tags \
+            | grep 'name.*v[0-9]' \
+            | head -n 1 \
+            | cut -d '"' -f 4)
+        echo "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-${QEMU_ARCH}-static.tar.gz"
+        curl -SL "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-${QEMU_ARCH}-static.tar.gz" \
+            | tar xzv -C "${DEST_DIR}"
+    fi
+}
+
+# Executes the QEMU register script
+# Usage: RegisterQemuHandlers
+RegisterQemuHandlers() {
+    docker run --rm --privileged multiarch/qemu-user-static:register --reset
+}
--- a/snap/local/compile_native_wheels.sh
+++ b/snap/local/compile_native_wheels.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Cross-compile cryptography and cffi native wheels for arm64 and armhf architectures,
+# on the versions required by the current pinning of Certbot dependencies.
+# Wheels are stored in snap/local/packages folder to speed up cross-compilation of Certbot snap.
+set -ex
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+TARGET_ARCHS="i386 arm64 armhf"
+
+rm -rf "${DIR}/packages/"*
+
+# shellcheck source=common.sh
+source "${DIR}/common.sh"
+
+RegisterQemuHandlers
+
+tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > "${DIR}/snap-constraints.txt"
+for SNAP_ARCH in ${TARGET_ARCHS}; do
+    ResolveArch "${SNAP_ARCH}"
+    DownloadQemuStatic "${QEMU_ARCH}" "${DIR}"
+
+    docker run \
+        --rm \
+        -v "${DIR}/qemu-${QEMU_ARCH}-static:/usr/bin/qemu-${QEMU_ARCH}-static" \
+        -v "${DIR}:/workspace" \
+        -w "/workspace" \
+        "${DOCKER_ARCH}/ubuntu:18.04" \
+        sh -c "\
+   apt-get update \
+&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-venv python3-dev libffi-dev libssl-dev gcc \
+&& mkdir -p /build \
+&& python3 -m venv /build/venv \
+&& /build/venv/bin/pip install wheel \
+&& /build/venv/bin/pip wheel cryptography cffi -c snap-constraints.txt -w /build \
+&& mkdir -p /workspace/packages/cffi /workspace/packages/cryptography \
+&& mv /build/cryptography-* /workspace/packages/cryptography \
+&& mv /build/cffi-* /workspace/packages/cffi \
+&& chmod 777 /workspace/packages /workspace/packages/cffi /workspace/packages/cryptography \
+&& chmod 666 /workspace/packages/cffi/* /workspace/packages/cryptography/*
+"
+done
--- a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl
+++ b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl
--- a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl
+++ b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl
--- a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl
+++ b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl
--- a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl
+++ b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl
--- a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl
+++ b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl
--- a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl
+++ b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -46,13 +46,15 @@ parts:
    plugin: python
    source: .
    source-subdir: acme
-    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
    python-version: python3
+    # To build cryptography and cffi if needed
+    build-packages: [libffi-dev, libssl-dev]
  certbot:
    plugin: python
    source: .
    source-subdir: certbot
-    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
    python-version: python3
    after: [acme]
    override-pull: |
@@ -65,7 +67,7 @@ parts:
    plugin: python
    source: .
    source-subdir: certbot-apache
-    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
    python-version: python3
    after: [python-augeas, certbot]
    stage-packages: [libaugeas0]
@@ -76,7 +78,7 @@ parts:
    plugin: python
    source: .
    source-subdir: certbot-nginx
-    constraints: [$SNAPCRAFT_PART_SRC/constraints.txt]
+    constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt]
    python-version: python3
    # This is the last step, compile pycache now as there should be no conflicts.
    override-prime: |