diff --git a/.gitignore b/.gitignore index 064e7fffe..34987d319 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ parts prime stage *.snap +snap-constraints.txt +qemu-* diff --git a/.travis.yml b/.travis.yml index afd656edf..624205f10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ before_script: # Use Travis retry feature for farm tests since they are flaky - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi' - export TOX_TESTENV_PASSENV=TRAVIS - - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh; fi' + - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh $TARGET_ARCH; fi' # Only build pushes to the master branch, PRs, and branches beginning with # `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or @@ -39,6 +39,45 @@ not-on-master: ¬-on-master extended-test-suite: &extended-test-suite if: type = cron OR (type = push AND branch != master) +# Common configuration for all snap tasks +snap-config: &snap-config + dist: bionic + addons: + apt: + packages: + - nginx-light + git: + # By default, Travis clones the repo to a depth of 50 commits which can + # break the ability to use `git describe` to set the version of the + # snap. This setting removes the --depth flag from git commands solving + # this problem. See + # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth + # for more info. + depth: false + deploy: + # This section relies on credentials stored in a SNAP_TOKEN environment + # variable in Travis. See + # https://docs.travis-ci.com/user/deployment/snaps/ for more info. + # This credential has a maximum lifetime of 1 year and the current + # credential will expire on 4/22/2021. The value of SNAP_TOKEN will + # need to be updated to use a new credential before then to prevent + # automated deploys from breaking. Remembering to do this is also + # tracked by https://github.com/certbot/certbot/issues/7931. + 'on': + # Deploy on release tags or nightly runs from any branch. We only try + # to deploy from the certbot/certbot repo to prevent errors if forks + # of this repo try to run tests. + all_branches: true + condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron + repo: certbot/certbot + provider: snap + snap: certbot_*.snap + channel: edge + # skip_cleanup is needed to prevent Travis from deleting the snaps we + # just built and tested. See + # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup. + skip_cleanup: true + matrix: include: # Main test suite @@ -231,49 +270,17 @@ matrix: - libaugeas0 <<: *extended-test-suite - stage: "Snap" - sudo: required - env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble - addons: - apt: - packages: - - nginx-light - snaps: - - name: snapcraft - channel: stable - confinement: classic - - name: lxd - channel: stable - git: - # By default, Travis clones the repo to a depth of 50 commits which can - # break the ability to use `git describe` to set the version of the - # snap. This setting removes the --depth flag from git commands solving - # this problem. See - # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth - # for more info. - depth: false - deploy: - # This section relies on credentials stored in a SNAP_TOKEN environment - # variable in Travis. See - # https://docs.travis-ci.com/user/deployment/snaps/ for more info. - # This credential has a maximum lifetime of 1 year and the current - # credential will expire on 4/22/2021. The value of SNAP_TOKEN will - # need to be updated to use a new credential before then to prevent - # automated deploys from breaking. Remembering to do this is also - # tracked by https://github.com/certbot/certbot/issues/7931. - 'on': - # Deploy on release tags or nightly runs from any branch. We only try - # to deploy from the certbot/certbot repo to prevent errors if forks - # of this repo try to run tests. - all_branches: true - condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron - repo: certbot/certbot - provider: snap - snap: certbot_*.snap - channel: edge - # skip_cleanup is needed to prevent Travis from deleting the snaps we - # just built and tested. See - # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup. - skip_cleanup: true + env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble TARGET_ARCH=amd64 + <<: *snap-config + <<: *extended-test-suite + - env: SNAP=true TARGET_ARCH=i386 + <<: *snap-config + <<: *extended-test-suite + - env: SNAP=true TARGET_ARCH=arm64 + <<: *snap-config + <<: *extended-test-suite + - env: SNAP=true TARGET_ARCH=armhf + <<: *snap-config <<: *extended-test-suite # container-based infrastructure @@ -304,7 +311,7 @@ install: 'tools/pip_install.py -I tox virtualenv' # script command. It is set only to `travis_retry` during farm tests, in # order to trigger the Travis retry feature, and compensate the inherent # flakiness of these specific tests. -script: '$TRAVIS_RETRY tox' +script: 'if [[ ! -z "$TOXENV" ]]; then $TRAVIS_RETRY tox; fi' notifications: email: false diff --git a/snap/local/build_and_install.sh b/snap/local/build_and_install.sh index f103477f1..b5883bc8a 100755 --- a/snap/local/build_and_install.sh +++ b/snap/local/build_and_install.sh @@ -1,20 +1,53 @@ #!/bin/bash +# Cross-compile the Certbot snap from local sources for the specified architecture, +# and install it if this architecture is also the the current machine one. +# This script is designed for CI tests purpose. +# Usage: build_and_install.sh [amd64,arm64,armhf] set -ex -if [[ -z "$TRAVIS" ]]; then +if [[ -z "${TRAVIS}" ]]; then echo "This script makes global changes to the system it is run on so should only be run in CI." exit 1 fi -# Add the current user to the lxd group so they can run `snapcraft --use-lxd` -# without sudo since running the command without sudo is required by newer -# versions of snapcraft. -sudo usermod -aG lxd "$USER" -sudo /snap/bin/lxd.migrate -yes -sudo /snap/bin/lxd waitready -sudo /snap/bin/lxd init --auto -tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > constraints.txt -# Run snapcraft with the lxd group since it has not been added to the current -# shell. -sg lxd -c 'snapcraft --use-lxd' -sudo snap install --dangerous --classic *.snap +SNAP_ARCH=$1 + +if [[ -z "${SNAP_ARCH}" ]]; then + echo "You need to specify the target architecture" + exit 1 +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")" + +# shellcheck source=common.sh +source "${DIR}/common.sh" + +RegisterQemuHandlers +ResolveArch "${SNAP_ARCH}" + +tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > snap-constraints.txt + +pushd "${DIR}/packages" +"${CERTBOT_DIR}/tools/simple_http_server.py" 8080 >/dev/null 2>&1 & +HTTP_SERVER_PID="$!" +popd + +function cleanup() { + kill "${HTTP_SERVER_PID}" +} + +trap cleanup EXIT + +docker run \ + --rm \ + --net=host \ + -v "${CERTBOT_DIR}:/certbot" \ + -w "/certbot" \ + -e "PIP_EXTRA_INDEX_URL=http://localhost:8080" \ + "adferrand/snapcraft:${DOCKER_ARCH}-stable" \ + snapcraft + +if [[ "$(arch)" == "${QEMU_ARCH}" ]]; then + sudo snap install --dangerous --classic *.snap +fi diff --git a/snap/local/common.sh b/snap/local/common.sh new file mode 100644 index 000000000..00a7614e8 --- /dev/null +++ b/snap/local/common.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Common bash functions useful for cross-compiling Certbot snaps. + +# Resolve the Snap architecture to Docker architecture (DOCKER_ARCH variable) +# and QEMU architecture (QEMU_ARCH variable). +# Usage: ResolveArch [amd64|i386|arm64|armhf] +ResolveArch() { + local SNAP_ARCH=$1 + + case "${SNAP_ARCH}" in + "amd64") + DOCKER_ARCH="amd64" + QEMU_ARCH="x86_64" + ;; + "i386") + DOCKER_ARCH="i386" + QEMU_ARCH="i386" + ;; + "arm64") + DOCKER_ARCH="arm64v8" + QEMU_ARCH="aarch64" + ;; + "armhf") + DOCKER_ARCH="arm32v7" + QEMU_ARCH="arm" + ;; + "*") + echo "Not supported build architecture '$1'." >&2 + exit 1 + esac +} + +# Downloads QEMU static binary file for architecture +# Usage: DownloadQemuStatic [x86_64|aarch64|arm] DEST_DIR +DownloadQemuStatic() { + local QEMU_ARCH=$1 + local DEST_DIR=$2 + local QEMU_DOWNLOAD_URL + local QEMU_LATEST_TAG + + if [ ! -f "${DIR}/qemu-${QEMU_ARCH}-static" ]; then + QEMU_DOWNLOAD_URL="https://github.com/multiarch/qemu-user-static/releases/download" + QEMU_LATEST_TAG=$(curl -s https://api.github.com/repos/multiarch/qemu-user-static/tags \ + | grep 'name.*v[0-9]' \ + | head -n 1 \ + | cut -d '"' -f 4) + echo "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-${QEMU_ARCH}-static.tar.gz" + curl -SL "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-${QEMU_ARCH}-static.tar.gz" \ + | tar xzv -C "${DEST_DIR}" + fi +} + +# Executes the QEMU register script +# Usage: RegisterQemuHandlers +RegisterQemuHandlers() { + docker run --rm --privileged multiarch/qemu-user-static:register --reset +} diff --git a/snap/local/compile_native_wheels.sh b/snap/local/compile_native_wheels.sh new file mode 100755 index 000000000..5dbb0f5d6 --- /dev/null +++ b/snap/local/compile_native_wheels.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Cross-compile cryptography and cffi native wheels for arm64 and armhf architectures, +# on the versions required by the current pinning of Certbot dependencies. +# Wheels are stored in snap/local/packages folder to speed up cross-compilation of Certbot snap. +set -ex + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +TARGET_ARCHS="i386 arm64 armhf" + +rm -rf "${DIR}/packages/"* + +# shellcheck source=common.sh +source "${DIR}/common.sh" + +RegisterQemuHandlers + +tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > "${DIR}/snap-constraints.txt" +for SNAP_ARCH in ${TARGET_ARCHS}; do + ResolveArch "${SNAP_ARCH}" + DownloadQemuStatic "${QEMU_ARCH}" "${DIR}" + + docker run \ + --rm \ + -v "${DIR}/qemu-${QEMU_ARCH}-static:/usr/bin/qemu-${QEMU_ARCH}-static" \ + -v "${DIR}:/workspace" \ + -w "/workspace" \ + "${DOCKER_ARCH}/ubuntu:18.04" \ + sh -c "\ + apt-get update \ +&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-venv python3-dev libffi-dev libssl-dev gcc \ +&& mkdir -p /build \ +&& python3 -m venv /build/venv \ +&& /build/venv/bin/pip install wheel \ +&& /build/venv/bin/pip wheel cryptography cffi -c snap-constraints.txt -w /build \ +&& mkdir -p /workspace/packages/cffi /workspace/packages/cryptography \ +&& mv /build/cryptography-* /workspace/packages/cryptography \ +&& mv /build/cffi-* /workspace/packages/cffi \ +&& chmod 777 /workspace/packages /workspace/packages/cffi /workspace/packages/cryptography \ +&& chmod 666 /workspace/packages/cffi/* /workspace/packages/cryptography/* +" +done diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl new file mode 100644 index 000000000..36dfa9ade Binary files /dev/null and b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl differ diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl new file mode 100644 index 000000000..26f107cc9 Binary files /dev/null and b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl differ diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl new file mode 100644 index 000000000..1e5491b6e Binary files /dev/null and b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl differ diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl new file mode 100644 index 000000000..3a969945a Binary files /dev/null and b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl differ diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl new file mode 100644 index 000000000..ea49f5dab Binary files /dev/null and b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl differ diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl new file mode 100644 index 000000000..18d7e6fd7 Binary files /dev/null and b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl differ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index fd5b908f6..88833bce2 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -46,13 +46,15 @@ parts: plugin: python source: . source-subdir: acme - constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] + constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt] python-version: python3 + # To build cryptography and cffi if needed + build-packages: [libffi-dev, libssl-dev] certbot: plugin: python source: . source-subdir: certbot - constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] + constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt] python-version: python3 after: [acme] override-pull: | @@ -65,7 +67,7 @@ parts: plugin: python source: . source-subdir: certbot-apache - constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] + constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt] python-version: python3 after: [python-augeas, certbot] stage-packages: [libaugeas0] @@ -76,7 +78,7 @@ parts: plugin: python source: . source-subdir: certbot-nginx - constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] + constraints: [$SNAPCRAFT_PART_SRC/snap-constraints.txt] python-version: python3 # This is the last step, compile pycache now as there should be no conflicts. override-prime: |