From 4e9d3afcc4104fc646c37899efc41cc4e62e5e09 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 18 Aug 2020 10:48:01 -0700 Subject: [PATCH] Docker build improvements (#8218) Fixes https://github.com/certbot/certbot/issues/8208. Fixes https://github.com/certbot/certbot/issues/8198. In addition to those two linked issues, this PR: * Splits both the build and deploy steps based on architecture for performance. The Docker builds should no longer be the bottleneck in any stage of the pipeline. * Skips building Docker images for ARM on `test-` branches like [we do for snaps](https://github.com/certbot/certbot/blob/e8a232297d6faedb3b0bffc197a2e98c2e59edcb/.azure-pipelines/templates/jobs/packaging-jobs.yml#L67-L71). I initially didn't want to do this, but the ARM builds take ~18 minutes which is significantly longer than any other job currently running on our `test-` branches. You can see tests running on my fork at: * [Release pipeline](https://dev.azure.com/bmw0523/bmw/_build/results?buildId=387&view=results) * [Test pipeline](https://dev.azure.com/bmw0523/bmw/_build/results?buildId=388&view=results) * [Nightly pipeline](https://dev.azure.com/bmw0523/bmw/_build/results?buildId=390&view=results) * update script intro * update readme * ParseRequestedArch * build all arch in Azure * Build docker images during testing/packaging. * require global variable? * Error if TAG_BASE is empty. * prepare build job * change variable syntax * Update deploy stage. * remove old dockerTag param * add displayName * fix docker images command * split docker_build by arch * Allow deploying a subset of architectures. * deploy in parallel * Skip ARM builds on test- branches. * fix spacing --- .azure-pipelines/advanced-test.yml | 5 +++ .azure-pipelines/nightly.yml | 3 ++ .azure-pipelines/release.yml | 4 ++- .../templates/jobs/packaging-jobs.yml | 31 +++++++++++++++++++ .../templates/stages/deploy-stage.yml | 22 +++++++++---- tools/docker/README.md | 11 ++++--- tools/docker/build.sh | 20 +++++++----- tools/docker/deploy.sh | 30 +++++++++++------- tools/docker/lib/common | 20 ++++++++++++ 9 files changed, 116 insertions(+), 30 deletions(-) diff --git a/.azure-pipelines/advanced-test.yml b/.azure-pipelines/advanced-test.yml index 7a9bb588b..e2342630f 100644 --- a/.azure-pipelines/advanced-test.yml +++ b/.azure-pipelines/advanced-test.yml @@ -5,5 +5,10 @@ trigger: - test-* pr: none +variables: + # We don't publish our Docker images in this pipeline, but when building them + # for testing, let's use the nightly tag. + dockerTag: nightly + stages: - template: templates/stages/test-and-package-stage.yml diff --git a/.azure-pipelines/nightly.yml b/.azure-pipelines/nightly.yml index 1d3261fb6..dd2930c41 100644 --- a/.azure-pipelines/nightly.yml +++ b/.azure-pipelines/nightly.yml @@ -9,6 +9,9 @@ schedules: - master always: true +variables: + dockerTag: nightly + stages: - template: templates/stages/test-and-package-stage.yml - template: templates/stages/deploy-stage.yml diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml index b00d4be78..eaa9be0fd 100644 --- a/.azure-pipelines/release.yml +++ b/.azure-pipelines/release.yml @@ -6,11 +6,13 @@ trigger: - v* pr: none +variables: + dockerTag: ${{variables['Build.SourceBranchName']}} + stages: - template: templates/stages/test-and-package-stage.yml - template: templates/stages/changelog-stage.yml - template: templates/stages/deploy-stage.yml parameters: snapReleaseChannel: beta - dockerTag: ${{variables['Build.SourceBranchName']}} - template: templates/stages/notify-failure-stage.yml diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 4ff5ff7b9..b0c7998cb 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -1,4 +1,35 @@ jobs: + - job: docker_build + pool: + vmImage: ubuntu-18.04 + strategy: + matrix: + amd64: + DOCKER_ARCH: amd64 + # Do not run the heavy non-amd64 builds for test branches + ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: + arm32v6: + DOCKER_ARCH: arm32v6 + arm64v8: + DOCKER_ARCH: arm64v8 + steps: + - bash: tools/docker/build.sh $(dockerTag) $DOCKER_ARCH + displayName: Build the Docker images + # We don't filter for the Docker Hub organization to continue to allow + # easy testing of these scripts on forks. + - bash: | + DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}') + docker save --output images.tar $DOCKER_IMAGES + displayName: Save the Docker images + # If the name of the tar file or artifact changes, the deploy stage will + # also need to be updated. + - bash: mv images.tar $(Build.ArtifactStagingDirectory) + displayName: Prepare Docker artifact + - task: PublishPipelineArtifact@1 + inputs: + path: $(Build.ArtifactStagingDirectory) + artifact: docker_$(DOCKER_ARCH) + displayName: Store Docker artifact - job: installer_build pool: vmImage: vs2017-win2016 diff --git a/.azure-pipelines/templates/stages/deploy-stage.yml b/.azure-pipelines/templates/stages/deploy-stage.yml index 8fc1ff4c6..e6cb6a0ba 100644 --- a/.azure-pipelines/templates/stages/deploy-stage.yml +++ b/.azure-pipelines/templates/stages/deploy-stage.yml @@ -5,9 +5,6 @@ parameters: values: - edge - beta -- name: dockerTag - type: string - default: nightly stages: - stage: Deploy @@ -65,7 +62,22 @@ stages: - job: publish_docker pool: vmImage: ubuntu-18.04 + strategy: + matrix: + amd64: + DOCKER_ARCH: amd64 + arm32v6: + DOCKER_ARCH: arm32v6 + arm64v8: + DOCKER_ARCH: arm64v8 steps: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: docker_$(DOCKER_ARCH) + path: $(Build.SourcesDirectory) + displayName: Retrieve Docker images + - bash: docker load --input $(Build.SourcesDirectory)/images.tar + displayName: Load Docker images - task: Docker@2 inputs: command: login @@ -81,7 +93,5 @@ stages: # Certbot organization on Docker Hub. containerRegistry: docker-hub displayName: Login to Docker Hub - - bash: tools/docker/build.sh ${{ parameters.dockerTag }} - displayName: Build the Docker images - - bash: tools/docker/deploy.sh ${{ parameters.dockerTag }} + - bash: tools/docker/deploy.sh $(dockerTag) $DOCKER_ARCH displayName: Deploy the Docker images diff --git a/tools/docker/README.md b/tools/docker/README.md index 02e61be9c..799ddcd0b 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -20,11 +20,12 @@ DNS plugin Docker images to Docker Hub. High-level behavior ------------------- -Running `./build.sh && ./deploy.sh ` causes the Docker images to be -built and deployed to Docker Hub where `` is the base of the tag that -should be given to the given images. The tag should either be `nightly` or a -git version tag like `v0.34.0`. The given tag is only the base of the tag -because the CPU architecture is also added to the tag. +Running `./build.sh all && ./deploy.sh all` causes the Docker +images to be built and deployed to Docker Hub for all supported architectures +where `` is the base of the tag that should be given to the given images. +The tag should either be `nightly` or a git version tag like `v0.34.0`. The +given tag is only the base of the tag because the CPU architecture is also +added to the tag. Configuration ------------- diff --git a/tools/docker/build.sh b/tools/docker/build.sh index 1b18c2c42..9fcc4473c 100755 --- a/tools/docker/build.sh +++ b/tools/docker/build.sh @@ -5,11 +5,12 @@ IFS=$'\n\t' # This script builds certbot docker and certbot dns plugins docker using the # local Certbot files. -# Usage: ./build.sh [TAG] -# with [TAG] corresponding the base of the tag to give the Docker images. -# Values should be something like `v0.34.0` or `nightly`. The given value is -# only the base of the tag because the things like the CPU architecture are -# also added to the full tag. +# Usage: ./build.sh [TAG] [all|amd64|arm32v6|arm64v8] +# with the [TAG] value corresponding the base of the tag to give the Docker +# images and the 2nd value being the architecture to build snaps for. +# Values for the tag should be something like `v0.34.0` or `nightly`. The +# given value is only the base of the tag because the things like the CPU +# architecture are also added to the full tag. # As of writing this, runs of this script consistently fail in Azure # Pipelines, but they are fixed by using Docker BuildKit. A log of the failures @@ -76,13 +77,18 @@ DownloadQemuStatic() { } TAG_BASE="$1" +if [ -z "$TAG_BASE" ]; then + echo "We cannot tag Docker images with an empty string!" >&2 + exit 1 +fi +ParseRequestedArch "${2}" # Register QEMU handlers docker run --rm --privileged multiarch/qemu-user-static:register --reset # Step 1: Certbot core Docker DOCKER_REPO="${DOCKER_HUB_ORG}/certbot" -for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do +for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do pushd "${REPO_ROOT}" DownloadQemuStatic "${TARGET_ARCH}" QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}") @@ -101,7 +107,7 @@ for plugin in "${CERTBOT_PLUGINS[@]}"; do pushd "${REPO_ROOT}/certbot-${plugin}" # Copy QEMU static binaries downloaded when building the core Certbot image cp ../qemu-*-static . - for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do + for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}") BASE_IMAGE="${DOCKER_HUB_ORG}/certbot:${TARGET_ARCH}-${TAG_BASE}" docker build \ diff --git a/tools/docker/deploy.sh b/tools/docker/deploy.sh index a280e3982..97feb7f8f 100755 --- a/tools/docker/deploy.sh +++ b/tools/docker/deploy.sh @@ -4,8 +4,9 @@ IFS=$'\n\t' # This script deploys new versions of Certbot and Certbot plugin docker images. -# Usage: ./deploy.sh [TAG] -# with [TAG] corresponding the base of the tag to give the Docker images. +# Usage: ./deploy.sh [TAG] [all|amd64|arm32v6|arm64v8] +# with the [TAG] value corresponding the base of the tag to give the Docker +# images and the 2nd value being the architecture to build snaps for. # Values should be something like `v0.34.0` or `nightly`. The given value is # only the base of the tag because the things like the CPU architecture are # also added to the full tag. @@ -13,12 +14,18 @@ IFS=$'\n\t' WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" TAG_BASE="$1" # Eg. v0.35.0 or nightly +if [ -z "$TAG_BASE" ]; then + echo "We cannot tag Docker images with an empty string!" >&2 + exit 1 +fi source "$WORK_DIR/lib/common" +ParseRequestedArch "${2}" -# Creates and pushes all Docker images aliases for all architectures. -# If the value of the global variable TAG_BASE is a version tag such as -# v0.35.0, the "latest" tag is also updated. Tags without the architecture part -# are also created for the default architecture. +# Creates and pushes all Docker images aliases for the requested architectures +# set in the environment variable ALL_REQUESTED_ARCH. If the value of the +# global variable TAG_BASE is a version tag such as v0.35.0, the "latest" tag +# is also updated. Tags without the architecture part are also created for the +# default architecture. # As an example, for amd64 (the default architecture) and the tag v0.35.0, the # following tags would be created: # - certbot/certbot:v0.35.0 @@ -30,14 +37,15 @@ source "$WORK_DIR/lib/common" # For other tags such as "nightly", aliases are only created for the default # architecture where the tag "nightly" would be used without an architecture # part. -# Usage: TagAndPushForAllArch [IMAGE NAME] +# Usage: TagAndPushForAllRequestedArch [IMAGE NAME] # where [IMAGE NAME] is the name of the Docker image in the Docker repository # such as "certbot" or "dns-cloudflare". # Read globals: # * TAG_BASE -TagAndPushForAllArch() { +# * ALL_REQUESTED_ARCH +TagAndPushForAllRequestedArch() { DOCKER_REPO="${DOCKER_HUB_ORG}/${1}" - for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do + for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do docker push "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}" if [[ "${TAG_BASE}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -56,9 +64,9 @@ TagAndPushForAllArch() { } # Step 1: Certbot core Docker -TagAndPushForAllArch "certbot" +TagAndPushForAllRequestedArch "certbot" # Step 2: Certbot DNS plugins Docker images for plugin in "${CERTBOT_PLUGINS[@]}"; do - TagAndPushForAllArch "${plugin}" + TagAndPushForAllRequestedArch "${plugin}" done diff --git a/tools/docker/lib/common b/tools/docker/lib/common index 1dd62aff4..4ba86cac0 100644 --- a/tools/docker/lib/common +++ b/tools/docker/lib/common @@ -31,3 +31,23 @@ export CERTBOT_PLUGINS=( "dns-linode" "dns-sakuracloud" ) + +# Parses the requested architecture string and sets ALL_REQUESTED_ARCH to +# result. +# Usage: ParseRequestedArch [all|amd64|arm32v6|arm64v8] +ParseRequestedArch() { + REQUESTED_ARCH="${1}" + if [[ "${REQUESTED_ARCH}" == "all" ]]; then + ALL_REQUESTED_ARCH=("${ALL_TARGET_ARCH[@]}") + return 0 + fi + for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do + if [[ "${TARGET_ARCH}" == "${REQUESTED_ARCH}" ]]; then + ALL_REQUESTED_ARCH=("${REQUESTED_ARCH}") + return 0 + fi + done + # If we didn't return above, REQUESTED_ARCH has an unexpected value. + echo "Unexpected target architecture \"${REQUESTED_ARCH}\"". >&2 + exit 1 +}