Docker build improvements (#8218)

Fixes https://github.com/certbot/certbot/issues/8208.
Fixes https://github.com/certbot/certbot/issues/8198.

In addition to those two linked issues, this PR:

* Splits both the build and deploy steps based on architecture for performance. The Docker builds should no longer be the bottleneck in any stage of the pipeline.
* Skips building Docker images for ARM on `test-` branches like [we do for snaps](e8a232297d/.azure-pipelines/templates/jobs/packaging-jobs.yml (L67-L71)). I initially didn't want to do this, but the ARM builds take ~18 minutes which is significantly longer than any other job currently running on our `test-` branches.

You can see tests running on my fork at:

* [Release pipeline](https://dev.azure.com/bmw0523/bmw/_build/results?buildId=387&view=results)
* [Test pipeline](https://dev.azure.com/bmw0523/bmw/_build/results?buildId=388&view=results)
* [Nightly pipeline](https://dev.azure.com/bmw0523/bmw/_build/results?buildId=390&view=results)

* update script intro

* update readme

* ParseRequestedArch

* build all arch in Azure

* Build docker images during testing/packaging.

* require global variable?

* Error if TAG_BASE is empty.

* prepare build job

* change variable syntax

* Update deploy stage.

* remove old dockerTag param

* add displayName

* fix docker images command

* split docker_build by arch

* Allow deploying a subset of architectures.

* deploy in parallel

* Skip ARM builds on test- branches.

* fix spacing

.azure-pipelines/advanced-test.yml

@@ -5,5 +5,10 @@ trigger:
   - test-*
 pr: none
 
+variables:
+  # We don't publish our Docker images in this pipeline, but when building them
+  # for testing, let's use the nightly tag.
+  dockerTag: nightly
+
 stages:
   - template: templates/stages/test-and-package-stage.yml

.azure-pipelines/nightly.yml

@@ -9,6 +9,9 @@ schedules:
       - master
     always: true
 
+variables:
+  dockerTag: nightly
+
 stages:
   - template: templates/stages/test-and-package-stage.yml
   - template: templates/stages/deploy-stage.yml

.azure-pipelines/release.yml

@@ -6,11 +6,13 @@ trigger:
       - v*
 pr: none
 
+variables:
+  dockerTag: ${{variables['Build.SourceBranchName']}}
+
 stages:
   - template: templates/stages/test-and-package-stage.yml
   - template: templates/stages/changelog-stage.yml
   - template: templates/stages/deploy-stage.yml
     parameters:
       snapReleaseChannel: beta
-      dockerTag: ${{variables['Build.SourceBranchName']}}
   - template: templates/stages/notify-failure-stage.yml

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -1,4 +1,35 @@
 jobs:
+  - job: docker_build
+    pool:
+      vmImage: ubuntu-18.04
+    strategy:
+      matrix:
+        amd64:
+          DOCKER_ARCH: amd64
+        # Do not run the heavy non-amd64 builds for test branches
+        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
+          arm32v6:
+            DOCKER_ARCH: arm32v6
+          arm64v8:
+            DOCKER_ARCH: arm64v8
+    steps:
+      - bash: tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
+        displayName: Build the Docker images
+      # We don't filter for the Docker Hub organization to continue to allow
+      # easy testing of these scripts on forks.
+      - bash: |
+          DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
+          docker save --output images.tar $DOCKER_IMAGES
+        displayName: Save the Docker images
+      # If the name of the tar file or artifact changes, the deploy stage will
+      # also need to be updated.
+      - bash: mv images.tar $(Build.ArtifactStagingDirectory)
+        displayName: Prepare Docker artifact
+      - task: PublishPipelineArtifact@1
+        inputs:
+          path: $(Build.ArtifactStagingDirectory)
+          artifact: docker_$(DOCKER_ARCH)
+        displayName: Store Docker artifact
   - job: installer_build
     pool:
       vmImage: vs2017-win2016

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -5,9 +5,6 @@ parameters:
   values:
   - edge
   - beta
-- name: dockerTag
-  type: string
-  default: nightly
 
 stages:
   - stage: Deploy
@@ -65,7 +62,22 @@ stages:
       - job: publish_docker
         pool:
           vmImage: ubuntu-18.04
+        strategy:
+          matrix:
+            amd64:
+              DOCKER_ARCH: amd64
+            arm32v6:
+              DOCKER_ARCH: arm32v6
+            arm64v8:
+              DOCKER_ARCH: arm64v8
         steps:
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              artifact: docker_$(DOCKER_ARCH)
+              path: $(Build.SourcesDirectory)
+            displayName: Retrieve Docker images
+          - bash: docker load --input $(Build.SourcesDirectory)/images.tar
+            displayName: Load Docker images
           - task: Docker@2
             inputs:
               command: login
@@ -81,7 +93,5 @@ stages:
               # Certbot organization on Docker Hub.
               containerRegistry: docker-hub
             displayName: Login to Docker Hub
-          - bash: tools/docker/build.sh  ${{ parameters.dockerTag }}
-            displayName: Build the Docker images
-          - bash: tools/docker/deploy.sh ${{ parameters.dockerTag }}
+          - bash: tools/docker/deploy.sh $(dockerTag) $DOCKER_ARCH
             displayName: Deploy the Docker images

tools/docker/README.md

@@ -20,11 +20,12 @@ DNS plugin Docker images to Docker Hub.
 High-level behavior
 -------------------
 
-Running `./build.sh <TAG> && ./deploy.sh <TAG>` causes the Docker images to be
-built and deployed to Docker Hub where `<TAG>` is the base of the tag that
-should be given to the given images. The tag should either be `nightly` or a
-git version tag like `v0.34.0`. The given tag is only the base of the tag
-because the CPU architecture is also added to the tag.
+Running `./build.sh <TAG> all && ./deploy.sh <TAG> all` causes the Docker
+images to be built and deployed to Docker Hub for all supported architectures
+where `<TAG>` is the base of the tag that should be given to the given images.
+The tag should either be `nightly` or a git version tag like `v0.34.0`. The
+given tag is only the base of the tag because the CPU architecture is also
+added to the tag.
 
 Configuration
 -------------

tools/docker/build.sh

@@ -5,11 +5,12 @@ IFS=$'\n\t'
 # This script builds certbot docker and certbot dns plugins docker using the
 # local Certbot files.
 
-# Usage: ./build.sh [TAG]
-#   with [TAG] corresponding the base of the tag to give the Docker images.
-#   Values should be something like `v0.34.0` or `nightly`. The given value is
-#   only the base of the tag because the things like the CPU architecture are
-#   also added to the full tag.
+# Usage: ./build.sh [TAG] [all|amd64|arm32v6|arm64v8]
+#   with the [TAG] value corresponding the base of the tag to give the Docker
+#   images and the 2nd value being the architecture to build snaps for.
+#   Values for the tag should be something like `v0.34.0` or `nightly`. The
+#   given value is only the base of the tag because the things like the CPU
+#   architecture are also added to the full tag.
 
 # As of writing this, runs of this script consistently fail in Azure
 # Pipelines, but they are fixed by using Docker BuildKit. A log of the failures
@@ -76,13 +77,18 @@ DownloadQemuStatic() {
 }
 
 TAG_BASE="$1"
+if [ -z "$TAG_BASE" ]; then
+    echo "We cannot tag Docker images with an empty string!" >&2
+    exit 1
+fi
+ParseRequestedArch "${2}"
 
 # Register QEMU handlers
 docker run --rm --privileged multiarch/qemu-user-static:register --reset
 
 # Step 1: Certbot core Docker
 DOCKER_REPO="${DOCKER_HUB_ORG}/certbot"
-for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
+for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
     pushd "${REPO_ROOT}"
     DownloadQemuStatic "${TARGET_ARCH}"
     QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}")
@@ -101,7 +107,7 @@ for plugin in "${CERTBOT_PLUGINS[@]}"; do
     pushd "${REPO_ROOT}/certbot-${plugin}"
     # Copy QEMU static binaries downloaded when building the core Certbot image
     cp ../qemu-*-static .
-    for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
+    for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
         QEMU_ARCH=$(GetQemuArch "${TARGET_ARCH}")
         BASE_IMAGE="${DOCKER_HUB_ORG}/certbot:${TARGET_ARCH}-${TAG_BASE}"
         docker build \

tools/docker/deploy.sh

@@ -4,8 +4,9 @@ IFS=$'\n\t'
 
 # This script deploys new versions of Certbot and Certbot plugin docker images.
 
-# Usage: ./deploy.sh [TAG]
-#   with [TAG] corresponding the base of the tag to give the Docker images.
+# Usage: ./deploy.sh [TAG] [all|amd64|arm32v6|arm64v8]
+#   with the [TAG] value corresponding the base of the tag to give the Docker
+#   images and the 2nd value being the architecture to build snaps for.
 #   Values should be something like `v0.34.0` or `nightly`. The given value is
 #   only the base of the tag because the things like the CPU architecture are
 #   also added to the full tag.
@@ -13,12 +14,18 @@ IFS=$'\n\t'
 WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 TAG_BASE="$1"  # Eg. v0.35.0 or nightly
+if [ -z "$TAG_BASE" ]; then
+    echo "We cannot tag Docker images with an empty string!" >&2
+    exit 1
+fi
 source "$WORK_DIR/lib/common"
+ParseRequestedArch "${2}"
 
-# Creates and pushes all Docker images aliases for all architectures.
-# If the value of the global variable TAG_BASE is a version tag such as
-# v0.35.0, the "latest" tag is also updated. Tags without the architecture part
-# are also created for the default architecture.
+# Creates and pushes all Docker images aliases for the requested architectures
+# set in the environment variable ALL_REQUESTED_ARCH.  If the value of the
+# global variable TAG_BASE is a version tag such as v0.35.0, the "latest" tag
+# is also updated. Tags without the architecture part are also created for the
+# default architecture.
 # As an example, for amd64 (the default architecture) and the tag v0.35.0, the
 # following tags would be created:
 #  - certbot/certbot:v0.35.0
@@ -30,14 +37,15 @@ source "$WORK_DIR/lib/common"
 # For other tags such as "nightly", aliases are only created for the default
 # architecture where the tag "nightly" would be used without an architecture
 # part.
-# Usage: TagAndPushForAllArch [IMAGE NAME]
+# Usage: TagAndPushForAllRequestedArch [IMAGE NAME]
 #   where [IMAGE NAME] is the name of the Docker image in the Docker repository
 #   such as "certbot" or "dns-cloudflare".
 # Read globals:
 # * TAG_BASE
-TagAndPushForAllArch() {
+# * ALL_REQUESTED_ARCH
+TagAndPushForAllRequestedArch() {
     DOCKER_REPO="${DOCKER_HUB_ORG}/${1}"
-    for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
+    for TARGET_ARCH in "${ALL_REQUESTED_ARCH[@]}"; do
         docker push "${DOCKER_REPO}:${TARGET_ARCH}-${TAG_BASE}"
 
         if [[ "${TAG_BASE}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
@@ -56,9 +64,9 @@ TagAndPushForAllArch() {
 }
 
 # Step 1: Certbot core Docker
-TagAndPushForAllArch "certbot"
+TagAndPushForAllRequestedArch "certbot"
 
 # Step 2: Certbot DNS plugins Docker images
 for plugin in "${CERTBOT_PLUGINS[@]}"; do
-    TagAndPushForAllArch "${plugin}"
+    TagAndPushForAllRequestedArch "${plugin}"
 done

tools/docker/lib/common

@@ -31,3 +31,23 @@ export CERTBOT_PLUGINS=(
     "dns-linode"
     "dns-sakuracloud"
 )
+
+# Parses the requested architecture string and sets ALL_REQUESTED_ARCH to
+# result.
+# Usage: ParseRequestedArch [all|amd64|arm32v6|arm64v8]
+ParseRequestedArch() {
+    REQUESTED_ARCH="${1}"
+    if [[ "${REQUESTED_ARCH}" == "all" ]]; then
+        ALL_REQUESTED_ARCH=("${ALL_TARGET_ARCH[@]}")
+        return 0
+    fi
+    for TARGET_ARCH in "${ALL_TARGET_ARCH[@]}"; do
+        if [[ "${TARGET_ARCH}" == "${REQUESTED_ARCH}" ]]; then
+            ALL_REQUESTED_ARCH=("${REQUESTED_ARCH}")
+            return 0
+        fi
+    done
+    # If we didn't return above, REQUESTED_ARCH has an unexpected value.
+    echo "Unexpected target architecture \"${REQUESTED_ARCH}\"". >&2
+    exit 1
+}