Extend release automation with GPG signing, assets & changelog merging (#3852)

* Tidy reusable release workflow

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add ability to include upstream changes

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add ability to upload assets and gpg sign them

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update relative composite actions

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Wire up validating release tarball signature

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Validate release has expected assets

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Paths

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Use gpg outputs for email instead of scraping it ourselves

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* v6

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Extract pre-release and post-merge-master scripts

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Reuse pre-release and post-merge-master scripts in gha

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Cull unused vars

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Revert

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove unused variables

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Simplify

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Simplify and fix merge-release-notes script

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Tidy release automation

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update release.sh

* Move environment

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* s/includes/contains/

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate uses syntax

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix action-repo calls

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix RELEASE_NOTES env

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix if check

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix gpg tag signing

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Cull stale params

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix sign-release-tarball paths being outside the workspace

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix gpg validation (of course wget uses `-O` and not `-o`)

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix expected asset assertion

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix release publish mode

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

.github/actions/sign-release-tarball/action.yml

@@ -0,0 +1,28 @@
+name: Sign Release Tarball
+description: Generates signature for release tarball and uploads it as a release asset
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the tarball.
+        required: true
+    upload-url:
+        description: GitHub release upload URL to upload the signature file to.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Generate tarball signature
+          shell: bash
+          run: |
+              git -c tar.tar.gz.command='gzip -cn' archive --format=tar.gz --prefix="${REPO#*/}-${VERSION#v}/" -o "/tmp/${VERSION}.tar.gz" "${VERSION}"
+              gpg -u "$GPG_FINGERPRINT" --armor --output "${VERSION}.tar.gz.asc" --detach-sig "/tmp/${VERSION}.tar.gz"
+              rm "/tmp/${VERSION}.tar.gz"
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              REPO: ${{ github.repository }}
+
+        - name: Upload tarball signature
+          if: ${{ inputs.upload-url }}
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ env.VERSION }}.tar.gz.asc

.github/actions/upload-release-assets/action.yml

@@ -0,0 +1,41 @@
+name: Upload release assets
+description: Uploads assets to an existing release and optionally signs them
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the assets, if any.
+        required: false
+    upload-url:
+        description: GitHub release upload URL to upload the assets to.
+        required: true
+    asset-path:
+        description: |
+            The path to the asset you want to upload, if any. You can use glob patterns here.
+            Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Sign assets
+          if: inputs.gpg-fingerprint
+          shell: bash
+          run: |
+              for FILE in $ASSET_PATH
+              do
+                  gpg -u "$GPG_FINGERPRINT" --armor --output "$FILE".asc --detach-sig "$FILE"
+              done
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              ASSET_PATH: ${{ inputs.asset-path }}
+
+        - name: Upload asset signatures
+          if: inputs.gpg-fingerprint
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}.asc
+
+        - name: Upload assets
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}

.github/workflows/release-action.yml

@@ -6,6 +6,10 @@ on:
                 required: true
             NPM_TOKEN:
                 required: false
+            GPG_PASSPHRASE:
+                required: false
+            GPG_PRIVATE_KEY:
+                required: false
         inputs:
             final:
                 description: Make final release
@@ -22,11 +26,39 @@ on:
                     `version` can be `"current"` to leave it at the current version.
                 type: string
                 required: false
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+            gpg-fingerprint:
+                description: Fingerprint of the GPG key to use for signing the git tag and assets, if any.
+                type: string
+                required: false
+            asset-path:
+                description: |
+                    The path to the asset you want to upload, if any. You can use glob patterns here.
+                    Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+                type: string
+                required: false
+            expected-asset-count:
+                description: The number of expected assets, including signatures, excluding generated zip & tarball.
+                type: number
+                required: false
 jobs:
     release:
         name: Release
         runs-on: ubuntu-latest
+        environment: Release
         steps:
+            - name: Load GPG key
+              id: gpg
+              if: inputs.gpg-fingerprint
+              uses: crazy-max/ghaction-import-gpg@82a020f1f7f605c65dd2449b392a52c3fcfef7ef # v6
+              with:
+                  gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+                  passphrase: ${{ secrets.GPG_PASSPHRASE }}
+                  fingerprint: ${{ inputs.gpg-fingerprint }}
+
             - name: Get draft release
               id: release
               uses: cardinalby/git-get-release-action@cedef2faf69cb7c55b285bad07688d04430b7ada # v1
@@ -49,11 +81,24 @@ jobs:
                   persist-credentials: false
                   path: .action-repo
                   sparse-checkout: |
+                      .github/actions
                       scripts/release
 
-            - name: Load version
+            - name: Prepare variables
-              run: echo "VERSION=$VERSION" >> $GITHUB_ENV
+              id: prepare
+              run: |
+                  echo "VERSION=$VERSION" >> $GITHUB_ENV
+                  {
+                      echo "RELEASE_NOTES<<EOF"
+                      echo $BODY
+                      echo "EOF"
+                  } >> $GITHUB_ENV
+
+                  HAS_DIST=0
+                  jq -e .scripts.dist package.json >/dev/null 2>&1 && HAS_DIST=1
+                  echo "has-dist-script=$HAS_DIST" >> $GITHUB_OUTPUT
               env:
+                  BODY: ${{ steps.release.outputs.body }}
                   VERSION: ${{ steps.release.outputs.tag_name }}
 
             - name: Finalise version
@@ -73,8 +118,10 @@ jobs:
               run: "yarn install --frozen-lockfile"
 
             - name: Update dependencies
+              id: update-dependencies
               if: inputs.dependencies
               run: |
+                  UPDATED=()
                   while IFS= read -r DEPENDENCY; do
                       [ -z "$DEPENDENCY" ] && continue
                       IFS="=" read -r PACKAGE UPDATE_VERSION <<< "$DEPENDENCY"
@@ -98,7 +145,11 @@ jobs:
                       yarn upgrade "$PACKAGE@$UPDATE_VERSION" --exact
                       git add -u
                       git commit -m "Upgrade $PACKAGE to $UPDATE_VERSION"
+                      UPDATED+=("$PACKAGE")
                   done <<< "$DEPENDENCIES"
+
+                  JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${UPDATED[@]}")
+                  echo "updated=$JSON" >> $GITHUB_OUTPUT
               env:
                   DEPENDENCIES: ${{ inputs.dependencies }}
 
@@ -115,6 +166,28 @@ jobs:
             - name: Bump package.json version
               run: yarn version --no-git-tag-version --new-version "$VERSION"
 
+            - name: Ingest upstream changes
+              if: |
+                  inputs.dependencies &&
+                  inputs.include-changes &&
+                  contains(fromJSON(steps.update-dependencies.outputs.updated), inputs.include-changes)
+              uses: actions/github-script@v6
+              env:
+                  RELEASE_ID: ${{ steps.upstream-release.outputs.body }}
+                  DEPENDENCY: ${{ inputs.include-changes }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: releaseId, DEPENDENCY } = process.env;
+                      const { owner, repo } = context.repo;
+                      const script = require("./.action-repo/scripts/release/merge-release-notes.js");
+                      const notes = await script({
+                          github,
+                          releaseId,
+                          dependencies: [DEPENDENCY],
+                      });
+                      core.exportVariable("RELEASE_NOTES", notes);
+
             - name: Add to CHANGELOG.md
               if: inputs.mode == 'final'
               run: |
@@ -125,25 +198,84 @@ jobs:
                       echo "$HEADER"
                       printf '=%.0s' $(seq ${#HEADER})
                       echo ""
-                      echo "$BODY"
+                      echo "$RELEASE_NOTES"
                       echo ""
                   } > CHANGELOG.md
 
                   cat CHANGELOG.md.old >> CHANGELOG.md
                   rm CHANGELOG.md.old
                   git add CHANGELOG.md
-              env:
-                  BODY: ${{ steps.release.outputs.body }}
 
             - name: Run pre-release script to update package.json fields
               run: |
                   ./.action-repo/scripts/release/pre-release.sh
                   git add package.json
 
-            - name: Commit and push changes
+            - name: Commit changes
+              run: git commit -m "$VERSION"
+
+            - name: Build assets
+              if: steps.prepare.outputs.has-dist-script
+              run: DIST_VERSION="$VERSION" yarn dist
+
+            - name: Upload release assets & signatures
+              if: inputs.asset-path
+              uses: ./.action-repo/.github/actions/upload-release-assets
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+                  asset-path: ${{ inputs.asset-path }}
+
+            - name: Create signed tag
+              if: inputs.gpg-fingerprint
               run: |
-                  git commit -m "$VERSION"
+                  GIT_COMMITTER_EMAIL="$SIGNING_ID" GPG_TTY=$(tty) git tag -u "$SIGNING_ID" -m "Release $VERSION" "$VERSION"
-                  git push origin staging
+              env:
+                  SIGNING_ID: ${{ steps.gpg.outputs.email }}
+
+            - name: Generate & upload tarball signature
+              if: inputs.gpg-fingerprint
+              uses: ./.action-repo/.github/actions/sign-release-tarball
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+
+            # We defer pushing changes until after the release assets are built,
+            # signed & uploaded to improve the atomicity of this action.
+            - name: Push changes to staging
+              run: |
+                  git push origin staging $TAG
+                  git reset --hard
+              env:
+                  TAG: ${{ inputs.gpg-fingerprint && env.VERSION || '' }}
+
+            - name: Validate tarball signature
+              if: inputs.gpg-fingerprint
+              run: |
+                  wget https://github.com/$GITHUB_REPOSITORY/archive/refs/tags/$VERSION.tar.gz
+                  gpg --verify "$VERSION.tar.gz.asc" "$VERSION.tar.gz"
+
+            - name: Validate release has expected assets
+              if: inputs.expected-asset-count
+              uses: actions/github-script@v6
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  EXPECTED_ASSET_COUNT: ${{ inputs.expected-asset-count }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: release_id, EXPECTED_ASSET_COUNT } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const { data: release } = await github.rest.repos.getRelease({
+                          owner,
+                          repo,
+                          release_id,
+                      });
+
+                      if (release.assets.length !== parseInt(EXPECTED_ASSET_COUNT, 10)) {
+                          core.setFailed(`Found ${release.assets.length} assets but expected ${EXPECTED_ASSET_COUNT}`);
+                      }
 
             - name: Merge to master
               if: inputs.final
@@ -154,15 +286,13 @@ jobs:
 
             - name: Publish release
               uses: actions/github-script@v6
-              id: my-script
               env:
                   RELEASE_ID: ${{ steps.release.outputs.id }}
                   FINAL: ${{ inputs.final }}
               with:
-                  result-encoding: string
                   retries: 3
                   script: |
-                      let { RELEASE_ID: release_id, VERSION, FINAL } = process.env;
+                      const { RELEASE_ID: release_id, RELEASE_NOTES, VERSION, FINAL } = process.env;
                       const { owner, repo } = context.repo;
 
                       const opts = {
@@ -172,6 +302,7 @@ jobs:
                           tag_name: VERSION,
                           name: VERSION,
                           draft: false,
+                          body: RELEASE_NOTES,
                       };
 
                       if (FINAL == "true") {

scripts/release/merge-release-notes.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+
+async function getRelease(github, dependency) {
+    const upstreamPackageJson = JSON.parse(fs.readFileSync(`./node_modules/${dependency}/package.json`, "utf8"));
+    const [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+    const tag = `v${upstreamPackageJson.version}`;
+
+    const response = await github.rest.repos.getReleaseByTag({
+        owner,
+        repo,
+        tag,
+    });
+    return response.data;
+}
+
+const main = async ({ github, releaseId, dependencies }) => {
+    const { GITHUB_REPOSITORY } = process.env;
+    const [owner, repo] = GITHUB_REPOSITORY.split("/");
+
+    const sections = new Map();
+    let heading = null;
+    for (const dependency of dependencies) {
+        const release = await getRelease(github, dependency);
+        for (const line of release.body.split("\n")) {
+            if (line.startsWith("#")) {
+                heading = line;
+                sections.set(heading, []);
+                continue;
+            }
+            if (heading && line) {
+                sections.get(heading).push(line);
+            }
+        }
+    }
+
+    const { data: release } = await github.rest.repos.getRelease({
+        owner,
+        repo,
+        release_id: releaseId,
+    });
+
+    heading = null;
+    const output = [];
+    for (const line of [...release.body.split("\n"), null]) {
+        if (line === null || line.startsWith("#")) {
+            if (heading && sections.has(heading)) {
+                const lastIsBlank = !output.at(-1)?.trim();
+                if (lastIsBlank) output.pop();
+                output.push(...sections.get(heading));
+                if (lastIsBlank) output.push("");
+            }
+            heading = line;
+        }
+        output.push(line);
+    }
+
+    return output.join("\n");
+};
+
+// This is just for testing locally
+// Needs environment variables GITHUB_TOKEN & GITHUB_REPOSITORY
+if (require.main === module) {
+    const { Octokit } = require("@octokit/rest");
+    const github = new Octokit({ auth: process.env.GITHUB_TOKEN });
+    if (process.argv.length < 4) {
+        // eslint-disable-next-line no-console
+        console.error("Usage: node merge-release-notes.js owner/repo:release_id npm-package-name ...");
+        process.exit(1);
+    }
+    const [releaseId, ...dependencies] = process.argv.slice(2);
+    main({ github, releaseId, dependencies }).then((output) => {
+        // eslint-disable-next-line no-console
+        console.log(output);
+    });
+}
+
+module.exports = main;