Merge pull request #11524 from padelsbach/ccache-auto-cleanup

CI: ccache performance improvements

.github/actions/ccache-setup/action.yml

@@ -102,8 +102,9 @@ runs:
 
     # read-only=false (default): restore + post-job save (the run_id in the
     # key never hits, so it always saves its contribution).
+    # Skip restore on reseed-only scheduled runs, cache built from scratch.
     - name: Restore + save ccache
-      if: inputs.read-only != 'true'
+      if: inputs.read-only != 'true' && github.event_name != 'schedule'
       uses: actions/cache@v5
       with:
         path: ~/.ccache
@@ -114,6 +115,13 @@ runs:
         restore-keys: |
           ccache-${{ inputs.workflow-id }}-${{ runner.os }}-${{ runner.arch }}-${{ inputs.config-hash }}-
           ccache-${{ inputs.workflow-id }}-${{ runner.os }}-${{ runner.arch }}-
+    # Scheduled reseed: no restore, save only
+    - name: Save ccache (scheduled reseed, no restore)
+      if: inputs.read-only != 'true' && github.event_name == 'schedule'
+      uses: actions/cache@v5
+      with:
+        path: ~/.ccache
+        key: ccache-${{ inputs.workflow-id }}-${{ runner.os }}-${{ runner.arch }}-${{ inputs.config-hash }}-${{ github.run_id }}-${{ github.run_attempt }}
     # read-only=true: restore the shared cache but never upload (PR runs).
     - name: Restore ccache (read-only)
       if: inputs.read-only == 'true'

.github/workflows/cache-janitor.yml

@@ -0,0 +1,83 @@
+name: Actions cache janitor
+
+# Scheduled workflow to delete stale/superseded cache entries.
+# Goal is to keep the repo cache usage under the 10 GB limit, but also to ensure
+# that stale entries are not restored. Without this cleanup, it's been observed
+# that a new cache entry can fall to the end of the LRU list and be evicted
+# before it is used.
+on:
+  schedule:
+    - cron: '30 1-23/2 * * *' # :30 past every odd numbered hour
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        description: Only list what would be deleted
+        type: boolean
+        default: false
+
+permissions:
+  actions: write
+
+jobs:
+  prune:
+    name: keep the newest entry per lineage
+    if: ${{ github.repository_owner == 'wolfssl' }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - name: List, group and delete superseded cache entries
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry-run && 'true' || 'false' }}
+        run: |
+          # Oldest first: saves land at the end and restores do not reorder
+          # the list, so a row cannot repeat across pages while paging.
+          gh api --paginate -X GET "repos/$REPO/actions/caches" \
+            -F per_page=100 -f sort=created_at -f direction=asc \
+            --jq '.actions_caches[]' | jq -s '.' > caches.json
+          python3 - <<'PY' > delete-ids.txt
+          import json, re, sys
+          entries = json.load(open("caches.json"))
+          def lineage(key):
+              key = re.sub(r"-[0-9a-f]{40}$", "", key)
+              key = re.sub(r"-\d+-\d+$", "", key)
+              return key
+          # Guard against a row repeated across pages.
+          entries = list({e["id"]: e for e in entries}.values())
+          groups = {}
+          for e in entries:
+              groups.setdefault((e["ref"], lineage(e["key"])), []).append(e)
+          keep = deleted = 0
+          for (ref, lin), group in sorted(groups.items()):
+              group.sort(key=lambda e: e["created_at"], reverse=True)
+              keep += group[0]["size_in_bytes"]
+              for e in group[1:]:
+                  deleted += e["size_in_bytes"]
+                  print(e["id"])
+                  print("delete %s %6.0f MB %s (%s), newest is %s" % (
+                      e["created_at"][:16], e["size_in_bytes"] / 1e6, e["key"],
+                      ref, group[0]["created_at"][:16]), file=sys.stderr)
+          print("entries: %d, keep %.1f GB, delete %.1f GB" % (
+              len(entries), keep / 1e9, deleted / 1e9), file=sys.stderr)
+          PY
+          echo "superseded entries: $(wc -l < delete-ids.txt)"
+          if [ "$DRY_RUN" = "true" ]; then
+            echo "dry run: nothing deleted"
+            exit 0
+          fi
+          # An entry can vanish between the list and the delete (eviction, an
+          # in-job prune, another janitor run); that is not a failure. Anything
+          # else, a 403 for instance, is.
+          failed=0
+          while read -r id; do
+            if err=$(gh api --method DELETE "repos/$REPO/actions/caches/$id" 2>&1 >/dev/null); then
+              echo "deleted $id"
+            elif [[ $err == *"HTTP 404"* ]]; then
+              echo "::warning::$id already gone"
+            else
+              echo "failed to delete $id: $err"; failed=1
+            fi
+          done < delete-ids.txt
+          exit $failed

.github/workflows/os-check.yml

@@ -3,7 +3,7 @@ name: Ubuntu-Macos-Windows Tests
 # START OF COMMON SECTION
 on:
   push:
-    branches: [ 'release/**' ]
+    branches: [ 'master', 'release/**' ]
     # Docs-only changes cannot affect the build/test matrix - skip the
     # run for them. Keep this list narrow (markdown + doc/ only);
     # do not add cert/test data extensions here.
@@ -16,19 +16,28 @@ on:
     paths-ignore:
       - '**/*.md'
       - 'doc/**'
-  # Weekday-morning cron (10:00 UTC) seeds the master-scoped ccache that PR runs restore: the
-  # linux and macOS jobs re-run --build-only (compile only, no tests) on the
-  # default branch, where their ccache writes are visible to every PR. Only
-  # Windows is skipped on a seed run (see its job `if`) - seeding the linux
-  # and macOS shards is where the cold-cache cost lives.
+
+  # Run the build and update the ccache on:
+  # - Scheduled: weekday mornings at 10:00 UTC to keep the ccache.
+  #   This reseeds both platforms from scratch (a manual run reseeds
+  #   linux from scratch, but only refreshes macOS - see below).
+  # - Any push to master that changed headers or the build system since the
+  #   last seed (linux shards only; macOS is refreshed by the cron and
+  #   manual runs, since its runner pool is small and its cold-cache cost
+  #   minor).
+  # Note that Windows builds are skipped on a seed run since it uses Visual
+  # Studio and has no ccache to seed.
   #
   # The two platforms do not seed identically, so do not read "the seed" as
-  # one mechanism. Linux sets CCACHE_RECACHE on a seed run and saves only on
-  # one, so its cache is rebuilt from scratch by seed runs alone. macOS does
-  # neither: it never sets CCACHE_RECACHE, so a macOS seed only accumulates
-  # deltas, and its ccache-setup step is read-only purely on pull_request, so
-  # any non-PR run writes it. Left as-is deliberately; the note is here so
-  # the difference is not rediscovered from a surprising cache.
+  # one mechanism. Linux wires ccache by hand: CCACHE_RECACHE and no restore
+  # on a cron or manual seed run (a push seed is incremental), and a save
+  # only on seed runs. macOS goes through the shared ccache-setup composite,
+  # which sets CCACHE_RECACHE and skips the restore only on the cron (not on
+  # a dispatch seed) and saves on every run that is not read-only,
+  # i.e. every non-PR run, release pushes included. So a dispatch refills an
+  # empty macOS cache but only layers onto a stale one; only the cron
+  # rebuilds macOS from scratch. Left as-is deliberately; the note is here
+  # so the difference is not rediscovered from a surprising cache.
   schedule:
     - cron: '0 10 * * 1-5'
   # Runs the same seed on demand. Needed whenever the cache is empty and
@@ -54,6 +63,64 @@ permissions:
 # END OF COMMON SECTION
 
 jobs:
+  # Determine if the merged changed anything which could significantly impact
+  # the ccache.
+  seed_gate:
+    name: seed needed
+    if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name == 'push') && (github.ref == 'refs/heads/master') }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      seed: ${{ steps.check.outputs.seed }}
+    steps:
+      - id: check
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          SHA: ${{ github.sha }}
+          BEFORE: ${{ github.event.before }}
+        run: |
+          # Compare with the commit of the oldest shard seed rather than the
+          # previous push, so a reseed cancelled by a later push (the master
+          # runs share one concurrency group) is redone by that push.
+          # The compare endpoint lists at most 300 changed files, so a push
+          # that large is treated as wide-reaching without looking further.
+          seed=true
+          last=$(gh api -X GET "repos/$REPO/actions/caches" \
+                   -f key=os-check-linux-ccache- -f ref=refs/heads/master -F per_page=100 \
+                   --jq '[.actions_caches[] | select(.key | test("^os-check-linux-ccache-[1-4]-master-[0-9a-f]{40}$"))]
+                         | group_by(.key | split("-")[4])
+                         | if length == 4 then map(max_by(.created_at)) | min_by(.created_at).key
+                           else "incomplete" end') || last=
+          base=${last##*-}
+          [[ $base =~ ^[0-9a-f]{40}$ ]] || base=$BEFORE
+          # Build inputs the ccache is sensitive to. Headers live outside
+          # wolfssl/ too: examples/configs holds the user_settings variants
+          # the matrix stages, and wolfcrypt/src has a few private ones.
+          wide='^((wolfssl|tests|wolfcrypt|examples|testsuite)/.*\.h'
+          wide+='|wolfcrypt/src/misc\.c|configure\.ac|m4/.*|.*\.am'
+          wide+='|\.github/configs/os-check-.*\.json'
+          wide+='|\.github/workflows/os-check\.yml)$'
+          if [ "$last" = incomplete ]; then
+            # Fewer than four shard seeds exist, so there is no complete set
+            # to compare against; comparing with the previous push instead
+            # would keep declining the seed that would refill it.
+            echo "incomplete shard seed set, seeding"
+          elif files=$(gh api "repos/$REPO/compare/$base...$SHA" --jq '.files[]?.filename' | sort -u); then
+            count=$(printf '%s\n' "$files" | grep -c . || true)
+            if [ "$count" -lt 300 ] && ! printf '%s\n' "$files" | grep -qE "$wide"; then
+              seed=false
+            fi
+            echo "last seed ${base:0:10}, changed files: $count, wide reach: $seed"
+          else
+            echo "compare unavailable, seeding to be safe"
+          fi
+          echo "seed=$seed" >> "$GITHUB_OUTPUT"
+
   # Ubuntu config matrix. macOS is covered separately by make_check_macos
   # below with a curated subset; configs here either have equivalent macOS
   # coverage there or exercise no Darwin-specific code.
@@ -72,11 +139,17 @@ jobs:
   # disable it).
   make_check_linux:
     name: make check linux
+    needs: [seed_gate]
+    permissions:
+      contents: read
+      actions: write # allows deletion of old caches
     strategy:
       fail-fast: false
       matrix:
         shard: [1, 2, 3, 4]
-    if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    # !cancelled() keeps this job alive when seed_gate is skipped (every
+    # event but a master push); a declined gate skips the seed.
+    if: ${{ !cancelled() && (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) && (github.event_name != 'push' || github.ref != 'refs/heads/master' || needs.seed_gate.outputs.seed == 'true') }}
     runs-on: ubuntu-24.04
     # Measured cold on a completely empty cache (2026-08-03): ~102
     # thread-minutes per shard / 4 worker threads, i.e. 24-27 min of wall
@@ -85,7 +158,7 @@ jobs:
     # run is normal whenever the seed has not run recently, and a shard
     # dying on the timeout would look like a test failure rather than a
     # cold cache.
-    timeout-minutes: 40
+    timeout-minutes: 60
     env:
       CCACHE_MAXSIZE: 500M
     steps:
@@ -118,8 +191,13 @@ jobs:
 
       # PRs restore the cache a seed run writes but never save it, so PR
       # runs add no per-shard ccache entries to the Actions cache. Only a
-      # seed run (the cron, or a manual dispatch) saves, below.
+      # seed run (a push to master, the cron, or a manual dispatch) saves,
+      # below.
+      # Don't restore on cron or manual since we will rebuild from scratch
+      # using CCACHE_RECACHE. Restore would bring the stale cache entry to the
+      # top of the Github LRU, above newly written entries.
       - name: Restore ccache
+        if: ${{ !(github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.seed != 'false')) }}
         uses: actions/cache/restore@v5
         with:
           path: ~/.cache/ccache
@@ -132,9 +210,6 @@ jobs:
             os-check-linux-ccache-${{ matrix.shard }}-
             os-check-linux-ccache-
 
-      # On a seed run, force clean recompiles (CCACHE_RECACHE) so the saved
-      # master ccache is reseeded from scratch rather than only accumulating
-      # deltas. PR/push runs leave it unset and keep their warm hits.
       - name: Force fresh compiles on a reseed
         if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.seed != 'false') }}
         run: echo "CCACHE_RECACHE=1" >> "$GITHUB_ENV"
@@ -164,21 +239,49 @@ jobs:
       - name: Build and make check this shard's configs (parallel, out-of-tree)
         run: |
           .github/scripts/parallel-make-check.py \
-            ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.seed != 'false')) && '--build-only' || '' }} \
+            ${{ (github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && github.event.inputs.seed != 'false')) && '--build-only' || '' }} \
             --shard "${{ matrix.shard }}/${{ strategy.job-total }}" \
             --cflags='-pedantic -Wdeclaration-after-statement -Wnull-dereference -Wno-overlength-strings -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE' \
             --private-dir=certs .github/configs/os-check-linux.json
 
-      # Seed runs (the cron, or a manual dispatch with `seed` ticked) refresh
-      # the master-scoped ccache that PR runs restore above; PR/push runs
-      # never save, so PRs add nothing.
+      # Seed runs (a push to master, the cron, or a manual dispatch with
+      # `seed` ticked) refresh the master-scoped ccache that PR runs restore
+      # above; PR and release-branch runs never save, so PRs add nothing.
       - name: Save ccache (seed only)
-        if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.seed != 'false') }}
+        if: ${{ github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && github.event.inputs.seed != 'false') }}
         uses: actions/cache/save@v5
         with:
           path: ~/.cache/ccache
           key: os-check-linux-ccache-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }}
 
+      # Delete old ccache seed entries. Note that cache-janitor.yml cleans
+      # entries which do not go through this flow.
+      - name: Prune this shard's stale seeds
+        continue-on-error: true
+        if: ${{ github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && github.event.inputs.seed != 'false') }}
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          REF: ${{ github.ref }}
+          PREFIX: os-check-linux-ccache-${{ matrix.shard }}-${{ github.ref_name }}-
+        run: |
+          # Housekeeping must not fail the build: an entry that vanished
+          # between the list and the delete is fine, anything else is
+          # surfaced as an annotation.
+          gh api -X GET "repos/$REPO/actions/caches" -f key="$PREFIX" -f ref="$REF" \
+            -f sort=created_at -f direction=desc -F per_page=100 \
+            --jq '.actions_caches[1:][] | "\(.id) \(.key)"' |
+          while read -r id key; do
+            if err=$(gh api -X DELETE "repos/$REPO/actions/caches/$id" 2>&1 >/dev/null); then
+              echo "deleted $key"
+            elif [[ $err == *"HTTP 404"* ]]; then
+              echo "$key already gone"
+            else
+              echo "::error::could not delete $key: $err"
+            fi
+          done
+
       - name: ccache stats
         if: always()
         run: ccache -s || true
@@ -212,7 +315,11 @@ jobs:
     # Runs on PRs/pushes and on a seed run (the cron or a manual dispatch),
     # where it --build-only-seeds the macOS ccache like the linux shards.
     # Windows is the only job skipped on a seed run (no ccache to seed).
-    if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    # Not seeded on master pushes: macOS runner concurrency is small, so
+    # per-merge seeds would queue PR macOS jobs behind them, and a fresh
+    # macOS seed saves only about two minutes per PR job. The cron refreshes
+    # it from scratch; a manual dispatch only layers onto what is there.
+    if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) && (github.event_name != 'push' || github.ref != 'refs/heads/master') }}
     runs-on: macos-latest
     # Serial checks: roughly the sum of the per-config minutes plus
     # one-time setup, with headroom for a cold ccache.
@@ -277,9 +384,9 @@ jobs:
 
   windows_build:
     name: Windows Build Test
-    # Skipped on any seed run - the cron, or a manual dispatch that left
-    # `seed` ticked: there is no ccache to seed here.
-    if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'schedule') && (github.event_name != 'workflow_dispatch' || github.event.inputs.seed == 'false') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
+    # Skipped on any seed run - a push to master, the cron, or a manual
+    # dispatch that left `seed` ticked: there is no ccache to seed here.
+    if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'schedule') && (github.event_name != 'push' || github.ref != 'refs/heads/master') && (github.event_name != 'workflow_dispatch' || github.event.inputs.seed == 'false') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }}
     runs-on: windows-latest
     strategy:
       fail-fast: false

.github/workflows/smoke-test.yml

@@ -50,6 +50,9 @@ permissions:
 
 jobs:
   smoke:
+    permissions:
+      contents: read
+      actions: write # allows deletion of old caches
     # Only run from the wolfssl org to avoid burning forks' CI minutes.
     if: github.repository_owner == 'wolfssl'
     runs-on: ubuntu-24.04
@@ -173,6 +176,32 @@ jobs:
           path: ~/.cache/ccache
           key: smoke-ccache-${{ github.ref_name }}-${{ github.sha }}
 
+      - name: Prune superseded seeds
+        continue-on-error: true
+        if: github.event_name != 'pull_request' && steps.merge_check.outputs.skip != 'true'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          REF: ${{ github.ref }}
+          PREFIX: smoke-ccache-${{ github.ref_name }}-
+        run: |
+          # Housekeeping must not fail the build: an entry that vanished
+          # between the list and the delete is fine, anything else is
+          # surfaced as an annotation.
+          gh api -X GET "repos/$REPO/actions/caches" -f key="$PREFIX" -f ref="$REF" \
+            -f sort=created_at -f direction=desc -F per_page=100 \
+            --jq '.actions_caches[1:][] | "\(.id) \(.key)"' |
+          while read -r id key; do
+            if err=$(gh api -X DELETE "repos/$REPO/actions/caches/$id" 2>&1 >/dev/null); then
+              echo "deleted $key"
+            elif [[ $err == *"HTTP 404"* ]]; then
+              echo "$key already gone"
+            else
+              echo "::error::could not delete $key: $err"
+            fi
+          done
+
       - name: ccache stats
         if: always() && steps.merge_check.outputs.skip != 'true'
         run: ccache -s || true