Rewrite conformance action to enable locking to spec version

Signed-off-by: Josh Dolitsky <josh@dolit.ski>

.github/workflows/conformance-action-pr.yml

@@ -0,0 +1,26 @@
+name: conformance-action-pr
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout source code
+        uses: actions/checkout@v3
+      - name: Start a test registry (zot)
+        run: |
+          set -x
+          make registry-ci
+      - name: Run OCI distribution-spec conformance
+        env:
+          OCI_ROOT_URL: http://localhost:5000
+          OCI_NAMESPACE: myorg/myrepo
+          OCI_TEST_PULL: 1
+          OCI_TEST_PUSH: 1
+          OCI_TEST_CONTENT_DISCOVERY: 1
+          OCI_TEST_CONTENT_MANAGEMENT: 1
+        uses: ./

.github/workflows/conformance-action.yml

@@ -0,0 +1,26 @@
+name: conformance-action
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout source code
+        uses: actions/checkout@v3
+      - name: Start a test registry (zot)
+        run: |
+          set -x
+          make registry-ci
+      - name: Run OCI distribution-spec conformance
+        env:
+          OCI_ROOT_URL: http://localhost:5000
+          OCI_NAMESPACE: myorg/myrepo
+          OCI_TEST_PULL: 1
+          OCI_TEST_PUSH: 1
+          OCI_TEST_CONTENT_DISCOVERY: 1
+          OCI_TEST_CONTENT_MANAGEMENT: 1
+        uses: ./

Makefile

@@ -1,5 +1,6 @@
 EPOCH_TEST_COMMIT	:= 91d6d8466e68f1efff7977b63ad6f48e72245e05
 CURRENT_COMMIT	:= $(shell git log --format="%H" -n 1)
+CONFORMANCE_VERSION ?= $(CURRENT_COMMIT)
 
 DOCKER	?= $(shell command -v docker 2>/dev/null)
 PANDOC	?= $(shell command -v pandoc 2>/dev/null)
@@ -94,12 +95,14 @@ conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
 TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.0.0-rc6@sha256:bf95a94849cd9c6f596fb10e5a2d03b74267e7886d1ba0b3dab33337d9e46e5c
 registry-ci:
 	docker rm -f oci-conformance && \
+		mkdir -p $(OUTPUT_DIRNAME) && \
 		echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
 		docker run -d \
 			-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
 			--name=oci-conformance \
 			-p 5000:5000 \
-			$(TEST_REGISTRY_CONTAINER)
+			$(TEST_REGISTRY_CONTAINER) && \
+		sleep 5
 
 conformance-ci:
 	export OCI_ROOT_URL="http://localhost:5000" && \
@@ -108,10 +111,9 @@ conformance-ci:
 		export OCI_TEST_PUSH=1 && \
 		export OCI_TEST_CONTENT_DISCOVERY=1 && \
 		export OCI_TEST_CONTENT_MANAGEMENT=1 && \
-		sleep 5 && \
 		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
 
 $(OUTPUT_DIRNAME)/conformance.test:
 	cd conformance && \
 		CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \
-			--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CURRENT_COMMIT)"
+			--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"

action.yml

@@ -7,5 +7,64 @@ branding:
 inputs: {}
 outputs: {}
 runs:
-  using: docker
-  image: "docker://ghcr.io/opencontainers/distribution-spec/conformance:v1.0.0@sha256:6302582c3b35169f7846282ad5b81471cd8814f30f7a958030e04fd82b19e17f"
+  using: composite
+  steps:
+    - name: Check required env vars for OCI distribution-spec conformance
+      shell: bash
+      id: check-conformance
+      run: |
+        set -x
+        required_env_vars=(
+          "OCI_ROOT_URL"
+          "OCI_NAMESPACE"
+        )
+        for v in ${required_env_vars[@]}; do
+          if [[ "${!v}" == "" ]]; then
+            echo "Error: the following environment variable is required: ${v}"
+            exit 1
+          fi
+        done
+
+    - name: Build OCI distribution-spec conformance binary
+      shell: bash
+      id: build-conformance
+      run: |
+        set -x
+
+        # Enter the directory containing the checkout of this action which is surpisingly hard to do (but we did it... #OCI)
+        cd "$(dirname $(find $(find ~/work/_actions -name distribution-spec -print -quit) -name Makefile -print -quit))"
+        
+        # The .git folder is not present, but the dirname is the requested action ref, so use this as the conformance version
+        conformance_version="$(basename "${PWD}")"
+        echo "conformance-version=${conformance_version}" >> $GITHUB_OUTPUT
+
+        # Build the conformance binary
+        CONFORMANCE_VERSION="${conformance_version}" OUTPUT_DIRNAME=bin make conformance-binary
+
+        # Add bin to the PATH so we can just run "conformance.test"
+        echo "${PWD}/bin" >> $GITHUB_PATH
+
+    - name: Run OCI distribution-spec conformance binary
+      shell: bash
+      id: run-conformance
+      run: |
+        set -x
+        set +e
+        conformance.test
+        conformance_rc="$?"
+        set -e
+        if [[ -f report.html ]]; then
+          echo "Found report.html."
+          echo "has-report=true" >> $GITHUB_OUTPUT
+        fi
+        echo "Conformance return code: ${conformance_rc}"
+        exit ${conformance_rc}
+
+    - name: Upload OCI distribution-spec conformance results as build artifact
+      if: always() && steps.run-conformance.outputs.has-report == 'true'
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      with:
+        name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.conformance-version }}
+        path: |
+          report.html
+          junit.xml

conformance/README.md

@@ -229,6 +229,8 @@ jobs:
     steps:
       - name: Run OCI Distribution Spec conformance tests
         uses: opencontainers/distribution-spec@main
+        # you can also run against a specific tag or commit instead
+        # uses: opencontainers/distribution-spec@v1.1.0 
         env:
           OCI_ROOT_URL: https://myreg.io
           OCI_NAMESPACE: mytestorg/mytestrepo
@@ -241,14 +243,6 @@ jobs:
           OCI_HIDE_SKIPPED_WORKFLOWS: 0
           OCI_DEBUG: 0
           OCI_DELETE_MANIFEST_BEFORE_BLOBS: 0
-      - run: mkdir -p .out/ && mv {report.html,junit.xml} .out/
-        if: always()
-      - name: Upload test results zip as build artifact
-        uses: actions/upload-artifact@v1
-        with:
-          name: oci-test-results-${{ github.sha }}
-          path: .out/
-        if: always()
 ```
 
 You can also add a badge pointing to list of runs for this action using the following markdown: