Build release images natively per architecture (fixes the v0.4.0-alpha arm64 failure)

.github/workflows/docker-publish.yml

@@ -11,13 +11,22 @@ permissions:
   contents: read
 
 jobs:
+  # The same native per-architecture builds the release runs (no QEMU
+  # emulation), without registry writes.
   build:
-    name: Build ${{ matrix.image }} (no publication)
-    runs-on: ubuntu-latest
+    name: Build ${{ matrix.image }} (${{ matrix.arch }}, no publication)
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
+        image: [web, api, neo4j]
+        arch: [amd64, arm64]
         include:
+          - arch: amd64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
           - image: web
             dockerfile: packages/web/Dockerfile
           - image: api
@@ -28,14 +37,13 @@ jobs:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
-      - uses: docker/setup-qemu-action@v3
       - uses: docker/setup-buildx-action@v3
-      - name: Validate both image architectures without registry writes
+      - name: Validate linux/${{ matrix.arch }} without registry writes
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/${{ matrix.arch }}
           push: false
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.arch }}

.github/workflows/release.yml

@@ -44,18 +44,27 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
+  # Each architecture builds natively on its own runner and pushes by digest
+  # only (no tag). QEMU emulation is not used: emulated arm64 `npm ci` crashed
+  # with "Illegal instruction" in the v0.4.0-alpha release run.
   images:
-    name: Publish ${{ matrix.image }} image
+    name: Build ${{ matrix.image }} (${{ matrix.arch }})
     needs: verify
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
     permissions:
       contents: read
       packages: write
     strategy:
       fail-fast: true
       matrix:
+        image: [web, api, neo4j]
+        arch: [amd64, arm64]
         include:
+          - arch: amd64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
           # Build arguments match the stack the production gate validated
           # (deployment/docker-compose.yml).
           - image: web
@@ -74,47 +83,90 @@ jobs:
         with:
           ref: ${{ needs.verify.outputs.sha }}
           persist-credentials: false
-      - uses: docker/setup-qemu-action@v3
       - uses: docker/setup-buildx-action@v3
       - uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ github.token }}
-      # Only the version tag here; `latest` moves for all images together in
-      # github-release, after every image has been published.
-      - name: Build and push (amd64 + arm64)
+      - name: Build and push linux/${{ matrix.arch }} by digest
         id: build
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ matrix.dockerfile }}
           build-args: ${{ matrix.build-args }}
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ghcr.io/graphdone/graphdone-${{ matrix.image }}:${{ needs.verify.outputs.version }}
+          platforms: linux/${{ matrix.arch }}
+          outputs: type=image,name=ghcr.io/graphdone/graphdone-${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true
           provenance: mode=max
           sbom: true
           labels: |
             org.opencontainers.image.source=https://github.com/${{ github.repository }}
             org.opencontainers.image.revision=${{ needs.verify.outputs.sha }}
             org.opencontainers.image.version=${{ needs.verify.outputs.version }}
             org.opencontainers.image.licenses=MIT
-      - name: Record digest
+      - name: Record platform digest
         env:
           DIGEST: ${{ steps.build.outputs.digest }}
         run: |
           mkdir -p digests
-          echo "ghcr.io/graphdone/graphdone-${{ matrix.image }}@$DIGEST" > "digests/${{ matrix.image }}.txt"
+          echo "$DIGEST" > "digests/${{ matrix.image }}-${{ matrix.arch }}"
       - uses: actions/upload-artifact@v4
         with:
-          name: digest-${{ matrix.image }}
+          name: digest-${{ matrix.image }}-${{ matrix.arch }}
           path: digests/
           if-no-files-found: error
 
+  # Join both architectures into the version tag. Only the version tag here;
+  # `latest` moves for all images together in github-release.
+  manifests:
+    name: Publish ${{ matrix.image }} ${{ needs.verify.outputs.version }}
+    needs: [verify, images]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: true
+      matrix:
+        image: [web, api, neo4j]
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: digest-${{ matrix.image }}-*
+          merge-multiple: true
+          path: digests
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Create the multi-architecture version tag
+        env:
+          IMAGE: ghcr.io/graphdone/graphdone-${{ matrix.image }}
+          VERSION: ${{ needs.verify.outputs.version }}
+        run: |
+          set -euo pipefail
+          refs=()
+          for file in digests/*; do refs+=("$IMAGE@$(cat "$file")"); done
+          [ "${#refs[@]}" -eq 2 ] || { echo "::error::expected 2 platform digests, found ${#refs[@]}"; exit 1; }
+          docker buildx imagetools create -t "$IMAGE:$VERSION" "${refs[@]}"
+          manifest=$(docker buildx imagetools inspect "$IMAGE:$VERSION" --format '{{json .Manifest}}')
+          platforms=$(echo "$manifest" | jq -r '[.manifests[].platform | select(.os != "unknown") | "\(.os)/\(.architecture)"] | sort | join(",")')
+          [ "$platforms" = "linux/amd64,linux/arm64" ] || { echo "::error::$IMAGE:$VERSION has platforms '$platforms'"; exit 1; }
+          mkdir -p published
+          echo "$IMAGE@$(echo "$manifest" | jq -r .digest)" > "published/${{ matrix.image }}.txt"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: image-${{ matrix.image }}
+          path: published/
+          if-no-files-found: error
+
   github-release:
     name: Create GitHub release
-    needs: [verify, images]
+    needs: [verify, manifests]
     runs-on: ubuntu-latest
     timeout-minutes: 15
     permissions:
@@ -151,9 +203,9 @@ jobs:
           done
       - uses: actions/download-artifact@v4
         with:
-          pattern: digest-*
+          pattern: image-*
           merge-multiple: true
-          path: digests
+          path: published
       - name: Publish release notes with image digests
         env:
           GH_TOKEN: ${{ github.token }}
@@ -166,7 +218,7 @@ jobs:
             echo
             echo "**Container images** (linux/amd64 + linux/arm64, with SBOM and provenance):"
             echo
-            for f in digests/*.txt; do echo "- \`$(cat "$f")\`"; done
+            for f in published/*.txt; do echo "- \`$(cat "$f")\`"; done
             echo
           } > notes.md
           gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --verify-tag \

docs/quality/RELEASE_GATES.md

@@ -9,7 +9,10 @@ that job; their other required jobs must still pass.
 ## Publishing a release
 
 Docker Build Validation builds the web, API and Neo4j images for amd64 and arm64
-on main-target PRs without registry writes.
+on main-target PRs without registry writes. Each architecture builds natively on
+its own runner (`ubuntu-24.04`, `ubuntu-24.04-arm`); QEMU emulation is not used
+because emulated arm64 `npm ci` crashed with "Illegal instruction" in the
+v0.4.0-alpha release run.
 
 A release is published only by pushing a version tag on a validated main commit:
 
@@ -33,9 +36,10 @@ closed unless:
   runs test a merge ref, so they never count.
 
 Only then do write-scoped jobs build `ghcr.io/graphdone/graphdone-{web,api,neo4j}`
-from the verified SHA for amd64 and arm64 (SBOM and provenance, the same web
-build arguments as the validated stack) and push the version tag. After every
-image is published, the final job decides from the published releases whether
+from the verified SHA, natively per architecture (SBOM and provenance, the same
+web build arguments as the validated stack), push each platform image by digest,
+and join the amd64 and arm64 digests into the version tag, refusing an index
+that lacks either architecture. After every image is published, the final job decides from the published releases whether
 this version is the highest — a pre-release never outranks a stable release —
 moves `:latest` on all three images together if so, and creates the GitHub
 release with the image digests. Each tag runs in its own concurrency group and a

tests/lib/release/workflows.test.mjs

@@ -34,35 +34,82 @@ test('release runs only for version tags, one at a time, with no default permiss
   assert.deepEqual(doc.concurrency, { group: 'release-${{ github.ref }}', 'cancel-in-progress': false });
 });
 
-test('images push only their version tag; latest moves for all images after every image published', () => {
+const NATIVE = { amd64: 'ubuntu-24.04', arm64: 'ubuntu-24.04-arm' };
+
+function expandMatrix(matrix) {
+  // GitHub semantics for the forms used here: the cross product of the list
+  // keys, then each include entry adds its keys to every matching combination.
+  const keys = Object.keys(matrix).filter(key => key !== 'include');
+  let combos = [{}];
+  for (const key of keys) combos = combos.flatMap(combo => matrix[key].map(value => ({ ...combo, [key]: value })));
+  for (const extra of matrix.include ?? []) {
+    for (const combo of combos) {
+      if (keys.every(key => !(key in extra) || extra[key] === combo[key])) {
+        for (const [key, value] of Object.entries(extra)) if (!keys.includes(key)) combo[key] = value;
+      }
+    }
+  }
+  return combos;
+}
+
+for (const name of ['release', 'docker-publish']) {
+  test(`${name} builds every image natively for both architectures, without QEMU`, () => {
+    const { jobs } = workflow(name);
+    const job = jobs[name === 'release' ? 'images' : 'build'];
+    assert.equal(job['runs-on'], '${{ matrix.runner }}');
+    const combos = expandMatrix(job.strategy.matrix);
+    assert.equal(combos.length, 6);
+    for (const combo of combos) {
+      assert.equal(combo.runner, NATIVE[combo.arch], `${combo.image}/${combo.arch} runner`);
+      assert.ok(combo.dockerfile, `${combo.image}/${combo.arch} dockerfile`);
+    }
+    const build = job.steps.find(step => step.uses?.startsWith('docker/build-push-action'));
+    assert.equal(build.with.platforms, 'linux/${{ matrix.arch }}');
+    for (const each of Object.values(jobs)) for (const step of each.steps ?? []) assert.ok(!step.uses?.startsWith('docker/setup-qemu-action'));
+  });
+}
+
+test('platform builds push by digest only; version tags join both digests; latest moves after every image', () => {
   const { jobs } = workflow('release');
   const build = jobs.images.steps.find(step => step.uses?.startsWith('docker/build-push-action'));
-  assert.equal(build.with.tags, 'ghcr.io/graphdone/graphdone-${{ matrix.image }}:${{ needs.verify.outputs.version }}');
+  assert.equal(build.with.tags, undefined, 'platform builds push no tag');
+  assert.equal(build.with.push, undefined);
+  assert.equal(build.with.outputs, 'type=image,name=ghcr.io/graphdone/graphdone-${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true');
   assert.equal(jobs.images.strategy['fail-fast'], true);
+  const web = expandMatrix(jobs.images.strategy.matrix).find(item => item.image === 'web');
+  assert.match(web['build-args'], /VITE_GRAPHQL_URL=https:\/\/localhost:3128\/api\/graphql/);
+
+  const join = jobs.manifests.steps.find(step => /imagetools create/.test(step.run || ''));
+  assert.match(join.run, /imagetools create -t "\$IMAGE:\$VERSION" "\$\{refs\[@\]\}"/);
+  assert.doesNotMatch(join.run, /latest/);
+  assert.match(join.run, /linux\/amd64,linux\/arm64/, 'the published index must hold both architectures');
+  assert.deepEqual(expandMatrix(jobs.manifests.strategy.matrix).map(item => item.image), ['web', 'api', 'neo4j']);
+
   const decide = jobs['github-release'].steps.findIndex(step => step.run === 'node scripts/verify-release.mjs --latest');
   const move = jobs['github-release'].steps.findIndex(step => /imagetools create/.test(step.run || ''));
   assert.ok(decide >= 0 && move > decide, 'latest is decided in the final job, then moved');
   assert.equal(jobs['github-release'].steps[move].if, "steps.latest.outputs.latest == 'true'");
-  const web = jobs.images.strategy.matrix.include.find(item => item.image === 'web');
-  assert.match(web['build-args'], /VITE_GRAPHQL_URL=https:\/\/localhost:3128\/api\/graphql/);
 });
 
 test('release writes happen only after the source gate, from the verified commit', () => {
   const { jobs } = workflow('release');
   assert.deepEqual(jobs.verify.permissions, { contents: 'read', actions: 'read' });
   assert.ok(jobs.verify.steps.some(step => step.run === 'node scripts/verify-release.mjs'));
   assert.deepEqual(jobs.images.permissions, { contents: 'read', packages: 'write' });
+  assert.deepEqual(jobs.manifests.permissions, { contents: 'read', packages: 'write' });
   assert.deepEqual(jobs['github-release'].permissions, { contents: 'write', packages: 'write' });
   assert.equal(jobs.images.needs, 'verify');
-  assert.deepEqual(jobs['github-release'].needs, ['verify', 'images']);
+  assert.deepEqual(jobs.manifests.needs, ['verify', 'images']);
+  assert.deepEqual(jobs['github-release'].needs, ['verify', 'manifests']);
   for (const [name, job] of Object.entries(jobs)) {
     for (const step of job.steps) {
       if (step.uses?.startsWith('actions/checkout')) {
         const expected = name === 'verify' ? '${{ github.sha }}' : '${{ needs.verify.outputs.sha }}';
         assert.equal(step.with.ref, expected, `${name} checks out the verified commit`);
       }
       if (step.uses?.startsWith('docker/build-push-action')) assert.equal(name, 'images');
-      if (/gh\s+release|imagetools/.test(step.run || '')) assert.equal(name, 'github-release');
+      if (/imagetools create/.test(step.run || '')) assert.ok(['manifests', 'github-release'].includes(name), name);
+      if (/gh\s+release/.test(step.run || '')) assert.equal(name, 'github-release');
     }
   }
 });