Show every Ruby's speed side by side on the results site

.github/workflows/benchmarks-cross-ruby.yml

@@ -0,0 +1,107 @@
+name: Benchmarks across Rubies
+
+# Every build on one machine, so the results site can compare i/s across Rubies (benchmarks.yml runs each Ruby on its own machine, which only allows comparing idioms inside one Ruby).
+# Weekly, because a full run takes about 3 hours per shard.
+on:
+  schedule:
+    # Sunday 03:00 UTC, away from the working day's PR runs.
+    - cron: '0 3 * * 0'
+  workflow_dispatch:
+
+jobs:
+  shard:
+    name: shard ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    # 35 passes of about 13 files each; the job limit is 6 hours.
+    timeout-minutes: 330
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Each shard gets every 6th benchmark file and runs all 26 builds on them.
+        shard: [0, 1, 2, 3, 4, 5]
+
+    # Read by docker/collect_results.rb.
+    env:
+      RESULTS_COMMIT: ${{ github.sha }}
+
+    steps:
+      - uses: actions/checkout@v4
+      # Each image is pulled or built before its Ruby's passes and removed after them; this shows how much disk the runner had to start with.
+      - name: Show free disk
+        run: df -h /
+      - name: Run every build on this machine
+        run: ruby script/run_cross_ruby.rb --shard ${{ matrix.shard }} --shards 6 --fresh-images --out results
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: cross-ruby-${{ matrix.shard }}
+          path: results/
+          if-no-files-found: warn
+          retention-days: 90
+
+  # Builds the whole site, both views, from every shard's results (also when a shard failed, so there is a preview).
+  site:
+    needs: shard
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cross-ruby-*
+          path: cross-ruby/
+          merge-multiple: true
+      - name: Build the results site
+        # A failed shard leaves benchmarks or Rubies out, so the page says so.
+        env:
+          RESULTS_INCOMPLETE: ${{ needs.shard.result != 'success' && '1' || '' }}
+        run: docker compose run --rm -T -e RESULTS_INCOMPLETE --entrypoint ruby ruby_4.0 script/build_results_site.rb --cross-ruby cross-ruby _site
+      - name: Upload the site preview
+        id: preview
+        uses: actions/upload-artifact@v4
+        with:
+          name: site-preview
+          path: _site/
+          overwrite: true
+      - name: Link the site preview in the run summary
+        env:
+          PREVIEW_URL: ${{ steps.preview.outputs.artifact-url }}
+        run: |
+          {
+            echo "### Results site preview"
+            echo ""
+            echo "[Download the site preview]($PREVIEW_URL) (a zip, needs a GitHub login), unzip it and open \`index.html\` in a browser."
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  # Publishes only a complete run of main; this is the only workflow that deploys the site.
+  deploy:
+    needs: [shard, site]
+    if: github.ref_name == 'main' && needs.shard.result == 'success' && needs.site.result == 'success'
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    # One deploy at a time; a newer run waits instead of cancelling one that is halfway through.
+    concurrency:
+      group: pages
+      cancel-in-progress: false
+
+    steps:
+      # The site job's preview is the whole site: publish it as it is.
+      - uses: actions/download-artifact@v4
+        with:
+          name: site-preview
+          path: _site/
+      - name: Upload the site for GitHub Pages
+        uses: actions/upload-pages-artifact@v5
+        with:
+          path: _site/
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v5

.github/workflows/benchmarks.yml

@@ -99,12 +99,9 @@ jobs:
           path: results/
           if-no-files-found: warn
 
-  # Builds the results site from every job's results, also when some jobs
-  # failed, so a PR gets a preview (the site-preview artifact, named outside
-  # the results-* pattern so a re-run never downloads it). Skipped when the
-  # benchmark jobs did not run (a lint failure). Only a push to
-  # main where every job passed publishes it, so a partial run never replaces
-  # the live site. Not required: a Pages problem never blocks a merge.
+  # Builds a preview of the "Does it hold?" view from every job's results, also when some jobs failed (the site-preview artifact, named outside the results-* pattern so a re-run never downloads it).
+  # Skipped when the benchmark jobs did not run (a lint failure).
+  # The published site comes from benchmarks-cross-ruby.yml, where every Ruby runs on one machine; this workflow never deploys.
   site:
     needs: [changes, rake]
     if: ${{ !cancelled() && needs.changes.outputs.run == 'true' && needs.rake.result != 'skipped' }}
@@ -140,57 +137,6 @@ jobs:
             echo ""
             echo "[Download the site preview]($PREVIEW_URL) (a zip, needs a GitHub login), unzip it and open \`index.html\` in a browser."
           } >> "$GITHUB_STEP_SUMMARY"
-      - name: Upload the site for GitHub Pages
-        if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success'
-        uses: actions/upload-pages-artifact@v5
-        with:
-          path: _site/
-
-  deploy:
-    needs: [rake, site]
-    if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success' && needs.site.result == 'success'
-    runs-on: ubuntu-latest
-    # Only this job can publish; the rest of the workflow keeps the default
-    # token permissions.
-    permissions:
-      pages: write
-      id-token: write
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    # One deploy at a time; a newer merge waits instead of cancelling one that
-    # is halfway through.
-    concurrency:
-      group: pages
-      cancel-in-progress: false
-
-    steps:
-      # Runs can finish out of order, so an older run must not put its results back over a newer one.
-      # When main moved on, ask pick-benchmarks.sh (the same rule CI uses) whether the newer commits run benchmarks.
-      # If they do, their own run deploys newer results; if not (a README-only merge), this run's results are still the newest.
-      - uses: actions/checkout@v4
-        with:
-          ref: main
-          fetch-depth: 0
-      - name: Check no newer run on main will deploy
-        id: newest
-        run: |
-          if [ "$(git rev-parse HEAD)" = "$GITHUB_SHA" ]; then
-            echo "deploy=true" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          GITHUB_EVENT_NAME=push GITHUB_OUTPUT=newer.txt .github/scripts/pick-benchmarks.sh "$GITHUB_SHA"
-          if grep -q '^run=true' newer.txt; then
-            echo "main moved on and its newer commits run benchmarks, so their run deploys; skipping."
-            echo "deploy=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "main moved on, but nothing since this run affects benchmarks; deploying."
-            echo "deploy=true" >> "$GITHUB_OUTPUT"
-          fi
-      - name: Deploy to GitHub Pages
-        id: deployment
-        if: steps.newest.outputs.deploy == 'true'
-        uses: actions/deploy-pages@v5
 
   # The check to require on main. Passes when every benchmark job passed, or
   # when there was nothing to benchmark.

.gitignore

@@ -3,3 +3,4 @@
 /Gemfile.lock
 /results/
 /_site/
+/cross-ruby/

CONTRIBUTING.md

@@ -119,8 +119,23 @@ into `_site/`, then open `_site/index.html` in a browser:
 docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site
 ```
 
-CI does the same after every run: the site is attached to the run as the
-`site-preview` artifact, and published to GitHub Pages from `main`.
+CI does the same after every benchmark run and attaches it as the `site-preview` artifact.
+The published site comes from the weekly run below.
+
+Those results only compare idioms inside one Ruby: in CI each Ruby runs on its own machine.
+The site's "Across Rubies" view compares Rubies, so every build has to run on one machine.
+`script/run_cross_ruby.rb` (Ruby 3 on your machine, it calls Docker) does that, and runs the newest released MRI again after every 3 builds, so the site can show how steady the machine was.
+For example, two files on three builds:
+
+```
+ruby script/run_cross_ruby.rb --files code/date/iso8601-vs-parse.rb,code/string/gsub-vs-tr.rb --builds ruby_3.4,ruby_3.4+yjit,ruby_4.0 --out cross-ruby
+docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb --cross-ruby cross-ruby _site
+```
+
+`ruby script/run_cross_ruby.rb --help` lists the options.
+The builds are the ones in the CI matrix (`.github/workflows/benchmarks.yml`), so a new Ruby added there and in `compose.yaml` is measured too, and once released it becomes both the reference and the Ruby the site opens on.
+CI runs every build on every file once a week, split over 6 machines (`.github/workflows/benchmarks-cross-ruby.yml`), builds both views of the site from that run and publishes it to GitHub Pages.
+On an Apple silicon Mac, leave out `ruby_2.1` and `jruby_9.1` (they run under emulation, so their numbers are not comparable) and `ruby_3.1+yjit` (Ruby 3.1's YJIT only exists on x86-64).
 
 ## Benchmarks that need a newer Ruby
 

compose.yaml

@@ -21,6 +21,11 @@ x-benchmark: &benchmark
     - RESULTS_LABEL
     - RESULTS_COMMIT
     - RESULTS_PR
+    # Cross-Ruby runs only (script/run_cross_ruby.rb).
+    - RESULTS_SHARD
+    - RESULTS_SHARDS
+    - RESULTS_PASS
+    - RESULTS_REFERENCE
     - GITHUB_RUN_ID
     - GITHUB_RUN_ATTEMPT
     - RUNNER_NAME

docker/collect_results.rb

@@ -52,6 +52,13 @@ def write(report)
         "run_id" => env("GITHUB_RUN_ID"),
         "run_attempt" => env("GITHUB_RUN_ATTEMPT"),
         "runner" => env("RUNNER_NAME"),
+        # Set by script/run_cross_ruby.rb, which runs every build on one machine and the reference build between them; nil otherwise.
+        "shard" => env("RESULTS_SHARD") && env("RESULTS_SHARD").to_i,
+        "shards" => env("RESULTS_SHARDS") && env("RESULTS_SHARDS").to_i,
+        "pass" => env("RESULTS_PASS") && env("RESULTS_PASS").to_i,
+        "reference" => env("RESULTS_REFERENCE") == "1",
+        # When this report finished, to place it between the reference passes.
+        "measured_at" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
         "environment" => environment,
         "entries" => report.data
       }))