diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d9a7914..636015f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,5 +1,7 @@ name: Benchmarks +# Checks that every benchmark runs on every Ruby (PRs: only the files they change). +# It publishes nothing: the results site comes from results-site.yml. # Always runs, so the benchmarks-ok check at the end always reports and can be # required. The changes job decides whether there is anything to benchmark. on: @@ -67,14 +69,12 @@ jobs: - { ruby: ruby_4.0, variant: zjit, flags: --zjit } - { ruby: ruby_head, variant: zjit, flags: --zjit } - # Read by docker/run-benchmarks.sh and docker/collect_results.rb. env: + # Read by docker/run-benchmarks.sh. RUBY_VARIANT: ${{ matrix.variant }} RUBY_VARIANT_FLAGS: ${{ matrix.flags }} - RESULTS_DIR: results - RESULTS_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} - RESULTS_PR: ${{ github.event.pull_request.number }} - RESULTS_LABEL: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }} + # The build's name in the step names and the failure report, for example ruby_3.4+yjit. + BUILD: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }} steps: # Plain Rubies only: shared reports do not record the Ruby or its flags, @@ -84,113 +84,13 @@ jobs: run: | echo "SHARE=1" >> "$GITHUB_ENV" - uses: actions/checkout@v4 - - name: Run benchmarks on ${{ env.RESULTS_LABEL }} + - name: Run benchmarks on ${{ env.BUILD }} env: FILES: ${{ needs.changes.outputs.files }} run: .github/scripts/run-benchmarks.sh ${{ matrix.ruby }} - name: Report failed benchmarks if: failure() - run: .github/scripts/report-failures.sh benchmarks.log ${{ env.RESULTS_LABEL }} - - name: Upload results - uses: actions/upload-artifact@v4 - if: always() - with: - name: results-${{ env.RESULTS_LABEL }} - path: results/ - if-no-files-found: warn - - # Builds the results site from every job's results, also when some jobs - # failed, so a PR gets a preview (the site-preview artifact, named outside - # the results-* pattern so a re-run never downloads it). Skipped when the - # benchmark jobs did not run (a lint failure). Only a push to - # main where every job passed publishes it, so a partial run never replaces - # the live site. Not required: a Pages problem never blocks a merge. - site: - needs: [changes, rake] - if: ${{ !cancelled() && needs.changes.outputs.run == 'true' && needs.rake.result != 'skipped' }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - pattern: results-* - path: results/ - merge-multiple: true - - name: Build the results site - # A failed benchmark job leaves no result file, like a benchmark that needs a newer Ruby, so the page says missing results may be crashes. - env: - RESULTS_INCOMPLETE: ${{ needs.rake.result != 'success' && '1' || '' }} - run: docker compose run --rm -T -e RESULTS_INCOMPLETE --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site - - name: Upload the site preview - id: preview - uses: actions/upload-artifact@v4 - with: - name: site-preview - path: _site/ - # A re-run replaces the earlier attempt's preview. - overwrite: true - # A one-click link on the run's Summary page; it needs no extra permissions, so it works for PRs from forks too. - - name: Link the site preview in the run summary - env: - PREVIEW_URL: ${{ steps.preview.outputs.artifact-url }} - run: | - { - echo "### Results site preview" - echo "" - echo "[Download the site preview]($PREVIEW_URL) (a zip, needs a GitHub login), unzip it and open \`index.html\` in a browser." - } >> "$GITHUB_STEP_SUMMARY" - - name: Upload the site for GitHub Pages - if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success' - uses: actions/upload-pages-artifact@v5 - with: - path: _site/ - - deploy: - needs: [rake, site] - if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success' && needs.site.result == 'success' - runs-on: ubuntu-latest - # Only this job can publish; the rest of the workflow keeps the default - # token permissions. - permissions: - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - # One deploy at a time; a newer merge waits instead of cancelling one that - # is halfway through. - concurrency: - group: pages - cancel-in-progress: false - - steps: - # Runs can finish out of order, so an older run must not put its results back over a newer one. - # When main moved on, ask pick-benchmarks.sh (the same rule CI uses) whether the newer commits run benchmarks. - # If they do, their own run deploys newer results; if not (a README-only merge), this run's results are still the newest. - - uses: actions/checkout@v4 - with: - ref: main - fetch-depth: 0 - - name: Check no newer run on main will deploy - id: newest - run: | - if [ "$(git rev-parse HEAD)" = "$GITHUB_SHA" ]; then - echo "deploy=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - GITHUB_EVENT_NAME=push GITHUB_OUTPUT=newer.txt .github/scripts/pick-benchmarks.sh "$GITHUB_SHA" - if grep -q '^run=true' newer.txt; then - echo "main moved on and its newer commits run benchmarks, so their run deploys; skipping." - echo "deploy=false" >> "$GITHUB_OUTPUT" - else - echo "main moved on, but nothing since this run affects benchmarks; deploying." - echo "deploy=true" >> "$GITHUB_OUTPUT" - fi - - name: Deploy to GitHub Pages - id: deployment - if: steps.newest.outputs.deploy == 'true' - uses: actions/deploy-pages@v5 + run: .github/scripts/report-failures.sh benchmarks.log ${{ env.BUILD }} # The check to require on main. Passes when every benchmark job passed, or # when there was nothing to benchmark. diff --git a/.github/workflows/results-site.yml b/.github/workflows/results-site.yml new file mode 100644 index 0000000..0dee86a --- /dev/null +++ b/.github/workflows/results-site.yml @@ -0,0 +1,95 @@ +name: Results site + +# Builds and publishes the results site (https://fastruby.github.io/fast-ruby/). +# Every Ruby runs each benchmark on the same machine, so the site can compare i/s across Rubies; benchmarks.yml runs each Ruby on its own machine and only checks that the benchmarks run. +# Weekly, because a full run takes about 3 hours per shard; run it by hand to publish sooner. +on: + schedule: + # Sunday 03:00 UTC, away from the working day's PR runs. + - cron: '0 3 * * 0' + workflow_dispatch: + +jobs: + # 6 machines, each running every 6th benchmark file on all 26 builds, with the newest released Ruby run again between them (script/run_cross_ruby.rb). + shard: + name: shard ${{ matrix.shard }} + runs-on: ubuntu-latest + # 35 passes of about 13 files each; the job limit is 6 hours. + timeout-minutes: 330 + + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5] + + # Read by docker/collect_results.rb. + env: + RESULTS_COMMIT: ${{ github.sha }} + + steps: + - uses: actions/checkout@v4 + - name: Run every build on this machine + run: ruby script/run_cross_ruby.rb --shard ${{ matrix.shard }} --shards 6 --fresh-images --out results + # Also when a pass failed: the shard's other results are kept, to see what went wrong (the site is not built from an incomplete run). + - name: Upload results + uses: actions/upload-artifact@v4 + if: always() + with: + name: results-shard-${{ matrix.shard }} + path: results/ + retention-days: 90 + # A re-run of a failed shard replaces its earlier attempt. + overwrite: true + + # Builds both views from all six shards and publishes them, only for a complete run on main. + site: + needs: shard + if: github.ref_name == 'main' + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + # Reads this workflow's earlier runs (the newest-results check below). + actions: read + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + # One deploy at a time; a newer run waits instead of cancelling one that is halfway through. + concurrency: + group: pages + cancel-in-progress: false + + steps: + - uses: actions/checkout@v4 + # Never put older results back: a re-run of an old run, or two runs finishing out of order, must not publish over a newer run's results. + # Run ids only grow and a re-run keeps its id, so a newer run has a higher id (and, since main cannot be force-pushed, a commit at least as new). + - name: Check no newer results are live + id: newest + env: + GH_TOKEN: ${{ github.token }} + run: | + live=$(gh run list --workflow results-site.yml --branch main --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty') + if [ -n "$live" ] && [ "$live" -gt "$GITHUB_RUN_ID" ]; then + echo "The live site has results from run $live, newer than this run ($GITHUB_RUN_ID); not publishing." + echo "publish=false" >> "$GITHUB_OUTPUT" + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/download-artifact@v4 + if: steps.newest.outputs.publish == 'true' + with: + pattern: results-shard-* + path: results/ + merge-multiple: true + - name: Build the results site + if: steps.newest.outputs.publish == 'true' + run: docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site + - name: Upload the site for GitHub Pages + if: steps.newest.outputs.publish == 'true' + uses: actions/upload-pages-artifact@v5 + with: + path: _site/ + - name: Deploy to GitHub Pages + id: deployment + if: steps.newest.outputs.publish == 'true' + uses: actions/deploy-pages@v5 diff --git a/.gitignore b/.gitignore index da0fa2b..6dfcfa6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /Gemfile.lock /results/ /_site/ +/cross-ruby/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 762b82c..74d4260 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,7 @@ These idioms list here are trying to satisfy following goals: - [Note on entry](#note-on-entry) - [Running it on other Rubies](#running-it-on-other-rubies) +- [The results site](#the-results-site) - [Benchmarks that need a newer Ruby](#benchmarks-that-need-a-newer-ruby) - [License](#license) @@ -104,23 +105,25 @@ RUBY_VARIANT=yjit RUBY_VARIANT_FLAGS=--yjit docker compose run --rm ruby_3.4 cod RUBY_VARIANT=zjit RUBY_VARIANT_FLAGS=--zjit docker compose run --rm ruby_4.0 code/your-new/entry.rb ``` -To keep the results, set `RESULTS_DIR`. Each benchmark then also writes its -report as JSON to `results/