diff --git a/CHANGELOG.md b/CHANGELOG.md index 397ba15..cdf4e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog -## Unreleased +## 1.1.0 + +- Track one grouped saved-Scenario Run with `test-run-id`, replacing retired sweep IDs. +- Wait through scenario preparation and approval before evaluating results. +- Document saved YAML Scenarios as the repeatable PR testing rules and report cleanup separately. - Assign preview URLs to a Composal project and track the PR number or source branch. - Auto-detect PR, workflow-run, and branch-push context; register branch previews even before an open PR exists. diff --git a/README.md b/README.md index af4f0bb..863eb38 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Test the product journeys affected by your PR against its deployed preview. Veri preview-url: ${{ steps.deploy.outputs.url }} ``` -Configure the Composal project in **Verify → Projects → your project → PR Testing**. Select its source repository and **Your CI**, your scenario pack, and an external Verify environment template containing your personas and safety policy. Add a Composal administrator API token as the `COMPOSAL_TOKEN` Actions secret and your organization slug as the `COMPOSAL_ORG` variable. +Configure the Composal project in **Verify → Projects → your project → PR Testing**. Select its source repository, **Your CI**, and an external Verify environment template containing your personas and safety policy. Verify selects relevant saved YAML Scenarios from that project and launches them together in one parallel Run. Add a Composal administrator API token as the `COMPOSAL_TOKEN` Actions secret and your organization slug as the `COMPOSAL_ORG` variable. Run this step after deploying **`github.event.pull_request.head.sha`** and waiting for the preview to become reachable. GitHub's `github.sha` can be a synthetic merge commit; Verify deliberately uses the PR head. Your existing deployment provider supplies the URL; this action does not build an arbitrary app. @@ -98,7 +98,7 @@ jobs: The fork condition avoids running a secret-dependent step when GitHub withholds secrets. Keep your deployment and secret use on trusted workflow events; do not switch to `pull_request_target` just to expose secrets to fork code. -By default the action waits up to 15 minutes. Passed or explicitly skipped requests succeed. Failed, blocked, cancelled, superseded, cleanup-blocked, and timed-out requests fail the step. A wait timeout also fails the step, with a link to the run that continues in Composal. Set `wait: 'false'` for an asynchronous handoff; a successful handoff means accepted, not passed. +By default the action waits up to 15 minutes. Passed or explicitly skipped requests succeed. Failed, blocked, cancelled, superseded, and timed-out scenario outcomes fail the step. Cleanup is reported separately and does not turn a passing product outcome into a failure. A wait timeout also fails the step, with a link to the run that continues in Composal. Set `wait: 'false'` for an asynchronous handoff; a successful handoff means accepted, not passed. ## Inputs and outputs @@ -116,7 +116,7 @@ By default the action waits up to 15 minutes. Passed or explicitly skipped reque | `timeout` | `'15'` | Wait limit in minutes, between 1 and 60. Set the job timeout higher. | | `api-url` | `https://composal.ai` | HTTPS API origin, for installations using another endpoint. | -Outputs: `url` (PR history or registered branch environment), `pull-request-id`, `run-id`, `sweep-id` (when started), `environment-id` (branch registration), and `status`. The workflow summary links to the full results. Critical/high issue recordings remain authenticated links in Verify. +Outputs: `url` (PR history or registered branch environment), `pull-request-id`, `run-id`, `test-run-id` (when started), `environment-id` (branch registration), and `status`. The workflow summary links to the full results. Critical/high issue recordings remain authenticated links in Verify. Retries within the same workflow attempt reuse the same verification request. Rerunning the workflow creates a fresh request. A workflow for an obsolete head cannot start a run for the newer head. diff --git a/action.yml b/action.yml index a792e01..0fff666 100644 --- a/action.yml +++ b/action.yml @@ -40,8 +40,8 @@ outputs: description: Composal PR identity. run-id: description: This verification request's identity. - sweep-id: - description: Browser sweep identity, when started. + test-run-id: + description: Grouped scenario Run identity, when started. status: description: Verification state or terminal outcome. queued means accepted, not passed. environment-id: diff --git a/verify.mjs b/verify.mjs index 6bc6734..2d0dc68 100644 --- a/verify.mjs +++ b/verify.mjs @@ -182,7 +182,7 @@ export async function verify( url: url.href, 'pull-request-id': pr.id || '', 'run-id': pr.requested_run_id || '', - 'sweep-id': '', + 'test-run-id': '', status: pr.requested_run_id ? 'queued' : pr.status || 'skipped', 'environment-id': pr.environment_id || '', } @@ -202,12 +202,20 @@ export async function verify( if (run.id !== pr.requested_run_id) throw new Error('A newer verification request superseded this handoff.') outputs.status = run.display_state - outputs['sweep-id'] = run.verify_sweep_id || '' + outputs['test-run-id'] = run.run_id || '' await publish(outputs) } if (!config.wait) return outputs const deadline = now() + config.timeoutMs - const active = new Set(['queued', 'waiting_for_preview', 'running', 'cancelling', 'terminal']) + const active = new Set([ + 'queued', + 'waiting_for_preview', + 'preparing', + 'prepared', + 'running', + 'cancelling', + 'terminal', + ]) while (now() < deadline) { const page = await api(`${requests}/${encodeURIComponent(pr.id)}`) const run = page.runs.find((candidate) => candidate.id === pr.requested_run_id) @@ -219,7 +227,7 @@ export async function verify( ) throw new Error('A newer PR revision or verification request superseded this run.') outputs.status = run.display_state - outputs['sweep-id'] = run.verify_sweep_id || '' + outputs['test-run-id'] = run.run_id || '' await publish(outputs) if (!active.has(outputs.status)) { if (!['passed', 'skipped'].includes(outputs.status)) diff --git a/verify.test.mjs b/verify.test.mjs index ea5c475..4cb8af1 100644 --- a/verify.test.mjs +++ b/verify.test.mjs @@ -32,7 +32,7 @@ const run = (status = 'passed') => ({ id: 'vprun_1', head_sha: sha, display_state: status, - verify_sweep_id: 'vsweep_1', + run_id: 'run_grouped_1', }) function transport(responses) { const calls = [] @@ -180,10 +180,13 @@ test('validates input authority and refuses ambiguous or missing PR context', () assert.equal(config({ pr: '#45', sha }, {}).number, 45) }) -test('hands a deployed URL to the exact request, waits, and publishes the sweep link', async () => { +test('hands a deployed URL to the exact request, waits, and publishes the grouped run identity', async () => { const api = transport([ pr, run('waiting_for_preview'), + { pull_request: pr, runs: [run('preparing')] }, + { pull_request: pr, runs: [run('prepared')] }, + { pull_request: pr, runs: [run('queued')] }, { pull_request: pr, runs: [run('running')] }, { pull_request: pr, runs: [run()] }, ]) @@ -209,11 +212,11 @@ test('hands a deployed URL to the exact request, waits, and publishes the sweep assert.equal(api.calls[0].headers.Authorization, 'Bearer private-token') assert.equal(api.calls[0].redirect, 'error') assert.equal(result.status, 'passed') - assert.equal(result['sweep-id'], 'vsweep_1') + assert.equal(result['test-run-id'], 'run_grouped_1') assert.equal(result.url, 'https://composal.ai/acme/verify/pull-requests/vpr_1') assert.deepEqual( updates.map((value) => value.status), - ['queued', 'waiting_for_preview', 'running', 'passed'] + ['queued', 'waiting_for_preview', 'preparing', 'prepared', 'queued', 'running', 'passed'] ) })