diff --git a/bench/explore/README.md b/bench/explore/README.md index fc6e87f5..932b3972 100644 --- a/bench/explore/README.md +++ b/bench/explore/README.md @@ -2,7 +2,8 @@ Measures `genclass_explore` (docs/runtime/AGENT.md section 5) the way an agent uses it: two fifo baselines and seeded reorderings of the same user steps, compared by the MCP server. Every number goes to -[RESULTS.md](../../docs/runtime/RESULTS.md) section 22 with its false-report rate next to it. +[RESULTS.md](../../docs/runtime/RESULTS.md) (sections 22 and "explore's verdict on real apps") with its false-report rate +next to it. ## What is here @@ -10,6 +11,8 @@ seeded reorderings of the same user steps, compared by the MCP server. Every num |---|---| | `summarise.mjs` | scores a directory of explore sessions: per reordered run, whether explore reported it as a race, against a DOM oracle the harness recorded for the same load (a run "differs" when its oracle equals neither baseline's). With `--old ` it scores the older comparison on the same recorded runs. | | `apps.mjs` | the two small apps below in headless Chromium with the real runtime and MCP server; writes sessions `summarise.mjs` reads | +| `planted.mjs` | small pages made of what the 2026-11-04 hunts and the review of the fix met (ages, read-only POSTs, double submits, a lost POST, generated field names, bugs that need only a slow response, a bare number beside a version, a new path per load, an optional click, wrong values that read like a time, clocks moving, a random tip, a save queue), each with a correct variant and, where it has one, a planted bug; scores each session itself against each load's own check, with `--old` the older comparison on the same runs | +| `stored.mjs` | rescoring reports a hunt kept only as text: parses the listed differences back into runs and gives them to `compareRuns`; its header says what the text cannot show. `--raw ` compares a bridge's recorded runs directly | | `apps/` | a search page on Redux Toolkit Query and one on TanStack Query (`src/rtk.jsx`, `src/tanstack.jsx`), each with a planted stale race (`?bug`) and a correct variant; installed on its own (`npm install` in `apps/`, not a workspace) | The Strapi sessions come from the hunt harness on the `claude/dazzling-cray-8tqzkn` branch @@ -28,6 +31,15 @@ node bench/explore/summarise.mjs /tmp/explore-apps # the same with the comparison of another checkout (a git worktree of main, built) node bench/explore/apps.mjs /tmp/explore-apps-before --rt ../before/packages/runtime/dist --mcp ../before/packages/mcp/dist --slow 300,600,900 node bench/explore/summarise.mjs /tmp/explore-apps-before --old ../before/packages/mcp/dist + +# the pages made of what the hunts met; shards run side by side, each with its own browser, and write into one directory +node bench/explore/planted.mjs /tmp/planted --shard 0/2 --port 41700 & +node bench/explore/planted.mjs /tmp/planted --shard 1/2 --port 41700 & +wait +node bench/explore/planted.mjs /tmp/planted --score --old ../before/packages/mcp/dist + +# reports a hunt kept as text +node bench/explore/stored.mjs ``` Not in CI: it needs Chromium, the apps' dependencies and (for Strapi) a running Strapi. diff --git a/bench/explore/apps.mjs b/bench/explore/apps.mjs index d1a1939b..2b0ae407 100644 --- a/bench/explore/apps.mjs +++ b/bench/explore/apps.mjs @@ -125,12 +125,14 @@ async function session(browser, name, bug, slow) { }; const result = { scenario: `${name}-${bug ? "bug" : "ok"}`, slow_ms: slow, runs: RUNS, loads: [] }; result.start = await tool("genclass_explore", { action: "start", runs: RUNS, slow_ms: slow }); - for (let i = 0; i < RUNS + 2; i++) result.loads.push(await load(i)); + // the plan says how many loads: 2 baselines, with slow_ms a third with no added delay, and the seeded ones + const planned = Number(/next (\d+) page loads/.exec(result.start)?.[1] ?? RUNS + 2); + for (let i = 0; i < planned; i++) result.loads.push(await load(i)); result.report = await tool("genclass_explore", { action: "report" }); const fid = /Race finding\(s\): (f\d+)/.exec(result.report)?.[1]; if (fid) { result.replay = { id: fid, arm: await tool("genclass_replay", { action: "arm", id: fid }) }; - result.replay.load = await load(RUNS + 2); + result.replay.load = await load(planned); result.replay.result = await tool("genclass_replay", { action: "result", id: fid }); } writeFileSync(join(OUT, `${tag}.json`), JSON.stringify(result, null, 2)); diff --git a/bench/explore/planted.mjs b/bench/explore/planted.mjs new file mode 100644 index 00000000..7eef8632 --- /dev/null +++ b/bench/explore/planted.mjs @@ -0,0 +1,469 @@ +// The explore bench on small pages made of what the 2026-11-04 hunts met (docs/runtime/RESULTS.md, "explore's verdict on +// real apps"): each scenario has a correct variant and, where it has one, a planted bug, in headless Chromium with the +// real runtime (script-tag build, observe, no model) and the real MCP server, driven like bench/explore/apps.mjs. Every +// recorded session is scored with this checkout's comparison and, with --old, another build's, on the same runs (without +// the baseline with no added delay, which the older plan did not have). Each load's own check is `wrong`: what the page +// shows, or what the server received, against what it should be. +// +// npm run build -w @genclass/runtime && npm run build -w @genclass/mcp +// node bench/explore/planted.mjs [--old ] [--slow 300,600,900] [--runs 6] [--only a,b] [--shard i/n] [--port 41700] +// node bench/explore/planted.mjs --score [--old ] # the tables, from the sessions written +// +// The pages were written by the author of the rules they measure: they show the mechanism and what each rule costs, not +// an independent recall figure. + +import { createServer } from "node:http"; +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, extname, join, resolve } from "node:path"; +import { PassThrough } from "node:stream"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { randomUUID } from "node:crypto"; + +const here = dirname(fileURLToPath(import.meta.url)); +const args = process.argv.slice(2); +const opt = (name, d) => (args.includes(name) ? args[args.indexOf(name) + 1] : d); +const OUT = resolve(args[0] ?? join(here, "out-planted")); +const rtDist = resolve(opt("--rt", join(here, "../../packages/runtime/dist"))); +const mcpDist = resolve(opt("--mcp", join(here, "../../packages/mcp/dist"))); +const SLOW = opt("--slow", "300,600,900").split(",").map(Number); +const RUNS = Number(opt("--runs", "6")); +const ONLY = opt("--only", "") ? opt("--only", "").split(",") : null; +const [SHARD, SHARDS] = opt("--shard", "0/1").split("/").map(Number); +const PORT = Number(opt("--port", "41700")); +const cur = await import(pathToFileURL(join(mcpDist, "index.js")).href); +mkdirSync(OUT, { recursive: true }); + +// ----------------------------------------------------------------------------------------------- the pages' own scripts +const SEARCH = (extra = "") => ` +

    ${extra} +`; +const typeReact = async (p) => p.getByLabel("Search").pressSequentially("react", { delay: 80 }); +const saveBody = (variant) => ` +

      +`; +const twoClicks = async (p) => { + await p.locator("#save").click(); + await p.waitForTimeout(120); + await p.locator("#save").click(); +}; +const serverSaved = (n) => async (p) => { + const load = new URL(p.url()).searchParams.get("load"); + return (await p.evaluate(async (l) => (await fetch("/api/saved?load=" + l)).json(), load)).n !== n; +}; + +// two picks in a row, the last answer to arrive shown (the correct variant keeps the latest pick's): a wrong value that +// reads like a time, which the comparison must not take for one that moved on its own +const slotBody = (a, b) => () => `

      Book a slot

      Pick a slot

      `; +const twoPicks = async (p) => { + await p.click("#a"); + await p.click("#b"); +}; + +/** Each scenario: the page's body, the user's steps (`i`: which load), its check, what a report should name when it has a bug. */ +const SCENARIOS = { + // a page full of ages on a clock twice as fast as a real one (a "minute" every 30 s, so one crosses in most sessions and + // moves no further than a session could): it never depends on the order, and a stale search can sit beside it + times: { + body: () => `
        ${SEARCH()}`, + steps: typeReact, + names: /Results for|react/, + }, + // what reads like a time and is taken for a value, moving on the same clock on a correct page: what that decision costs + clocks: { + bugless: true, + body: () => `
          ${SEARCH()}`, + steps: typeReact, + }, + // read-only POSTs whose number depends on arrival order; the page shows the same + counts: { + bugless: true, + body: () => `

          `, + steps: async (p) => p.locator("#go").click(), + }, + // double submits, each as far from the page as a real one can be + "dbl-shown": { body: () => saveBody("shown"), steps: twoClicks, wrong: serverSaved(1), names: /Order|POST \/api\/save/, load: true }, + "dbl-silent": { body: () => saveBody("silent"), steps: twoClicks, wrong: serverSaved(1), names: /POST \/api\/save/, load: true }, + "dbl-body": { body: () => saveBody("body"), steps: twoClicks, wrong: serverSaved(1), names: /POST \/api\/save/, load: true }, + "dbl-chain": { body: () => saveBody("chain"), steps: twoClicks, wrong: serverSaved(1), names: /POST \/api\/save/, load: true }, + "dbl-timer": { body: () => saveBody("timer"), steps: twoClicks, wrong: serverSaved(1), names: /POST \/api\/save/, load: true }, + // fields named for this load only (no label), one filled from the answers in arrival order when the bug is planted + fields: { + body: () => ` + +`, + steps: async (p) => p.locator("#go").click(), + names: /\*|default/, + }, + // needs only a slow response: an answer that lands while the next question is out is added to the list (Open WebUI's Notes) + slowonly: { + body: () => `
            `, + steps: async (p) => p.getByLabel("Search Notes").pressSequentially("Mango", { delay: 80 }), + names: /note/, + }, + // needs only a slow response too: a refresh pressed while the first read is out is lost (OpenCode's file view); a bare + // number that differs in every load sits beside it (the digit blanking used to take "version-1" with it) + fileslow: { + body: () => `

            ${Math.floor(Math.random() * 900) + 100}

            `,
            +    steps: async (p) => p.locator("#refresh").click(),
            +    settleFirst: false,
            +    names: /version-/,
            +    load: true,
            +  },
            +  // the older answer lands last, and a bare number differs between the baselines (it used to hide the version line)
            +  counter: {
            +    body: () => `

            ${Math.floor(Math.random() * 900) + 100}

            `,
            +    steps: async (p) => p.getByLabel("Search").pressSequentially("ab", { delay: 60 }),
            +    names: /version-/,
            +  },
            +  // a new path on every load (a fresh session, a new document)
            +  route: { body: () => SEARCH(), steps: typeReact, names: /Results for|react/, freshPath: true },
            +  "slot-clock": { body: slotBody("Your appointment: 3:30 PM", "Your appointment: 4:00 PM"), steps: twoPicks, names: /appointment/ },
            +  "slot-day": { body: slotBody("Arrives today", "Arrives tomorrow"), steps: twoPicks, names: /Arrives/ },
            +  "slot-in": { body: slotBody("Ships in 2 days", "Ships in 5 days"), steps: twoPicks, names: /Ships/ },
            +  "slot-ago": { body: slotBody("Last synced 3 days ago", "Last synced just now"), steps: twoPicks, names: /synced/ },
            +  "slot-bare": { body: slotBody("3d", "5d"), steps: twoPicks, names: /3d|5d/ },
            +  // a save sent behind two answers: the planted page sends it only when one answer beat the other, so it can be lost, and
            +  // the page shows the same either way
            +  "lost-post": {
            +    body: () => `

            Notes

            Autosaved notes stay on this page

            `, + steps: async (p) => p.locator("#s").click(), + wrong: serverSaved(1), + names: /POST \/api\/save/, + load: true, + }, + // correct pages that vary by themselves: a tip picked at random, a save queue that sends less often when answers are slow + tip: { + bugless: true, + body: () => `

            Home

            ${Math.random() < 0.5 ? "Press save often to keep your work" : "Drag and drop files onto the page"}

            none

            `, + steps: async (p) => p.locator("#s").click(), + }, + queue: { + bugless: true, + body: () => `

            Doc

            All changes are kept on the server

            `, + steps: async (p) => { + for (const c of "abcd") { + await p.locator("#t").type(c); + await p.waitForTimeout(120); + } + }, + }, + // a dialog to dismiss in only some loads + optional: { + body: () => `${SEARCH()}`, + steps: async (p, i) => { + if (i % 2) await p.locator("#dismiss").click(); + await typeReact(p); + }, + names: /Results for|react/, + }, +}; + +// ------------------------------------------------------------------------------------------------------ the app server +const TYPES = { ".js": "text/javascript", ".map": "application/json", ".wasm": "application/wasm" }; +const htmlOf = (name, bodies) => `${name} + + + +${bodies}`; +const perLoad = new Map(); +const json = (res, v, ms = 40) => setTimeout(() => res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(v)), ms); +const app = createServer((req, res) => { + const url = new URL(req.url, "http://localhost"); + const load = url.searchParams.get("load") ?? ""; + const state = perLoad.get(load) ?? { saved: 0, version: 1 }; + perLoad.set(load, state); + req.resume(); + switch (url.pathname) { + case "/api/search": + return json(res, { items: [`${url.searchParams.get("q")} one`, `${url.searchParams.get("q")} two`] }); + case "/api/notes": + return json(res, { items: [`${url.searchParams.get("q")} note 1`, `${url.searchParams.get("q")} note 2`] }); + case "/api/a": + case "/api/b": + case "/api/token": + case "/api/doc": + return json(res, { ok: true }); + case "/api/slot": + return json(res, { line: url.searchParams.get("v") }, 20); + case "/api/counts": + return json(res, { n: 3 }); + case "/api/save": + state.saved++; + return json(res, { rows: Array.from({ length: state.saved }, (_, i) => i + 1) }); + case "/api/saved": + return json(res, { n: state.saved }, 0); + case "/api/defaults": + return json(res, { value: url.searchParams.get("n") === "1" ? "default one" : "default two" }); + case "/api/version": + return json(res, { text: `version-${url.searchParams.get("n")}` }); + case "/api/file": + return json(res, { text: `version-${state.version}` }); + case "/api/bump": + state.version = 2; + return json(res, { ok: true }); + } + const m = /^\/p\/([a-z-]+)\//.exec(url.pathname); + if (m && SCENARIOS[m[1]]) return res.writeHead(200, { "content-type": "text/html" }).end(htmlOf(m[1], SCENARIOS[m[1]].body())); + if (url.pathname.startsWith("/rt/")) { + const file = resolve(rtDist, url.pathname.slice(4)); + if (file.startsWith(rtDist) && existsSync(file)) return res.writeHead(200, { "content-type": TYPES[extname(file)] ?? "application/octet-stream" }).end(readFileSync(file)); + } + res.writeHead(404).end(); +}); + +// ------------------------------------------------------------------------------------------------- one explore session +async function session(browser, base, name, bug, slow) { + const sc = SCENARIOS[name]; + const tag = `${name}-${bug ? "bug" : "ok"}-${slow}`; + const input = new PassThrough(); + const output = new PassThrough(); + const lines = []; + let buf = ""; + output.setEncoding("utf8"); + output.on("data", (c) => { + buf += c; + let nl; + while ((nl = buf.indexOf("\n")) >= 0) { + lines.push(JSON.parse(buf.slice(0, nl))); + buf = buf.slice(nl + 1); + } + }); + const bridgePort = PORT + 10 + SHARD; + const served = await cur.serve({ cwd: OUT, env: {}, stdin: input, stdout: output, stderr: process.stderr, port: bridgePort, stateDir: join(OUT, `state-${tag}`) }); + let id = 0; + const rpc = async (method, params = {}) => { + const my = ++id; + input.write(`${JSON.stringify({ jsonrpc: "2.0", id: my, method, params })}\n`); + for (let i = 0; i < 1000; i++) { + const m = lines.find((x) => x.id === my); + if (m) return m; + await new Promise((r) => setTimeout(r, 10)); + } + throw new Error(`no reply to ${method}`); + }; + const tool = async (t, a = {}) => (await rpc("tools/call", { name: t, arguments: a })).result.content[0].text; + await rpc("initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "planted-bench", version: "1" } }); + const bridge = `http://127.0.0.1:${served.port}`; + const load = async (i) => { + const ctx = await browser.newContext(); + const p = await ctx.newPage(); + await p.addInitScript((u) => (window.__GENCLASS_AGENT_URL__ = u), bridge); + const out = { i }; + try { + const path = sc.freshPath ? `/p/${name}/${randomUUID()}/` : `/p/${name}/`; + const q = [bug ? "bug" : "", sc.load ? `load=${randomUUID()}` : ""].filter(Boolean).join("&"); + await p.goto(`${base}${path}${q ? `?${q}` : ""}`); + await p.waitForFunction(() => window.__GENCLASS_AGENT__?.connected === true, null, { timeout: 15_000 }); + out.run = await p.evaluate(() => window.__GENCLASS_AGENT__.run); + out.schedule = await p.evaluate(() => window.__GENCLASS_AGENT__.schedule); + if (sc.settleFirst === false) await p.waitForTimeout(150); + else await p.evaluate(() => window.__GENCLASS_AGENT__.whenSettled(15_000)); + await sc.steps(p, i); + out.settled = await p.evaluate(() => window.__GENCLASS_AGENT__.whenSettled(15_000)); + out.wrong = sc.wrong ? await sc.wrong(p) : await p.evaluate(() => (window.__wrong ? window.__wrong() : false)); + await p.waitForTimeout(400); // the last batch reaches the bridge + } catch (e) { + out.error = String(e?.message ?? e).slice(0, 300); + } + await ctx.close(); + return out; + }; + const result = { scenario: name, bug, slow_ms: slow, runs: RUNS, loads: [] }; + result.start = await tool("genclass_explore", { action: "start", runs: RUNS, slow_ms: slow }); + const planned = Number(/next (\d+) page loads/.exec(result.start)?.[1] ?? RUNS + 3); + for (let i = 0; i < planned; i++) result.loads.push(await load(i)); + result.report = await tool("genclass_explore", { action: "report" }); + writeFileSync(join(OUT, `${tag}.json`), JSON.stringify(result, null, 2)); + await served.close(); + const wrong = result.loads.map((l) => (l.error ? "E" : l.wrong ? "D" : "s")).join(""); + console.log(`${tag}: ${wrong} ${result.report.split("\n").filter((l) => /^(RACE|No race found|NO VERDICT|No verdict|Compared:)/.test(l)).map((l) => l.slice(0, 90)).join(" | ")}`); +} + +// ------------------------------------------------------------------------------------------------------------ the score +async function score() { + const old = opt("--old") ? await import(pathToFileURL(join(resolve(opt("--old")), "index.js")).href) : null; + const rows = []; + for (const f of readdirSync(OUT).filter((x) => /-(ok|bug)-\d+\.json$/.test(x)).sort()) { + const d = JSON.parse(readFileSync(join(OUT, f), "utf8")); + const tag = f.replace(/\.json$/, ""); + const st = JSON.parse(readFileSync(join(OUT, `state-${tag}`, "state.json"), "utf8")); + const byId = new Map(st.runs.map((r) => [r.id, r])); + const x = st.explores.find((e) => e.runs.length) ?? st.explores[0]; + const runs = x.runs.map((id) => byId.get(id)).filter(Boolean); + const wrong = new Map(d.loads.filter((l) => l.run).map((l) => [l.run, !!l.wrong])); + const plain = (r) => r.schedule?.order === "fifo" && !r.schedule?.delay; + const tally = (reports, withDelay) => { + const seeds = runs.filter((r) => r.schedule?.order === "seeded"); + const raced = new Set(reports.flatMap((g) => g.diffs.filter((y) => (y.tier ?? "visible") === "visible" && g.baselines.length >= 2).map((y) => y.run))); + const delay = withDelay ? reports.find((g) => g.delay)?.delay : undefined; + const compared = reports.reduce((n, g) => n + (g.compared ?? (g.baselines.length >= 2 ? g.runs - g.baselines.length : 0)), 0); + return { apart: reports.some((g) => g.look), compared, seeds: seeds.length, races: raced.size, tp: seeds.filter((r) => raced.has(r.id) && wrong.get(r.id)).length, fp: seeds.filter((r) => raced.has(r.id) && !wrong.get(r.id)).length, fn: seeds.filter((r) => !raced.has(r.id) && wrong.get(r.id)).length, delay: !!delay, verdict: raced.size > 0 || !!delay }; + }; + const after = tally(cur.compareRuns(runs), true); + const before = old ? tally(old.compareRuns(runs.filter((r) => !plain(r))), false) : null; + // what the report's headline names, by this checkout's comparison (the first group that has something to say) + const told = cur.compareRuns(runs).flatMap((g) => [...(g.delay ? [g.delay] : []), ...cur.strongestFirst(g.diffs.filter((y) => y.tier === "visible"))]); + const headline = told.length ? cur.nameDifference(told[0]) : ""; + const names = SCENARIOS[d.scenario]?.names; + rows.push({ tag, scenario: d.scenario, bug: d.bug, slow: d.slow_ms, check: d.loads.map((l) => (l.error ? "E" : l.wrong ? "D" : "s")).join(""), before, after, named: !!(names && names.test(headline)), headline: headline.slice(0, 200), anyWrong: d.loads.some((l) => l.wrong) }); + } + writeFileSync(join(OUT, "score.json"), JSON.stringify(rows, null, 2)); + const groups = new Map(); + for (const r of rows) { + const k = `${r.scenario}-${r.bug ? "bug" : "ok"}`; + groups.set(k, [...(groups.get(k) ?? []), r]); + } + const sum = (list, side, key) => list.reduce((n, r) => n + (r[side]?.[key] ?? 0), 0); + const cell = (list, side) => `${list.filter((r) => r[side]?.verdict).length}/${list.length}; ${sum(list, side, "races")} of ${sum(list, side, "compared")} (true ${sum(list, side, "tp")}, false ${sum(list, side, "fp")}, missed ${sum(list, side, "fn")})`; + console.log("| scenario | a load's own check wrong | before: sessions with a RACE; runs reported of compared | after: the same | after: sessions naming a slow response alone | after: headline names the line | after: worth a look |"); + console.log("|---|---|---|---|---|---|---|"); + for (const [k, list] of groups) console.log(`| ${k} | ${list.filter((r) => r.anyWrong).length}/${list.length} | ${old ? cell(list, "before") : ""} | ${cell(list, "after")} | ${list.filter((r) => r.after.delay).length} | ${list.filter((r) => r.named).length} | ${list.filter((r) => r.after.apart).length} |`); + for (const bug of [false, true]) { + const list = rows.filter((r) => r.bug === bug); + console.log(`\n${bug ? "planted bug" : "correct"}: ${old ? `before ${cell(list, "before")}; ` : ""}after ${cell(list, "after")}`); + } +} + +if (args.includes("--score")) { + await score(); + process.exit(0); +} +const { chromium } = await import("@playwright/test"); +await new Promise((r) => app.listen(PORT + SHARD, "127.0.0.1", r)); +const base = `http://localhost:${PORT + SHARD}`; +const browser = await chromium.launch(); +const jobs = []; +for (const slow of SLOW) for (const name of Object.keys(SCENARIOS)) for (const bug of SCENARIOS[name].bugless ? [false] : [false, true]) if (!ONLY || ONLY.includes(name)) jobs.push([name, bug, slow]); +for (const [i, [name, bug, slow]] of jobs.entries()) { + if (i % SHARDS !== SHARD) continue; + if (existsSync(join(OUT, `${name}-${bug ? "bug" : "ok"}-${slow}.json`))) continue; + await session(browser, base, name, bug, slow); +} +await browser.close(); +app.close(); +process.exit(0); diff --git a/bench/explore/stored.mjs b/bench/explore/stored.mjs new file mode 100644 index 00000000..ade785aa --- /dev/null +++ b/bench/explore/stored.mjs @@ -0,0 +1,205 @@ +// Scores this checkout's comparison on explore reports a hunt kept only as text (docs/runtime/RESULTS.md, "explore's verdict on real apps"): +// each report's groups are parsed back into recorded runs (what the report listed per run, over one synthetic baseline) +// and given to compareRuns() from packages/mcp/dist. The harness's own page check is read from the same file's lines +// `load N: DIFFER|same|ERROR ...`. +// +// node bench/explore/stored.mjs [--json ] +// +// It can only move what a report listed: a run's first 4 page lines per side and first 6 fields. So +// - a line hidden by those caps stays hidden: runs whose list hit the cap and came out clean are counted `capped` +// (their verdict is not known from the text), not as cleared; +// - a baseline that shared a bug, or a difference filed as noise by shape, cannot come back from the text: fixes that +// need a new load (the baseline with no added delay) or the raw lines are measured on real pages instead; +// - a report has no send steps, so a POST count here never has "sent directly by a user action" as evidence; +// - a run's list was cut at 6 fields and 4 lines a side, and a partner filed as noise by the old rule is not listed: so a +// field with a generated name listed as absent on one side (its partner was noise, or cut) is taken to be there with +// the same value, and a line that holds an age is dropped from a list that hit its cap or a report whose noise holds +// an age. Both are this script's reading of the text, not the comparison: the raw runs a hunt kept check the first +// (`--raw ` compares a bridge's recorded runs directly); +// - the runs here have no times, so two ages are one line within one minute, the least a real session gets. + +import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const args = process.argv.slice(2); +const dir = resolve(args[0] ?? "."); +const opt = (name) => (args.includes(name) ? args[args.indexOf(name) + 1] : undefined); +const here = new URL(".", import.meta.url); +const { compareRuns, shownField, fieldShape, agesIn } = await import(pathToFileURL(resolve(opt("--dist") ?? new URL("../../packages/mcp/dist", here).pathname, "index.js")).href); + +const walk = (d) => readdirSync(d).flatMap((f) => (statSync(join(d, f)).isDirectory() ? walk(join(d, f)) : [join(d, f)])); +const strings = (s) => [...s.matchAll(/"((?:[^"\\]|\\.)*)"/g)].map((m) => JSON.parse(`"${m[1]}"`)); + +/** One report's groups as written: its scenario, its noise line and, per listed run, what it listed. */ +function parseReport(text) { + const groups = []; + let g = null; + let section = null; + let run = null; + for (const line of text.split("\n")) { + let m; + if ((m = /^## (.*?) · (\d+) run\(s\) · scenario/.exec(line))) { + g = { route: m[1], runs: Number(m[2]), scenario: [], races: 0, compared: 0, listed: [], noVerdict: false }; + groups.push(g); + section = "scenario"; + run = null; + continue; + } + if (!g) continue; + if (section === "scenario" && line.startsWith(" ")) { + g.scenario.push(line.trim()); + continue; + } + if ((m = /^RACE: (\d+) of (\d+)/.exec(line))) { + section = "visible"; + g.races = Number(m[1]); + g.compared = Number(m[2]); + continue; + } + if ((m = /^No race found: none of the (\d+)/.exec(line))) { + section = "none"; + g.compared = Number(m[1]); + continue; + } + if (line.startsWith("Ignored as noise") && /\bminutes? ago|\bhours? ago|an hour ago|a minute ago/.test(line)) g.timeNoise = true; + if (/^(No verdict|Only one baseline|No baseline run)/.test(line)) { + g.noVerdict = true; + section = "none"; + continue; + } + if (line.startsWith("Internal state only")) { + section = "internal"; + continue; + } + if (line.startsWith("Along the way only")) { + section = "transient"; + continue; + } + if ((m = /^- run (\S+) \((.*?)\)/.exec(line))) { + run = { id: m[1], label: m[2], tier: section, onlyBase: [], onlyRun: [], fields: [], errors: [], witnesses: [] }; + g.listed.push(run); + continue; + } + if (!run || !line.startsWith(" ")) continue; + if ((m = /^ {4}page text only in the baseline: (.*)$/.exec(line))) run.onlyBase = strings(m[1]); + else if ((m = /^ {4}page text only in this run: (.*)$/.exec(line))) run.onlyRun = strings(m[1]); + else if ((m = /^ {4}(field|state|requests sent to) ("(?:[^"\\]|\\.)*"): baseline ("(?:[^"\\]|\\.)*"), this run ("(?:[^"\\]|\\.)*")/.exec(line))) run.fields.push({ where: m[1] === "requests sent to" ? "requests" : m[1], name: JSON.parse(m[2]), baseline: JSON.parse(m[3]), run: JSON.parse(m[4]) }); + else if ((m = /^ {4}error only here: (.*)$/.exec(line))) run.errors.push(JSON.parse(m[1])); + else if ((m = /^ {4}witness only here: (.*)$/.exec(line))) run.witnesses.push(m[1]); + } + return groups; +} + +/** A group's listed runs back into recorded runs: two equal baselines and one run per seed, their lines and fields as listed. */ +function runsOf(g, plan) { + const page = { url: `http://localhost:1${g.route}`, route: g.route }; + const steps = [{ k: "act", i: 0, t: 0, kind: "click", target: 'button "x"' }, ...writes]; + // what the listing's caps and the old noise rule hid: see the header + for (const r of g.listed) { + r.capped = r.onlyBase.length >= 4 || r.onlyRun.length >= 4; + if (r.capped || g.timeNoise) { + const partnerHidden = (l) => !!agesIn(l); + r.onlyBase = r.onlyBase.filter((l) => !partnerHidden(l)); + r.onlyRun = r.onlyRun.filter((l) => !partnerHidden(l)); + } + r.fields = r.fields.filter((f) => !(f.where === "field" && fieldShape(f.name) !== f.name && (f.baseline === "(absent)" || f.run === "(absent)"))); + } + const count = new Map(); + const fields = new Map(); + for (const r of g.listed) { + const n = new Map(); + for (const l of r.onlyBase) n.set(l, (n.get(l) ?? 0) + 1); + for (const [l, c] of n) count.set(l, Math.max(count.get(l) ?? 0, c)); + for (const f of r.fields) if (f.baseline !== "(absent)") fields.set(`${f.where}|${f.name}`, f.baseline); + } + const baseText = [...count].flatMap(([l, c]) => Array(c).fill(l)); + const snapshotOf = (r) => { + const text = [...baseText]; + for (const l of r?.onlyBase ?? []) text.splice(text.indexOf(l), 1); + text.push(...(r?.onlyRun ?? [])); + const s = { t: 1000, state: {}, text, fields: {}, requests: Object.fromEntries(writes.map((w) => [w.by, 1])), witnesses: 0, errors: 0, forced: 0 }; + const put = (where, name, v) => { + if (where === "state") s.state[name] = v; + else if (where === "field") s.fields[name] = v; + else s.requests[name] = Number(v); + }; + for (const [k, v] of fields) put(k.slice(0, k.indexOf("|")), k.slice(k.indexOf("|") + 1), v); + for (const f of r?.fields ?? []) { + if (f.run !== "(absent)") put(f.where, f.name, f.run); + else if (f.where === "state") delete s.state[f.name]; + else if (f.where === "field") delete s.fields[f.name]; + else delete s.requests[f.name]; + } + return s; + }; + const run = (id, order, seed, r) => ({ id, started: 0, lastSeen: 0, page, schedule: { order, ...(seed ? { seed } : {}), label: r?.label }, steps, lost: 0, order: [], forced: 0, witnesses: [], errors: (r?.errors ?? []).map((message) => ({ t: 1, name: "Error", message, chain: [] })), detections: [], interventions: [], checks: [], expectations: [], snapshot: snapshotOf(r), snapshots: 1, ended: true }); + const seeds = plan.map((label, i) => { + const listed = g.listed.find((r) => r.label === label); + return run(listed?.id ?? `s${i}`, "seeded", i + 1, listed ?? { label }); + }); + return [run("b1", "fifo"), run("b2", "fifo"), ...seeds]; +} + +const rows = []; +const files = walk(dir).filter((f) => f.endsWith(".txt")); +// the writes other sites' responses made (a finding's explanation lists them, in some reports of a hunt): the same fields in all +const writes = [...new Map(files.flatMap((f) => [...readFileSync(f, "utf8").matchAll(/ write (\S+) by ([A-Z]+ [^\s/]+\.[^\s/]+\/[^\s:#]*)/g)]).map((m) => [m[1], { k: "write", t: 1, store: "x", paths: [m[1]], by: m[2] }])).values()]; +if (opt("--raw")) { + const st = JSON.parse(readFileSync(opt("--raw"), "utf8")); + const byId = new Map(st.runs.map((r) => [r.id, r])); + for (const x of st.explores) { + const runs = x.runs.map((id) => byId.get(id)).filter(Boolean); + for (const g of compareRuns(runs)) console.log(`${x.id} ${g.route}: compared ${g.compared}, ${g.diffs.map((d) => `${d.label}: ${d.tier}`).join(", ") || "no differences"}; noise ${g.noise.filter((n) => !n.startsWith("state")).slice(0, 6).join(", ")}`); + } + process.exit(0); +} +for (const file of files) { + const text = readFileSync(file, "utf8"); + if (!/^Explore x\d+: /m.test(text)) continue; + const planned = [...text.matchAll(/^ {2}\d+\. (seed \d+)$/gm)].map((m) => m[1]); + const loads = [...text.matchAll(/^load (\d+): (DIFFER|same|ERROR)/gm)].map((m) => m[2]); + const oracleOf = (label) => (/^seed (\d+)$/.test(label) ? loads[Number(label.slice(5)) + 1] : undefined); + const baselineWrong = loads.slice(0, 2).filter((l) => l === "DIFFER").length; + const groups = parseReport(text); + const row = { file: relative(dir, file), oracle: loads.map((l) => (l === "DIFFER" ? "D" : l === "ERROR" ? "E" : "s")).join(""), compared: 0, seeds: 0, wrongSeeds: 0, baselineWrong, before: { races: 0, tp: 0, fp: 0, fn: 0 }, after: { races: 0, tp: 0, fp: 0, fn: 0 }, capped: 0, cleared: [], left: [] }; + for (const g of groups) { + if (g.noVerdict || !g.compared) continue; + row.compared += g.compared; + const plan = planned.slice(0, g.compared); + const report = compareRuns(runsOf(g, plan))[0]; + const raced = new Map((report?.diffs ?? []).filter((d) => d.tier === "visible").map((d) => [d.label, d])); + for (const label of plan) { + const o = oracleOf(label); + const before = g.listed.find((r) => r.label === label); + const wasRace = before?.tier === "visible"; + const isRace = wasRace && raced.has(label); + row.seeds++; + if (o === "DIFFER") row.wrongSeeds++; + const tally = (t, hit) => { + if (hit) t.races++; + if (hit && o === "DIFFER") t.tp++; + if (hit && o === "same") t.fp++; + if (!hit && o === "DIFFER") t.fn++; + }; + tally(row.before, wasRace); + tally(row.after, isRace); + if (wasRace && !isRace) { + if (before.capped) row.capped++; + row.cleared.push(label); + } + if (isRace) { + const d = raced.get(label); + row.left.push(...d.textOnlyRun, ...d.textOnlyBaseline, ...d.fields.filter(shownField).map((f) => f.name)); + } + } + } + row.left = [...new Set(row.left)].slice(0, 8); + rows.push(row); +} +rows.sort((a, b) => a.file.localeCompare(b.file)); +if (opt("--json")) writeFileSync(opt("--json"), JSON.stringify(rows, null, 2)); +console.log("| report | page check per load | compared | before: races (true, false, missed) | after: races (true, false, missed) | cleared with a capped list | still listed |"); +console.log("|---|---|---|---|---|---|---|"); +const cell = (t) => `${t.races} (${t.tp}, ${t.fp}, ${t.fn})`; +for (const r of rows) console.log(`| ${r.file} | ${r.oracle} | ${r.compared} | ${cell(r.before)} | ${cell(r.after)} | ${r.capped} | ${r.left.map((l) => JSON.stringify(l.slice(0, 40))).join(", ")} |`); diff --git a/bench/explore/summarise.mjs b/bench/explore/summarise.mjs index e697a6b5..bafa08b5 100644 --- a/bench/explore/summarise.mjs +++ b/bench/explore/summarise.mjs @@ -10,7 +10,7 @@ // // A reordered run "differs" when its DOM oracle equals neither baseline's (a title the scenario makes unique per run // has its digits blanked). A run explore "reports" is one in a group with two baselines whose difference reaches the -// race verdict: any difference for the old comparison (it reported every one as "RACE"), the visible tier for the new. +// race verdict: the visible tier (any difference for a comparison older than tiers, which reported every one as "RACE"). import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; @@ -41,8 +41,8 @@ function groupsOf(impl, runs) { return [...m.values()]; } -/** Per reordered run: compared with two baselines, reported as a race, and its tier. `isNew`: only the visible tier is a race. */ -function score(impl, runs, isNew) { +/** Per reordered run: compared with two baselines, reported as a race, and its tier. Only the visible tier is a race (a comparison older than tiers: any difference). */ +function score(impl, runs) { const out = new Map(); const reports = impl.compareRuns(runs); const groups = groupsOf(impl, runs); @@ -53,7 +53,7 @@ function score(impl, runs, isNew) { if (r.schedule?.order === "fifo") continue; const d = diffs.get(r.id); const tier = !d ? "same" : d.tier ?? (d.fields.length || d.textOnlyBaseline.length || d.textOnlyRun.length || d.errors.length ? "ended differently" : "witness only"); - out.set(r.id, { compared: two, reported: two && !!d && (!isNew || d.tier === "visible"), tier, grouping: g.grouping }); + out.set(r.id, { compared: two, reported: two && !!d && (d.tier === undefined || d.tier === "visible"), tier, grouping: g.grouping }); } }); return out; @@ -71,11 +71,13 @@ for (const f of readdirSync(dir).filter((x) => /^[a-z-]+-\d+(-[a-z]+)?\.json$/.t const runs = x.runs.map((id) => byId.get(id)).filter(Boolean); // the DOM oracle per run (the harness's own check, secondary) const oracle = new Map((d.loads ?? []).filter((l) => l.run).map((l) => [l.run, l])); + // the slow baselines and the one with no added delay: a run like any of them ended in a state a baseline did const baseKeys = new Set((d.loads ?? []).filter((l) => l.schedule?.order === "fifo").map((l) => key(l.oracle))); const seeded = runs.filter((r) => r.schedule?.order !== "fifo"); const truth = new Map(seeded.map((r) => [r.id, oracle.has(r.id) ? !baseKeys.has(key(oracle.get(r.id).oracle)) : null])); - const sNew = score(cur, runs, true); - const sOld = old ? score(old, runs, false) : null; + const sNew = score(cur, runs); + // the older comparison, on what its plan recorded: without the baseline with no added delay (it would read as a baseline) + const sOld = old ? score(old, runs.filter((r) => !(r.schedule?.order === "fifo" && !r.schedule?.delay) || !runs.some((o) => o.schedule?.order === "fifo" && o.schedule?.delay))) : null; const tally = (s) => { if (!s) return null; let compared = 0, reported = 0, tp = 0, fp = 0, fn = 0; diff --git a/bench/hunt/README.md b/bench/hunt/README.md index 9ee3a730..16082218 100644 --- a/bench/hunt/README.md +++ b/bench/hunt/README.md @@ -10,7 +10,7 @@ What one run does: 1. starts the GenClass MCP server in-process (`@genclass/mcp`, its bridge on a free loopback port) and headless Chromium; 2. arms `genclass_explore` (two baselines with responses in request order, then N seeded orders, optionally slower, - like a bad network) and, for each planned load, opens a fresh browser context, puts `@genclass/runtime` on the page + like a bad network, which adds a third baseline with no added delay) and, for each planned load, opens a fresh browser context, puts `@genclass/runtime` on the page (see "How the injection works"), connects the agent link to the bridge, waits for the page's first quiet point (as the generated regression test does), runs the scenario's `steps(page)`, waits for the next quiet point (`whenSettled`), and records the scenario's `oracle(page)`; diff --git a/bench/hunt/hunt.mjs b/bench/hunt/hunt.mjs index ece027fc..1aa04a1f 100644 --- a/bench/hunt/hunt.mjs +++ b/bench/hunt/hunt.mjs @@ -368,9 +368,12 @@ function verdictOf(rep, loads, session) { let compared = 0; let ended = 0; let along = 0; + let slow = 0; for (const g of judged) { - compared += g.runs - g.baselines.length; - for (const d of g.diffs) endedDifferently(d) ? ended++ : along++; + compared += g.compared ?? g.runs - g.baselines.length; // the baseline with no added delay is neither + // what reached the page or the writes; store values alone (internal) are no race, whatever they ended on + for (const d of g.diffs) d.tier ? (d.tier === "visible" ? ended++ : d.tier === "transient" ? along++ : 0) : endedDifferently(d) ? ended++ : along++; + if (g.delay) slow++; } const explored = loads.filter((l) => l.phase === "explore"); const armed = explored.filter((l) => l.schedule?.id === session); @@ -385,7 +388,7 @@ function verdictOf(rep, loads, session) { } for (const l of armed) if (!l.reorder && l.run) l.reorder = reorderOf(mcp.store.runs.get(l.run)); const out = { compared, notCompared: Math.max(0, seededLoads.length - compared), seeded: seededLoads.length, armed: armed.length, reordered, groups: groups.length, findingIds }; - if (ended || along) return { kind: "race", ended, along, of: compared, ...out }; + if (ended || along || slow) return { kind: "race", ended, along, slow, of: compared, ...out }; let reason; const baseFailed = armed.filter((l) => l.schedule?.order === "fifo" && (l.stepsError || l.error)); if (explored.length && explored.every((l) => l.error)) reason = `every page load failed (first: ${explored[0].error})`; diff --git a/docs/runtime/AGENT.md b/docs/runtime/AGENT.md index 6e4f1969..0fa4206c 100644 --- a/docs/runtime/AGENT.md +++ b/docs/runtime/AGENT.md @@ -197,7 +197,7 @@ after 30 minutes and are not restored after a restart. | `genclass_status` | connected pages (route, schedule, model state), counts of findings, the explore session, armed replays | | `genclass_findings` | ranked, deduplicated findings with counts and the last replay verdict | | `genclass_explain` | one finding: the evidence in causal order, the recorded steps and delivery order, the model's last situation, how to replay, and a Playwright regression test | -| `genclass_explore` | `start` (two baselines + N seeded loads, optional `slow_ms`, `focus`, `route`), `report` (the verdict per scenario, race findings with a replayable seed), `stop` | +| `genclass_explore` | `start` (two baselines + N seeded loads, optional `focus`, `route`, `slow_ms`, which adds a third baseline with no added delay), `report` (the verdict per scenario, race findings with a replayable seed), `stop` | | `genclass_replay` | `arm` the next page load with a finding's order; `result`: reproduced, not reproduced, or inconclusive (and why) | | `genclass_analytics` | `local`: per-endpoint counts, median and p95 latency, error rates, failure streaks, per-page runs, errors and overwritten fields, learned relations and flows; `production`: the GenClass dashboard's sessions, detections by diagnosis and page, latest decisions (`GENCLASS_DASHBOARD`) | | `genclass_setup` | the pinned install line and the lines to add for this project's framework (read from `package.json`), all behind a development check; changes no file, and tells the agent to show them and wait for a yes | @@ -218,9 +218,9 @@ test when the user only asked what broke. The skill (`packages/mcp/plugin/skills/genclass/SKILL.md`; the plugin installs it, `genclass-mcp init` writes it to `.claude/skills/genclass/SKILL.md`, and `skills/genclass/SKILL.md` at the repository root is the identical copy skill directories read) carries the "how", which always-loaded context cannot afford: the explore arguments -(`runs: 6, slow_ms: 600`), the Playwright pairing (`--repeat-each 8 --workers 1`), how to read a report -("Compared: 6 of 6" against "No verdict" and "0 compared"; "Internal state only" and "Along the way only" are not -races), what a witness, an explore race and a model detection each are, the reporting rule ("reproduced", +(`runs: 6, slow_ms: 600`), the Playwright pairing (`--repeat-each 9 --workers 1`), how to read a report +("Compared: 6 of 6" against "NO VERDICT", "No verdict" and "0 compared"; "Internal state only" and "Along the way only" +are not races), what a witness, an explore race and a model detection each are, the reporting rule ("reproduced", "not reproduced" or "no race in N orders", never "fixed" unless a replay or explore showed it), where it stops and waits for a yes, and what to do when the `genclass_*` tools are missing (the one-line install, or `npx @genclass/runtime@ scan --json --local` for the visual checks alone). Its frontmatter description repeats the triggers, since that is what a client matches a skill @@ -297,19 +297,40 @@ the agent does with each (Fig. 6); by default the dashboard cannot know what the Runs of one explore session are grouped by page and **scenario** (their user actions, in order, typing bursts with their final value). In each group, the two `fifo` runs are the baseline; with `slow_ms` they get the same extra latency as the -seeded runs, so timing alone never separates them. Anything that differs between the two baselines is **noise** and is -ignored: a store field whose value differs; the records under a store object whose keys differ (new ids on every run: -compared by their values, not their keys); a page line whose shape differs (tokens holding a digit blanked: times, -counters, ids). A value the runtime redacted (`[redacted]`) equals any value, alone or inside a summary: whether an +seeded runs, so timing alone never separates them, and a third `fifo` run gets none (the baseline with no added delay, +below). Anything that differs between the two baselines is **noise** and is ignored: a store field whose value differs; +the records under a store object whose keys differ (new ids on every run: compared by their values, not their keys); in +a page line, the tokens that differ at their position (a token holding a digit: times, counters, ids), so "12 items, +version-1" against "13 items, version-1" leaves "version-1" compared; a page line only one of them shows. A token that +varied stands for tokens of its own shape (a bare number does not stand for "version-2"). Another number on that line may +drift a little, at most 2 or 10% ("CPU 12% memory 40%" against "CPU 13% memory 41%": a second live metric), and no further +("Order 123 total 40" is not "Order 125 total 90", and "$1,000" is one number, not "$3,000"). A line both baselines show the same is never noise, whatever varied +elsewhere on the page. A value the runtime redacted (`[redacted]`) equals any value, alone or inside a summary: whether an opaque id is redacted depends on the id itself (one with a digit looks like a credential, one without does not), so a redaction is never a difference. +Two shapes are compared whatever the baselines did, because they move on their own and two baselines can agree by luck. +**Ages** in a page line ("13 minutes ago", "an hour ago", "a few seconds ago", "just now", "10m ago"; `src/text.ts` +`agesIn`) are one line when the later load is older by about the time between the two, from the runs' own load and snapshot +times, plus the minute an age rolls over in: "26 minutes ago" then "27 minutes ago" is no difference, "just now" against "3 +days ago" is, and so is "12 minutes ago" then "2 minutes ago" ten minutes on. Rows pair oldest with oldest, so a list whose +rows all moved a minute is the same list and one wrong row stands out. Rounded phrases read as what a library shows just +before them ("an hour ago" from 45 minutes, not "40 days" for "a month", not "9 years" for "a few years"). Nothing else +counts as moving on its own. A clock ("Your appointment: 3:30 PM"), "today", "yesterday", "tomorrow", a future form ("Ships +in 2 days") and a bare "3d" are values a page can get wrong, compared as they are unless the baselines themselves show them +varying. **Generated field names** (`fieldShape`) have their random part blanked when some baseline lacks the name and a +partner of its shape is in another: `el-id-2476-32` is `el-id-*-32`, `-input` is `*-input`, and only those names pool +their values, as records under fresh ids do. A name every baseline has is a name, whatever it looks like and whatever row +beside it is new: `qty-100` and `qty-200` swapping values is a difference. Among names made for each load, one swapping +values with another cannot be told from nothing (section 8). These are rules about the shape of text, ages in English, and +say nothing about any bug. + What a seeded run ends with beyond that is sorted by where it shows, from the evidence the run recorded: | tier | what differs | reported as | |---|---|---| -| visible | what the page shows (a page line outside a live region), a form value, the number of unsafe requests (POST, PUT, PATCH, DELETE) to a signature, or an error only this run had | **RACE**, a race finding with a replayable seed | -| internal | store values, the number of safe requests (GET, HEAD, OPTIONS: a cache refetching once more changes nothing the user sees or the server keeps), or lines of ARIA live regions (status messages, alerts, toasts: a slower run lets a toast expire), while the page showed the same as the baseline | "Internal state only", listed with the fields, no finding | +| visible | what the page shows (a page line outside a live region), a form value, the number of writes to a signature (PUT, PATCH, DELETE; a POST one side never sent, or a different count with evidence, below), or an error only this run had | **RACE**, a race finding with a replayable seed | +| internal | store values, the number of safe requests (GET, HEAD, OPTIONS: a cache refetching once more changes nothing the user sees or the server keeps), a different non-zero number of a POST with nothing recorded that makes it a write, or lines of ARIA live regions (status messages, alerts, toasts: a slower run lets a toast expire), while the page showed the same as the baseline | "Internal state only", listed with the fields, no finding | | transient | nothing at the end, but an `overwrote-newer` witness the baselines did not have (a later response repaired it) | "Along the way only", listed with the witness, no finding | The internal tier exists because a data-fetching cache keeps bookkeeping whose final value depends on arrival order by @@ -322,6 +343,57 @@ state (a button left disabled, a spinner without text) is not in the page text, (section 8). No ignore list was added: the tiers and the rules below left no false report on the measured sessions, so a per-library default would have been code without a measured purpose. +**POST counts.** POST is the one unsafe method that often only reads (`POST /api/worktree/refresh`, +`POST /rest/workflow-dependencies/counts`, `POST /api/v1/authz/me/permissions`, GraphQL). A POST one side never sent is a +lost or an extra write and stays visible, however it was sent: a page that saves only when one answer beats another +loses its save behind two requests, and shows nothing. A different non-zero count, with the page the same, is a difference +that shows only when the run recorded what makes it a write: two identical requests in flight together that the baseline +did not have (or the reverse), or a user action sending it directly (no other request or timer in between) a different +number of times. Decided from the recorded `send` steps and witnesses, with no list of URLs. What it costs: a duplicate +POST sent behind another request or a timer, never overlapping and with the page unchanged, is internal. Measured in +RESULTS.md, "explore's verdict on real apps". + +**The baseline with no added delay.** Every slow run shares `slow_ms`, so a bug that needs only a slow response can +show in both baselines: seeded runs that ended right then read as racing and the bug itself as the reference (Notes in +Open WebUI, a refresh dropped in OpenCode's file view; each was right without the added latency and wrong with it). With +`slow_ms` the plan is two slow baselines, then one `fifo` run without it, then the seeded runs: `runs + 3` loads, against +`runs + 2` without `slow_ms`. It is one load, so it says little on its own, and it never decides what is noise: + +- what **both** slow baselines show the same and it does not, a page line, a form value or an error, is reported first, as + what a slow response alone does: `RACE: a slow response alone changes the page`, a race finding that replays the slow + baseline's schedule (`delay` in the group), with what it shows beside it. A seeded run is then not listed for showing what + it shows on those same lines: that is one of the two endings the headline already names. It is listed for anything else. + When no reordered run sides with it either, only that one load differs, which is what an error page it landed on looks + like and also what a bug that makes every slow load wrong looks like. It stays reported, and says so: `RACE, on one + load's word`, and to load the page once more with no added delay; +- what **one** slow baseline did against it is one load against one load, which chance can do (a random tip, a toast + outside a live region). That stays noise, is no race and makes no finding, and is named apart: "worth a look", with how + many reordered runs show it (`look` in the group), lines (a price too, not a symbol or a counter that moved a little), + errors, form values and writes; +- also under "worth a look": lines only it shows, with nothing from the slow side (an error page it landed on); a different + count of writes with both sides having sent some and no evidence the slow side sent more (a save queue sends less often + when answers are slow); and a number a little off in both slow baselines with the rest of the line the same (a counter + between two loads). Where the slow baselines show two values and it shows a third, nothing is said: no two loads agree + on anything. + +Without `slow_ms` nothing changes: the two baselines have no added delay already. + +**Other sites.** Safe requests to another site (a signature that starts with a host: `GET api.github.com/repos/x`) leave +the counts unless a user action led to them or a request to the page's own origin came from their answer; the page's own +machine and network (a backend on another port, `host.docker.internal`, a private address, a name with no dot, `.local`, +`.internal`) and its sibling hosts (`api.acme.com` from `app.acme.com`) never do. Writes stay in the counts wherever they +go: from a page on localhost, a hosted API of its own and someone else's look alike. The report names what it left out. A +page line that differs is internal, not a race, only when it is a bare count ("156k", not "$4.00") whose digits are those of +a store field exactly one request wrote, the same one in both runs, to another site, requested once, with no overwrite +recorded on that field: that site answered differently (a rate limit's `null`). Anything else, such as the load's `GET +/cart` landing over a click's `POST /cart/add`, is the question being asked, and stays a race. These requests still join the +schedule and can make holds run out: leaving them out of it needs the runtime's agent entry, and was not done (section 8). + +**The report.** Page lines come in reading order, real content (two words or more) before short labels before counts +and symbols, in each run's list and in which runs are shown first; the headline names the most telling difference +(content, then an error, a form value, a label, a write, a count), and so does the finding's title. Beyond what shows, +a race line lists at most two store values or refetch counts and says how many more there are. + **Scenarios.** Runs are grouped by their actions with element names normalised: digits blanked in typed values and names (a title made unique per run, "page 3"), parenthesised counts dropped (`combobox "author (1)"` is `combobox "author"`: the count arrives with a response), and consecutive typing into one field taken as one burst (the @@ -329,13 +401,18 @@ recording splits a burst when the field's label changes under it). A label can s column header renders its name in another element), so when that grouping leaves reordered runs without two baselines to compare with, the session is grouped by action kinds and element roles (`click button`; a `div` or `span` is a generic element), then by action kinds alone: the finest grouping that compares as many reordered runs as any does. -The report says which grouping it used, and how many reordered runs it compared ("Compared: 6 of 6"); a group whose -baselines had no reordered run beside them says "No verdict: 0 reordered runs were compared", never "No race found". +When a load still did an action the others did not (a dialog to dismiss, a tooltip clicked), the last level leaves out up +to two actions, at the kinds level, for a load that did at least one. Routes are grouped by shape (`routeShape`: numbers, +UUIDs, long ids), so `/session/` and `/c/` are one page to a schedule, a scenario group and a finding key, also +when armed and replayed. The report says which grouping it used, and how many reordered runs it compared ("Compared: 6 of +6"); a group whose baselines had no reordered run beside them says "No verdict: 0 reordered runs were compared", never +"No race found", and a report that compared nothing at all, or that has no run at a quiet point yet, says "NO VERDICT". What the baselines alone show is a verdict of its own: a report with a slow response's race and 0 reordered runs says "No verdict on reordering", so a race finding and "NO VERDICT" never stand together. A group with only one baseline gets no verdict (its differences are listed unfiltered), and the page of the first armed load is the page under test unless `route` says otherwise, so other pages and tabs do not take a slot. A snapshot that -left store fields out (section 2) says so in the report, and a field missing from such a snapshot is not compared. No -pattern is matched anywhere: the only question is whether what the page shows depends on the delivery order. +left store fields out (section 2) says so in the report, and a field missing from such a snapshot is not compared. The +shapes above are all the matching there is, and none of them is a bug pattern: the only question is whether what the page +shows depends on the timing of its responses. ### 5.2 Replays (`store.ts` `replayResult`) @@ -344,7 +421,8 @@ replays its run's seeded schedule (with its delay); anything recorded in a natur order as `given`. When the recorded run is gone, it refuses to arm rather than replay something else. `result` compares the replay run with the finding: the same witness (field by shape, signature), the same error (by name and shape), the failing run's page text, form values, request counts or error again = reproduced; the baseline's = not reproduced -(page lines count: a line the page always shows once and the failing run showed twice, a product card and a stale +(a race a slow response alone caused replays that slow baseline's schedule, and its "baseline" is the run with no added +delay) (ages within the time since the failing run, and fields by name or generated shape, as above) (page lines count: a line the page always shows once and the failing run showed twice, a product card and a stale result, is reproduced only when the replay shows it twice too; the finding's evidence keeps the counts, `textCounts`) (store values decide only for a finding recorded with nothing else, and a redaction matches any value); a replay whose element names changed with the order still counts as the same steps when its actions match by kind and role. It is @@ -361,7 +439,8 @@ matches by its prefix; a target without a name is flagged for checking; the reco `dblclick`), waits for a quiet point, and asserts the fact that made the finding: no `overwrote-newer` on a field of that shape, no `identical-in-flight` on that signature, no recurrence of the error (by name and shape), the baseline's page lines, form values and request counts for a race and none of the failing run's (compared with the agent's own -snapshot; store values only when nothing visible differed, so a cache's bookkeeping cannot keep a fixed page failing). A finding it +snapshot; store values only when nothing visible differed, so a cache's bookkeeping cannot keep a fixed page failing; a +line with an age in it is no assertion, a field with a generated name is found by the rest of its name, and a field with nothing but a generated name gets a note instead of a line). A finding it cannot give a generic assertion produces a test that fails until the domain assertion is written: it never passes while asserting nothing. Page strings enter the file only as JSON string literals. @@ -404,7 +483,7 @@ the agent's steps driven through the real MCP server and bridge; the user types | step | buggy page | fixed page (keeps only the latest answer) | |---|---|---| | natural typing (the two baselines) | right: the race never shows on this fast backend | right | -| `genclass_explore` (2 fifo baselines + 6 seeded, `slow_ms: 600`) | **race in 5 of 6** reordered runs (final results for "reac", "rea" or "r" instead of "react"); the 6th ended right after an older response overwrote newer data along the way | **no race**: 0 of 6 reordered runs differ (no false report) | +| `genclass_explore` (2 fifo baselines + 6 seeded, `slow_ms: 600`; a third baseline with no added delay since) | **race in 5 of 6** reordered runs (final results for "reac", "rea" or "r" instead of "react"); the 6th ended right after an older response overwrote newer data along the way | **no race**: 0 of 6 reordered runs differ (no false report) | | `genclass_replay` of the race | reproduced | not reproduced | | the generated Playwright test | **fails** (expected the results for "react", received "reac") | **passes** | @@ -428,6 +507,11 @@ race-like verdict (62 reordered runs, all false by a DOM oracle) and a correct R 18 of 18 reordered runs; after, 0 of 21 and 0 of 18, with the real 403 race still found in 6 of 6 reordered runs at 300, 600 and 900 ms (0 false) and every planted stale race found (32 of 32 runs) with the same e2e result as above. +On the apps hunted on 2026-11-04 (OpenCode, n8n, Open WebUI, Langflow) the fixed builds read as racing in 85 of 120 +reordered runs, by their own page checks all false, and in 0 of 120 after the rules above, rescored from the stored +reports; on OpenCode's own recorded runs the correct scenarios went from race to no race and the real file bug to a slow +response's race (RESULTS.md, "explore's verdict on real apps", which also says what that rescoring cannot see). + This is one page and one bug class, a proof of the mechanism, not a recall figure. The demos' and real apps' numbers (with the false-report rate next to each) are the next measurement (section 8). @@ -435,12 +519,28 @@ This is one page and one bug class, a proof of the mechanism, not a recall figur - Explore needs the same steps repeated, one page load each, and each run must reach a quiet point. A scenario whose requests come in a different order on each run gets different keys and is not compared reliably. Runs whose recorded - actions differ even by kind (a click that a page only offers in some orders) are not compared; the report says - "0 compared" for them. + actions differ by more than two actions are not compared; the report says "0 compared" for them. A load grouped with + the others by leaving an action out may end differently because of that action: the report says which grouping it used. +- The baseline with no added delay is one load. Both slow baselines landing on one value of a random line, and it on + another, reads as what a slow response does (a two-valued tip: 1 session in 4). A number that happens to be far off in + it the same way. What one slow baseline alone does, and a bare number that varies between them, stay noise: a bug that + shows only that way in the baselines is named under "worth a look" at best (Langflow's New Flow at 600 ms), and a line + only that load shows, a bug that only loses a line under a slow response, is named there too. +- Ages are recognised in English; other languages fall to the baselines' own noise. Two ages of one form that moved by the + time between two loads, plus a minute, are one line, so "Deployed 3 minutes ago" then "Deployed 4 minutes ago" is hidden. + The reverse costs a false report: an age in hours or longer that rolls over during a session, and any clock, "Today" or + bare "7m" that moved once between loads while the two baselines happened to agree, read as a difference. +- Field names made for each load pool their values, so two of them swapping values, or a wrong value in a name with its + value in it (`total-1000`, `total-1001`, `total-9999`), read as nothing when the pool holds the same values. +- A POST sent behind another request or a timer, with no overlap, with the page unchanged and a different non-zero count, + is not counted as a write (above). "No race found" says so when it happened, and to check what the server holds. Two POSTs from one action through one request (Langflow's New Flow) are the case seen + on a real app: that page shows the second flow, which is how it is found. +- Third-party requests leave the counts, not the schedule: they can still make holds run out, and a page line that shows + their answer (other than a bare count that stands for the field its one response wrote) can still differ. - The verdict looks at what the page shows through its text and form values, the writes it sends and its errors. A difference that shows only in a control's state (a button left disabled, a spinner without text, a style), beyond the snapshot's 300 page lines, only inside a live region (an alert that an order made persist), or only in the - number of GET requests is reported as internal state only, not as a race. Store fields beyond the snapshot's + number of GET requests or POST requests without evidence is reported as internal state only, not as a race. Store fields beyond the snapshot's budget (1,000 fields) are not compared; the report says when that happened. - An overwrite that a later response repaired (a flicker) is reported along the way, not as a race; the stale-write findings each witness makes stay as they were, so on an app whose cache writes its own bookkeeping (Redux Toolkit diff --git a/docs/runtime/RESULTS.md b/docs/runtime/RESULTS.md index a97efdd1..0803349d 100644 --- a/docs/runtime/RESULTS.md +++ b/docs/runtime/RESULTS.md @@ -1435,3 +1435,128 @@ telemetry opt-out key. No false-intervention rate applies: nothing here is a det Not measured: Turbopack, Parcel, and a plain webpack project. `packages/mcp/test/mcp.test.ts` repeats the esbuild row on the exact lines `setupText` prints for Vite and Next.js, whenever the runtime is built. + +## 27. Explore's verdict on real apps: false reports and recall before and after (development Mac, Chromium through Playwright 1.63.0, 2026-11-05, branch `fix/explore-verdict-noise`) + +[AGENT.md](AGENT.md) section 5.1. On 2026-11-04 four hunts ran `genclass_explore` next to their own check of the page, on released apps and on fixed builds. This measures the comparison of `origin/main` (`91139b92`, "before") against this branch ("after") on those reports, on OpenCode's own recorded runs and on small pages made of what they met. Two independent reviews of earlier versions of this branch found rules that hid real races `main` reports (a wrong value that reads like a time, a lost POST, a stale write from a hosted API, a swap beside a new row) and one that reported from a single sample; every number here is measured after those fixes unless it says otherwise. + +**Headline.** Reordered runs reported as a race on fixed builds, where the hunts' own check said the page was right: **85 of 120 before, 0 of 120 after**, of which only **36** rest on evidence the reports list (below). On every session of the four apps whose check was right in all its loads: **211 of 466 before, 24 of 466 after**. Recall on the bugs the reports could show: sessions with a wrong run and a race verdict go from 35 of 37 to 25 of 37; the 10 are 9 OpenCode sessions whose "RACE" was a read-only POST count (the same verdict on the fixed build, so never evidence) and Notes at 600 ms, which needs the new load. On OpenCode's recorded runs the correct scenarios went from race to no race (0 of 6 where `main` said 3, 6, 6 and 5 of 6) and the real file bug from an indistinct count to a slow response's race. On planted pages: false reports on correct variants **49 of 360 runs (12 of 66 sessions) before, 18 of 396 (6 of 66) after**, sessions with a planted bug given a race verdict **38 of 54 before, 48 of 54 after**, with 0 false runs among them after. The 18 false runs left are a save queue's PUT count and clocks moving, both false on `main` too. + +**Stored reports.** `bench/explore/stored.mjs` parses the 136 explore reports kept as text back into runs (what each listed, over one synthetic baseline) and gives them to `compareRuns`; the page check is each file's `load N:` lines. "Listed cap" counts cleared runs whose list hit its cap of 4 page lines a side. + +| reports | sessions | reordered runs compared | reported before (all false by the page check) | reported after | listed cap | +|---|---|---|---|---|---| +| OpenCode, fixed file refresh | 3 | 12 | 9 | **0** | 0 | +| OpenCode, fixed file reopen | 3 | 12 | 9 | **0** | 0 | +| n8n, fixed Execute Sub-workflow picker | 3 | 18 | 18 | **0** | 0 | +| Open WebUI, fixed searches | 9 | 54 | 43 | **0** | 43 | +| Langflow, fixed New Flow and paging | 4 | 24 | 6 | **0** | 6 | +| **fixed builds** | 22 | **120** | **85** | **0** | 49 | +| OpenCode, every session whose check was right in all loads | 16 | 68 | 52 | 8 | 0 | +| n8n, the same | 28 | 98 | 48 | 0 | 5 | +| Open WebUI, the same | 27 | 162 | 60 | 13 | 43 | +| Langflow, the same (the 51 of 138 of the hunt) | 23 | 138 | 51 | 3 | 32 | +| **all sessions whose check was right in all loads** | 94, of which 7 compared nothing (2 OpenCode, 5 n8n) | **466** | **211** | **24** | 80 | + +What this rescoring can and cannot say: +- **49 of the 85 cleared runs had capped lists** (Open WebUI 43 of 43, Langflow 6 of 6). A line behind the cap is unknown, so what the listed evidence clears is **36 of 85**. The other 49 listed only ages, and their page check said the page was right. +- **The "sent directly" evidence for a POST cannot fire here:** a report has no send steps. So a double submit with a different non-zero count reads as internal in this rescoring even when a live run would keep it. No stored run depended on it. +- **No earlier row measured a time-shaped wrong value or a lost POST:** none of the stored reports or earlier benches has one, which is how the first version of this branch passed them. They are measured below, on planted pages and the review's inputs. +- A field with a generated name listed as absent on one side is taken to be there with the same value (a reading of the text), checked against the only raw session of that hunt kept. Runs here have no times, so two ages are one line within one minute, the least a real session gets. It has no baseline with no added delay. + +The 24 left, by what they list: OpenCode `new-session` (4) and `double-enter` (4), a stub's reply caught mid-stream at the quiet point and session labels that came with it; Open WebUI `rename-two` (5: "Rename two" against "Renamed second", which may be a real bug its check does not read), `settings-toggle` (3: "Title Auto-Generation") and `note-autosave` (5: "Today at 3:27 PM" against 3:28, a clock moving on a correct page, which the first version hid along with real wrong values and is now reported: the cost of not taking clocks for ages); Langflow "New Project (1)" (2), "Delta" (1). Causes that fell away: ages, read-only POST counts, generated names, the star count. + +| sessions with a run the page check called wrong | sessions | wrong runs | race verdict before, after | wrong runs reported before, after | right runs reported before, after | +|---|---|---|---|---|---| +| OpenCode | 15 | 55 | 15, 6 | 39, 12 | 13, 5 | +| n8n | 10 | 27 | 10, 10 | 27, 27 | 6, 0 | +| Open WebUI | 4 | 9 | 4, 3 | 9, 8 | 8, 3 | +| Langflow | 8 | 35 | 6, 6 | 20, 15 | 2, 0 | +| **all** | **37** | **126** | **35, 25** | **95, 62** | **29, 8** | + +What the drop is. OpenCode, 27 runs in 10 sessions (9 lost their verdict): each rested on a count such as `POST /api/worktree/refresh`, and the fixed builds got the same verdict from the same line, so these were coincidence, not recall. Four of the sessions are wrong the same way in all their loads (file reopen, file open), which depends on no timing and no order comparison can see; the other six are the dropped refresh, the new load's case, measured below. Open WebUI, 1 run: Notes at 600 ms. Langflow, 5 runs: their only listed difference was the star count `156k`; those runs' real difference was a second flow or variable on the server, not in the list. On the same reports the real lines stay reported: n8n's "helper task 29", "row-040", "Total 41"; Langflow's "New Flow (1)"; OpenCode's "MARKER running-2 question". + +**What each rule costs, on the stored reports.** +- *POST.* A different non-zero count, with the page the same and nothing that makes it a write, is demoted; a POST one side never sent stays visible. Of the 113 POST lines in the reports, **106 stay demoted**, all of them non-zero: `POST /api/worktree/refresh` 89, `opencode.goal/get` 9, `authz/me/permissions` 4, `workflow-dependencies/counts` 4. **7 stay visible**, one-sided: `POST /api/session/:id/view`, 3 runs in `new-session` and 4 in `send-two`, runs already reported for page lines, so no verdict moves. Keeping lost POSTs visible costs **0 false runs** here. On the live pages below it is what brings `lost-post` back. +- *Ages only.* Taking clocks, days and bare forms as values adds **5 false runs** on correct sessions (`note-autosave`) and **3** on a session with a real bug (Notes at 900 ms: "2m" against "3m", on runs its check called right), and brings back **2 true ones** there. Against that, all 17 hidden inputs below report. +- *Both slow baselines.* The report does not have the load, so this has no cost here; the planted pages and OpenCode's runs below have it. + +**OpenCode 2.2.5, recorded runs.** The review ran OpenCode itself (`slow_ms` 600, 6 seeds, runtime 0.3.0-beta.5, its page checks), on the commit before these fixes; the "now" column is this comparison on the same recorded runs, no new load. + +| scenario | sessions | page check | `main` | on the commit before these fixes | now | +|---|---|---|---|---|---| +| `file-refresh` (a real bug) | 2 | stale in 2 of 2 slow baselines, 0 of 1 no-delay, 5 of 6 seeded | 5 of 6 in both, on request counts | RACE, a slow response alone | **RACE, a slow response alone**, "draft-one", 2 of 2; seeded 0 of 6 | +| `switch` | 3 | right in 27 of 27 loads | 3, 6 and 6 of 6 | no race in 3 of 3 | **no race in 3 of 3** | +| `palette` | 1 | right in 9 of 9 | 5 of 6 | no race | **no race** | +| `new-session` | 1 | right in 9 of 9 by a narrow check | 6 of 6 | RACE, a slow response alone, plus 4 of 6 | **RACE, a slow response alone, plus 4 of 6**: the stub's reply and the composer's text; it may be a false report, and the check reads only that one session was created | +| `mention` | 1 | wrong in 9 of 9 by its check, the same way in all | 5 of 6 | no race | **no race**: nothing depends on timing | + +"Worth a look" prints on 6 of the 8 sessions (`POST /api/session/:id/view` absent without the delay, or `opencode.goal/get` sent 4 times against 2 or 3): read-only POSTs still make noise there, under that line. + +**The review's inputs** (`scratchpad/review-explore/adv.mjs`, `adv2.mjs`, `hidden.mjs`; each is now a test in `packages/mcp/test/compare.test.ts`). `compareRuns` on the same synthetic runs: + +| inputs | `main` reports a race on | the commit before these fixes | now | +|---|---|---|---| +| a wrong value that reads like a time (17: "Your appointment: 3:30 PM" against 4:00 PM, "Arrives today", "Ships in 2 days", "Last synced 3 days ago" against "just now", "3d" against "5d", ...) | 17 | 1 | **17** | +| a wrong value in a field with a generated-looking name (6: `qty-100` and `qty-200` swapped, `:r1:` and `:r2:` swapped, ...) | 6 | 3 | **6** | +| noise beside a real difference (6) | 1 | 4 | 4 (all of `main`'s, and "version-2", "Order 125 total 90", "Build 123 status 9") | +| POST counts (9) | 9 | 4 | **5**: a lost POST by either route, direct double submit, in flight, PUT. The 4 not reported are a different non-zero count behind a request or a timer, or with no steps recorded | +| correct pages with `slow_ms` (12), races given | 3 on seeded runs | 4 on seeded runs, plus 8 "slow response alone" | 2 on seeded runs, both also on `main`, plus 1 "slow response alone" (a load time equal by chance in both slow baselines) | +| another site from localhost (3) | 3 | 1 | **2**: the stale count and its labelled form. The POST sent twice stays in the counts, as internal, under the POST rule | + +Decided for the localhost page with a hosted API: only safe requests leave the counts, since from localhost a hosted API of the page's own and someone else's look alike, and the page's own network never counts as another site (`host.docker.internal`, a private address, a name with no dot, `.local`, `.internal`). A bare count is internal only when it stands for a store field exactly one request wrote, the same one in both runs, to another site, requested once, with no overwrite recorded on the field, and its digits are that field's: so a stale cart count stays a race, and Langflow's star count (one request, a rate limit's `null`) does not. In Chromium (`hidden.mjs`, `slow_ms` 300, the page wrong in 4 of 6 loads): `slot-clock`, `slot-day` and `slot-days` went from no race to **RACE in 4 of 6**, as `main`; `lost-post` from no race to **RACE**, 2 of 6 with `slow_ms` and 2 of 4 without, as `main` on the same runs; the random `tip`, on the first version a false "slow response alone" with 2 of 6, here read 3 of 6 on seeded runs, which `main` gives on the same runs: all three baselines landed on one value, and nothing can tell that from a race; `queue-put` still RACE in 4 of 6, as `main`. + +**The second review's inputs** (`scratchpad/review2/a1.mjs`, `a2.mjs`, `a6.mjs`, `live.mjs`; each is now a test). Races reported on the same synthetic runs: + +| inputs | `main` | the commit before these fixes | now | what stays unreported | +|---|---|---|---|---| +| a load's GET over a POST's answer from a hosted API, two GETs writing one field, a price, a backend on `localhost` (4) | 3 of the 3 it can read | 1 | **4** | nothing | +| a swap or a wrong value beside a new row (`qty-100`, `qty-200`, `qty-9001`) (3) | 3 | 0 | **3** | nothing | +| ages, both directions and rounded phrases (19) | 19 | 4 | **16** | "2 minutes" then "3 minutes" in 10 s, "a few minutes" against "just now", "over a year" against "a year": each an age that can have moved | +| parsing ("2 hours and 5 minutes", "1,000 days") (9) | 9 | 4 | **7** | "1,000 minutes" then "1,001", "1,000 seconds" then "1,030": the same | +| field names (10) | 10 | 5 | **7** | names made for each load swapping values, a noisy one beside a wrong one, or holding a value (`total-9999`): not told from nothing | +| POST rule (6) | 5 | 3 | **3** | a different non-zero count behind a timer or with no action recorded, named by "No race found" | + +Also: five adjacent `[redacted]` in a state value against 199 characters took over 15 s and now about 4 ms; the plain load alone on an error page gives no race and no finding when nothing from the slow side differs, and `RACE, on one load's word` when one line stands in for another and no other load sides with it; a direct double submit no longer prints under both RACE and "worth a look". None of the numbers in this section's other tables moved with these fixes. + +**Planted pages.** `bench/explore/planted.mjs`: small pages in headless Chromium with the real runtime and MCP server, each scenario with a correct variant and, where it has one, a planted bug; `slow_ms` 300, 600 and 900, 2 slow baselines, 1 with no added delay, 6 seeded orders (9 loads a session, 120 sessions, 2 browsers). Each load has its own check of the page or of what the server received. Before and after are scored on the same recorded runs (before without the third baseline, which its plan did not have). "Reported" counts seeded runs: a run that ends like the slow baselines is not reported after, because the session's headline is that difference, so read the sessions column first. Written by the author of the rules they measure: they show the mechanism and what each rule costs, not an independent recall figure. + +| scenario | sessions with a race verdict, before | after | runs reported of compared, before (true, false) | after (true, false) | after: names a slow response alone | after: worth a look | +|---|---|---|---|---|---|---| +| `times`: ages crossing a "minute" every 30 s, correct | 0 of 3 | 0 of 3 | 0 of 18 (0, 0) | 0 of 18 (0, 0) | 0 | 0 | +| `times`: ages crossing a "minute" every 30 s, planted bug | 3 of 3 | 3 of 3 | 16 of 18 (16, 0) | 16 of 18 (16, 0) | 0 | 0 | +| `clocks`: a clock, "today", a bare "7m" moving on that clock, correct | 2 of 3 | 2 of 3 | 6 of 18 (0, 6) | 6 of 18 (0, 6) | 0 | 1 | +| `counts`: read-only POSTs whose number depends on order, correct | 3 of 3 | 0 of 3 | 9 of 18 (0, 9) | 0 of 18 (0, 0) | 0 | 0 | +| `fields`: generated names, correct | 3 of 3 | 0 of 3 | 18 of 18 (0, 18) | 0 of 18 (0, 0) | 0 | 0 | +| `fields`: generated names, planted bug | 3 of 3 | 3 of 3 | 18 of 18 (9, 9) | 9 of 18 (9, 0) | 0 | 0 | +| `slowonly`: needs only a slow response (Notes), planted bug | 3 of 3 | 3 of 3 | 18 of 18 (18, 0) | 17 of 18 (17, 0) | 3 | 0 | +| `fileslow`: a refresh dropped during a slow read (OpenCode), a bare counter beside it, planted bug | 0 of 3 | 3 of 3 | 0 of 18 (0, 0) | 4 of 18 (4, 0) | 1 | 0 | +| `counter`: "version-1" against "version-2" beside a bare counter, planted bug | 0 of 3 | 3 of 3 | 0 of 18 (0, 0) | 3 of 18 (3, 0) | 0 | 0 | +| `route`: a new path on every load, planted bug | 0 of 3 | 3 of 3 | 0 of 0 (0, 0) | 16 of 18 (16, 0) | 0 | 0 | +| `optional`: a click only some loads did, planted bug | 0 of 3 | 3 of 3 | 0 of 0 (0, 0) | 16 of 18 (16, 0) | 0 | 0 | +| `dbl-shown`: a double submit the page lists, planted bug | 3 of 3 | 3 of 3 | 11 of 18 (6, 5) | 0 of 18 (0, 0) | 3 | 3 | +| `dbl-silent`: the page does not show it, planted bug | 3 of 3 | 3 of 3 | 5 of 18 (0, 5) | 0 of 18 (0, 0) | 3 | 3 | +| `dbl-body`: the same with a body that differs, planted bug | 3 of 3 | 3 of 3 | 5 of 18 (0, 5) | 0 of 18 (0, 0) | 3 | 3 | +| `dbl-chain`: sent behind another request, planted bug | 0 of 3 | 0 of 3 | 0 of 18 (0, 0) | 0 of 18 (0, 0) | 0 | 3 | +| `dbl-timer`: sent behind a 30 ms timer, planted bug | 2 of 3 | 0 of 3 | 4 of 18 (0, 4) | 0 of 18 (0, 0) | 0 | 3 | +| `lost-post`: a save sent only when one answer beat another, planted bug | 3 of 3 | 3 of 3 | 9 of 18 (9, 0) | 9 of 18 (9, 0) | 0 | 0 | +| `slot-clock`: "Your appointment: 3:30 PM" against 4:00 PM, planted bug | 3 of 3 | 3 of 3 | 12 of 18 (12, 0) | 12 of 18 (12, 0) | 0 | 0 | +| `slot-day`: "Arrives today" against tomorrow, planted bug | 3 of 3 | 3 of 3 | 12 of 18 (12, 0) | 12 of 18 (12, 0) | 0 | 0 | +| `slot-in`: "Ships in 2 days" against 5, planted bug | 3 of 3 | 3 of 3 | 12 of 18 (12, 0) | 12 of 18 (12, 0) | 0 | 0 | +| `slot-ago`: "Last synced 3 days ago" against just now, planted bug | 3 of 3 | 3 of 3 | 12 of 18 (12, 0) | 12 of 18 (12, 0) | 0 | 0 | +| `slot-bare`: "3d" against "5d", planted bug | 3 of 3 | 3 of 3 | 12 of 18 (12, 0) | 12 of 18 (12, 0) | 0 | 0 | +| `tip`: a random two-value line, correct | 1 of 3 | 1 of 3 | 4 of 18 (0, 4) | 0 of 18 (0, 0) | 1 | 2 | +| `queue`: a save queue, fewer PUTs when answers are slow, correct | 3 of 3 | 3 of 3 | 12 of 18 (0, 12) | 12 of 18 (0, 12) | 0 | 1 | +| the other 16 correct variants (`slowonly`, `fileslow`, `counter`, `route`, `optional`, five `dbl-*`, `lost-post`, five `slot-*`) | 0 of 48 | 0 of 48 | none (0, 0); `route` and `optional` compared 0 | 0 of 288 (0, 0) | 0 | 0 | +| **correct variants** | 12 of 66 | 6 of 66 | 49 of 360 (0, 49) | 18 of 396 (0, 18) | 1 | 4 | +| **planted bugs** | 38 of 54 | 48 of 54 | 146 of 288 (118, 28) | 150 of 324 (150, 0) | 13 | 15 | + +**What the POST rule costs, measured.** "No race found" now says when a POST count differed, that a duplicate write may have happened with the page unchanged, and to check what the server holds. On the planted pages a lost POST is back (`lost-post` 3 of 3 sessions, 9 of 18 runs, 0 false). The direct rule keeps the double submits sent from a click handler (`dbl-shown`, `dbl-silent`, `dbl-body`: 9 of 9 sessions, each as a slow response's race, 2 of 2 baselines; before, the same sessions got their verdict with 15 of 21 reported runs false, correct runs listed against wrong baselines). It loses the double submit behind another request or a timer that never overlaps with the page unchanged: `dbl-timer` **2 of 3 sessions before, 0 of 3 after** (the 2 before rested on 4 correct runs reported as racing; `dbl-chain` was 0 of 3 both ways), all 6 named under "worth a look", none a finding. `identical-in-flight` rarely helped on these pages (3 of 101 runs that sent two POSTs): the witness counts a request's own start and end, the answers came back in 40 ms, and two clicks were 120 ms apart; only the schedule held them longer. On the hunts: of the recorded sends in the stored explanations, every read-only POST that made a false report was sent `via` another request (`/api/worktree/refresh` 60 of 60, `opencode.goal/get` 29, `workflow-dependencies/counts` 38, `authz/me/permissions` 40), and Langflow's real `POST /api/v1/variables/` double submit directly from two actions (4 of 4). Langflow's other one, `POST /api/v1/flows/`, was seen both ways (3 direct, 5 via `GET /api/v1/variables/` from one action): when the page shows no second flow it is internal now. + +**What requiring both slow baselines costs.** A bug that shows in one slow baseline and not the other is noise, as on `main`, and named under "worth a look" with how many reordered runs show it, not a race: OpenCode's file refresh when its baselines split (the first hunt's `file-refresh-600`: stale, right), and Langflow's New Flow at 600 ms. Both slow baselines landing on one value of a random line, and the plain load on another, is still a false "slow response alone" (`tip`: 1 of 3 sessions; by chance 1 in 4). + +**Small apps and the end-to-end checks, for regressions.** `bench/explore/apps.mjs` (Redux Toolkit Query, TanStack Query; section 22's bench, 12 sessions, now 9 loads each): recall on the planted stale race **32 of 32** runs before and after, with **0 false** both; false reports on the correct variants **0 of 36** both; every replay reproduced. `packages/mcp/test/e2e/race.e2e.mjs`: race in **5 of 6** reordered runs on the buggy page, 0 of 6 on the fixed one, replay reproduced and not reproduced, the generated test failed and passed. `bench/hunt` e2e pages: race on the buggy page, no race on the fixed one, and the form that sends one POST per click a `duplicate-request` finding with no race, as before. + +**Report text.** On the hunts' Notes at 900 ms the sample before named "2m", "8", "3m", "Loading..." and never the differing "Maple note" row; the lists are now in reading order and the headline names the row. On planted pages the headline names the planted line in **48 of 48** sessions with a verdict (that it shows "Last synced 3 days ago", which the baseline does not; that form field "el-id-*-33" is "default one"; that requests sent to "POST /api/save" are "(absent)", "1" in the baseline). The Notes report itself cannot be shown again without its app. + +Caveats: one machine, seeds 1 to 6 only, other agents running on it (load 4 to 11). The stored rescoring reads text, not the runs: the limits above apply, and it says nothing about the new baseline. A page check is the hunt's own and narrow: a run it calls right can differ for a real reason (`rename-two`, `new-session`), and "false" means by that check. Real apps with the new baseline: OpenCode only, one build, the review's four scenarios. Not run: n8n, Open WebUI and Langflow with the new load (no installs, no room), the demos (each has its own runtime and shell on the page, so the harness's injected one would make two), the model suite, `realapps/`. diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index f8118d6e..27b05c8d 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -26,6 +26,39 @@ the check would do. An app that already runs GenClass keeps its setup: `init` returns the running runtime. - **The READMEs and manifests say what can leave the machine**, instead of "sends nothing" (no code change). +- **`genclass_explore`'s verdict on real apps.** On 2026-11-04 four hunts ran explore next to their own page checks; on + fixed builds where the page was right it said RACE in 85 of 120 reordered runs (OpenCode request counts, n8n's generated + field names, Open WebUI's relative times and counts), and it missed real bugs. Rescored from those reports: 0 of 120 + ([RESULTS.md](https://github.com/genclass-dev/GenClass/blob/main/docs/runtime/RESULTS.md), "explore's verdict on real + apps", with what the rescoring cannot see and the recall next to it). What changed: + - An age ("13 minutes ago", "an hour ago", "just now", "10m ago") is one line when the later load is older by about the + time between the two, plus a minute, whatever the baselines did. Nothing else moves on its own: "3:30 PM", "today", "in 2 days" and a bare "3d" are + values a page can get wrong, compared as they are unless the baselines show them varying. + - A different non-zero count of a POST, with the page the same, is no race without two identical requests in flight or + a user action sending it directly a different number of times. A POST one side never sent stays a race, however it was + sent. PUT, PATCH and DELETE counts are as before. **Cost:** a duplicate POST behind another request or a timer, not + overlapping, with the page unchanged, is internal now (measured there), and "No race found" says a duplicate write may + have happened and to check what the server holds. + - A field name some baseline lacks compares with itself by its shape (`el-id-*-32`, `*-input`); a name every baseline + has is a name (`qty-100`), whatever row beside it is new. + - **With `slow_ms` the plan has one more load**: a baseline with no added delay after the two slow ones (`runs + 3` + loads; `--repeat-each 9` for `runs: 6`). What both slow baselines show the same and it does not is reported first + (`RACE: a slow response alone changes the page`, a race finding replaying the slow schedule; `RACE, on one load's word` + when no other load sides with it). What one slow baseline alone does, and a line only it shows, is named under "worth + a look", and is no race. + - Noise blanks only the tokens that varied, at their position: "version-1" against "version-2" stays a difference when + a number elsewhere varied, and another number on that line may drift a little and no further. + - Report text: page content before labels before counts, in each run's list and in which runs come first; the + headline and the finding's title name that line; store values and refetch counts capped with a count of the rest. + - Routes group by shape (`/session/`, `/c/`: a new path on every load also gets its armed schedule), an + action only some loads did no longer splits a session, and a report that compared nothing says `NO VERDICT`, never + beside a race finding. + - Safe requests to other sites leave the counts unless a user action or the page's own origin led to them, and a bare + count on the page is internal only when it stands for a store field one such response alone wrote, with no overwrite + recorded on it. They still join the schedule (that needs the runtime's agent entry). + - `sameValue` no longer backtracks on many redactions in a row: a page could hang the bridge for seconds with one value. + - A race finding's report says to fix only if the user asked for a fix. + - **The plugin's repository vendors the server, and the lockfile is gone.** In [genclass-dev/genclass-plugin](https://github.com/genclass-dev/genclass-plugin), `.mcp.json` and the hook run `node ${CLAUDE_PLUGIN_ROOT}/server/genclass-mcp.mjs` instead of `npx -y @genclass/mcp@`: `server/cli.mjs` diff --git a/packages/mcp/README.md b/packages/mcp/README.md index b87d295e..04640163 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -340,13 +340,20 @@ agent: genclass_findings → f1 [stale-write] /search: the response to GET /api genclass_replay arm f1 → (loads the page, types) → result: REPRODUCED (fixes the handler to keep only the latest answer, adds the test) genclass_replay arm f1 → result: not reproduced - genclass_explore start {slow_ms: 600} → 8 runs → report: no race + genclass_explore start {slow_ms: 600} → 9 runs → report: no race ``` On a real page in headless Chromium (the package's end-to-end check), explore found that race in 5 of 6 reordered runs of a page that never showed it on the developer's fast backend, the generated test failed on the buggy page and passed on the fixed one, and explore on the fixed page reported nothing. Details: [docs/runtime/AGENT.md](https://github.com/genclass-dev/GenClass/blob/main/docs/runtime/AGENT.md). +What it calls a race is what the page shows or writes, not the noise of a running page: an age that moved by the time +the session took ("13 minutes ago", "14 minutes ago"), a field name made for each load and a different count of a POST +that only reads are not differences by themselves; "3:30 PM" against "4:00 PM" and a save that never went out are. With +`slow_ms` one load runs with no added delay (`runs + 3` loads: 2 slow baselines, that one, and the reordered runs): what +both slow baselines show and it does not is named as what a slow response alone does, and what only one of them does is +"worth a look". A report that compared nothing says `NO VERDICT`, not "no race". + ## Tools | tool | | diff --git a/packages/mcp/plugin/skills/genclass/SKILL.md b/packages/mcp/plugin/skills/genclass/SKILL.md index 7780d9bd..d6e8d488 100644 --- a/packages/mcp/plugin/skills/genclass/SKILL.md +++ b/packages/mcp/plugin/skills/genclass/SKILL.md @@ -41,17 +41,20 @@ shows which pages and endpoints the GenClass dashboard flagged. ## 2. Explore response orders 1. `genclass_explore {action:"start", runs: 6, slow_ms: 600}` (add `route` to limit it to one page). -2. Run the same scenario once per armed page load (2 baselines + 6), each a fresh load: with Playwright - `--repeat-each 8 --workers 1`, or with a browser tool. After the steps, wait for +2. Run the same scenario once per armed page load (3 baselines, one with no added delay, + 6), each a fresh load: + with Playwright `--repeat-each 9 --workers 1`, or with a browser tool. After the steps, wait for `window.__GENCLASS_AGENT__.whenSettled()`. Keep a script you write for this out of the repo, and wait for a yes before adding Playwright to a project that has none. 3. `genclass_explore {action:"report"}` and read the verdict per scenario: - "No race found" with the reordered runs compared ("Compared: 6 of 6"): done. - - "No verdict" or "0 compared": nothing was compared, so it is not a negative result; run it again with `route` - set and the same steps on every load. - - "RACE": `genclass_explain` the race finding and go to section 1, step 3. + - "NO VERDICT", "No verdict" or "0 compared": nothing was compared, so it is not a negative result; run it again + with `route` set and the same steps on every load. + - "RACE" (or "a slow response alone changes the page": no reordering needed): `genclass_explain` the race finding + and go to section 1, step 3. - "Internal state only" and "Along the way only" are not races: store values the page did not show (a cache's own bookkeeping depends on arrival order by construction), and overwrites a later response repaired. + - "Worth a look" is no race either: one slow baseline against the load with no added delay, which chance can do. + Check the page it names, and the server for a write it names, before you call the page clear. ## 3. Scan a page diff --git a/packages/mcp/src/compare.ts b/packages/mcp/src/compare.ts index ad11f6b2..3b19257e 100644 --- a/packages/mcp/src/compare.ts +++ b/packages/mcp/src/compare.ts @@ -1,38 +1,60 @@ // The explore verdict (docs/runtime/AGENT.md section 5). Runs of the same scenario (the same user actions on the same -// route) differ only in when their responses were delivered. The two fifo runs are the baseline: responses in the -// order their requests started, as on a fast, serial network. Whatever differs between the two baselines is noise and -// is ignored, by shape as well as by value: -// - a store field whose value differs between the baselines; -// - the children of a store object whose keys differ between the baselines (new records get new ids on every run): -// there, records are compared by their values, not by their keys; -// - a page line whose shape (tokens holding digits blanked: times, counters, ids) differs between the baselines. +// route) differ only in when their responses were delivered. The fifo runs are the baselines: responses in the order +// their requests started, as on a fast, serial network. With added latency (`slow_ms`) there are two slow baselines and +// one with no added delay. Whatever differs between the two slow baselines is noise and is ignored, by shape as well as +// by value: +// - a store field whose value differs between them; +// - the children of a store object whose keys differ between them (new records get new ids on every run): there, +// records are compared by their values, not by their keys; +// - in a page line, the tokens that differ at their position (times, counters, ids); another number on that line may +// drift a little (a second live metric), and a word with a digit in it stays exact; +// - a page line only one of them shows. // A value the runtime redacted ("[redacted]") equals any value: whether an opaque id is redacted depends on the id // itself, so a redaction is never a difference. // -// Whatever a seeded run ends with that differs from the baseline beyond that depends on network timing. It is sorted by -// where the difference shows, from the evidence the run recorded: -// visible what the page shows (its text, its form values), the unsafe requests it sent (POST, PUT, ...), or an -// error only it had: a race; -// internal store values, counts of safe requests (GET: a cache refetching once more) or announcements (lines in -// ARIA live regions: a toast a slower run let expire) only, while the page showed the same as the -// baseline: reported apart, never as a race (a cache's bookkeeping, such as request ids and lists of -// cache keys, depends on arrival order by construction and never reaches the page); +// Both slow baselines showing the same thing the baseline with no added delay does not is what a slow response alone does +// (GroupReport.delay), and a run like that baseline is not listed for it. One slow baseline unlike it, or a line only it +// shows, is one load against one load, which chance can do: it is named apart, "worth a look" (GroupReport.look), and is +// no race. A seeded run's +// difference from the slow baselines beyond that depends on network timing. It is sorted by where it shows, from the +// evidence the run recorded: +// visible what the page shows (its text, its form values), the writes it sent, or an error only it had: a race; +// internal store values, counts of safe requests (GET: a cache refetching once more), a different non-zero count of a +// POST with nothing recorded that makes it a write (POST often reads) or announcements (lines in ARIA live +// regions: a toast a slower run let expire) only, while the page showed the same as the baseline: reported +// apart, never as a race (a cache's bookkeeping, such as request ids and lists of cache keys, depends on +// arrival order by construction and never reaches the page); // transient the same final state, but an older response overwrote newer data along the way (a later response // repaired it): reported apart, never as a race. -// Scenarios are grouped by their recorded actions: element names with digits blanked and parenthesised counts dropped, -// consecutive typing into one field as one burst; when that leaves reordered runs without two baselines to compare -// with (a label that changed with the order), by action kinds and element roles, then by action kinds alone. A group -// without two baselines cannot tell noise from a race and gives no verdict. No pattern of any kind is matched here. +// A POST a run never sends, or only it sends, is a lost or an extra write and stays visible. A different non-zero count +// counts as a write only when the run recorded two identical requests in flight together that the baseline did not, or +// a user action sent it directly a different number of times. PUT, PATCH and DELETE counts always do. +// Safe requests to another site leave the counts unless a user action or a request to the page's own origin led to them. +// +// Scenarios are grouped by route (id-like segments blanked) and recorded actions: element names with digits blanked and +// parenthesised counts dropped, consecutive typing into one field as one burst; when that leaves reordered runs without +// two baselines to compare with (a label that changed with the order), by action kinds and element roles, then by action +// kinds alone, then with up to two actions only some loads did (a dialog to dismiss). A group without two baselines +// cannot tell noise from a race and gives no verdict. +// +// Text shapes are matched here, and none of them is a bug pattern: ages ("13 minutes ago": text.ts agesIn(), one line when +// the later load is older by about the time between the two, whatever the baselines did, because they move with the +// clock), generated field names (fieldShape(), only for names some baseline lacks) and route segments (routeShape()), +// tokens holding digits. Whether anything is a bug is only ever asked of the page itself: does what it shows depend on the timing of its +// responses? import type { ActStep, Run, Snapshot } from "./types.js"; -import { pathShape, sigOf } from "./text.js"; +import { ageless, agesIn, fieldShape, lineWeight, pathShape, q, routeShape, sameAges, sigOf, type Elapsed } from "./text.js"; /** The runtime's redaction placeholder (packages/runtime/src/util.ts REDACTED). */ export const REDACTED = "[redacted]"; -/** How a session's runs were grouped: by element names, by action kinds and element roles, or by action kinds only. */ -export type Grouping = "names" | "roles" | "kinds"; -const GROUPINGS: Grouping[] = ["names", "roles", "kinds"]; +/** + * How a session's runs were grouped: by element names, by action kinds and element roles, by action kinds only, or by + * action kinds with up to two actions only some loads did. + */ +export type Grouping = "names" | "roles" | "kinds" | "optional"; +const GROUPINGS: Grouping[] = ["names", "roles", "kinds", "optional"]; /** Where a seeded run's difference from the baseline shows (see the header). */ export type Tier = "visible" | "internal" | "transient"; @@ -42,9 +64,9 @@ const blankDigits = (s: string): string => s.replace(/\d+/g, "N"); /** Tags whose ARIA role is "generic": which of them takes a click can depend on what the page rendered first. */ const GENERIC = new Set(["div", "span", "b", "i", "u", "s", "small", "bdi", "bdo", "samp", "data"]); -/** A target's name for grouping: parenthesised counts dropped ('combobox "author (1)"' → 'combobox "author"'), digits blanked. */ +/** A target's name for grouping: ages as one token, parenthesised counts dropped ('combobox "author (1)"' → 'combobox "author"'), digits blanked. */ function targetName(t: string | undefined): string { - return blankDigits((t ?? "").replace(/\s*\(\d+\)/g, "")).trim(); + return blankDigits(ageless(t ?? "").replace(/\s*\(\d+\)/g, "")).trim(); } /** A target's role: 'button "Save"' → button; a generic container (div, span, ...) → element. */ @@ -101,15 +123,37 @@ export function sameValue(a: string | undefined, b: string | undefined): boolean return fits(a, b) || fits(b, a); } -/** One value inside a summary, as describe() writes it: a quoted string, a bare token, or another redaction. */ -const ONE_VALUE = String.raw`(?:"(?:[^"\\]|\\.)*"|[^,\s{}\[\]]+|\[redacted\])`; +/** Where one value inside a summary, as describe() writes it, can end from `at`: a quoted string, a redaction, a bare token. */ +function valueEnds(s: string, at: number): number[] { + if (s[at] === '"') { + for (let i = at + 1; i < s.length; i++) { + if (s[i] === "\\") i++; + else if (s[i] === '"') return [i + 1]; + } + return []; + } + const ends: number[] = s.startsWith(REDACTED, at) ? [at + REDACTED.length] : []; + for (let i = at; i < s.length && !/[,\s{}[\]]/.test(s[i]); i++) ends.push(i + 1); + return ends; +} -/** `pattern` with its "[redacted]" standing for the whole value, or each one inside it for one value, matches `s`. */ +/** + * `pattern` with its "[redacted]" standing for the whole value, or each one inside it for one value, matches `s`. Walks the + * positions each redaction can reach, once each: a page can send many of them in a row, and this runs on what it sends. + */ function fits(pattern: string, s: string): boolean { if (pattern === REDACTED) return true; if (!pattern.includes(REDACTED)) return false; - const re = new RegExp(`^${pattern.split(REDACTED).map((p) => p.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join(ONE_VALUE)}$`); - return re.test(s); + const [first, ...pieces] = pattern.split(REDACTED); + if (!s.startsWith(first)) return false; + let at = new Set([first.length]); + for (const piece of pieces) { + const next = new Set(); + for (const p of at) for (const e of valueEnds(s, p)) if (s.startsWith(piece, e)) next.add(e + piece.length); + if (!next.size) return false; + at = next; + } + return at.has(s.length); } /** Two lists of values are the same multiset, a redacted value matching any one value. */ @@ -135,6 +179,10 @@ export interface FieldDiff { name: string; baseline: string; run: string; + /** A POST count: what the run recorded that makes the difference a write (without it, POST counts are internal). */ + evidence?: string; + /** A field name's generated shape ("qty-*"): the names of that shape the baselines all have, which are not of the pool. */ + except?: string[]; } export interface RunDiff { @@ -161,6 +209,8 @@ export interface RunDiff { witnesses: string[]; /** Responses this run delivered only because their hold ran out (it may not have followed its schedule). */ forced: number; + /** Its page text differs only in a bare count, beside a store field another site's response wrote: internal, not a race. */ + otherSite?: boolean; } /** @@ -176,6 +226,15 @@ export function stateValue(state: Record, name: string): string return list.length === 0 ? undefined : list.length === 1 ? list[0] : `[${list.sort().join(" | ")}]`; } +/** The value keyed() gives a form field in a raw snapshot: by its name, or by its generated shape, fields of one shape pooled. */ +export function fieldValue(fields: Record | undefined, name: string, except: string[] = []): string | undefined { + if (fields?.[name] !== undefined) return fields[name]; + const list = Object.entries(fields ?? {}) + .filter(([k]) => fieldShape(k) === name && !except.includes(k)) + .map(([, v]) => v); + return list.length === 0 ? undefined : list.length === 1 ? list[0] : `[${list.sort().join(" | ")}]`; +} + /** The run's snapshot was taken after its last user action (a final state, not the page as it loaded). */ export function settledAfterActs(r: Run): boolean { if (!r.snapshot) return false; @@ -192,8 +251,17 @@ export function endedDifferently(d: RunDiff): boolean { /** Requests with a safe method (RFC 9110: GET, HEAD, OPTIONS) change nothing the server keeps: a cache refetching once more. */ const SAFE = /^(GET|HEAD|OPTIONS)\s/; -/** A difference the page shows or sends: a form value, the count of an unsafe request (POST, PUT, ...). */ -export const shownField = (f: FieldDiff): boolean => f.where === "field" || (f.where === "requests" && !SAFE.test(f.name)); +/** POST is the one unsafe method that often only reads ("POST /counts", GraphQL): how many times it went says little. */ +const POST = /^POST\s/; + +/** A request sent by one side and never by the other: a lost write, or one only that side sent, whatever the method says. */ +const onOneSide = (f: Pick): boolean => f.baseline === "(absent)" || f.run === "(absent)"; + +/** + * A difference the page shows or sends: a form value, or the count of a write. PUT, PATCH and DELETE always; a POST when + * one side never sent it, or with evidence that a different non-zero count is a write. + */ +export const shownField = (f: FieldDiff): boolean => f.where === "field" || (f.where === "requests" && !SAFE.test(f.name) && (!POST.test(f.name) || onOneSide(f) || !!f.evidence)); /** * Did the difference reach what the page shows or sends (its text outside announcements, form values, unsafe @@ -203,6 +271,38 @@ export function visibleDiff(d: Pick 0 || d.textOnlyRun.length > 0 || d.errors.length > 0; } +/** The weight of the most telling line a run's difference shows (real content before labels before counts). */ +function bestLine(d: Pick): number { + return Math.max(-1, ...[...d.textOnlyBaseline, ...d.textOnlyRun].map(lineWeight)); +} + +/** Runs in the order their evidence should be read: real page content first, then errors, form values, labels, writes, counts. */ +export function strongestFirst>(diffs: T[]): T[] { + const score = (d: T): number => (bestLine(d) === 2 ? 5 : d.errors.length ? 4 : d.fields.some((f) => f.where === "field") ? 3 : bestLine(d) === 1 ? 2 : d.fields.some(shownField) ? 1 : 0); + return diffs.map((d, i) => ({ d, i, s: score(d) })).sort((x, y) => y.s - x.s || x.i - y.i).map((x) => x.d); +} + +/** What a run's difference comes down to, in one phrase (the most telling evidence first; page strings quoted). */ +export function nameDifference(d: Pick): string { + const line = (weight: number): string | null => { + const inRun = d.textOnlyRun.find((l) => lineWeight(l) === weight); + if (inRun !== undefined) return `it shows ${q(inRun, 80)}, which the baseline does not`; + const inBase = d.textOnlyBaseline.find((l) => lineWeight(l) === weight); + return inBase !== undefined ? `it does not show ${q(inBase, 80)}, which the baseline does` : null; + }; + const field = d.fields.find((f) => f.where === "field"); + const write = d.fields.find(shownField); + return ( + line(2) ?? + (d.errors.length ? `an error only there: ${q(d.errors[0], 100)}` : null) ?? + (field ? `form field ${q(field.name, 60)} is ${q(field.run, 60)}, ${q(field.baseline, 60)} in the baseline` : null) ?? + line(1) ?? + (write ? `requests sent to ${q(write.name, 80)}: ${q(write.run, 20)}, ${q(write.baseline, 20)} in the baseline` : null) ?? + line(0) ?? + "what it sent or showed differs" + ); +} + export interface GroupReport { route: string; /** The scenario as the group's first baseline (else its first run) recorded it. */ @@ -226,6 +326,27 @@ export interface GroupReport { unfiltered: RunDiff[]; /** Runs whose snapshot left store fields out (its budget), and the most left out of how many. */ truncated?: { runs: number; left: number; total: number }; + /** The baseline with no added delay, when the session has one in this group. */ + plainBaseline?: string; + /** + * What a slow response alone does: what every slow baseline shows that the baseline with no added delay does not, and + * the reverse beside it (`textOnlyBaseline` is what that baseline shows, `textOnlyRun` what the slow ones show), in the + * first slow baseline's words. Only when it came from the slow side: a page line, a form value or an error they have and + * it has not, or a write one side never sent or one with evidence. A different count of writes alone can be a queue + * saving less often when answers are slow, and a line only it shows can be an error page it landed on (`look`). + * `oneLoad`: no reordered run sides with that baseline on any of it, so it is the only load that says so, and it may be + * the odd one out (the report says so): still reported, since a bug can make every slow load wrong. + */ + delay?: RunDiff & { of: number; differing: number; oneLoad?: boolean }; + /** + * Worth a look, not a race: what only one slow baseline did against the baseline with no added delay, and lines only that + * baseline shows (one load against one load, which chance can do; an error page it landed on), a number both slow + * baselines show a little off from it (`drift`: slow line, its line, a counter that moved between loads), and counts of + * writes that changed with the delay and nothing else. + */ + look?: { lines: { text: string; only: "slow" | "plain" | "plain-alone"; runs: number }[]; drift: [string, string][]; errors: string[]; fields: FieldDiff[]; writes: FieldDiff[] }; + /** Requests to other sites left out of the counts (nothing the user did or the page's own origin asked for led to them). */ + thirdParty?: string[]; } export function multiset(lines: Iterable): Map { @@ -240,8 +361,143 @@ function onlyIn(a: Map, b: Map): string[] { return out; } +/** A token that holds a digit; a comma before three digits is inside it ("$1,000" is one number, not 1 and 000). */ +/** How much later `y`'s lines were read than `x`'s, on the bridge's clock: each run from its hello to its snapshot. */ +function elapsed(x: Run, y: Run): Elapsed { + return { min: y.started - (x.started + (x.snapshot?.t ?? 0)), max: y.started + (y.snapshot?.t ?? 0) - x.started }; +} + +const DIGIT_TOKEN = /[\w.:/-]*\d(?:[\w.:/-]|,(?=\d{3}(?!\d)))*/g; + /** A page line with tokens that hold a digit blanked: "Order #A7F3KQ placed 12:01:04" → "Order #N placed N". */ -export const lineShape = (l: string): string => l.replace(/[\w.:/-]*\d[\w.:/-]*/g, "N"); +export const lineShape = (l: string): string => l.replace(DIGIT_TOKEN, "N"); + +const tokensOf = (l: string): string[] => l.match(DIGIT_TOKEN) ?? []; +const tokenShape = (t: string): string => t.replace(/\d[\d,]*/g, "N"); +const kindOf = (l: string): string => l.replace(DIGIT_TOKEN, tokenShape); + +/** + * What may stand at one digit-holding position of a noisy line: this very token, a number near this one, or any token of + * a shape ("": any). + */ +type Slot = { exact: string } | { near: number; was: string } | { shape: string }; + +const NUMBER = /^\d[\d,]*(?:\.\d+)?$/; +const numberOf = (t: string): number => Number(t.replace(/,/g, "")); + +/** + * Which tokens varied between two lines of one shape. A word with a digit that did not vary stays exact ("12 items, + * version-1"); a number that did not vary, on a line where another did, may drift a little ("CPU 12% memory 40%") and no + * further ("Order 123 total 40" is not "Order 125 total 90"). + */ +function slotsOf(a: string[], b: string[]): Slot[] { + return a.map((t, i) => (t !== b[i] ? { shape: tokenShape(t) === tokenShape(b[i]) ? tokenShape(t) : "" } : NUMBER.test(t) ? { near: numberOf(t), was: t } : { exact: t })); +} + +const nearBy = (n: number, t: string): boolean => NUMBER.test(t) && Math.abs(numberOf(t) - n) <= Math.max(2, Math.abs(n) * 0.1); + +const fitsSlots = (slots: Slot[], tokens: string[]): boolean => slots.every((s, i) => ("exact" in s ? s.exact === tokens[i] : "near" in s ? nearBy(s.near, tokens[i]) : !s.shape || tokenShape(tokens[i]) === s.shape)); + +/** A noisy line for the report: the tokens that varied as "N" or "*", the others as they are. */ +function slotsText(line: string, slots: Slot[]): string { + let i = 0; + return line.replace(DIGIT_TOKEN, (t) => { + const s = slots[i++]; + return !s ? t : "exact" in s ? s.exact : "near" in s ? s.was : s.shape || "*"; + }); +} + +/** + * Lines only in `a` and only in `b`, as counts, after a line with an age met its like: "26 minutes ago" then "27 minutes + * ago" is no difference, "just now" against "3 days ago" is; `later` says how much later `b` was read. Oldest with oldest, + * so a list whose rows all moved a minute pairs row by row. + */ +function lineDiff(a: Map, b: Map, later: Elapsed): { onlyA: string[]; onlyB: string[] } { + const onlyA = onlyIn(a, b); + const onlyB = onlyIn(b, a); + if (!onlyA.length || !onlyB.length) return { onlyA, onlyB }; + const aged = (list: string[]) => { + const m = new Map>["ages"] }[]>(); + list.forEach((l, i) => { + const x = agesIn(l); + if (x) m.set(x.form, [...(m.get(x.form) ?? []), { i, ages: x.ages }]); + }); + for (const v of m.values()) v.sort((p, q) => p.ages[0].lo - q.ages[0].lo); + return m; + }; + const inB = aged(onlyB); + const dropA = new Set(); + const dropB = new Set(); + for (const [form, xs] of aged(onlyA)) { + const ys = inB.get(form) ?? []; + let i = 0; + let j = 0; + while (i < xs.length && j < ys.length) { + if (sameAges(xs[i].ages, ys[j].ages, later)) { + dropA.add(xs[i++].i); + dropB.add(ys[j++].i); + } else if (xs[i].ages[0].lo + later.min < ys[j].ages[0].lo) i++; + else j++; + } + } + return { onlyA: onlyA.filter((_, i) => !dropA.has(i)), onlyB: onlyB.filter((_, i) => !dropB.has(i)) }; +} + +/** Page lines the baselines say nothing about: lines only some of them show, and lines with a token that varied. */ +interface LineNoise { + exact: Set; + slots: Map; + /** The noisy lines as the report writes them. */ + labels: Set; + /** Lines every baseline shows the same: a token that varied elsewhere does not make them noise. */ + stable: Set; + /** One value in place of another between two baselines (no digit to pair them by); a third anywhere else is not news. */ + swapped: [string, string][]; + /** Lines paired by shape whose tokens are far apart or not numbers ("$12.00", "$45.00"): the rest are counters. */ + apart: [string, string][]; +} + +const isNoiseLine = (n: LineNoise, l: string): boolean => { + if (n.exact.has(l)) return true; + if (n.stable.has(l)) return false; + const tokens = tokensOf(l); + return !!n.slots.get(lineShape(l))?.some((s) => fitsSlots(s, tokens)); +}; + +/** What differs between the baselines' lines. */ +function lineNoise(base: Run[], texts: Map[]): LineNoise { + const n: LineNoise = { exact: new Set(), slots: new Map(), labels: new Set(), stable: new Set(), swapped: [], apart: [] }; + const b0 = texts[0]; + for (const l of b0.keys()) if (texts.every((b) => b.has(l))) n.stable.add(l); + for (let k = 1; k < texts.length; k++) { + const { onlyA: fromFirst, onlyB } = lineDiff(b0, texts[k], elapsed(base[0], base[k])); + const unpaired = new Set(fromFirst.keys()); + const left: string[] = []; + for (const y of onlyB) { + // its own kind first ("3" with "4", not with "version-1"), then any line of the shape (generated ids of two forms) + const i = [...unpaired].find((k) => kindOf(fromFirst[k]) === kindOf(y)) ?? [...unpaired].find((k) => lineShape(fromFirst[k]) === lineShape(y)); + if (i === undefined) { + left.push(y); + continue; + } + unpaired.delete(i); + const x = fromFirst[i]; + const shape = lineShape(x); + const list = n.slots.get(shape) ?? []; + const slots = slotsOf(tokensOf(x), tokensOf(y)); + list.push(slots); + n.slots.set(shape, list); + n.labels.add(slotsText(x, slots)); + if (!drifted(x, y)) n.apart.push([x, y]); + } + const rest = [...unpaired].map((k) => fromFirst[k]); + for (const l of [...rest, ...left]) n.exact.add(l); + if (rest.length === left.length) rest.forEach((x, i) => n.swapped.push([x, left[i]])); + } + return n; +} + +const byWeight = (lines: string[]): string[] => lines.map((l, i) => ({ l, i, w: lineWeight(l) })).sort((a, b) => b.w - a.w || a.i - b.i).map((x) => x.l); /** Parents ("todos.entities") whose child keys differ between two key sets (record ids made fresh each run). */ function noisyParents(a: Iterable, b: Iterable): Set { @@ -285,7 +541,7 @@ function underNoisy(path: string, noisy: Set): string | null { * A snapshot as comparable entries: `state `, `field