diff --git a/docs/src/content/docs/guide/inspect.mdx b/docs/src/content/docs/guide/inspect.mdx index 5d1bfea..3078b37 100644 --- a/docs/src/content/docs/guide/inspect.mdx +++ b/docs/src/content/docs/guide/inspect.mdx @@ -8,7 +8,7 @@ import { Aside, Steps } from '@astrojs/starlight/components'; *SHRINE Inspect is a local, read-only command-line tool. It measures your AI coding setup and session history against SHRINE and reports what to change first. It changes nothing itself.* - The first release reads one harness: Claude Code; other harnesses come later @@ -35,6 +35,7 @@ This is a trial release. Get the newest `inspect-v*` release from the [releases At T1 only, 12 scrubbed episode excerpts go to your model provider through your own coding agent login, one call each. - Each excerpt holds your messages, the start of each reply, the end of the last reply, and how the episode ended +- Code adds end signals: an open question for you in the last reply, a later revert or fixup, a later episode on the same files within a day - Code searches all your loaded instruction files and sends only the lines that match your corrections - When no line matches, that search counts as evidence; "could not tell" is kept for files that may have changed since or could not be read in full @@ -53,9 +54,9 @@ It checks that it changed nothing it read. If anything changed, the report shows 2. Download the three release files into a new folder ```sh - mkdir -p ~/shrine-inspect-0.1.2 && cd ~/shrine-inspect-0.1.2 - base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.2 - curl -fLO "$base/shrine-inspect-0.1.2.tgz" + mkdir -p ~/shrine-inspect-0.1.3 && cd ~/shrine-inspect-0.1.3 + base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.3 + curl -fLO "$base/shrine-inspect-0.1.3.tgz" curl -fLO "$base/install.sh" curl -fLO "$base/SHA256SUMS" ``` diff --git a/inspect/README.md b/inspect/README.md index d8e0b94..850c5bd 100644 --- a/inspect/README.md +++ b/inspect/README.md @@ -79,11 +79,13 @@ Status: pre-release trial (background runs, a thin skill, a checksummed release) - The consent, with the approved estimate, is recorded in `facts.json`; the judge command is refused until it is there; every run asks again - Sample: 12 episodes with a correction candidate, seeded (seed in the report footer), recency thirds, at most 4 per project unless one project holds them all, 3x weight for sessions with 2 or more corrections - Fewer than 12 eligible: all are judged, labelled thin evidence; under 10 sessions: T1 is not offered -- Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 300 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), and the instruction lines that match the flagged messages - - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 1,600 characters) - - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 1,500 tokens per pack at most - - Over the cap: replies cut first, then middle messages that were not flagged (the first and last always stay), then long messages, then instruction lines, then flagged middle messages -- Gate B estimate: pack size (bytes / 4) plus a per-call overhead and output measured on Claude Code 2.1.318, then 10% retry reserve and 30% margin on the top +- Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 400 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), end signals computed by code, and the instruction lines that match the flagged messages + - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 2,400 characters) + - End signals: the last reply asks you something (a question, lettered choices, a questions list) and no tool call follows, a later revert or fixup commit, a later episode on the same files within a day + - The judge uses these and the last reply to pick resolved, abandoned (left open), or reworked; "could not tell" is for an episode none of these fits + - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 4,000 tokens per pack at most (the cap, at 2.5 bytes per token) + - Over the cap: middle replies cut first, then middle messages that were not flagged (the first and last always stay, the last reply whole with its closing), then instruction lines, then flagged middle messages, then the first and last messages +- Gate B estimate: pack size (bytes / 2.5) plus the prompt, a per-call overhead and output measured on Claude Code 2.1.318, then 10% retry reserve and 30% margin on the top - Minutes: 10 seconds per call, judging only (not the scan or time at gates); real time varies by machine, login, and network - The judging time limit (the estimate plus 50%) is the only fixed time figure - Session text is data for the judge, never instructions: it sits inside a fence named by a random nonce, and the judge prompt says so @@ -146,8 +148,9 @@ T1 trial (recorded, not fixed): - At 12 episodes the top of the Gate B range is tens of thousands of tokens at any pack size: the fixed cost per call (prompt, overhead, output) times 12, plus reserve and margin (`test/judge/workorder.test.mjs`) - A correction you later reversed still gives a before and after from your own words - The harness version shown at Gate B comes from your newest session log, not from the installed command -- A session whose loaded-instructions snapshot is logged only after an episode starts is read as "may have changed", so its corrections stay "could not tell" -- Instruction lines are matched by shared words (common words left out), not meaning: a rule worded with other words is not found, and its correction reads as yours +- The loaded-instructions snapshot logged right after an episode's first message counts as loaded for it; a first snapshot logged only after a reply is read as "may have changed", so its corrections stay "could not tell" +- End signals are word and pattern checks: a reply that asks something without a question mark, lettered choices, or a questions heading is not seen as asking +- Instruction lines are matched by shared words (common words and simple plural or tense endings left out), not meaning: a rule worded with other words is not found, and its correction reads as yours. Sending whole files would fix this but sends far more of your config, so it is not done Leak check, `check-share` and the dry run (`test/release/leak-known-limits.test.mjs`): diff --git a/inspect/TRIAL.md b/inspect/TRIAL.md index 08034fb..a7818fb 100644 --- a/inspect/TRIAL.md +++ b/inspect/TRIAL.md @@ -13,12 +13,12 @@ A local health check of your Claude Code setup and session history, measured aga node --version ``` -2. Download the 3 release files into a new folder. With the GitHub CLI, use `gh release download inspect-v0.1.2 -R stablekernel/SHRINE` in place of the `curl` lines. +2. Download the 3 release files into a new folder. With the GitHub CLI, use `gh release download inspect-v0.1.3 -R stablekernel/SHRINE` in place of the `curl` lines. ```sh - mkdir -p ~/shrine-inspect-0.1.2 && cd ~/shrine-inspect-0.1.2 - base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.2 - curl -fLO "$base/shrine-inspect-0.1.2.tgz" + mkdir -p ~/shrine-inspect-0.1.3 && cd ~/shrine-inspect-0.1.3 + base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.3 + curl -fLO "$base/shrine-inspect-0.1.3.tgz" curl -fLO "$base/install.sh" curl -fLO "$base/SHA256SUMS" ``` @@ -33,7 +33,7 @@ A local health check of your Claude Code setup and session history, measured aga ``` install.sh: OK - shrine-inspect-0.1.2.tgz: OK + shrine-inspect-0.1.3.tgz: OK ``` 4. Install. Run it from the folder that holds the files. @@ -43,9 +43,9 @@ A local health check of your Claude Code setup and session history, measured aga ``` - Usage: `sh install.sh [--skill] [path/to/tgz]`. - - With no path, it uses `shrine-inspect-0.1.2.tgz` next to `install.sh`, not in your current folder. + - With no path, it uses `shrine-inspect-0.1.3.tgz` next to `install.sh`, not in your current folder. - It checks Node, then checks the tarball against the SHA-256 written in `install.sh`, then extracts it. A mismatch installs nothing. - - The CLI goes to `~/.local/share/shrine-inspect/0.1.2/`, linked as `~/.local/bin/shrine-inspect`. + - The CLI goes to `~/.local/share/shrine-inspect/0.1.3/`, linked as `~/.local/bin/shrine-inspect`. - `--skill` copies the Claude Code skill to `~/.claude/skills/shrine-inspect/`. Leave it out to use the CLI only. - To install somewhere else, set the install folder and the link folder first: @@ -68,7 +68,7 @@ A local health check of your Claude Code setup and session history, measured aga ``` ``` - shrine-inspect 0.1.2 + shrine-inspect 0.1.3 ``` - All commands: `shrine-inspect --help`. @@ -187,7 +187,7 @@ shrine-inspect check-share - T0: nothing leaves your machine. No network, no model calls. - T1: 12 short episode excerpts go to your own Claude Code login, one call each, billed to your account like normal use. Secrets are removed and names, paths, and hosts are masked first. -- Each excerpt holds your messages, the start of each reply, the end of the last reply, how the episode ended, and only the instruction lines that match your corrections. Code searches all your loaded instruction files for those lines; the rest stay on your machine. +- Each excerpt holds your messages, the start of each reply, the end of the last reply, how the episode ended (with end signals code computed, such as an open question for you), and only the instruction lines that match your corrections. Code searches all your loaded instruction files for those lines; the rest stay on your machine. - Nothing goes to SHRINE, Stable Kernel, or any other service. No telemetry. ## Read the report diff --git a/inspect/package-lock.json b/inspect/package-lock.json index 8b25c44..a7aadb6 100644 --- a/inspect/package-lock.json +++ b/inspect/package-lock.json @@ -1,12 +1,12 @@ { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "bin": { "shrine-inspect": "bin/shrine-inspect.mjs" diff --git a/inspect/package.json b/inspect/package.json index 7a3b68d..b765013 100644 --- a/inspect/package.json +++ b/inspect/package.json @@ -1,6 +1,6 @@ { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "description": "Health check of an AI coding setup against the SHRINE docs. Local, read-only, no network.", "license": "Apache-2.0", "author": "Stable Kernel", diff --git a/inspect/skill/.claude-plugin/plugin.json b/inspect/skill/.claude-plugin/plugin.json index 419f4ca..c70462c 100644 --- a/inspect/skill/.claude-plugin/plugin.json +++ b/inspect/skill/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "description": "Thin skill that starts and watches the shrine-inspect CLI. Holds no logic; install the CLI from the same release.", "license": "Apache-2.0", "author": { "name": "Stable Kernel" }, diff --git a/inspect/skill/skills/shrine-inspect/SKILL.md b/inspect/skill/skills/shrine-inspect/SKILL.md index 9fd4555..e44c923 100644 --- a/inspect/skill/skills/shrine-inspect/SKILL.md +++ b/inspect/skill/skills/shrine-inspect/SKILL.md @@ -6,7 +6,7 @@ allowed-tools: Bash(node --version), Bash(shrine-inspect version:*), Bash(shrine # SHRINE Inspect -Skill version: 0.1.2 +Skill version: 0.1.3 This skill only starts the `shrine-inspect` CLI and watches it. The CLI does every read, every gate, and every judge call. @@ -31,7 +31,7 @@ This skill only starts the `shrine-inspect` CLI and watches it. The CLI does eve 2. Check the CLI. Run: ```bash - shrine-inspect version --skill-version 0.1.2 + shrine-inspect version --skill-version 0.1.3 ``` If the command is not found, or it stops with `E_VERSION`, stop. Tell the user to install the CLI and this skill from the same release (see TRIAL.md in the release). @@ -39,7 +39,7 @@ This skill only starts the `shrine-inspect` CLI and watches it. The CLI does eve 3. Start a run in the background. Use `--tier T1` only when the user asked for judging ("Pulse"); else use `--tier T0`. Session id: `${CLAUDE_SESSION_ID}`. If that id is empty or still shows as a placeholder, leave out `--session-id` and its value. Never leave `--session-id` with nothing after it. ```bash - shrine-inspect start --tier T0 --session-id ${CLAUDE_SESSION_ID} --skill-version 0.1.2 --json + shrine-inspect start --tier T0 --session-id ${CLAUDE_SESSION_ID} --skill-version 0.1.3 --json ``` Keep the `run_id` from the output. @@ -47,7 +47,7 @@ This skill only starts the `shrine-inspect` CLI and watches it. The CLI does eve 4. Watch the run. Run this about every 15 seconds while the state is `running`: ```bash - shrine-inspect status --skill-version 0.1.2 --json + shrine-inspect status --skill-version 0.1.3 --json ``` - `running`: give the user one short line with the phase. Check again. diff --git a/inspect/src/adapters/claude-code/transcript.mjs b/inspect/src/adapters/claude-code/transcript.mjs index 4a0a619..682f675 100644 --- a/inspect/src/adapters/claude-code/transcript.mjs +++ b/inspect/src/adapters/claude-code/transcript.mjs @@ -6,8 +6,9 @@ import { readSession } from './events.mjs'; // Events of one episode: from its root human turn (matched by uuid, else by start // time) through its end time. Subagent events are left out. The live config is -// the newest instruction snapshot at or before the episode start, else the first -// one in the session (loaded at start, before the first prompt was logged). +// the snapshot logged right after the root turn (loaded for that prompt), else +// the newest one at or before the episode start, else the first one in the +// session, marked late. export function sliceEpisode(timeline, ep) { const main = timeline.filter((e) => !e.sub); let from = main.findIndex((e) => e.kind === 'human' && e.uuid && e.uuid === ep.root_turn_id); @@ -21,6 +22,12 @@ export function sliceEpisode(timeline, ep) { events.push(e); } const snaps = main.filter((e) => e.kind === 'instructions'); + // The harness logs the files it loads for a prompt just after that prompt. A + // snapshot logged after the root turn, with nothing else between them, is the + // config loaded for the root prompt, so it counts as loaded at the start. + let loaded = null; + for (let i = from + 1; i < main.length && main[i].kind === 'instructions'; i++) loaded = main[i]; + if (loaded) return { events, snapshot: loaded.files, snapshot_late: false }; const before = snaps.filter((e) => e.t && e.t <= ep.start).at(-1); if (before) return { events, snapshot: before.files, snapshot_late: false }; // A snapshot logged after the episode started may hold files loaded later on, so diff --git a/inspect/src/judge/bundle.mjs b/inspect/src/judge/bundle.mjs index e254554..9f80115 100644 --- a/inspect/src/judge/bundle.mjs +++ b/inspect/src/judge/bundle.mjs @@ -1,6 +1,8 @@ // Evidence pack for the Episode Classifier (plan T1 task 2). One episode per // bundle: human turns (text), assistant replies (opening only, plus the closing -// of the last reply), tool names (no arguments), how the episode ended, and the +// of the last reply), tool names (no arguments), how the episode ended plus +// code-computed end signals (an open question for the person, a later revert, +// fixup, or episode on the same files), and the // lines of the live config that match the flagged turns. Code searches the whole // config; only matching lines are sent. Every string goes through scrubSecrets, // then mask with the known identifiers, always: secrets, names, and paths never @@ -9,18 +11,19 @@ import { randomBytes } from 'node:crypto'; import { scrubSecrets } from '../scrub/secrets.mjs'; import { mask, maskPath } from '../scrub/mask.mjs'; import { isCorrectionCue } from '../probes/usage.mjs'; +import { endsWithQuestion } from '../episodes/extract.mjs'; export const JUDGE_MARKER = 'SHRINE inspect judge v1:'; export const DEFAULT_CAPS = Object.freeze({ human_turn_chars: 1200, assistant_chars: 400, - closing_chars: 300, + closing_chars: 400, turns: 12, - bundle_tokens: 1500, - config_excerpt_chars: 1600, + bundle_tokens: 4000, + config_excerpt_chars: 2400, config_line_chars: 240, config_lines_per_turn: 6, - bytes_per_token: 4, + bytes_per_token: 2.5, tools_per_turn: 10, }); @@ -37,6 +40,7 @@ export function payloadOf(b) { return { episode_id: b.episode_id, episode_end: b.episode_end ?? null, + end_signals: b.end_signals ?? null, omitted_turns: b.omitted_turns ?? 0, turns: b.turns.map(turn), config_searched: b.config_searched === true, @@ -56,6 +60,39 @@ const stem = (w) => w.replace(/(?:ing|ed|es|s)$/u, (m) => (w.length - m.length > export const keywords = (s) => new Set((String(s).toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}_-]{3,}/gu) ?? []).filter((w) => !STOP.has(w)).map(stem)); const HEADING = /^\s{0,3}#{1,6}\s+\S/; +// The end of a reply asks the person something: it ends with a question, offers +// lettered choices (A) and B)), or opens a questions list. Only the last 600 +// characters are read, so the check stays linear and cheap. +const CHOICE_A = /(?:^|\s)A\)\s/u; +const CHOICE_B = /(?:^|\s)B\)\s/u; +const QUESTIONS = /^[\s#*_-]{0,6}(?:open\s)?questions?[\s:*_]{0,4}$/imu; +export function asksPerson(text) { + const tail = String(text ?? '').slice(-600); + return endsWithQuestion(tail) || (CHOICE_A.test(tail) && CHOICE_B.test(tail)) || QUESTIONS.test(tail); +} + +// End signals computed by code, never by a model: what was still open when the +// episode ended, and what git and later episodes show about the same work. +// reverted and fixup_later: the linker's flags when present, else the scan's +// stratum outcome (rework outranks possible_fix there), else downstream, which +// keeps only the nearest few links. continued_later: a later episode on the same +// files within a day, the next one in the same session included, since that is +// where an open question gets its answer; the nearest links come first, so +// downstream holds it when there is one. +export function endSignals(episode, lastAssistant, lastReplyText) { + const down = Array.isArray(episode.downstream) ? episode.downstream : []; + const outcome = episode.stratum?.outcome; + const has = (kind) => down.some((d) => d.kind === kind); + const reverted = typeof episode.rework === 'boolean' ? episode.rework : outcome === 'rework' || has('revert'); + const fixup = typeof episode.possible_fix === 'boolean' ? episode.possible_fix : outcome === 'possible_fix' || has('fixup'); + return { + asks_person: Boolean(lastAssistant) && asksPerson(lastReplyText), + reverted, + fixup_later: fixup, + continued_later: down.some((d) => d.kind === 'episode' && d.delta_hours <= 24), + }; +} + // Lines of the whole config that match a flagged turn. files: [{path, text}]; // turns: [{id, words}]. A word found on more than a quarter of the lines (in a // config of 8 or more lines) is too common to count. Score: 2 per matching word @@ -144,8 +181,11 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n } if (e.kind === 'assistant_text') { if (!assistant.text) assistant.text = clean(e.head ?? e.text, known).slice(0, caps.assistant_chars); - lastReply = { turn: assistant, text: e.text }; + lastReply = { turn: assistant, text: e.text, tool_after: false }; } + // A tool call after the text means the agent went on: the question, if any, + // was not left for the person (the episode cutter reads it the same way). + if (e.kind === 'tool_use' && lastReply?.turn === assistant) lastReply.tool_after = true; if (e.kind === 'tool_use' && assistant.tools.length < caps.tools_per_turn && !assistant.tools.includes(e.name)) assistant.tools.push(clean(e.name, known).slice(0, 64)); } lastKind = e.kind; @@ -192,10 +232,12 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n chosen.push(r); used += n; } + const lastTurn = turns.at(-1); const b = { episode_id: episode.id, nonce, episode_end: episode.terminal ?? null, + end_signals: endSignals(episode, lastTurn?.role === 'assistant' ? lastTurn : null, lastReply && lastTurn === lastReply.turn && !lastReply.tool_after ? lastReply.text : ''), omitted_turns: 0, turns, config: [], @@ -212,14 +254,15 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n }; setConfig(); - // Size cap (bytes / 4). Order: shorten replies, drop middle turns that are not - // flagged (first and last always stay), shorten turns, trim config to a floor, - // drop flagged middle turns, then the rest of the config, then cut every turn. + // Size cap (bytes / bytes_per_token). Order: shorten middle replies, drop middle + // turns that are not flagged (first and last always stay, the last reply at full + // length with its closing), trim config to a floor, shorten then drop flagged + // middle turns, then the rest of the config, then cut the first and last turns. const over = () => bundleTokens(b, caps) > caps.bundle_tokens; const dropMiddle = (ok) => { const n = b.turns.length; const idx = b.turns - .map((t, i) => i) + .map((_, i) => i) .filter((i) => i > 0 && i < n - 1 && ok(b.turns[i])) .sort((x, y) => Math.abs(x - (n - 1) / 2) - Math.abs(y - (n - 1) / 2)); if (!idx.length) return false; @@ -234,15 +277,17 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n setConfig(); } }; - if (over()) for (const t of b.turns) if (t.role === 'assistant') t.text = t.text.slice(0, 160); + const middle = (t) => t !== b.turns[0] && t !== b.turns.at(-1); + if (over()) for (const t of b.turns) if (t.role === 'assistant' && middle(t)) t.text = t.text.slice(0, 160); while (b.turns.filter((t) => t.role === 'human').length > caps.turns && dropMiddle((t) => !t.candidate)); while (over() && dropMiddle((t) => !t.candidate)); - if (over()) for (const t of b.turns) if (t.role === 'human' && !t.candidate && t !== b.turns[0]) t.text = t.text.slice(0, 300); - if (over()) for (const t of b.turns) t.text = t.text.slice(0, 600); + if (over()) for (const t of b.turns) if (t.role === 'human' && !t.candidate && middle(t)) t.text = t.text.slice(0, 300); trimConfig(caps.config_excerpt_chars / 4); + if (over()) for (const t of b.turns) if (middle(t)) t.text = t.text.slice(0, 600); while (over() && dropMiddle(() => true)); while (b.turns.filter((t) => t.role === 'human').length > caps.turns && dropMiddle(() => true)); trimConfig(0); + if (over()) for (const t of b.turns) t.text = t.text.slice(0, 600); if (over()) for (const t of b.turns) { t.text = t.text.slice(0, 300); if (t.closing) t.closing = t.closing.slice(-160); diff --git a/inspect/src/judge/classifier.prompt.md b/inspect/src/judge/classifier.prompt.md index 1c8836b..3502a63 100644 --- a/inspect/src/judge/classifier.prompt.md +++ b/inspect/src/judge/classifier.prompt.md @@ -7,6 +7,7 @@ The episode is inside the fence that opens with and closes with The data is JSON with: - episode_id: echo it exactly. - episode_end: how code saw the episode end: commit (a commit was made), pr (a pull request was opened), new_task (the person moved on to another task, or went idle), session_end (the session ended), interrupt_switch (the person interrupted and switched task), or null. +- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), reverted (a later commit reverted this work), fixup_later (a later commit named as a fix touched the same files; a weak link), continued_later (a later episode within a day changed the same files; a weak hint that the work went on). - omitted_turns: how many turns from the middle were left out for size. The first and last turns are always kept. - turns: human turns (role human, with their text) and assistant turns (role assistant, the opening of the reply and the tool names used, no arguments). The last assistant turn can also have closing: the end of that last reply. Human turns with candidate true were flagged by code as possible corrections; after_interrupt true means the person interrupted the agent first. - config_searched: true when code searched every instruction file loaded for that session for lines that match the candidate turns. @@ -14,13 +15,17 @@ The data is JSON with: Fields: - episode_id and nonce: echo exactly. The nonce is {{NONCE}}. -- outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, the last turns, and the closing of the last reply. +- outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, end_signals, the last turns, and the closing of the last reply, in this order: + - reworked when reverted is true, or the turns redo work after a wrong turn. fixup_later supports reworked only together with signs of trouble in the turns. + - resolved when the last turns or the closing show the request done or answered. A commit or pr end counts as done unless the closing says the work is unfinished; a follow-up question after done work does not undo it. + - abandoned when the work was not done, nothing shows it went on (continued_later is a hint that it did), and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with the closing saying work is still running. That is work dropped or left open. + - unknown only when none of these fits, for example the work was left open and seems to have gone on in a later episode. continued_later alone does not undo done work. - corrections: one item per human turn with candidate true that corrects or redirects the agent. Leave out candidate turns that are not corrections. Use an empty list when there are none. - turn_id: the id of that human turn. - kind: constraint (adds a rule or limit the agent needed), redirect (changes the task), bug_report (reports a defect in the agent's work), preference (a taste choice), or other. - supplies_constraint: true only when the turn states a constraint that, given in the first message, would have avoided the correction. - constraint_present_in: where that constraint already existed before this turn: prompt (an earlier human turn in this episode), instruction_file (a config block), skill, none, or unknown. - - attribution: label user (the constraint arrived late from the person), model (it was available and not followed), harness (the setup did not load it), or unknown; confidence from 0 to 1. + - attribution: label user (the constraint arrived late from the person, or the turn changes the request or makes a taste choice that nothing earlier could have covered), model (it was available and not followed), harness (the setup did not load it), or unknown; confidence from 0 to 1. - sufficiency: enough, thin, or none: how well the data supports your reading. - quotes: up to 3 items {turn_id, text}; text is a short exact copy (12 to 200 characters) from that turn's text. Never paraphrase. - config_ref: only when constraint_present_in is instruction_file: {config_id, text}, where text is an exact copy (12 to 200 characters) from that config block. diff --git a/inspect/src/judge/workorder.mjs b/inspect/src/judge/workorder.mjs index e025c16..5e075f7 100644 --- a/inspect/src/judge/workorder.mjs +++ b/inspect/src/judge/workorder.mjs @@ -24,7 +24,7 @@ export function createWorkOrder({ runId, seed, bundles, estimate, caps }) { }; } -// Token range and minutes for Gate B: bundle bytes / 4 plus a fixed per-call +// Token range and minutes for Gate B: bundle tokens (bytes / bytes_per_token) plus a fixed per-call // overhead and output allowance, a retry reserve, then a margin for the top. export function estimateOrder(bundleTokenCounts, e) { const calls = bundleTokenCounts.length; diff --git a/inspect/src/scorer/score.mjs b/inspect/src/scorer/score.mjs index b136119..c58d8aa 100644 --- a/inspect/src/scorer/score.mjs +++ b/inspect/src/scorer/score.mjs @@ -7,7 +7,7 @@ import { underHome } from '../scrub/mask.mjs'; import { THRESHOLDS } from '../probes/config.mjs'; import { controlTest, usageConfidence, usageSeverity } from './stats.mjs'; -export const TOOL_VERSION = '0.1.2'; +export const TOOL_VERSION = '0.1.3'; const SEV_RANK = { high: 3, medium: 2, low: 1 }; const KIND_ORDER = ['blanket_permissions', 'stale_instructions', 'oversized_instructions', 'done_without_test', 'no_owner_or_review', 'stack_gap', 'short_retention']; diff --git a/inspect/src/scorer/thresholds.json b/inspect/src/scorer/thresholds.json index fea9de4..3897288 100644 --- a/inspect/src/scorer/thresholds.json +++ b/inspect/src/scorer/thresholds.json @@ -45,8 +45,8 @@ "bar_min": 2, "anchor_turns": [1, 2], "confidence": { "base": 0.6, "per_hit": 0.05, "agree_bonus": 0.15, "disagree_cap": 0.5, "candidates_max": 0.74 }, - "caps": { "human_turn_chars": 1200, "assistant_chars": 400, "closing_chars": 300, "turns": 12, "bundle_tokens": 1500, "config_excerpt_chars": 1600, "config_line_chars": 240, "config_lines_per_turn": 6, "bytes_per_token": 4, "tools_per_turn": 10 }, - "estimate": { "overhead_tokens_per_call": 3000, "output_tokens_per_call": 400, "retry_reserve": 0.1, "margin": 0.3, "seconds_per_call": 10, "measured_on": "2.1.318" }, + "caps": { "human_turn_chars": 1200, "assistant_chars": 400, "closing_chars": 400, "turns": 12, "bundle_tokens": 4000, "config_excerpt_chars": 2400, "config_line_chars": 240, "config_lines_per_turn": 6, "bytes_per_token": 2.5, "tools_per_turn": 10 }, + "estimate": { "overhead_tokens_per_call": 1800, "output_tokens_per_call": 400, "retry_reserve": 0.1, "margin": 0.3, "seconds_per_call": 10, "measured_on": "2.1.318" }, "abort": { "error_rate": 0.1, "time_box_factor": 1.5 }, "per_call_timeout_ms": 180000, "max_stdout_bytes": 262144, diff --git a/inspect/test/golden/bad-setup/findings.json b/inspect/test/golden/bad-setup/findings.json index 3d2e09c..0056bc5 100644 --- a/inspect/test/golden/bad-setup/findings.json +++ b/inspect/test/golden/bad-setup/findings.json @@ -3,7 +3,7 @@ "run": { "id": "", "started_at": "2026-11-04T12:00:00.000Z", - "tool_version": "0.1.2", + "tool_version": "0.1.3", "harness": { "name": "Claude Code", "version": null diff --git a/inspect/test/golden/zero-history-good/findings.json b/inspect/test/golden/zero-history-good/findings.json index b5b92c8..e80da3a 100644 --- a/inspect/test/golden/zero-history-good/findings.json +++ b/inspect/test/golden/zero-history-good/findings.json @@ -3,7 +3,7 @@ "run": { "id": "", "started_at": "2026-11-04T12:00:00.000Z", - "tool_version": "0.1.2", + "tool_version": "0.1.3", "harness": { "name": "Claude Code", "version": null diff --git a/inspect/test/judge/bundle.test.mjs b/inspect/test/judge/bundle.test.mjs index 187468a..624de5a 100644 --- a/inspect/test/judge/bundle.test.mjs +++ b/inspect/test/judge/bundle.test.mjs @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { buildBundle, bundleTokens, DEFAULT_CAPS, judgePrompt, JUDGE_MARKER, payloadOf } from '../../src/judge/bundle.mjs'; +import { asksPerson, buildBundle, bundleTokens, DEFAULT_CAPS, judgePrompt, JUDGE_MARKER, payloadOf } from '../../src/judge/bundle.mjs'; import { sliceEpisode } from '../../src/adapters/claude-code/transcript.mjs'; import { buildIdentifiers } from '../../src/scrub/mask.mjs'; import { HOUSEKEEPING } from '../../src/adapters/claude-code/projects.mjs'; @@ -201,3 +201,106 @@ test('a snapshot logged only after the episode started is kept but marked may_ha const b = buildBundle({ episode, transcript: s, known, nonce: 'n' }); assert.equal(b.config_may_have_changed, true); }); + +test('a snapshot logged right after the root turn, before any reply, is the config loaded for that prompt', () => { + const [root, ...rest] = events(); + // Logged a second after the prompt, so it is after the episode start. + const soon = new Date(Date.parse(at(0)) + 1000).toISOString(); + const tl = [root, { t: soon, kind: 'instructions', files: snapshot }, ...rest]; + const s = sliceEpisode(tl, episode); + assert.equal(s.snapshot_late, false); + assert.deepEqual(s.snapshot, snapshot); + const b = buildBundle({ episode, transcript: s, known, nonce: 'n' }); + assert.equal(b.config_may_have_changed, false); + // After the agent has replied, a first snapshot stays "may have changed". + const late = sliceEpisode([root, rest[0], rest[1], { t: at(1), kind: 'instructions', files: snapshot }, ...rest.slice(2)], episode); + assert.equal(late.snapshot_late, true); +}); + +test('asksPerson: a question, lettered choices, or a questions list at the end; a plain report is not', () => { + assert.ok(asksPerson('All done. Should I open the PR?')); + assert.ok(asksPerson('Two options:\n1. Layout: A) grid B) list\n2. Color: A) teal B) gray')); + assert.ok(asksPerson('Done with the draft.\n\nQuestions:\n1. Keep the old name')); + assert.ok(!asksPerson('All tests pass and the change is committed.')); + assert.ok(!asksPerson('')); + assert.ok(!asksPerson(`Questions:\n${'filler line\n'.repeat(100)}Done.`), 'only the end of the reply counts'); + assert.ok(!asksPerson('Questions answered above. All done.'), 'a line that only starts with the word is not a questions list'); + assert.ok(asksPerson('Draft is up.\n\n**Open questions:**\n- Naming')); +}); + +test('end signals: computed by code from the last reply and later git and episode links', () => { + const ev = [...events(), { t: at(9), kind: 'tool_use', name: 'Agent', id: 'a1' }, { t: at(9), kind: 'assistant_text', text: 'Pick one: A) keep B) drop', head: 'Two ways.' }]; + const b = buildBundle({ episode: { ...episode, downstream: [{ kind: 'revert', ref: 'x', delta_hours: 2, link: 'hard' }, { kind: 'episode', ref: 'e-2', delta_hours: 1, link: 'soft' }] }, transcript: { events: ev, snapshot }, known, nonce: 'n' }); + assert.deepEqual(payloadOf(b).end_signals, { asks_person: true, reverted: true, fixup_later: false, continued_later: true }); + // Ending on a human turn: nothing in a reply is open. + const h = buildBundle({ episode, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.deepEqual(payloadOf(h).end_signals, { asks_person: false, reverted: false, fixup_later: false, continued_later: false }); + assert.match(TEMPLATE, /end_signals/); + // A tool call after the question: the agent went on, so nothing was left open. + const went = buildBundle({ episode, transcript: { events: [...events(), { t: at(9), kind: 'assistant_text', text: 'Should I run the tests?' }, { t: at(9), kind: 'tool_use', name: 'Bash', id: 'b1' }], snapshot }, known, nonce: 'n' }); + assert.equal(payloadOf(went).end_signals.asks_person, false); + // The linker's flags cover every link; downstream keeps only the nearest few. + const near = Array.from({ length: 5 }, (_, i) => ({ kind: 'episode', ref: `e-${i}`, delta_hours: i, link: 'soft' })); + const f = buildBundle({ episode: { ...episode, downstream: near, rework: true, possible_fix: true, possible_rework: true }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.equal(payloadOf(f).end_signals.reverted, true); + assert.equal(payloadOf(f).end_signals.fixup_later, true); + // Episodes from facts carry no linker flags, only the stratum outcome. + const s = buildBundle({ episode: { ...episode, downstream: near, stratum: { outcome: 'rework' } }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.equal(payloadOf(s).end_signals.reverted, true); + const p = buildBundle({ episode: { ...episode, downstream: near, stratum: { outcome: 'possible_fix' } }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.deepEqual([payloadOf(p).end_signals.reverted, payloadOf(p).end_signals.fixup_later], [false, true]); +}); + +test('trimming shortens middle replies first; the last reply keeps its full opening and closing', () => { + const ev = []; + for (let i = 0; i < 12; i++) { + ev.push({ t: at(i), kind: 'human', uuid: `u${i}`, text: `next step ${'word '.repeat(150)}${i}` }); + ev.push({ t: at(i), kind: 'assistant_text', text: `${'tail '.repeat(100)}Final status line.`, head: `${'reply '.repeat(80)}` }); + } + const b = buildBundle({ episode: { ...episode, root_turn_id: 'u0' }, transcript: { events: ev, snapshot: null }, known, nonce: 'n' }); + assert.ok(bundleTokens(b) <= DEFAULT_CAPS.bundle_tokens); + const last = b.turns.at(-1); + assert.equal(last.role, 'assistant'); + assert.equal(last.text.length, DEFAULT_CAPS.assistant_chars); + assert.ok(last.closing.endsWith('Final status line.')); + assert.equal(b.turns[0].text.length, `next step ${'word '.repeat(150)}0`.length, 'the root prompt stays whole'); + assert.ok(b.turns.slice(1, -1).filter((t) => t.role === 'assistant').every((t) => t.text.length <= 160)); +}); + +test('config: four flagged turns with six best lines each all reach the judge', () => { + const words = ['migration', 'semicolon', 'deploy', 'fixture']; + const ev = [{ t: at(0), kind: 'human', uuid: 'u1', text: 'Start the work.' }]; + words.forEach((w, i) => { + ev.push({ t: at(2 * i + 1), kind: 'assistant_text', text: 'ok' }, { t: at(2 * i + 1), kind: 'interrupt' }); + ev.push({ t: at(2 * i + 2), kind: 'human', uuid: `v${i}`, text: `Stop, the ${w} rule matters here.` }); + }); + const lines = words.flatMap((w) => Array.from({ length: 6 }, (_, k) => `- Rule ${k} about the ${w} step, written out with enough plain words to be a long line.`)); + const b = buildBundle({ episode, transcript: { events: ev, snapshot: [{ path: '/r/CLAUDE.md', content: `# Rules\n${lines.join('\n')}\n` }] }, known, nonce: 'n' }); + for (const [id, m] of Object.entries(b.meta.config_lines)) assert.deepEqual(m, { matched: 6, best: 6, sent: 6 }, id); + assert.ok(bundleTokens(b) <= DEFAULT_CAPS.bundle_tokens); +}); + +test('continued_later: a later episode on the same files within a day; one days later does not count', () => { + const ep = (h) => ({ ...episode, possible_rework: true, downstream: [{ kind: 'episode', ref: 'e-2', delta_hours: h, link: 'soft' }] }); + assert.equal(payloadOf(buildBundle({ episode: ep(3), transcript: { events: events(), snapshot }, known, nonce: 'n' })).end_signals.continued_later, true); + assert.equal(payloadOf(buildBundle({ episode: ep(50), transcript: { events: events(), snapshot }, known, nonce: 'n' })).end_signals.continued_later, false); +}); + +test('over the cap, instruction lines go before the first message or a flagged message is cut', () => { + const first = `Build the export ${'word '.repeat(220)}end`; + const ev = [{ t: at(0), kind: 'human', uuid: 'u1', text: first }]; + const topics = ['migration', 'deploy', 'fixture']; + topics.forEach((w, i) => { + ev.push({ t: at(i + 1), kind: 'assistant_text', text: 'ok', head: 'reply '.repeat(60) }, { t: at(i + 1), kind: 'interrupt' }); + ev.push({ t: at(i + 1), kind: 'human', uuid: `v${i}`, text: `Actually the ${w} step ${'detail '.repeat(150)}` }); + }); + ev.push({ t: at(9), kind: 'assistant_text', text: 'Done.', head: 'Done.' }); + const filler = Array.from({ length: 40 }, (_, k) => `- Filler ${k} about lint and style.`); + const cfg = [...filler, ...topics.flatMap((w) => Array.from({ length: 6 }, (_, k) => `- The ${w} item ${k} needs a dry run, written with enough plain words to make a long line.`))].join('\n'); + const caps = { ...DEFAULT_CAPS, bundle_tokens: 2400 }; + const b = buildBundle({ episode, transcript: { events: ev, snapshot: [{ path: '/r/CLAUDE.md', content: `# Rules\n${cfg}\n` }] }, known, nonce: 'n', caps }); + assert.ok(bundleTokens(b, caps) <= caps.bundle_tokens, `tokens ${bundleTokens(b, caps)}`); + assert.ok(Object.values(b.meta.config_lines).some((m) => m.sent < m.best), 'instruction lines were trimmed'); + assert.equal(b.turns[0].text, first.slice(0, DEFAULT_CAPS.human_turn_chars), 'the first message stays whole'); + assert.ok(b.turns.filter((t) => t.candidate).every((t) => t.text.length > 600), 'flagged messages stay whole'); +}); diff --git a/inspect/test/judge/workorder.test.mjs b/inspect/test/judge/workorder.test.mjs index 99f7f9a..14ccc05 100644 --- a/inspect/test/judge/workorder.test.mjs +++ b/inspect/test/judge/workorder.test.mjs @@ -45,7 +45,7 @@ test('estimate: bundle tokens plus per-call overhead and output, then a retry re assert.ok(r.minutes >= 1); }); -// The Gate B estimate is computed from the packs as built: bytes / 4 per pack plus +// The Gate B estimate is computed from the packs as built: bytes / 2.5 per pack plus // the prompt, a fixed per-call overhead and output allowance, then the retry // reserve and margin on the top. Judging is bounded by its time box, not by size. test('estimate: computed from the pack sizes; 12 packs at the cap fit the time box', () => { @@ -61,6 +61,31 @@ test('estimate: computed from the pack sizes; 12 packs at the cap fit the time b assert.ok(r.minutes <= 5, `minutes ${r.minutes}`); }); +// Measured on 12 real judge calls (Claude Code 2.1.318, a judge prompt of 3,505 +// bytes): pack payload bytes, and the input tokens each call reported. These +// packs ran near 2.5 bytes per token; the fixed input near 3,100 tokens. +const MEASURED_PROMPT_BYTES = 3505; +const MEASURED = [ + [5870, 5347], [4786, 5039], [5743, 5486], [5502, 5240], [5863, 5368], [5081, 5226], + [5902, 5488], [5844, 5351], [3584, 4539], [3904, 4669], [5957, 5556], [5788, 5580], +]; +test('estimate: the per-call input estimate brackets measured calls, and stays honest at the larger cap', () => { + const J = THRESHOLDS.judged; + const est = (bytes, promptBytes) => Math.ceil(bytes / J.caps.bytes_per_token) + Math.ceil(promptBytes / J.caps.bytes_per_token) + J.estimate.overhead_tokens_per_call; + for (const [bytes, real] of MEASURED) { + const e = est(bytes, MEASURED_PROMPT_BYTES); + assert.ok(e >= real * 0.9 && e <= real * 1.1, `${bytes} bytes: estimate ${e} vs measured ${real}`); + } + // A full pack at the cap with today's prompt, against the measured slope and + // intercept plus the prompt growth: the low end within 10%, the top above it. + const capBytes = J.caps.bundle_tokens * J.caps.bytes_per_token; + const promptBytes = Buffer.byteLength(TEMPLATE, 'utf8'); + const expected = 3136 + 0.396 * (capBytes + promptBytes - MEASURED_PROMPT_BYTES); + const e = est(capBytes, promptBytes); + assert.ok(e >= expected * 0.9 && e <= expected * 1.1, `cap estimate ${e} vs ${Math.round(expected)}`); + assert.ok((e + J.estimate.output_tokens_per_call) * (1 + J.estimate.retry_reserve) * (1 + J.estimate.margin) > expected * 1.2); +}); + test('one call per item; every line records tokens, model, and the validated output only', async () => { const calls = []; const r = await run(order(3), async (input) => {