From 2d27a85c021affbd70deaefc5a7cc15df644cf4c Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 01:50:23 -0500 Subject: [PATCH 1/8] fix(inspect): count the snapshot logged right after the root prompt as loaded - The harness logs loaded instruction files just after the prompt - Only a first snapshot logged after a reply stays may-have-changed --- inspect/src/adapters/claude-code/transcript.mjs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/inspect/src/adapters/claude-code/transcript.mjs b/inspect/src/adapters/claude-code/transcript.mjs index 4a0a6192..682f6750 100644 --- a/inspect/src/adapters/claude-code/transcript.mjs +++ b/inspect/src/adapters/claude-code/transcript.mjs @@ -6,8 +6,9 @@ import { readSession } from './events.mjs'; // Events of one episode: from its root human turn (matched by uuid, else by start // time) through its end time. Subagent events are left out. The live config is -// the newest instruction snapshot at or before the episode start, else the first -// one in the session (loaded at start, before the first prompt was logged). +// the snapshot logged right after the root turn (loaded for that prompt), else +// the newest one at or before the episode start, else the first one in the +// session, marked late. export function sliceEpisode(timeline, ep) { const main = timeline.filter((e) => !e.sub); let from = main.findIndex((e) => e.kind === 'human' && e.uuid && e.uuid === ep.root_turn_id); @@ -21,6 +22,12 @@ export function sliceEpisode(timeline, ep) { events.push(e); } const snaps = main.filter((e) => e.kind === 'instructions'); + // The harness logs the files it loads for a prompt just after that prompt. A + // snapshot logged after the root turn, with nothing else between them, is the + // config loaded for the root prompt, so it counts as loaded at the start. + let loaded = null; + for (let i = from + 1; i < main.length && main[i].kind === 'instructions'; i++) loaded = main[i]; + if (loaded) return { events, snapshot: loaded.files, snapshot_late: false }; const before = snaps.filter((e) => e.t && e.t <= ep.start).at(-1); if (before) return { events, snapshot: before.files, snapshot_late: false }; // A snapshot logged after the episode started may hold files loaded later on, so From 4ada68d17848dcc93a2aa1949ddeeff1af10f274 Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 01:50:24 -0500 Subject: [PATCH 2/8] feat(inspect): end signals, outcome rules, and a larger T1 pack - Code-computed end signals: open question, subagent started, revert, fixup, later episode - Outcome rules in the judge prompt use them; unknown only when none fits - Trimming keeps the last reply whole with its closing; middle replies cut first - Pack cap and Gate B estimate at 2.5 bytes per token, checked against measured calls --- inspect/src/judge/bundle.mjs | 55 +++++++++++++++++++++----- inspect/src/judge/classifier.prompt.md | 7 +++- inspect/src/judge/workorder.mjs | 2 +- inspect/src/scorer/thresholds.json | 4 +- inspect/test/judge/bundle.test.mjs | 52 +++++++++++++++++++++++- inspect/test/judge/workorder.test.mjs | 27 ++++++++++++- 6 files changed, 131 insertions(+), 16 deletions(-) diff --git a/inspect/src/judge/bundle.mjs b/inspect/src/judge/bundle.mjs index e2545544..f51b3413 100644 --- a/inspect/src/judge/bundle.mjs +++ b/inspect/src/judge/bundle.mjs @@ -1,6 +1,8 @@ // Evidence pack for the Episode Classifier (plan T1 task 2). One episode per // bundle: human turns (text), assistant replies (opening only, plus the closing -// of the last reply), tool names (no arguments), how the episode ended, and the +// of the last reply), tool names (no arguments), how the episode ended plus +// code-computed end signals (an open question for the person, an agent started +// in the last reply, a later revert, fixup, or episode on the same files), and the // lines of the live config that match the flagged turns. Code searches the whole // config; only matching lines are sent. Every string goes through scrubSecrets, // then mask with the known identifiers, always: secrets, names, and paths never @@ -9,18 +11,19 @@ import { randomBytes } from 'node:crypto'; import { scrubSecrets } from '../scrub/secrets.mjs'; import { mask, maskPath } from '../scrub/mask.mjs'; import { isCorrectionCue } from '../probes/usage.mjs'; +import { endsWithQuestion } from '../episodes/extract.mjs'; export const JUDGE_MARKER = 'SHRINE inspect judge v1:'; export const DEFAULT_CAPS = Object.freeze({ human_turn_chars: 1200, assistant_chars: 400, - closing_chars: 300, + closing_chars: 400, turns: 12, - bundle_tokens: 1500, + bundle_tokens: 4000, config_excerpt_chars: 1600, config_line_chars: 240, config_lines_per_turn: 6, - bytes_per_token: 4, + bytes_per_token: 2.5, tools_per_turn: 10, }); @@ -37,6 +40,7 @@ export function payloadOf(b) { return { episode_id: b.episode_id, episode_end: b.episode_end ?? null, + end_signals: b.end_signals ?? null, omitted_turns: b.omitted_turns ?? 0, turns: b.turns.map(turn), config_searched: b.config_searched === true, @@ -56,6 +60,32 @@ const stem = (w) => w.replace(/(?:ing|ed|es|s)$/u, (m) => (w.length - m.length > export const keywords = (s) => new Set((String(s).toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}_-]{3,}/gu) ?? []).filter((w) => !STOP.has(w)).map(stem)); const HEADING = /^\s{0,3}#{1,6}\s+\S/; +// The end of a reply asks the person something: it ends with a question, offers +// lettered choices (A) and B)), or opens a questions list. Only the last 600 +// characters are read, so the check stays linear and cheap. +const CHOICE_A = /(?:^|\s)A\)\s/u; +const CHOICE_B = /(?:^|\s)B\)\s/u; +const QUESTIONS = /^[\s#*_-]{0,6}(?:open\s)?questions?\b/imu; +export function asksPerson(text) { + const tail = String(text ?? '').slice(-600); + return endsWithQuestion(tail) || (CHOICE_A.test(tail) && CHOICE_B.test(tail)) || QUESTIONS.test(tail); +} +const AGENT_TOOLS = new Set(['Task', 'Agent']); + +// End signals computed by code, never by a model: what was still open when the +// episode ended, and what git and later episodes show about the same work. +export function endSignals(episode, lastAssistant, lastReplyText) { + const down = Array.isArray(episode.downstream) ? episode.downstream : []; + const lastIsReply = Boolean(lastAssistant); + return { + asks_person: lastIsReply && asksPerson(lastReplyText), + started_agent: lastIsReply && lastAssistant.tools.some((n) => AGENT_TOOLS.has(n)), + reverted: down.some((d) => d.kind === 'revert'), + fixup_later: down.some((d) => d.kind === 'fixup'), + continued_later: down.some((d) => d.kind === 'episode'), + }; +} + // Lines of the whole config that match a flagged turn. files: [{path, text}]; // turns: [{id, words}]. A word found on more than a quarter of the lines (in a // config of 8 or more lines) is too common to count. Score: 2 per matching word @@ -192,10 +222,12 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n chosen.push(r); used += n; } + const lastTurn = turns.at(-1); const b = { episode_id: episode.id, nonce, episode_end: episode.terminal ?? null, + end_signals: endSignals(episode, lastTurn?.role === 'assistant' ? lastTurn : null, lastReply && lastTurn === lastReply.turn ? lastReply.text : ''), omitted_turns: 0, turns, config: [], @@ -212,14 +244,15 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n }; setConfig(); - // Size cap (bytes / 4). Order: shorten replies, drop middle turns that are not - // flagged (first and last always stay), shorten turns, trim config to a floor, - // drop flagged middle turns, then the rest of the config, then cut every turn. + // Size cap (bytes / bytes_per_token). Order: shorten middle replies, drop middle + // turns that are not flagged (first and last always stay, the last reply at full + // length with its closing), shorten middle turns, trim config to a floor, drop + // flagged middle turns, then the rest of the config, then cut every turn. const over = () => bundleTokens(b, caps) > caps.bundle_tokens; const dropMiddle = (ok) => { const n = b.turns.length; const idx = b.turns - .map((t, i) => i) + .map((_, i) => i) .filter((i) => i > 0 && i < n - 1 && ok(b.turns[i])) .sort((x, y) => Math.abs(x - (n - 1) / 2) - Math.abs(y - (n - 1) / 2)); if (!idx.length) return false; @@ -234,10 +267,12 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n setConfig(); } }; - if (over()) for (const t of b.turns) if (t.role === 'assistant') t.text = t.text.slice(0, 160); + const middle = (t) => t !== b.turns[0] && t !== b.turns.at(-1); + if (over()) for (const t of b.turns) if (t.role === 'assistant' && middle(t)) t.text = t.text.slice(0, 160); while (b.turns.filter((t) => t.role === 'human').length > caps.turns && dropMiddle((t) => !t.candidate)); while (over() && dropMiddle((t) => !t.candidate)); - if (over()) for (const t of b.turns) if (t.role === 'human' && !t.candidate && t !== b.turns[0]) t.text = t.text.slice(0, 300); + if (over()) for (const t of b.turns) if (t.role === 'human' && !t.candidate && middle(t)) t.text = t.text.slice(0, 300); + if (over()) for (const t of b.turns) if (middle(t)) t.text = t.text.slice(0, 600); if (over()) for (const t of b.turns) t.text = t.text.slice(0, 600); trimConfig(caps.config_excerpt_chars / 4); while (over() && dropMiddle(() => true)); diff --git a/inspect/src/judge/classifier.prompt.md b/inspect/src/judge/classifier.prompt.md index 1c8836b5..84e99c69 100644 --- a/inspect/src/judge/classifier.prompt.md +++ b/inspect/src/judge/classifier.prompt.md @@ -7,6 +7,7 @@ The episode is inside the fence that opens with and closes with The data is JSON with: - episode_id: echo it exactly. - episode_end: how code saw the episode end: commit (a commit was made), pr (a pull request was opened), new_task (the person moved on to another task, or went idle), session_end (the session ended), interrupt_switch (the person interrupted and switched task), or null. +- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), started_agent (the last reply started a subagent), reverted (a later commit reverted this work), fixup_later (a later session fixed it up), continued_later (a later episode changed the same files). - omitted_turns: how many turns from the middle were left out for size. The first and last turns are always kept. - turns: human turns (role human, with their text) and assistant turns (role assistant, the opening of the reply and the tool names used, no arguments). The last assistant turn can also have closing: the end of that last reply. Human turns with candidate true were flagged by code as possible corrections; after_interrupt true means the person interrupted the agent first. - config_searched: true when code searched every instruction file loaded for that session for lines that match the candidate turns. @@ -14,7 +15,11 @@ The data is JSON with: Fields: - episode_id and nonce: echo exactly. The nonce is {{NONCE}}. -- outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, the last turns, and the closing of the last reply. +- outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, end_signals, the last turns, and the closing of the last reply, in this order: + - reworked when reverted or fixup_later is true, or the turns redo work after a wrong turn. + - resolved when the last turns or the closing show the request done or answered. A commit or pr end counts as done unless the closing says the work is unfinished; a follow-up question after done work does not undo it. + - abandoned when the work was not done and the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person), or with the closing saying work is still running, and continued_later is false. That is work left open. + - unknown only when none of these fits, for example the work was left open but continued_later is true. - corrections: one item per human turn with candidate true that corrects or redirects the agent. Leave out candidate turns that are not corrections. Use an empty list when there are none. - turn_id: the id of that human turn. - kind: constraint (adds a rule or limit the agent needed), redirect (changes the task), bug_report (reports a defect in the agent's work), preference (a taste choice), or other. diff --git a/inspect/src/judge/workorder.mjs b/inspect/src/judge/workorder.mjs index e025c16e..5e075f7e 100644 --- a/inspect/src/judge/workorder.mjs +++ b/inspect/src/judge/workorder.mjs @@ -24,7 +24,7 @@ export function createWorkOrder({ runId, seed, bundles, estimate, caps }) { }; } -// Token range and minutes for Gate B: bundle bytes / 4 plus a fixed per-call +// Token range and minutes for Gate B: bundle tokens (bytes / bytes_per_token) plus a fixed per-call // overhead and output allowance, a retry reserve, then a margin for the top. export function estimateOrder(bundleTokenCounts, e) { const calls = bundleTokenCounts.length; diff --git a/inspect/src/scorer/thresholds.json b/inspect/src/scorer/thresholds.json index fea9de4e..007482d9 100644 --- a/inspect/src/scorer/thresholds.json +++ b/inspect/src/scorer/thresholds.json @@ -45,8 +45,8 @@ "bar_min": 2, "anchor_turns": [1, 2], "confidence": { "base": 0.6, "per_hit": 0.05, "agree_bonus": 0.15, "disagree_cap": 0.5, "candidates_max": 0.74 }, - "caps": { "human_turn_chars": 1200, "assistant_chars": 400, "closing_chars": 300, "turns": 12, "bundle_tokens": 1500, "config_excerpt_chars": 1600, "config_line_chars": 240, "config_lines_per_turn": 6, "bytes_per_token": 4, "tools_per_turn": 10 }, - "estimate": { "overhead_tokens_per_call": 3000, "output_tokens_per_call": 400, "retry_reserve": 0.1, "margin": 0.3, "seconds_per_call": 10, "measured_on": "2.1.318" }, + "caps": { "human_turn_chars": 1200, "assistant_chars": 400, "closing_chars": 400, "turns": 12, "bundle_tokens": 4000, "config_excerpt_chars": 1600, "config_line_chars": 240, "config_lines_per_turn": 6, "bytes_per_token": 2.5, "tools_per_turn": 10 }, + "estimate": { "overhead_tokens_per_call": 1800, "output_tokens_per_call": 400, "retry_reserve": 0.1, "margin": 0.3, "seconds_per_call": 10, "measured_on": "2.1.318" }, "abort": { "error_rate": 0.1, "time_box_factor": 1.5 }, "per_call_timeout_ms": 180000, "max_stdout_bytes": 262144, diff --git a/inspect/test/judge/bundle.test.mjs b/inspect/test/judge/bundle.test.mjs index 187468a7..a102ed3d 100644 --- a/inspect/test/judge/bundle.test.mjs +++ b/inspect/test/judge/bundle.test.mjs @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { buildBundle, bundleTokens, DEFAULT_CAPS, judgePrompt, JUDGE_MARKER, payloadOf } from '../../src/judge/bundle.mjs'; +import { asksPerson, buildBundle, bundleTokens, DEFAULT_CAPS, judgePrompt, JUDGE_MARKER, payloadOf } from '../../src/judge/bundle.mjs'; import { sliceEpisode } from '../../src/adapters/claude-code/transcript.mjs'; import { buildIdentifiers } from '../../src/scrub/mask.mjs'; import { HOUSEKEEPING } from '../../src/adapters/claude-code/projects.mjs'; @@ -201,3 +201,53 @@ test('a snapshot logged only after the episode started is kept but marked may_ha const b = buildBundle({ episode, transcript: s, known, nonce: 'n' }); assert.equal(b.config_may_have_changed, true); }); + +test('a snapshot logged right after the root turn, before any reply, is the config loaded for that prompt', () => { + const [root, ...rest] = events(); + // Logged a second after the prompt, so it is after the episode start. + const soon = new Date(Date.parse(at(0)) + 1000).toISOString(); + const tl = [root, { t: soon, kind: 'instructions', files: snapshot }, ...rest]; + const s = sliceEpisode(tl, episode); + assert.equal(s.snapshot_late, false); + assert.deepEqual(s.snapshot, snapshot); + const b = buildBundle({ episode, transcript: s, known, nonce: 'n' }); + assert.equal(b.config_may_have_changed, false); + // After the agent has replied, a first snapshot stays "may have changed". + const late = sliceEpisode([root, rest[0], rest[1], { t: at(1), kind: 'instructions', files: snapshot }, ...rest.slice(2)], episode); + assert.equal(late.snapshot_late, true); +}); + +test('asksPerson: a question, lettered choices, or a questions list at the end; a plain report is not', () => { + assert.ok(asksPerson('All done. Should I open the PR?')); + assert.ok(asksPerson('Two options:\n1. Layout: A) grid B) list\n2. Color: A) teal B) gray')); + assert.ok(asksPerson('Done with the draft.\n\nQuestions:\n1. Keep the old name')); + assert.ok(!asksPerson('All tests pass and the change is committed.')); + assert.ok(!asksPerson('')); + assert.ok(!asksPerson(`Questions:\n${'filler line\n'.repeat(100)}Done.`), 'only the end of the reply counts'); +}); + +test('end signals: computed by code from the last reply and later git and episode links', () => { + const ev = [...events(), { t: at(9), kind: 'assistant_text', text: 'Pick one: A) keep B) drop', head: 'Two ways.' }, { t: at(9), kind: 'tool_use', name: 'Agent', id: 'a1' }]; + const b = buildBundle({ episode: { ...episode, downstream: [{ kind: 'revert', ref: 'x', delta_hours: 2, link: 'hard' }, { kind: 'episode', ref: 'e-2', delta_hours: 1, link: 'soft' }] }, transcript: { events: ev, snapshot }, known, nonce: 'n' }); + assert.deepEqual(payloadOf(b).end_signals, { asks_person: true, started_agent: true, reverted: true, fixup_later: false, continued_later: true }); + // Ending on a human turn: nothing in a reply is open. + const h = buildBundle({ episode, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.deepEqual(payloadOf(h).end_signals, { asks_person: false, started_agent: false, reverted: false, fixup_later: false, continued_later: false }); + assert.match(TEMPLATE, /end_signals/); +}); + +test('trimming shortens middle replies first; the last reply keeps its full opening and closing', () => { + const ev = []; + for (let i = 0; i < 12; i++) { + ev.push({ t: at(i), kind: 'human', uuid: `u${i}`, text: `next step ${'word '.repeat(150)}${i}` }); + ev.push({ t: at(i), kind: 'assistant_text', text: `${'tail '.repeat(100)}Final status line.`, head: `${'reply '.repeat(80)}` }); + } + const b = buildBundle({ episode: { ...episode, root_turn_id: 'u0' }, transcript: { events: ev, snapshot: null }, known, nonce: 'n' }); + assert.ok(bundleTokens(b) <= DEFAULT_CAPS.bundle_tokens); + const last = b.turns.at(-1); + assert.equal(last.role, 'assistant'); + assert.equal(last.text.length, DEFAULT_CAPS.assistant_chars); + assert.ok(last.closing.endsWith('Final status line.')); + assert.equal(b.turns[0].text.length, `next step ${'word '.repeat(150)}0`.length, 'the root prompt stays whole'); + assert.ok(b.turns.slice(1, -1).filter((t) => t.role === 'assistant').every((t) => t.text.length <= 160)); +}); diff --git a/inspect/test/judge/workorder.test.mjs b/inspect/test/judge/workorder.test.mjs index 99f7f9a4..14ccc05e 100644 --- a/inspect/test/judge/workorder.test.mjs +++ b/inspect/test/judge/workorder.test.mjs @@ -45,7 +45,7 @@ test('estimate: bundle tokens plus per-call overhead and output, then a retry re assert.ok(r.minutes >= 1); }); -// The Gate B estimate is computed from the packs as built: bytes / 4 per pack plus +// The Gate B estimate is computed from the packs as built: bytes / 2.5 per pack plus // the prompt, a fixed per-call overhead and output allowance, then the retry // reserve and margin on the top. Judging is bounded by its time box, not by size. test('estimate: computed from the pack sizes; 12 packs at the cap fit the time box', () => { @@ -61,6 +61,31 @@ test('estimate: computed from the pack sizes; 12 packs at the cap fit the time b assert.ok(r.minutes <= 5, `minutes ${r.minutes}`); }); +// Measured on 12 real judge calls (Claude Code 2.1.318, a judge prompt of 3,505 +// bytes): pack payload bytes, and the input tokens each call reported. These +// packs ran near 2.5 bytes per token; the fixed input near 3,100 tokens. +const MEASURED_PROMPT_BYTES = 3505; +const MEASURED = [ + [5870, 5347], [4786, 5039], [5743, 5486], [5502, 5240], [5863, 5368], [5081, 5226], + [5902, 5488], [5844, 5351], [3584, 4539], [3904, 4669], [5957, 5556], [5788, 5580], +]; +test('estimate: the per-call input estimate brackets measured calls, and stays honest at the larger cap', () => { + const J = THRESHOLDS.judged; + const est = (bytes, promptBytes) => Math.ceil(bytes / J.caps.bytes_per_token) + Math.ceil(promptBytes / J.caps.bytes_per_token) + J.estimate.overhead_tokens_per_call; + for (const [bytes, real] of MEASURED) { + const e = est(bytes, MEASURED_PROMPT_BYTES); + assert.ok(e >= real * 0.9 && e <= real * 1.1, `${bytes} bytes: estimate ${e} vs measured ${real}`); + } + // A full pack at the cap with today's prompt, against the measured slope and + // intercept plus the prompt growth: the low end within 10%, the top above it. + const capBytes = J.caps.bundle_tokens * J.caps.bytes_per_token; + const promptBytes = Buffer.byteLength(TEMPLATE, 'utf8'); + const expected = 3136 + 0.396 * (capBytes + promptBytes - MEASURED_PROMPT_BYTES); + const e = est(capBytes, promptBytes); + assert.ok(e >= expected * 0.9 && e <= expected * 1.1, `cap estimate ${e} vs ${Math.round(expected)}`); + assert.ok((e + J.estimate.output_tokens_per_call) * (1 + J.estimate.retry_reserve) * (1 + J.estimate.margin) > expected * 1.2); +}); + test('one call per item; every line records tokens, model, and the validated output only', async () => { const calls = []; const r = await run(order(3), async (input) => { From 4680170b29440d844d3bd54fe8e306d3add6df7f Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 01:50:24 -0500 Subject: [PATCH 3/8] docs(inspect): end signals, pack trimming, snapshot timing, config matching limits --- docs/src/content/docs/guide/inspect.mdx | 1 + inspect/README.md | 15 +++++++++------ inspect/TRIAL.md | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/src/content/docs/guide/inspect.mdx b/docs/src/content/docs/guide/inspect.mdx index 5d1bfea0..458c7356 100644 --- a/docs/src/content/docs/guide/inspect.mdx +++ b/docs/src/content/docs/guide/inspect.mdx @@ -35,6 +35,7 @@ This is a trial release. Get the newest `inspect-v*` release from the [releases At T1 only, 12 scrubbed episode excerpts go to your model provider through your own coding agent login, one call each. - Each excerpt holds your messages, the start of each reply, the end of the last reply, and how the episode ended +- Code adds end signals: an open question for you in the last reply, a subagent it started, a later revert or fixup, a later episode on the same files - Code searches all your loaded instruction files and sends only the lines that match your corrections - When no line matches, that search counts as evidence; "could not tell" is kept for files that may have changed since or could not be read in full diff --git a/inspect/README.md b/inspect/README.md index d8e0b940..f101c709 100644 --- a/inspect/README.md +++ b/inspect/README.md @@ -79,11 +79,13 @@ Status: pre-release trial (background runs, a thin skill, a checksummed release) - The consent, with the approved estimate, is recorded in `facts.json`; the judge command is refused until it is there; every run asks again - Sample: 12 episodes with a correction candidate, seeded (seed in the report footer), recency thirds, at most 4 per project unless one project holds them all, 3x weight for sessions with 2 or more corrections - Fewer than 12 eligible: all are judged, labelled thin evidence; under 10 sessions: T1 is not offered -- Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 300 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), and the instruction lines that match the flagged messages +- Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 400 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), end signals computed by code, and the instruction lines that match the flagged messages - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 1,600 characters) - - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 1,500 tokens per pack at most - - Over the cap: replies cut first, then middle messages that were not flagged (the first and last always stay), then long messages, then instruction lines, then flagged middle messages -- Gate B estimate: pack size (bytes / 4) plus a per-call overhead and output measured on Claude Code 2.1.318, then 10% retry reserve and 30% margin on the top + - End signals: the last reply asks you something (a question, lettered choices, a questions list), the last reply started a subagent, a later revert or fixup commit, a later episode on the same files + - The judge uses these and the last reply to pick resolved, abandoned (left open), or reworked; "could not tell" is for an episode none of these fits + - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 4,000 tokens per pack at most (the cap, at 2.5 bytes per token) + - Over the cap: middle replies cut first, then middle messages that were not flagged (the first and last always stay, the last reply whole with its closing), then long middle messages, then instruction lines, then flagged middle messages +- Gate B estimate: pack size (bytes / 2.5) plus the prompt, a per-call overhead and output measured on Claude Code 2.1.318, then 10% retry reserve and 30% margin on the top - Minutes: 10 seconds per call, judging only (not the scan or time at gates); real time varies by machine, login, and network - The judging time limit (the estimate plus 50%) is the only fixed time figure - Session text is data for the judge, never instructions: it sits inside a fence named by a random nonce, and the judge prompt says so @@ -146,8 +148,9 @@ T1 trial (recorded, not fixed): - At 12 episodes the top of the Gate B range is tens of thousands of tokens at any pack size: the fixed cost per call (prompt, overhead, output) times 12, plus reserve and margin (`test/judge/workorder.test.mjs`) - A correction you later reversed still gives a before and after from your own words - The harness version shown at Gate B comes from your newest session log, not from the installed command -- A session whose loaded-instructions snapshot is logged only after an episode starts is read as "may have changed", so its corrections stay "could not tell" -- Instruction lines are matched by shared words (common words left out), not meaning: a rule worded with other words is not found, and its correction reads as yours +- The loaded-instructions snapshot logged right after an episode's first message counts as loaded for it; a first snapshot logged only after a reply is read as "may have changed", so its corrections stay "could not tell" +- End signals are word and pattern checks: a reply that asks something without a question mark, lettered choices, or a questions heading is not seen as asking +- Instruction lines are matched by shared words (common words and simple plural or tense endings left out), not meaning: a rule worded with other words is not found, and its correction reads as yours. Sending whole files would fix this but sends far more of your config, so it is not done Leak check, `check-share` and the dry run (`test/release/leak-known-limits.test.mjs`): diff --git a/inspect/TRIAL.md b/inspect/TRIAL.md index 08034fbe..cd17f4b6 100644 --- a/inspect/TRIAL.md +++ b/inspect/TRIAL.md @@ -187,7 +187,7 @@ shrine-inspect check-share - T0: nothing leaves your machine. No network, no model calls. - T1: 12 short episode excerpts go to your own Claude Code login, one call each, billed to your account like normal use. Secrets are removed and names, paths, and hosts are masked first. -- Each excerpt holds your messages, the start of each reply, the end of the last reply, how the episode ended, and only the instruction lines that match your corrections. Code searches all your loaded instruction files for those lines; the rest stay on your machine. +- Each excerpt holds your messages, the start of each reply, the end of the last reply, how the episode ended (with end signals code computed, such as an open question for you), and only the instruction lines that match your corrections. Code searches all your loaded instruction files for those lines; the rest stay on your machine. - Nothing goes to SHRINE, Stable Kernel, or any other service. No telemetry. ## Read the report From 04dc27006bc7a13ba190afe9489d182b6abd967c Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 01:54:51 -0500 Subject: [PATCH 4/8] fix(inspect): end signal flags from every link, outcome rules, config excerpt cap - reverted, fixup_later, continued_later come from the linker flags, not the nearest links only - fixup_later and continued_later are weak links; only a revert makes reworked - Dropped work with no open question counts as abandoned - A questions list must be a heading or label line - Config excerpt up to 2,400 characters so four flagged turns keep their best lines --- inspect/README.md | 2 +- inspect/src/judge/bundle.mjs | 15 +++++++++------ inspect/src/judge/classifier.prompt.md | 8 ++++---- inspect/src/scorer/thresholds.json | 2 +- inspect/test/judge/bundle.test.mjs | 20 ++++++++++++++++++++ 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/inspect/README.md b/inspect/README.md index f101c709..e13c2b24 100644 --- a/inspect/README.md +++ b/inspect/README.md @@ -80,7 +80,7 @@ Status: pre-release trial (background runs, a thin skill, a checksummed release) - Sample: 12 episodes with a correction candidate, seeded (seed in the report footer), recency thirds, at most 4 per project unless one project holds them all, 3x weight for sessions with 2 or more corrections - Fewer than 12 eligible: all are judged, labelled thin evidence; under 10 sessions: T1 is not offered - Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 400 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), end signals computed by code, and the instruction lines that match the flagged messages - - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 1,600 characters) + - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 2,400 characters) - End signals: the last reply asks you something (a question, lettered choices, a questions list), the last reply started a subagent, a later revert or fixup commit, a later episode on the same files - The judge uses these and the last reply to pick resolved, abandoned (left open), or reworked; "could not tell" is for an episode none of these fits - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 4,000 tokens per pack at most (the cap, at 2.5 bytes per token) diff --git a/inspect/src/judge/bundle.mjs b/inspect/src/judge/bundle.mjs index f51b3413..b69788d2 100644 --- a/inspect/src/judge/bundle.mjs +++ b/inspect/src/judge/bundle.mjs @@ -20,7 +20,7 @@ export const DEFAULT_CAPS = Object.freeze({ closing_chars: 400, turns: 12, bundle_tokens: 4000, - config_excerpt_chars: 1600, + config_excerpt_chars: 2400, config_line_chars: 240, config_lines_per_turn: 6, bytes_per_token: 2.5, @@ -65,7 +65,7 @@ const HEADING = /^\s{0,3}#{1,6}\s+\S/; // characters are read, so the check stays linear and cheap. const CHOICE_A = /(?:^|\s)A\)\s/u; const CHOICE_B = /(?:^|\s)B\)\s/u; -const QUESTIONS = /^[\s#*_-]{0,6}(?:open\s)?questions?\b/imu; +const QUESTIONS = /^[\s#*_-]{0,6}(?:open\s)?questions?[\s:*_]{0,4}$/imu; export function asksPerson(text) { const tail = String(text ?? '').slice(-600); return endsWithQuestion(tail) || (CHOICE_A.test(tail) && CHOICE_B.test(tail)) || QUESTIONS.test(tail); @@ -73,16 +73,19 @@ export function asksPerson(text) { const AGENT_TOOLS = new Set(['Task', 'Agent']); // End signals computed by code, never by a model: what was still open when the -// episode ended, and what git and later episodes show about the same work. +// episode ended, and what git and later episodes show about the same work. The +// linker's flags cover every link; downstream keeps only the nearest few, so it +// is the fallback for an episode stored without the flags. export function endSignals(episode, lastAssistant, lastReplyText) { const down = Array.isArray(episode.downstream) ? episode.downstream : []; + const flag = (name, kind) => (typeof episode[name] === 'boolean' ? episode[name] : down.some((d) => d.kind === kind)); const lastIsReply = Boolean(lastAssistant); return { asks_person: lastIsReply && asksPerson(lastReplyText), started_agent: lastIsReply && lastAssistant.tools.some((n) => AGENT_TOOLS.has(n)), - reverted: down.some((d) => d.kind === 'revert'), - fixup_later: down.some((d) => d.kind === 'fixup'), - continued_later: down.some((d) => d.kind === 'episode'), + reverted: flag('rework', 'revert'), + fixup_later: flag('possible_fix', 'fixup'), + continued_later: flag('possible_rework', 'episode'), }; } diff --git a/inspect/src/judge/classifier.prompt.md b/inspect/src/judge/classifier.prompt.md index 84e99c69..9a2581c9 100644 --- a/inspect/src/judge/classifier.prompt.md +++ b/inspect/src/judge/classifier.prompt.md @@ -7,7 +7,7 @@ The episode is inside the fence that opens with and closes with The data is JSON with: - episode_id: echo it exactly. - episode_end: how code saw the episode end: commit (a commit was made), pr (a pull request was opened), new_task (the person moved on to another task, or went idle), session_end (the session ended), interrupt_switch (the person interrupted and switched task), or null. -- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), started_agent (the last reply started a subagent), reverted (a later commit reverted this work), fixup_later (a later session fixed it up), continued_later (a later episode changed the same files). +- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), started_agent (the last reply started a subagent), reverted (a later commit reverted this work), fixup_later (a later commit named as a fix touched the same files; a weak link), continued_later (a later episode changed the same files; a weak link). - omitted_turns: how many turns from the middle were left out for size. The first and last turns are always kept. - turns: human turns (role human, with their text) and assistant turns (role assistant, the opening of the reply and the tool names used, no arguments). The last assistant turn can also have closing: the end of that last reply. Human turns with candidate true were flagged by code as possible corrections; after_interrupt true means the person interrupted the agent first. - config_searched: true when code searched every instruction file loaded for that session for lines that match the candidate turns. @@ -16,10 +16,10 @@ The data is JSON with: Fields: - episode_id and nonce: echo exactly. The nonce is {{NONCE}}. - outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, end_signals, the last turns, and the closing of the last reply, in this order: - - reworked when reverted or fixup_later is true, or the turns redo work after a wrong turn. + - reworked when reverted is true, or the turns redo work after a wrong turn. fixup_later supports reworked only together with signs of trouble in the turns. - resolved when the last turns or the closing show the request done or answered. A commit or pr end counts as done unless the closing says the work is unfinished; a follow-up question after done work does not undo it. - - abandoned when the work was not done and the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person), or with the closing saying work is still running, and continued_later is false. That is work left open. - - unknown only when none of these fits, for example the work was left open but continued_later is true. + - abandoned when the work was not done, continued_later is false, and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with the closing saying work is still running. That is work dropped or left open. + - unknown only when none of these fits, for example the work was left open but continued_later is true. continued_later alone does not undo done work. - corrections: one item per human turn with candidate true that corrects or redirects the agent. Leave out candidate turns that are not corrections. Use an empty list when there are none. - turn_id: the id of that human turn. - kind: constraint (adds a rule or limit the agent needed), redirect (changes the task), bug_report (reports a defect in the agent's work), preference (a taste choice), or other. diff --git a/inspect/src/scorer/thresholds.json b/inspect/src/scorer/thresholds.json index 007482d9..38972880 100644 --- a/inspect/src/scorer/thresholds.json +++ b/inspect/src/scorer/thresholds.json @@ -45,7 +45,7 @@ "bar_min": 2, "anchor_turns": [1, 2], "confidence": { "base": 0.6, "per_hit": 0.05, "agree_bonus": 0.15, "disagree_cap": 0.5, "candidates_max": 0.74 }, - "caps": { "human_turn_chars": 1200, "assistant_chars": 400, "closing_chars": 400, "turns": 12, "bundle_tokens": 4000, "config_excerpt_chars": 1600, "config_line_chars": 240, "config_lines_per_turn": 6, "bytes_per_token": 2.5, "tools_per_turn": 10 }, + "caps": { "human_turn_chars": 1200, "assistant_chars": 400, "closing_chars": 400, "turns": 12, "bundle_tokens": 4000, "config_excerpt_chars": 2400, "config_line_chars": 240, "config_lines_per_turn": 6, "bytes_per_token": 2.5, "tools_per_turn": 10 }, "estimate": { "overhead_tokens_per_call": 1800, "output_tokens_per_call": 400, "retry_reserve": 0.1, "margin": 0.3, "seconds_per_call": 10, "measured_on": "2.1.318" }, "abort": { "error_rate": 0.1, "time_box_factor": 1.5 }, "per_call_timeout_ms": 180000, diff --git a/inspect/test/judge/bundle.test.mjs b/inspect/test/judge/bundle.test.mjs index a102ed3d..18575805 100644 --- a/inspect/test/judge/bundle.test.mjs +++ b/inspect/test/judge/bundle.test.mjs @@ -224,6 +224,8 @@ test('asksPerson: a question, lettered choices, or a questions list at the end; assert.ok(!asksPerson('All tests pass and the change is committed.')); assert.ok(!asksPerson('')); assert.ok(!asksPerson(`Questions:\n${'filler line\n'.repeat(100)}Done.`), 'only the end of the reply counts'); + assert.ok(!asksPerson('Questions answered above. All done.'), 'a line that only starts with the word is not a questions list'); + assert.ok(asksPerson('Draft is up.\n\n**Open questions:**\n- Naming')); }); test('end signals: computed by code from the last reply and later git and episode links', () => { @@ -234,6 +236,11 @@ test('end signals: computed by code from the last reply and later git and episod const h = buildBundle({ episode, transcript: { events: events(), snapshot }, known, nonce: 'n' }); assert.deepEqual(payloadOf(h).end_signals, { asks_person: false, started_agent: false, reverted: false, fixup_later: false, continued_later: false }); assert.match(TEMPLATE, /end_signals/); + // The linker's flags cover every link; downstream keeps only the nearest few. + const near = Array.from({ length: 5 }, (_, i) => ({ kind: 'episode', ref: `e-${i}`, delta_hours: i, link: 'soft' })); + const f = buildBundle({ episode: { ...episode, downstream: near, rework: true, possible_fix: true, possible_rework: true }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.equal(payloadOf(f).end_signals.reverted, true); + assert.equal(payloadOf(f).end_signals.fixup_later, true); }); test('trimming shortens middle replies first; the last reply keeps its full opening and closing', () => { @@ -251,3 +258,16 @@ test('trimming shortens middle replies first; the last reply keeps its full open assert.equal(b.turns[0].text.length, `next step ${'word '.repeat(150)}0`.length, 'the root prompt stays whole'); assert.ok(b.turns.slice(1, -1).filter((t) => t.role === 'assistant').every((t) => t.text.length <= 160)); }); + +test('config: four flagged turns with six best lines each all reach the judge', () => { + const words = ['migration', 'semicolon', 'deploy', 'fixture']; + const ev = [{ t: at(0), kind: 'human', uuid: 'u1', text: 'Start the work.' }]; + words.forEach((w, i) => { + ev.push({ t: at(2 * i + 1), kind: 'assistant_text', text: 'ok' }, { t: at(2 * i + 1), kind: 'interrupt' }); + ev.push({ t: at(2 * i + 2), kind: 'human', uuid: `v${i}`, text: `Stop, the ${w} rule matters here.` }); + }); + const lines = words.flatMap((w) => Array.from({ length: 6 }, (_, k) => `- Rule ${k} about the ${w} step, written out with enough plain words to be a long line.`)); + const b = buildBundle({ episode, transcript: { events: ev, snapshot: [{ path: '/r/CLAUDE.md', content: `# Rules\n${lines.join('\n')}\n` }] }, known, nonce: 'n' }); + for (const [id, m] of Object.entries(b.meta.config_lines)) assert.deepEqual(m, { matched: 6, best: 6, sent: 6 }, id); + assert.ok(bundleTokens(b) <= DEFAULT_CAPS.bundle_tokens); +}); From cc04c81347462544d578175d52b6e6f45bd66303 Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 02:01:13 -0500 Subject: [PATCH 5/8] fix(inspect): trim instruction lines before the first or a flagged message - Flagged middle messages and the first and last turns are cut only after instruction lines - continued_later counts a later episode on the same files within a day, as a hint - A redirect or taste choice no earlier text could cover is labelled the person's --- inspect/README.md | 4 ++-- inspect/src/judge/bundle.mjs | 14 ++++++++------ inspect/src/judge/classifier.prompt.md | 8 ++++---- inspect/test/judge/bundle.test.mjs | 25 +++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/inspect/README.md b/inspect/README.md index e13c2b24..fcba4569 100644 --- a/inspect/README.md +++ b/inspect/README.md @@ -81,10 +81,10 @@ Status: pre-release trial (background runs, a thin skill, a checksummed release) - Fewer than 12 eligible: all are judged, labelled thin evidence; under 10 sessions: T1 is not offered - Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 400 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), end signals computed by code, and the instruction lines that match the flagged messages - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 2,400 characters) - - End signals: the last reply asks you something (a question, lettered choices, a questions list), the last reply started a subagent, a later revert or fixup commit, a later episode on the same files + - End signals: the last reply asks you something (a question, lettered choices, a questions list), the last reply started a subagent, a later revert or fixup commit, a later episode on the same files within a day - The judge uses these and the last reply to pick resolved, abandoned (left open), or reworked; "could not tell" is for an episode none of these fits - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 4,000 tokens per pack at most (the cap, at 2.5 bytes per token) - - Over the cap: middle replies cut first, then middle messages that were not flagged (the first and last always stay, the last reply whole with its closing), then long middle messages, then instruction lines, then flagged middle messages + - Over the cap: middle replies cut first, then middle messages that were not flagged (the first and last always stay, the last reply whole with its closing), then instruction lines, then flagged middle messages, then the first and last messages - Gate B estimate: pack size (bytes / 2.5) plus the prompt, a per-call overhead and output measured on Claude Code 2.1.318, then 10% retry reserve and 30% margin on the top - Minutes: 10 seconds per call, judging only (not the scan or time at gates); real time varies by machine, login, and network - The judging time limit (the estimate plus 50%) is the only fixed time figure diff --git a/inspect/src/judge/bundle.mjs b/inspect/src/judge/bundle.mjs index b69788d2..a0a24e86 100644 --- a/inspect/src/judge/bundle.mjs +++ b/inspect/src/judge/bundle.mjs @@ -75,7 +75,9 @@ const AGENT_TOOLS = new Set(['Task', 'Agent']); // End signals computed by code, never by a model: what was still open when the // episode ended, and what git and later episodes show about the same work. The // linker's flags cover every link; downstream keeps only the nearest few, so it -// is the fallback for an episode stored without the flags. +// is the fallback for an episode stored without the flags. continued_later is a +// later episode on the same files within a day: the nearest links come first, so +// downstream holds it when there is one. export function endSignals(episode, lastAssistant, lastReplyText) { const down = Array.isArray(episode.downstream) ? episode.downstream : []; const flag = (name, kind) => (typeof episode[name] === 'boolean' ? episode[name] : down.some((d) => d.kind === kind)); @@ -85,7 +87,7 @@ export function endSignals(episode, lastAssistant, lastReplyText) { started_agent: lastIsReply && lastAssistant.tools.some((n) => AGENT_TOOLS.has(n)), reverted: flag('rework', 'revert'), fixup_later: flag('possible_fix', 'fixup'), - continued_later: flag('possible_rework', 'episode'), + continued_later: down.some((d) => d.kind === 'episode' && d.delta_hours <= 24), }; } @@ -249,8 +251,8 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n // Size cap (bytes / bytes_per_token). Order: shorten middle replies, drop middle // turns that are not flagged (first and last always stay, the last reply at full - // length with its closing), shorten middle turns, trim config to a floor, drop - // flagged middle turns, then the rest of the config, then cut every turn. + // length with its closing), trim config to a floor, shorten then drop flagged + // middle turns, then the rest of the config, then cut the first and last turns. const over = () => bundleTokens(b, caps) > caps.bundle_tokens; const dropMiddle = (ok) => { const n = b.turns.length; @@ -275,12 +277,12 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n while (b.turns.filter((t) => t.role === 'human').length > caps.turns && dropMiddle((t) => !t.candidate)); while (over() && dropMiddle((t) => !t.candidate)); if (over()) for (const t of b.turns) if (t.role === 'human' && !t.candidate && middle(t)) t.text = t.text.slice(0, 300); - if (over()) for (const t of b.turns) if (middle(t)) t.text = t.text.slice(0, 600); - if (over()) for (const t of b.turns) t.text = t.text.slice(0, 600); trimConfig(caps.config_excerpt_chars / 4); + if (over()) for (const t of b.turns) if (middle(t)) t.text = t.text.slice(0, 600); while (over() && dropMiddle(() => true)); while (b.turns.filter((t) => t.role === 'human').length > caps.turns && dropMiddle(() => true)); trimConfig(0); + if (over()) for (const t of b.turns) t.text = t.text.slice(0, 600); if (over()) for (const t of b.turns) { t.text = t.text.slice(0, 300); if (t.closing) t.closing = t.closing.slice(-160); diff --git a/inspect/src/judge/classifier.prompt.md b/inspect/src/judge/classifier.prompt.md index 9a2581c9..3be8cc3f 100644 --- a/inspect/src/judge/classifier.prompt.md +++ b/inspect/src/judge/classifier.prompt.md @@ -7,7 +7,7 @@ The episode is inside the fence that opens with and closes with The data is JSON with: - episode_id: echo it exactly. - episode_end: how code saw the episode end: commit (a commit was made), pr (a pull request was opened), new_task (the person moved on to another task, or went idle), session_end (the session ended), interrupt_switch (the person interrupted and switched task), or null. -- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), started_agent (the last reply started a subagent), reverted (a later commit reverted this work), fixup_later (a later commit named as a fix touched the same files; a weak link), continued_later (a later episode changed the same files; a weak link). +- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), started_agent (the last reply started a subagent), reverted (a later commit reverted this work), fixup_later (a later commit named as a fix touched the same files; a weak link), continued_later (a later episode within a day changed the same files; a weak hint that the work went on). - omitted_turns: how many turns from the middle were left out for size. The first and last turns are always kept. - turns: human turns (role human, with their text) and assistant turns (role assistant, the opening of the reply and the tool names used, no arguments). The last assistant turn can also have closing: the end of that last reply. Human turns with candidate true were flagged by code as possible corrections; after_interrupt true means the person interrupted the agent first. - config_searched: true when code searched every instruction file loaded for that session for lines that match the candidate turns. @@ -18,14 +18,14 @@ Fields: - outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, end_signals, the last turns, and the closing of the last reply, in this order: - reworked when reverted is true, or the turns redo work after a wrong turn. fixup_later supports reworked only together with signs of trouble in the turns. - resolved when the last turns or the closing show the request done or answered. A commit or pr end counts as done unless the closing says the work is unfinished; a follow-up question after done work does not undo it. - - abandoned when the work was not done, continued_later is false, and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with the closing saying work is still running. That is work dropped or left open. - - unknown only when none of these fits, for example the work was left open but continued_later is true. continued_later alone does not undo done work. + - abandoned when the work was not done, nothing shows it went on (continued_later is a hint that it did), and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with the closing saying work is still running. That is work dropped or left open. + - unknown only when none of these fits, for example the work was left open and seems to have gone on in a later episode. continued_later alone does not undo done work. - corrections: one item per human turn with candidate true that corrects or redirects the agent. Leave out candidate turns that are not corrections. Use an empty list when there are none. - turn_id: the id of that human turn. - kind: constraint (adds a rule or limit the agent needed), redirect (changes the task), bug_report (reports a defect in the agent's work), preference (a taste choice), or other. - supplies_constraint: true only when the turn states a constraint that, given in the first message, would have avoided the correction. - constraint_present_in: where that constraint already existed before this turn: prompt (an earlier human turn in this episode), instruction_file (a config block), skill, none, or unknown. - - attribution: label user (the constraint arrived late from the person), model (it was available and not followed), harness (the setup did not load it), or unknown; confidence from 0 to 1. + - attribution: label user (the constraint arrived late from the person, or the turn changes the request or makes a taste choice that nothing earlier could have covered), model (it was available and not followed), harness (the setup did not load it), or unknown; confidence from 0 to 1. - sufficiency: enough, thin, or none: how well the data supports your reading. - quotes: up to 3 items {turn_id, text}; text is a short exact copy (12 to 200 characters) from that turn's text. Never paraphrase. - config_ref: only when constraint_present_in is instruction_file: {config_id, text}, where text is an exact copy (12 to 200 characters) from that config block. diff --git a/inspect/test/judge/bundle.test.mjs b/inspect/test/judge/bundle.test.mjs index 18575805..9204dd70 100644 --- a/inspect/test/judge/bundle.test.mjs +++ b/inspect/test/judge/bundle.test.mjs @@ -271,3 +271,28 @@ test('config: four flagged turns with six best lines each all reach the judge', for (const [id, m] of Object.entries(b.meta.config_lines)) assert.deepEqual(m, { matched: 6, best: 6, sent: 6 }, id); assert.ok(bundleTokens(b) <= DEFAULT_CAPS.bundle_tokens); }); + +test('continued_later: a later episode on the same files within a day; one days later does not count', () => { + const ep = (h) => ({ ...episode, possible_rework: true, downstream: [{ kind: 'episode', ref: 'e-2', delta_hours: h, link: 'soft' }] }); + assert.equal(payloadOf(buildBundle({ episode: ep(3), transcript: { events: events(), snapshot }, known, nonce: 'n' })).end_signals.continued_later, true); + assert.equal(payloadOf(buildBundle({ episode: ep(50), transcript: { events: events(), snapshot }, known, nonce: 'n' })).end_signals.continued_later, false); +}); + +test('over the cap, instruction lines go before the first message or a flagged message is cut', () => { + const first = `Build the export ${'word '.repeat(220)}end`; + const ev = [{ t: at(0), kind: 'human', uuid: 'u1', text: first }]; + const topics = ['migration', 'deploy', 'fixture']; + topics.forEach((w, i) => { + ev.push({ t: at(i + 1), kind: 'assistant_text', text: 'ok', head: 'reply '.repeat(60) }, { t: at(i + 1), kind: 'interrupt' }); + ev.push({ t: at(i + 1), kind: 'human', uuid: `v${i}`, text: `Actually the ${w} step ${'detail '.repeat(150)}` }); + }); + ev.push({ t: at(9), kind: 'assistant_text', text: 'Done.', head: 'Done.' }); + const filler = Array.from({ length: 40 }, (_, k) => `- Filler ${k} about lint and style.`); + const cfg = [...filler, ...topics.flatMap((w) => Array.from({ length: 6 }, (_, k) => `- The ${w} item ${k} needs a dry run, written with enough plain words to make a long line.`))].join('\n'); + const caps = { ...DEFAULT_CAPS, bundle_tokens: 2400 }; + const b = buildBundle({ episode, transcript: { events: ev, snapshot: [{ path: '/r/CLAUDE.md', content: `# Rules\n${cfg}\n` }] }, known, nonce: 'n', caps }); + assert.ok(bundleTokens(b, caps) <= caps.bundle_tokens, `tokens ${bundleTokens(b, caps)}`); + assert.ok(Object.values(b.meta.config_lines).some((m) => m.sent < m.best), 'instruction lines were trimmed'); + assert.equal(b.turns[0].text, first.slice(0, DEFAULT_CAPS.human_turn_chars), 'the first message stays whole'); + assert.ok(b.turns.filter((t) => t.candidate).every((t) => t.text.length > 600), 'flagged messages stay whole'); +}); From e65f8365f696ab6986af47e67139446091adc45b Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 02:05:50 -0500 Subject: [PATCH 6/8] fix(inspect): a tool call after the last question clears asks_person - Matches how the episode cutter reads a question the agent moved past - The abandoned rule names started_agent as support for work still running --- inspect/README.md | 2 +- inspect/src/judge/bundle.mjs | 7 +++++-- inspect/src/judge/classifier.prompt.md | 2 +- inspect/test/judge/bundle.test.mjs | 5 ++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/inspect/README.md b/inspect/README.md index fcba4569..0d60b404 100644 --- a/inspect/README.md +++ b/inspect/README.md @@ -81,7 +81,7 @@ Status: pre-release trial (background runs, a thin skill, a checksummed release) - Fewer than 12 eligible: all are judged, labelled thin evidence; under 10 sessions: T1 is not offered - Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 400 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), end signals computed by code, and the instruction lines that match the flagged messages - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 2,400 characters) - - End signals: the last reply asks you something (a question, lettered choices, a questions list), the last reply started a subagent, a later revert or fixup commit, a later episode on the same files within a day + - End signals: the last reply asks you something (a question, lettered choices, a questions list) and no tool call follows, the last reply started a subagent, a later revert or fixup commit, a later episode on the same files within a day - The judge uses these and the last reply to pick resolved, abandoned (left open), or reworked; "could not tell" is for an episode none of these fits - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 4,000 tokens per pack at most (the cap, at 2.5 bytes per token) - Over the cap: middle replies cut first, then middle messages that were not flagged (the first and last always stay, the last reply whole with its closing), then instruction lines, then flagged middle messages, then the first and last messages diff --git a/inspect/src/judge/bundle.mjs b/inspect/src/judge/bundle.mjs index a0a24e86..d8310bb6 100644 --- a/inspect/src/judge/bundle.mjs +++ b/inspect/src/judge/bundle.mjs @@ -179,8 +179,11 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n } if (e.kind === 'assistant_text') { if (!assistant.text) assistant.text = clean(e.head ?? e.text, known).slice(0, caps.assistant_chars); - lastReply = { turn: assistant, text: e.text }; + lastReply = { turn: assistant, text: e.text, tool_after: false }; } + // A tool call after the text means the agent went on: the question, if any, + // was not left for the person (the episode cutter reads it the same way). + if (e.kind === 'tool_use' && lastReply?.turn === assistant) lastReply.tool_after = true; if (e.kind === 'tool_use' && assistant.tools.length < caps.tools_per_turn && !assistant.tools.includes(e.name)) assistant.tools.push(clean(e.name, known).slice(0, 64)); } lastKind = e.kind; @@ -232,7 +235,7 @@ export function buildBundle({ episode, transcript, fallbackConfig = [], known, n episode_id: episode.id, nonce, episode_end: episode.terminal ?? null, - end_signals: endSignals(episode, lastTurn?.role === 'assistant' ? lastTurn : null, lastReply && lastTurn === lastReply.turn ? lastReply.text : ''), + end_signals: endSignals(episode, lastTurn?.role === 'assistant' ? lastTurn : null, lastReply && lastTurn === lastReply.turn && !lastReply.tool_after ? lastReply.text : ''), omitted_turns: 0, turns, config: [], diff --git a/inspect/src/judge/classifier.prompt.md b/inspect/src/judge/classifier.prompt.md index 3be8cc3f..358fc61f 100644 --- a/inspect/src/judge/classifier.prompt.md +++ b/inspect/src/judge/classifier.prompt.md @@ -18,7 +18,7 @@ Fields: - outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, end_signals, the last turns, and the closing of the last reply, in this order: - reworked when reverted is true, or the turns redo work after a wrong turn. fixup_later supports reworked only together with signs of trouble in the turns. - resolved when the last turns or the closing show the request done or answered. A commit or pr end counts as done unless the closing says the work is unfinished; a follow-up question after done work does not undo it. - - abandoned when the work was not done, nothing shows it went on (continued_later is a hint that it did), and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with the closing saying work is still running. That is work dropped or left open. + - abandoned when the work was not done, nothing shows it went on (continued_later is a hint that it did), and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with work still running (the closing says so; started_agent supports it). That is work dropped or left open. - unknown only when none of these fits, for example the work was left open and seems to have gone on in a later episode. continued_later alone does not undo done work. - corrections: one item per human turn with candidate true that corrects or redirects the agent. Leave out candidate turns that are not corrections. Use an empty list when there are none. - turn_id: the id of that human turn. diff --git a/inspect/test/judge/bundle.test.mjs b/inspect/test/judge/bundle.test.mjs index 9204dd70..d8932ca2 100644 --- a/inspect/test/judge/bundle.test.mjs +++ b/inspect/test/judge/bundle.test.mjs @@ -229,13 +229,16 @@ test('asksPerson: a question, lettered choices, or a questions list at the end; }); test('end signals: computed by code from the last reply and later git and episode links', () => { - const ev = [...events(), { t: at(9), kind: 'assistant_text', text: 'Pick one: A) keep B) drop', head: 'Two ways.' }, { t: at(9), kind: 'tool_use', name: 'Agent', id: 'a1' }]; + const ev = [...events(), { t: at(9), kind: 'tool_use', name: 'Agent', id: 'a1' }, { t: at(9), kind: 'assistant_text', text: 'Pick one: A) keep B) drop', head: 'Two ways.' }]; const b = buildBundle({ episode: { ...episode, downstream: [{ kind: 'revert', ref: 'x', delta_hours: 2, link: 'hard' }, { kind: 'episode', ref: 'e-2', delta_hours: 1, link: 'soft' }] }, transcript: { events: ev, snapshot }, known, nonce: 'n' }); assert.deepEqual(payloadOf(b).end_signals, { asks_person: true, started_agent: true, reverted: true, fixup_later: false, continued_later: true }); // Ending on a human turn: nothing in a reply is open. const h = buildBundle({ episode, transcript: { events: events(), snapshot }, known, nonce: 'n' }); assert.deepEqual(payloadOf(h).end_signals, { asks_person: false, started_agent: false, reverted: false, fixup_later: false, continued_later: false }); assert.match(TEMPLATE, /end_signals/); + // A tool call after the question: the agent went on, so nothing was left open. + const went = buildBundle({ episode, transcript: { events: [...events(), { t: at(9), kind: 'assistant_text', text: 'Should I run the tests?' }, { t: at(9), kind: 'tool_use', name: 'Bash', id: 'b1' }], snapshot }, known, nonce: 'n' }); + assert.equal(payloadOf(went).end_signals.asks_person, false); // The linker's flags cover every link; downstream keeps only the nearest few. const near = Array.from({ length: 5 }, (_, i) => ({ kind: 'episode', ref: `e-${i}`, delta_hours: i, link: 'soft' })); const f = buildBundle({ episode: { ...episode, downstream: near, rework: true, possible_fix: true, possible_rework: true }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); From 8646c6f1ff3a90a4b8deca5fcf0f40f38896eea9 Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 02:07:47 -0500 Subject: [PATCH 7/8] chore(inspect): version 0.1.3 --- docs/src/content/docs/guide/inspect.mdx | 8 ++++---- inspect/TRIAL.md | 16 ++++++++-------- inspect/package-lock.json | 4 ++-- inspect/package.json | 2 +- inspect/skill/.claude-plugin/plugin.json | 2 +- inspect/skill/skills/shrine-inspect/SKILL.md | 8 ++++---- inspect/src/scorer/score.mjs | 2 +- inspect/test/golden/bad-setup/findings.json | 2 +- .../test/golden/zero-history-good/findings.json | 2 +- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/src/content/docs/guide/inspect.mdx b/docs/src/content/docs/guide/inspect.mdx index 458c7356..1b15a5cd 100644 --- a/docs/src/content/docs/guide/inspect.mdx +++ b/docs/src/content/docs/guide/inspect.mdx @@ -8,7 +8,7 @@ import { Aside, Steps } from '@astrojs/starlight/components'; *SHRINE Inspect is a local, read-only command-line tool. It measures your AI coding setup and session history against SHRINE and reports what to change first. It changes nothing itself.* - The first release reads one harness: Claude Code; other harnesses come later @@ -54,9 +54,9 @@ It checks that it changed nothing it read. If anything changed, the report shows 2. Download the three release files into a new folder ```sh - mkdir -p ~/shrine-inspect-0.1.2 && cd ~/shrine-inspect-0.1.2 - base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.2 - curl -fLO "$base/shrine-inspect-0.1.2.tgz" + mkdir -p ~/shrine-inspect-0.1.3 && cd ~/shrine-inspect-0.1.3 + base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.3 + curl -fLO "$base/shrine-inspect-0.1.3.tgz" curl -fLO "$base/install.sh" curl -fLO "$base/SHA256SUMS" ``` diff --git a/inspect/TRIAL.md b/inspect/TRIAL.md index cd17f4b6..a7818fb0 100644 --- a/inspect/TRIAL.md +++ b/inspect/TRIAL.md @@ -13,12 +13,12 @@ A local health check of your Claude Code setup and session history, measured aga node --version ``` -2. Download the 3 release files into a new folder. With the GitHub CLI, use `gh release download inspect-v0.1.2 -R stablekernel/SHRINE` in place of the `curl` lines. +2. Download the 3 release files into a new folder. With the GitHub CLI, use `gh release download inspect-v0.1.3 -R stablekernel/SHRINE` in place of the `curl` lines. ```sh - mkdir -p ~/shrine-inspect-0.1.2 && cd ~/shrine-inspect-0.1.2 - base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.2 - curl -fLO "$base/shrine-inspect-0.1.2.tgz" + mkdir -p ~/shrine-inspect-0.1.3 && cd ~/shrine-inspect-0.1.3 + base=https://github.com/stablekernel/SHRINE/releases/download/inspect-v0.1.3 + curl -fLO "$base/shrine-inspect-0.1.3.tgz" curl -fLO "$base/install.sh" curl -fLO "$base/SHA256SUMS" ``` @@ -33,7 +33,7 @@ A local health check of your Claude Code setup and session history, measured aga ``` install.sh: OK - shrine-inspect-0.1.2.tgz: OK + shrine-inspect-0.1.3.tgz: OK ``` 4. Install. Run it from the folder that holds the files. @@ -43,9 +43,9 @@ A local health check of your Claude Code setup and session history, measured aga ``` - Usage: `sh install.sh [--skill] [path/to/tgz]`. - - With no path, it uses `shrine-inspect-0.1.2.tgz` next to `install.sh`, not in your current folder. + - With no path, it uses `shrine-inspect-0.1.3.tgz` next to `install.sh`, not in your current folder. - It checks Node, then checks the tarball against the SHA-256 written in `install.sh`, then extracts it. A mismatch installs nothing. - - The CLI goes to `~/.local/share/shrine-inspect/0.1.2/`, linked as `~/.local/bin/shrine-inspect`. + - The CLI goes to `~/.local/share/shrine-inspect/0.1.3/`, linked as `~/.local/bin/shrine-inspect`. - `--skill` copies the Claude Code skill to `~/.claude/skills/shrine-inspect/`. Leave it out to use the CLI only. - To install somewhere else, set the install folder and the link folder first: @@ -68,7 +68,7 @@ A local health check of your Claude Code setup and session history, measured aga ``` ``` - shrine-inspect 0.1.2 + shrine-inspect 0.1.3 ``` - All commands: `shrine-inspect --help`. diff --git a/inspect/package-lock.json b/inspect/package-lock.json index 8b25c447..a7aadb67 100644 --- a/inspect/package-lock.json +++ b/inspect/package-lock.json @@ -1,12 +1,12 @@ { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "bin": { "shrine-inspect": "bin/shrine-inspect.mjs" diff --git a/inspect/package.json b/inspect/package.json index 7a3b68da..b7650135 100644 --- a/inspect/package.json +++ b/inspect/package.json @@ -1,6 +1,6 @@ { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "description": "Health check of an AI coding setup against the SHRINE docs. Local, read-only, no network.", "license": "Apache-2.0", "author": "Stable Kernel", diff --git a/inspect/skill/.claude-plugin/plugin.json b/inspect/skill/.claude-plugin/plugin.json index 419f4ca7..c70462c3 100644 --- a/inspect/skill/.claude-plugin/plugin.json +++ b/inspect/skill/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "shrine-inspect", - "version": "0.1.2", + "version": "0.1.3", "description": "Thin skill that starts and watches the shrine-inspect CLI. Holds no logic; install the CLI from the same release.", "license": "Apache-2.0", "author": { "name": "Stable Kernel" }, diff --git a/inspect/skill/skills/shrine-inspect/SKILL.md b/inspect/skill/skills/shrine-inspect/SKILL.md index 9fd45559..e44c923e 100644 --- a/inspect/skill/skills/shrine-inspect/SKILL.md +++ b/inspect/skill/skills/shrine-inspect/SKILL.md @@ -6,7 +6,7 @@ allowed-tools: Bash(node --version), Bash(shrine-inspect version:*), Bash(shrine # SHRINE Inspect -Skill version: 0.1.2 +Skill version: 0.1.3 This skill only starts the `shrine-inspect` CLI and watches it. The CLI does every read, every gate, and every judge call. @@ -31,7 +31,7 @@ This skill only starts the `shrine-inspect` CLI and watches it. The CLI does eve 2. Check the CLI. Run: ```bash - shrine-inspect version --skill-version 0.1.2 + shrine-inspect version --skill-version 0.1.3 ``` If the command is not found, or it stops with `E_VERSION`, stop. Tell the user to install the CLI and this skill from the same release (see TRIAL.md in the release). @@ -39,7 +39,7 @@ This skill only starts the `shrine-inspect` CLI and watches it. The CLI does eve 3. Start a run in the background. Use `--tier T1` only when the user asked for judging ("Pulse"); else use `--tier T0`. Session id: `${CLAUDE_SESSION_ID}`. If that id is empty or still shows as a placeholder, leave out `--session-id` and its value. Never leave `--session-id` with nothing after it. ```bash - shrine-inspect start --tier T0 --session-id ${CLAUDE_SESSION_ID} --skill-version 0.1.2 --json + shrine-inspect start --tier T0 --session-id ${CLAUDE_SESSION_ID} --skill-version 0.1.3 --json ``` Keep the `run_id` from the output. @@ -47,7 +47,7 @@ This skill only starts the `shrine-inspect` CLI and watches it. The CLI does eve 4. Watch the run. Run this about every 15 seconds while the state is `running`: ```bash - shrine-inspect status --skill-version 0.1.2 --json + shrine-inspect status --skill-version 0.1.3 --json ``` - `running`: give the user one short line with the phase. Check again. diff --git a/inspect/src/scorer/score.mjs b/inspect/src/scorer/score.mjs index b136119b..c58d8aad 100644 --- a/inspect/src/scorer/score.mjs +++ b/inspect/src/scorer/score.mjs @@ -7,7 +7,7 @@ import { underHome } from '../scrub/mask.mjs'; import { THRESHOLDS } from '../probes/config.mjs'; import { controlTest, usageConfidence, usageSeverity } from './stats.mjs'; -export const TOOL_VERSION = '0.1.2'; +export const TOOL_VERSION = '0.1.3'; const SEV_RANK = { high: 3, medium: 2, low: 1 }; const KIND_ORDER = ['blanket_permissions', 'stale_instructions', 'oversized_instructions', 'done_without_test', 'no_owner_or_review', 'stack_gap', 'short_retention']; diff --git a/inspect/test/golden/bad-setup/findings.json b/inspect/test/golden/bad-setup/findings.json index 3d2e09c9..0056bc55 100644 --- a/inspect/test/golden/bad-setup/findings.json +++ b/inspect/test/golden/bad-setup/findings.json @@ -3,7 +3,7 @@ "run": { "id": "", "started_at": "2026-11-04T12:00:00.000Z", - "tool_version": "0.1.2", + "tool_version": "0.1.3", "harness": { "name": "Claude Code", "version": null diff --git a/inspect/test/golden/zero-history-good/findings.json b/inspect/test/golden/zero-history-good/findings.json index b5b92c8b..e80da3ac 100644 --- a/inspect/test/golden/zero-history-good/findings.json +++ b/inspect/test/golden/zero-history-good/findings.json @@ -3,7 +3,7 @@ "run": { "id": "", "started_at": "2026-11-04T12:00:00.000Z", - "tool_version": "0.1.2", + "tool_version": "0.1.3", "harness": { "name": "Claude Code", "version": null From d2006d63ac0ff6ad45413be08e80d0eb7956da5c Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 6 Nov 2026 02:12:22 -0500 Subject: [PATCH 8/8] fix(inspect): end signals from the scan's stratum outcome; drop started_agent - Facts episodes carry no linker flags, so reverted and fixup_later read the stratum outcome first - started_agent could not tell a finished subagent from a running one, so it is gone - continued_later keeps the next episode of the same session: that is where an open question is answered --- docs/src/content/docs/guide/inspect.mdx | 2 +- inspect/README.md | 2 +- inspect/src/judge/bundle.mjs | 28 +++++++++++++------------ inspect/src/judge/classifier.prompt.md | 4 ++-- inspect/test/judge/bundle.test.mjs | 9 ++++++-- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/src/content/docs/guide/inspect.mdx b/docs/src/content/docs/guide/inspect.mdx index 1b15a5cd..3078b370 100644 --- a/docs/src/content/docs/guide/inspect.mdx +++ b/docs/src/content/docs/guide/inspect.mdx @@ -35,7 +35,7 @@ This is a trial release. Get the newest `inspect-v*` release from the [releases At T1 only, 12 scrubbed episode excerpts go to your model provider through your own coding agent login, one call each. - Each excerpt holds your messages, the start of each reply, the end of the last reply, and how the episode ended -- Code adds end signals: an open question for you in the last reply, a subagent it started, a later revert or fixup, a later episode on the same files +- Code adds end signals: an open question for you in the last reply, a later revert or fixup, a later episode on the same files within a day - Code searches all your loaded instruction files and sends only the lines that match your corrections - When no line matches, that search counts as evidence; "could not tell" is kept for files that may have changed since or could not be read in full diff --git a/inspect/README.md b/inspect/README.md index 0d60b404..850c5bdf 100644 --- a/inspect/README.md +++ b/inspect/README.md @@ -81,7 +81,7 @@ Status: pre-release trial (background runs, a thin skill, a checksummed release) - Fewer than 12 eligible: all are judged, labelled thin evidence; under 10 sessions: T1 is not offered - Evidence pack per episode: your messages (up to 1,200 characters, 12 messages), the opening 400 characters of each reply, the closing 400 characters of the last reply, tool names without arguments, how the episode ended (commit, pull request, next task or idle, session end), end signals computed by code, and the instruction lines that match the flagged messages - Instruction lines: code searches every instruction file loaded for that session for words from each flagged message and sends only matching lines, under their heading (at most 2,400 characters) - - End signals: the last reply asks you something (a question, lettered choices, a questions list) and no tool call follows, the last reply started a subagent, a later revert or fixup commit, a later episode on the same files within a day + - End signals: the last reply asks you something (a question, lettered choices, a questions list) and no tool call follows, a later revert or fixup commit, a later episode on the same files within a day - The judge uses these and the last reply to pick resolved, abandoned (left open), or reworked; "could not tell" is for an episode none of these fits - Every string is scrubbed for secrets, then masked (names, paths, hosts) before a judge or a file sees it; about 4,000 tokens per pack at most (the cap, at 2.5 bytes per token) - Over the cap: middle replies cut first, then middle messages that were not flagged (the first and last always stay, the last reply whole with its closing), then instruction lines, then flagged middle messages, then the first and last messages diff --git a/inspect/src/judge/bundle.mjs b/inspect/src/judge/bundle.mjs index d8310bb6..9f801153 100644 --- a/inspect/src/judge/bundle.mjs +++ b/inspect/src/judge/bundle.mjs @@ -1,8 +1,8 @@ // Evidence pack for the Episode Classifier (plan T1 task 2). One episode per // bundle: human turns (text), assistant replies (opening only, plus the closing // of the last reply), tool names (no arguments), how the episode ended plus -// code-computed end signals (an open question for the person, an agent started -// in the last reply, a later revert, fixup, or episode on the same files), and the +// code-computed end signals (an open question for the person, a later revert, +// fixup, or episode on the same files), and the // lines of the live config that match the flagged turns. Code searches the whole // config; only matching lines are sent. Every string goes through scrubSecrets, // then mask with the known identifiers, always: secrets, names, and paths never @@ -70,23 +70,25 @@ export function asksPerson(text) { const tail = String(text ?? '').slice(-600); return endsWithQuestion(tail) || (CHOICE_A.test(tail) && CHOICE_B.test(tail)) || QUESTIONS.test(tail); } -const AGENT_TOOLS = new Set(['Task', 'Agent']); // End signals computed by code, never by a model: what was still open when the -// episode ended, and what git and later episodes show about the same work. The -// linker's flags cover every link; downstream keeps only the nearest few, so it -// is the fallback for an episode stored without the flags. continued_later is a -// later episode on the same files within a day: the nearest links come first, so +// episode ended, and what git and later episodes show about the same work. +// reverted and fixup_later: the linker's flags when present, else the scan's +// stratum outcome (rework outranks possible_fix there), else downstream, which +// keeps only the nearest few links. continued_later: a later episode on the same +// files within a day, the next one in the same session included, since that is +// where an open question gets its answer; the nearest links come first, so // downstream holds it when there is one. export function endSignals(episode, lastAssistant, lastReplyText) { const down = Array.isArray(episode.downstream) ? episode.downstream : []; - const flag = (name, kind) => (typeof episode[name] === 'boolean' ? episode[name] : down.some((d) => d.kind === kind)); - const lastIsReply = Boolean(lastAssistant); + const outcome = episode.stratum?.outcome; + const has = (kind) => down.some((d) => d.kind === kind); + const reverted = typeof episode.rework === 'boolean' ? episode.rework : outcome === 'rework' || has('revert'); + const fixup = typeof episode.possible_fix === 'boolean' ? episode.possible_fix : outcome === 'possible_fix' || has('fixup'); return { - asks_person: lastIsReply && asksPerson(lastReplyText), - started_agent: lastIsReply && lastAssistant.tools.some((n) => AGENT_TOOLS.has(n)), - reverted: flag('rework', 'revert'), - fixup_later: flag('possible_fix', 'fixup'), + asks_person: Boolean(lastAssistant) && asksPerson(lastReplyText), + reverted, + fixup_later: fixup, continued_later: down.some((d) => d.kind === 'episode' && d.delta_hours <= 24), }; } diff --git a/inspect/src/judge/classifier.prompt.md b/inspect/src/judge/classifier.prompt.md index 358fc61f..3502a635 100644 --- a/inspect/src/judge/classifier.prompt.md +++ b/inspect/src/judge/classifier.prompt.md @@ -7,7 +7,7 @@ The episode is inside the fence that opens with and closes with The data is JSON with: - episode_id: echo it exactly. - episode_end: how code saw the episode end: commit (a commit was made), pr (a pull request was opened), new_task (the person moved on to another task, or went idle), session_end (the session ended), interrupt_switch (the person interrupted and switched task), or null. -- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), started_agent (the last reply started a subagent), reverted (a later commit reverted this work), fixup_later (a later commit named as a fix touched the same files; a weak link), continued_later (a later episode within a day changed the same files; a weak hint that the work went on). +- end_signals: facts code computed about the end, or null: asks_person (the last reply ends with a question, lettered choices, or a questions list for the person), reverted (a later commit reverted this work), fixup_later (a later commit named as a fix touched the same files; a weak link), continued_later (a later episode within a day changed the same files; a weak hint that the work went on). - omitted_turns: how many turns from the middle were left out for size. The first and last turns are always kept. - turns: human turns (role human, with their text) and assistant turns (role assistant, the opening of the reply and the tool names used, no arguments). The last assistant turn can also have closing: the end of that last reply. Human turns with candidate true were flagged by code as possible corrections; after_interrupt true means the person interrupted the agent first. - config_searched: true when code searched every instruction file loaded for that session for lines that match the candidate turns. @@ -18,7 +18,7 @@ Fields: - outcome: resolved (the work reached what the person wanted), abandoned (dropped or left open), reworked (redone after a wrong turn), or unknown. Use episode_end, end_signals, the last turns, and the closing of the last reply, in this order: - reworked when reverted is true, or the turns redo work after a wrong turn. fixup_later supports reworked only together with signs of trouble in the turns. - resolved when the last turns or the closing show the request done or answered. A commit or pr end counts as done unless the closing says the work is unfinished; a follow-up question after done work does not undo it. - - abandoned when the work was not done, nothing shows it went on (continued_later is a hint that it did), and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with work still running (the closing says so; started_agent supports it). That is work dropped or left open. + - abandoned when the work was not done, nothing shows it went on (continued_later is a hint that it did), and either the person dropped it (interrupt_switch, or new_task with the request unmet) or the episode ended (new_task, session_end, interrupt_switch) with a question or choice still open for the person (asks_person) or with the closing saying work is still running. That is work dropped or left open. - unknown only when none of these fits, for example the work was left open and seems to have gone on in a later episode. continued_later alone does not undo done work. - corrections: one item per human turn with candidate true that corrects or redirects the agent. Leave out candidate turns that are not corrections. Use an empty list when there are none. - turn_id: the id of that human turn. diff --git a/inspect/test/judge/bundle.test.mjs b/inspect/test/judge/bundle.test.mjs index d8932ca2..624de5a2 100644 --- a/inspect/test/judge/bundle.test.mjs +++ b/inspect/test/judge/bundle.test.mjs @@ -231,10 +231,10 @@ test('asksPerson: a question, lettered choices, or a questions list at the end; test('end signals: computed by code from the last reply and later git and episode links', () => { const ev = [...events(), { t: at(9), kind: 'tool_use', name: 'Agent', id: 'a1' }, { t: at(9), kind: 'assistant_text', text: 'Pick one: A) keep B) drop', head: 'Two ways.' }]; const b = buildBundle({ episode: { ...episode, downstream: [{ kind: 'revert', ref: 'x', delta_hours: 2, link: 'hard' }, { kind: 'episode', ref: 'e-2', delta_hours: 1, link: 'soft' }] }, transcript: { events: ev, snapshot }, known, nonce: 'n' }); - assert.deepEqual(payloadOf(b).end_signals, { asks_person: true, started_agent: true, reverted: true, fixup_later: false, continued_later: true }); + assert.deepEqual(payloadOf(b).end_signals, { asks_person: true, reverted: true, fixup_later: false, continued_later: true }); // Ending on a human turn: nothing in a reply is open. const h = buildBundle({ episode, transcript: { events: events(), snapshot }, known, nonce: 'n' }); - assert.deepEqual(payloadOf(h).end_signals, { asks_person: false, started_agent: false, reverted: false, fixup_later: false, continued_later: false }); + assert.deepEqual(payloadOf(h).end_signals, { asks_person: false, reverted: false, fixup_later: false, continued_later: false }); assert.match(TEMPLATE, /end_signals/); // A tool call after the question: the agent went on, so nothing was left open. const went = buildBundle({ episode, transcript: { events: [...events(), { t: at(9), kind: 'assistant_text', text: 'Should I run the tests?' }, { t: at(9), kind: 'tool_use', name: 'Bash', id: 'b1' }], snapshot }, known, nonce: 'n' }); @@ -244,6 +244,11 @@ test('end signals: computed by code from the last reply and later git and episod const f = buildBundle({ episode: { ...episode, downstream: near, rework: true, possible_fix: true, possible_rework: true }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); assert.equal(payloadOf(f).end_signals.reverted, true); assert.equal(payloadOf(f).end_signals.fixup_later, true); + // Episodes from facts carry no linker flags, only the stratum outcome. + const s = buildBundle({ episode: { ...episode, downstream: near, stratum: { outcome: 'rework' } }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.equal(payloadOf(s).end_signals.reverted, true); + const p = buildBundle({ episode: { ...episode, downstream: near, stratum: { outcome: 'possible_fix' } }, transcript: { events: events(), snapshot }, known, nonce: 'n' }); + assert.deepEqual([payloadOf(p).end_signals.reverted, payloadOf(p).end_signals.fixup_later], [false, true]); }); test('trimming shortens middle replies first; the last reply keeps its full opening and closing', () => {