diff --git a/src/engine/resume.ts b/src/engine/resume.ts index ebb93d1..66e23ac 100644 --- a/src/engine/resume.ts +++ b/src/engine/resume.ts @@ -74,7 +74,7 @@ export function resumeRun(loop: Loop, runId: string): Run { const replayed = meta.keyVersion === KEY_VERSION ? view - : { ...view, completed: new Map() } + : { ...view, started: new Map(), terminal: new Map(), completed: new Map() } return { loop, ledger: new Ledger(path), state, replayed } } diff --git a/src/engine/tick.ts b/src/engine/tick.ts index 408978a..eaa58f6 100644 --- a/src/engine/tick.ts +++ b/src/engine/tick.ts @@ -15,7 +15,7 @@ import { hashObserver, type EffectObservation, type EffectsObserver } from "../k import type { Decision, Observation } from "../kernel/policy.js" import type { Executor, Loop, MemoryStore, RunContext, Step, StepResult, StepSkill, StepSpec } from "../kernel/types.js" import { derivedOutputSchema, zeroUsage } from "../kernel/types.js" -import { journalPath, Ledger, resolveLedgerRoot, resumeKey, KEY_VERSION, type Replay, type StepResultEntry } from "../ledger/ledger.js" +import { assertSafePathComponent, journalPath, Ledger, resolveLedgerRoot, resumeKey, KEY_VERSION, type Replay, type StepResultEntry, type StepStartedEntry } from "../ledger/ledger.js" import { embedSkillsInPrompt, nativeSkillsDirective, skillBody, snapshotSkills } from "../skills/skills.js" export type RunStatus = "running" | "done" | "needs_human" | "stopped" @@ -221,6 +221,7 @@ function applyFinalOutputGuard( /** Fresh run id: loopId + timestamp + entropy. Exposed so a driver can lease the run dir BEFORE the first journal write. */ export function newRunId(loop: Loop): string { + assertSafePathComponent(loop.id, "loop id") return `${loop.id}-${now().replace(/[-:T]/g, "").slice(0, 14)}-${randomBytes(3).toString("hex")}` } @@ -271,13 +272,15 @@ export async function tick(run: Run, deps: EngineDeps): Promise { const inputs = step.signature.input.parse(state.values) as Record const key = resumeKey(step.id, inputs, state.iteration, state.attempt) - // RESUME IS REPLAY: when the ledger already holds a completed result for + // RESUME IS REPLAY: when the ledger already holds a terminal result for // this exact (stepId, iteration, attempt, inputs) slot — a crash landed // after the step_result but before its decision — the slot is replayed // from the ledger, never re-executed. LLM steps are non-deterministic and // side-effecting steps (codex writes, `remember`) must not double-apply. - const journaled = run.replayed?.completed.get(key) + const journaled = run.replayed?.terminal.get(key) if (journaled) return replayTick(run, deps, step, key, journaled) + const started = run.replayed?.started.get(key) + if (started) return recoverStartedTick(run, deps, step, key, started) const executor = deps.executors.get(step.executor) if (!executor) throw new Error(`Unknown executor id \`${step.executor}\` for step \`${step.id}\`.`) @@ -373,7 +376,18 @@ export async function tick(run: Run, deps: EngineDeps): Promise { workdir: deps.workdir, ...(deps.memory ? { memory: deps.memory } : {}), }) - const effects: EffectObservation = await observer.assess(deps.workdir, before, step.effects) + let effects: EffectObservation + try { + effects = await observer.assess(deps.workdir, before, step.effects) + } catch (error) { + effects = { + changed: [], + allowed: false, + unexpected: [""], + observed: false, + reason: `effect observer failed after executor: ${error instanceof Error ? error.message : String(error)}`, + } + } // 5. Project engine-observed fields over the executor's output (e.g. an // artifact path from effect attribution — the projection wins on @@ -417,6 +431,7 @@ export async function tick(run: Run, deps: EngineDeps): Promise { contractFailedChecks: contractResult ? failedCheckMessages(contractResult) : [], effectsAllowed: effects.allowed, unexpectedChanges: effects.unexpected, + effectsObserved: effects.observed !== false, output: validatedOutput, } const decision = applyFinalOutputGuard(loop, state, loop.policy(observation), validatedOutput) @@ -435,14 +450,14 @@ export async function tick(run: Run, deps: EngineDeps): Promise { * (signature parse; the contract, when its entry is missing) are recomputed; * non-deterministic and side-effecting pieces are never re-run. */ -function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journaled: StepResultEntry): TickOutcome { +function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journaled: StepResultEntry, recoveredEffects?: EffectObservation): TickOutcome { const { loop, ledger, state } = run // Tolerant lookup: replaying must not require the executor to be wired. const executorId = deps.executors.get(step.executor)?.id ?? step.executor const output = journaled.output const outputParse = step.signature.output.safeParse(output) - const outputValid = outputParse.success + const outputValid = journaled.status === "completed" && journaled.outputValid && outputParse.success let contractResult: ContractResult | null = null if (step.contract) { @@ -465,9 +480,13 @@ function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journal // Effects: replay the ledgered observation. If the crash landed before the // effects entry was written, the before-snapshot is gone and observation is - // impossible — assume a clean scope rather than re-executing the step. + // impossible. Fail loud instead of assuming a clean scope. + const ledgeredEffects = run.replayed?.effects.get(key)?.observation const effects: EffectObservation = - run.replayed?.effects.get(key)?.observation ?? { changed: [], allowed: true, unexpected: [] } + ledgeredEffects ?? recoveredEffects ?? { changed: [], allowed: false, unexpected: [""], observed: false, reason: "crash before effects journal entry" } + if (!ledgeredEffects) { + ledger.append({ type: "effects", key, stepId: step.id, attempt: state.attempt, iteration: state.iteration, observation: effects, at: now() }) + } const validatedOutput = outputValid ? (outputParse.data as Record) : null const observation: Observation = { @@ -488,6 +507,7 @@ function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journal contractFailedChecks: contractResult ? failedCheckMessages(contractResult) : [], effectsAllowed: effects.allowed, unexpectedChanges: effects.unexpected, + effectsObserved: effects.observed !== false, output: validatedOutput, } const decision = applyFinalOutputGuard(loop, state, loop.policy(observation), validatedOutput) @@ -496,6 +516,32 @@ function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journal return { state: run.state, decision } } +function recoverStartedTick(run: Run, deps: EngineDeps, step: Step, key: string, started: StepStartedEntry): TickOutcome { + const { ledger } = run + const journaled: StepResultEntry = { + type: "step_result", + key, + stepId: step.id, + attempt: started.attempt, + iteration: started.iteration, + status: "interrupted", + output: { error: "crash after step_started before step_result" }, + outputValid: false, + evidence: [], + usage: zeroUsage(), + at: now(), + } + const effects: EffectObservation = { + changed: [], + allowed: false, + unexpected: [""], + observed: false, + reason: "crash after step_started before step_result", + } + ledger.append(journaled) + return replayTick(run, deps, step, key, journaled, effects) +} + /** The journal-to-state projection. Exported for the resume fold (engine/resume.ts); not a public API. */ export function nextState(loop: Loop, state: RunState, decision: Decision, output: Record): RunState { const values = { ...state.values, ...output } diff --git a/src/kernel/contract.ts b/src/kernel/contract.ts index 639af3d..5ce20f8 100644 --- a/src/kernel/contract.ts +++ b/src/kernel/contract.ts @@ -3,8 +3,8 @@ // ContractResult / registry). A `path`-valued output field whose file // content gets validated remains the common case (run-trace.v1 below). -import { existsSync, readFileSync } from "node:fs" -import { isAbsolute, join } from "node:path" +import { closeSync, constants, fstatSync, openSync, readSync, realpathSync } from "node:fs" +import { isAbsolute, relative, resolve } from "node:path" export interface ContractCheck { readonly label: string @@ -60,6 +60,40 @@ export class ContractRegistry { export const RUN_TRACE_V1 = "run-trace.v1" +const MAX_TRACE_BYTES = 1_000_000 + +function safeReadWorkdirRelativeTrace(tracePath: string, workdir: string): { exists: boolean; text: string; detail: string } { + if (!tracePath) return { exists: false, text: "", detail: "expected a readable trace file at ``" } + if (isAbsolute(tracePath)) return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to be relative to the workdir` } + const root = resolve(workdir) + const absolute = resolve(root, tracePath) + const rel = relative(root, absolute) + if (rel === "" || rel.startsWith("..") || rel.includes(":") || rel.startsWith("/")) { + return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to stay inside the workdir` } + } + try { + const realRoot = realpathSync(root) + const realAbsolute = realpathSync(absolute) + const realRel = relative(realRoot, realAbsolute) + if (realRel === "" || realRel.startsWith("..") || realRel.includes(":") || realRel.startsWith("/")) { + return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to stay inside the real workdir` } + } + const fd = openSync(absolute, constants.O_RDONLY | constants.O_NOFOLLOW) + try { + const stat = fstatSync(fd) + if (!stat.isFile()) return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to be a regular file` } + if (stat.size > MAX_TRACE_BYTES) return { exists: false, text: "", detail: `expected trace file \`${tracePath}\` to be <= ${MAX_TRACE_BYTES} bytes` } + const buffer = Buffer.alloc(Math.min(stat.size, MAX_TRACE_BYTES)) + const bytes = readSync(fd, buffer, 0, buffer.length, 0) + return { exists: true, text: buffer.subarray(0, bytes).toString("utf8"), detail: `expected a readable trace file at \`${tracePath}\`` } + } finally { + closeSync(fd) + } + } catch { + return { exists: false, text: "", detail: `expected a readable trace file at \`${tracePath}\`` } + } +} + function metadataLineContains(text: string, field: string, value: string): boolean { const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") return new RegExp(`^.*\`${escape(field)}\`.*${escape(value)}.*$`, "m").test(text) @@ -69,12 +103,12 @@ export const runTraceV1: Contract = { id: RUN_TRACE_V1, validate(output, ctx) { const tracePath = typeof output.trace === "string" ? output.trace : "" - const absolute = tracePath && !isAbsolute(tracePath) ? join(ctx.workdir, tracePath) : tracePath - const exists = Boolean(absolute) && existsSync(absolute) - const text = exists ? readFileSync(absolute, "utf8") : "" + const trace = safeReadWorkdirRelativeTrace(tracePath, ctx.workdir) + const exists = trace.exists + const text = trace.text const checks: ContractCheck[] = [ - { label: "trace file exists", passed: exists, detail: `expected a readable trace file at \`${tracePath || ""}\`` }, + { label: "trace file exists", passed: exists, detail: trace.detail }, { label: "trace heading recorded", passed: text.trimStart().startsWith("# Trace:"), detail: "expected a `# Trace:` markdown heading" }, { label: "trace id recorded", passed: text.includes(ctx.traceId), detail: `expected trace id \`${ctx.traceId}\`` }, { label: "loop id recorded", passed: metadataLineContains(text, "loop_id", ctx.loopId), detail: `expected loop id \`${ctx.loopId}\` in trace metadata` }, diff --git a/src/kernel/effects.ts b/src/kernel/effects.ts index 0dd9c17..8c78570 100644 --- a/src/kernel/effects.ts +++ b/src/kernel/effects.ts @@ -4,7 +4,7 @@ // worktree.ts only answers "did anything change?" — this attributes. import { createHash } from "node:crypto" -import { readdirSync, readFileSync } from "node:fs" +import { lstatSync, readdirSync, readFileSync, readlinkSync, realpathSync, statSync } from "node:fs" import { join, relative } from "node:path" import type { EffectScope, OutputProjection } from "./types.js" @@ -19,10 +19,13 @@ export function snapshotDir(root: string): Snapshot { for (const entry of readdirSync(dir, { withFileTypes: true })) { if (SKIP_DIRS.has(entry.name)) continue const full = join(dir, entry.name) + const rel = relative(root, full).split("\\").join("/") if (entry.isDirectory()) walk(full) + else if (entry.isSymbolicLink()) out.set(rel, symlinkDigest(full)) else if (entry.isFile()) { - const rel = relative(root, full).split("\\").join("/") out.set(rel, createHash("sha256").update(readFileSync(full)).digest("hex")) + } else { + out.set(rel, `type:${entryType(full)}`) } } } @@ -30,6 +33,49 @@ export function snapshotDir(root: string): Snapshot { return out } +function symlinkDigest(path: string, seen = new Set()): string { + const target = readlinkSync(path) + try { + const stat = statSync(path) + if (stat.isFile()) return `symlink:${target}:file:${createHash("sha256").update(readFileSync(path)).digest("hex")}` + if (stat.isDirectory()) return `symlink:${target}:directory:${directoryDigest(path, seen)}` + return `symlink:${target}:target:${entryType(path, false)}` + } catch (error) { + const reason = error instanceof Error ? ((error as NodeJS.ErrnoException).code ?? error.message) : String(error) + return `symlink:${target}:unreadable:${reason}` + } +} + +function directoryDigest(path: string, seen: Set): string { + const real = realpathSync(path) + if (seen.has(real)) return "cycle" + seen.add(real) + const entries = readdirSync(path, { withFileTypes: true }) + .filter((entry) => !SKIP_DIRS.has(entry.name)) + .sort((a, b) => a.name.localeCompare(b.name)) + const parts = entries.map((entry) => { + const full = join(path, entry.name) + if (entry.isDirectory()) return `${entry.name}/:${directoryDigest(full, seen)}` + if (entry.isSymbolicLink()) return `${entry.name}:${symlinkDigest(full, seen)}` + if (entry.isFile()) return `${entry.name}:file:${createHash("sha256").update(readFileSync(full)).digest("hex")}` + return `${entry.name}:type:${entryType(full)}` + }) + seen.delete(real) + return createHash("sha256").update(JSON.stringify(parts)).digest("hex") +} + +function entryType(path: string, noFollow = true): string { + const stat = noFollow ? lstatSync(path) : statSync(path) + if (stat.isFile()) return "file" + if (stat.isDirectory()) return "directory" + if (stat.isSymbolicLink()) return "symlink" + if (stat.isFIFO()) return "fifo" + if (stat.isSocket()) return "socket" + if (stat.isBlockDevice()) return "block-device" + if (stat.isCharacterDevice()) return "character-device" + return "unknown" +} + export function changedFiles(before: Snapshot, after: Snapshot): string[] { const changed = new Set() for (const [path, digest] of after) if (before.get(path) !== digest) changed.add(path) @@ -46,6 +92,9 @@ export interface EffectObservation { readonly changed: readonly string[] readonly allowed: boolean readonly unexpected: readonly string[] + /** False only when a crash made post-step effect observation unknowable. Missing means observed=true for old journals. */ + readonly observed?: boolean + readonly reason?: string } export function assessChanges(before: Snapshot, after: Snapshot, scope: EffectScope): EffectObservation { diff --git a/src/kernel/policy.ts b/src/kernel/policy.ts index 4534eae..c298e06 100644 --- a/src/kernel/policy.ts +++ b/src/kernel/policy.ts @@ -44,6 +44,8 @@ export interface Observation { readonly contractValid: boolean readonly contractFailedChecks: readonly string[] readonly effectsAllowed: boolean + /** False when post-step effects are unknowable because the run crashed or observation failed. */ + readonly effectsObserved?: boolean readonly unexpectedChanges: readonly string[] /** The signature-validated output value of this step; null when outputValid is false. */ readonly output: Readonly> | null @@ -80,6 +82,17 @@ export function decideNextStep(obs: Observation): Decision { } } + if (obs.effectsObserved === false) { + const reason = `step \`${obs.stepId}\` has unknown post-step effects.` + return { + kind: "escalate", + classification: "failure", + summary: `${reason} Human review is required before continuing.`, + notes: [reason], + improvement: "Recover or rerun in a clean worktree so effect attribution is explicit before promotion.", + } + } + if (obs.stepStatus !== "completed") { const reason = `executor \`${obs.executorId}\` ${obs.stepStatus === "interrupted" ? "was interrupted" : "failed"} on step \`${obs.stepId}\`.` return { @@ -106,6 +119,7 @@ export function decideNextStep(obs: Observation): Decision { } } + if (!obs.effectsAllowed || obs.unexpectedChanges.length > 0) { const notes: string[] = [] if (!obs.effectsAllowed) notes.push("At least one attributed change was outside the allowed effect scope.") diff --git a/src/ledger/evidence.ts b/src/ledger/evidence.ts index 2868bb8..f3cbfa3 100644 --- a/src/ledger/evidence.ts +++ b/src/ledger/evidence.ts @@ -110,7 +110,7 @@ export function projectRunEvidence(input: ProjectRunEvidenceInput): RunEvidenceP const contractStatus = contractEvidence(contract) const effectsStatus = effectEvidence(effects) const unexpectedEffects = effects?.observation.unexpected.map(String) ?? [] - const observedEffects = effects ? true : null + const observedEffects = effects ? effects.observation.observed !== false : null steps.push({ key: result.key, @@ -232,7 +232,7 @@ function contractEvidence(contract: { readonly result: { readonly valid: boolean } function effectEvidence(effects: EffectsEntry | undefined): EffectEvidenceStatus { - if (!effects) return "unknown" + if (!effects || effects.observation.observed === false) return "unknown" return effects.observation.allowed ? "passed" : "failed" } diff --git a/src/ledger/ledger.ts b/src/ledger/ledger.ts index 8e92c2a..a6580b4 100644 --- a/src/ledger/ledger.ts +++ b/src/ledger/ledger.ts @@ -16,8 +16,8 @@ // ledger entries, not just agent results. import { createHash } from "node:crypto" -import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs" -import { dirname, join, resolve } from "node:path" +import { appendFileSync, existsSync, lstatSync, mkdirSync, readFileSync, realpathSync } from "node:fs" +import { dirname, join, relative, resolve } from "node:path" import type { ContractResult } from "../kernel/contract.js" import type { EffectObservation } from "../kernel/effects.js" import type { Decision } from "../kernel/policy.js" @@ -163,8 +163,41 @@ export function resolveLedgerRoot(spec: LedgerSpec): string { return resolve(spec.root ?? process.env.VERNIER_HOME ?? join(process.cwd(), ".vernier")) } +export function assertSafePathComponent(value: string, label = "path component"): void { + const windowsDevice = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$/i + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) || value.endsWith(".") || windowsDevice.test(value)) { + throw new Error(`${label} must be a portable safe path component (letters, numbers, dot, underscore, dash; no separators, trailing dot, or Windows device names), got \`${value}\`.`) + } +} + +function assertContained(parent: string, child: string): void { + const rel = relative(parent, child) + if (rel === "" || rel.startsWith("..") || rel.includes(":") || rel.startsWith("/")) { + throw new Error(`journal path escaped ledger root: \`${child}\` is outside \`${parent}\`.`) + } +} + +function assertNoSymlink(path: string, label: string): void { + if (existsSync(path) && lstatSync(path).isSymbolicLink()) { + throw new Error(`${label} must not be a symlink: \`${path}\`.`) + } +} + +function assertRealContained(parent: string, child: string): void { + if (!existsSync(parent) || !existsSync(child)) return + assertContained(realpathSync(parent), realpathSync(child)) +} + export function journalPath(root: string, runId: string): string { - return join(root, "runs", runId, "journal.jsonl") + assertSafePathComponent(runId, "run id") + const runsRoot = resolve(root, "runs") + const runDir = resolve(runsRoot, runId) + assertNoSymlink(runsRoot, "ledger runs root") + assertNoSymlink(runDir, "ledger run directory") + assertRealContained(runsRoot, runDir) + const path = resolve(runDir, "journal.jsonl") + assertContained(runsRoot, path) + return path } export class Ledger { @@ -200,6 +233,11 @@ export class Ledger { */ export interface Replay { readonly meta?: RunMetaEntry + /** Started slots that have no terminal result yet are unsafe to re-run on resume. */ + readonly started: ReadonlyMap + /** Terminal step results in this exact execution slot, regardless of status. */ + readonly terminal: ReadonlyMap + /** Completed results only, for summaries that care about successful outputs. */ readonly completed: ReadonlyMap readonly contracts: ReadonlyMap readonly effects: ReadonlyMap @@ -210,13 +248,19 @@ export interface Replay { export function replay(entries: readonly LedgerEntry[]): Replay { let meta: RunMetaEntry | undefined let lastDecision: DecisionEntry | undefined + const started = new Map() + const terminal = new Map() const completed = new Map() const contracts = new Map() const effects = new Map() const decisions = new Map() for (const entry of entries) { if (entry.type === "meta") meta = entry - else if (entry.type === "step_result" && entry.status === "completed") completed.set(entry.key, entry) + else if (entry.type === "step_started") started.set(entry.key, entry) + else if (entry.type === "step_result") { + terminal.set(entry.key, entry) + if (entry.status === "completed") completed.set(entry.key, entry) + } else if (entry.type === "contract") contracts.set(entry.key, entry) else if (entry.type === "effects") effects.set(entry.key, entry) else if (entry.type === "decision") { @@ -224,6 +268,6 @@ export function replay(entries: readonly LedgerEntry[]): Replay { lastDecision = entry } } - const out: Replay = { completed, contracts, effects, decisions, ...(meta && { meta }), ...(lastDecision && { lastDecision }) } + const out: Replay = { started, terminal, completed, contracts, effects, decisions, ...(meta && { meta }), ...(lastDecision && { lastDecision }) } return out } diff --git a/test/contract.test.ts b/test/contract.test.ts new file mode 100644 index 0000000..83aca65 --- /dev/null +++ b/test/contract.test.ts @@ -0,0 +1,73 @@ +import { mkdtempSync, symlinkSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { RUN_TRACE_V1, runTraceV1 } from "../src/kernel/contract.js" + +function workdir(): string { + return mkdtempSync(join(tmpdir(), "vernier-contract-")) +} + +function validTrace(root: string, name = "trace.md"): string { + const path = join(root, name) + writeFileSync( + path, + [ + "# Trace: run-1", + "`loop_id`: loop-a", + "`loop_version`: 1.0.0", + "`result.classification`: success", + "`improvement_candidate.summary`: keep it simple", + "run-1", + "", + ].join("\n"), + "utf8", + ) + return name +} + +function validate(root: string, trace: string) { + return runTraceV1.validate({ trace }, { + traceId: "run-1", + loopId: "loop-a", + loopVersion: "1.0.0", + workdir: root, + executorId: "script", + runDir: join(root, "..", "run"), + }) +} + +describe(RUN_TRACE_V1, () => { + it("accepts readable workdir-relative traces", () => { + const root = workdir() + const result = validate(root, validTrace(root)) + expect(result.valid).toBe(true) + }) + + it("rejects absolute trace paths", () => { + const root = workdir() + const absolute = join(root, validTrace(root)) + const result = validate(root, absolute) + expect(result.valid).toBe(false) + expect(result.checks[0]?.detail).toContain("relative to the workdir") + }) + + it("rejects parent-directory escapes", () => { + const root = workdir() + const result = validate(root, "../trace.md") + expect(result.valid).toBe(false) + expect(result.checks[0]?.detail).toContain("inside the workdir") + }) + + it("rejects symlinks that resolve outside the workdir", () => { + const root = workdir() + const outside = join(root, "..", "outside.md") + writeFileSync(outside, "# Trace: run-1\n", "utf8") + symlinkSync(outside, join(root, "trace-link.md")) + + const result = validate(root, "trace-link.md") + + expect(result.valid).toBe(false) + expect(result.checks[0]?.detail).toContain("inside the real workdir") + }) +}) diff --git a/test/effects.test.ts b/test/effects.test.ts new file mode 100644 index 0000000..8fd63a9 --- /dev/null +++ b/test/effects.test.ts @@ -0,0 +1,66 @@ +import { mkdirSync, mkdtempSync, symlinkSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { assessChanges, snapshotDir } from "../src/kernel/effects.js" +import { fsScope } from "../src/kernel/types.js" + +function temp(): string { + return mkdtempSync(join(tmpdir(), "vernier-effects-")) +} + +describe("snapshotDir", () => { + it("records symlink topology changes", () => { + const root = temp() + writeFileSync(join(root, "a.txt"), "a", "utf8") + writeFileSync(join(root, "b.txt"), "b", "utf8") + symlinkSync("a.txt", join(root, "link.txt")) + const before = snapshotDir(root) + + symlinkSync("b.txt", join(root, "next-link.txt")) + const afterCreate = snapshotDir(root) + expect(assessChanges(before, afterCreate, fsScope("link.txt", "next-link.txt")).changed).toContain("next-link.txt") + }) + + it("records writes through in-workdir symlinks", () => { + const root = temp() + const outside = join(root, "..", "outside.txt") + writeFileSync(outside, "before", "utf8") + symlinkSync(outside, join(root, "linked.txt")) + const before = snapshotDir(root) + + writeFileSync(join(root, "linked.txt"), "after", "utf8") + const observation = assessChanges(before, snapshotDir(root), fsScope("linked.txt")) + + expect(observation.changed).toEqual(["linked.txt"]) + expect(observation.allowed).toBe(true) + }) + + it("records writes through directory symlinks", () => { + const root = temp() + const outsideDir = mkdtempSync(join(tmpdir(), "vernier-outside-dir-")) + writeFileSync(join(outsideDir, "note.txt"), "before", "utf8") + symlinkSync(outsideDir, join(root, "linked-dir")) + const before = snapshotDir(root) + + writeFileSync(join(root, "linked-dir", "note.txt"), "after", "utf8") + const observation = assessChanges(before, snapshotDir(root), fsScope("linked-dir")) + + expect(observation.changed).toEqual(["linked-dir"]) + }) + + it("flags symlink write-through outside declared scope", () => { + const root = temp() + mkdirSync(join(root, "allowed")) + const outside = join(root, "..", "outside.txt") + writeFileSync(outside, "before", "utf8") + symlinkSync(outside, join(root, "linked.txt")) + const before = snapshotDir(root) + + writeFileSync(join(root, "linked.txt"), "after", "utf8") + const observation = assessChanges(before, snapshotDir(root), fsScope("allowed/**")) + + expect(observation.allowed).toBe(false) + expect(observation.unexpected).toEqual(["linked.txt"]) + }) +}) diff --git a/test/evidence-projection.test.ts b/test/evidence-projection.test.ts index 5b1ed00..85798f7 100644 --- a/test/evidence-projection.test.ts +++ b/test/evidence-projection.test.ts @@ -52,14 +52,14 @@ function contract(stepId: string, valid: boolean, key = `${stepId}-1`): Contract } } -function effects(stepId: string, allowed: boolean, key = `${stepId}-1`): EffectsEntry { +function effects(stepId: string, allowed: boolean, key = `${stepId}-1`, observation: Partial = {}): EffectsEntry { return { type: "effects", key, stepId, iteration: 1, attempt: 1, - observation: { changed: ["docs/a.md"], allowed, unexpected: allowed ? [] : ["secret.txt"] }, + observation: { changed: ["docs/a.md"], allowed, unexpected: allowed ? [] : ["secret.txt"], ...observation }, at: at(4), } } @@ -177,6 +177,22 @@ describe("projectRunEvidence", () => { expect(projection.diagnostics).toContainEqual({ severity: "degraded", code: "MISSING_EFFECTS", detail: "completed step write has no effects entry" }) }) + it("marks explicitly unobserved effects as degraded/unknown evidence", () => { + const projection = projectRunEvidence({ + entries: [ + meta(), + started("write"), + result("write"), + effects("write", false, "write-1", { observed: false, reason: "crash before effects journal entry", unexpected: [""] }), + decision("write", "escalate", "failure"), + ], + }) + + expect(projection.strict.usableForTrust).toBe(false) + expect(projection.totals).toMatchObject({ effectsUnknown: 1, effectsFailed: 0 }) + expect(projection.steps[0]).toMatchObject({ effects: "unknown", observedEffects: false, unexpectedEffects: [""] }) + }) + it("keeps missing usage honest rather than fabricating availability or cost", () => { const resultWithoutUsage: StepResultEntry = JSON.parse(JSON.stringify(result("route", "route-1"), (_key, value) => (_key === "usage" ? undefined : value))) const projection = projectRunEvidence({ diff --git a/test/ledger.test.ts b/test/ledger.test.ts index a32b37b..566e8d3 100644 --- a/test/ledger.test.ts +++ b/test/ledger.test.ts @@ -1,9 +1,10 @@ -import { appendFileSync, mkdtempSync } from "node:fs" +import { appendFileSync, mkdirSync, mkdtempSync, symlinkSync } from "node:fs" import { tmpdir } from "node:os" -import { join } from "node:path" +import { join, resolve } from "node:path" import { describe, expect, it } from "vitest" import { canonical, + journalPath, Ledger, replay, resumeKey, @@ -115,3 +116,29 @@ describe("Ledger", () => { expect(view.lastDecision?.decision.kind).toBe("stop") }) }) + +describe("journalPath", () => { + it("accepts a safe run id under the ledger runs directory", () => { + const root = mkdtempSync(join(tmpdir(), "vernier-ledger-root-")) + expect(journalPath(root, "loop-20260628-a1b2c3")).toBe(resolve(root, "runs", "loop-20260628-a1b2c3", "journal.jsonl")) + }) + + it("rejects run ids that are not a single portable path component", () => { + const root = mkdtempSync(join(tmpdir(), "vernier-ledger-root-")) + for (const bad of ["../escape", "nested/run", "trailing.", "CON", "nul.txt"]) { + expect(() => journalPath(root, bad)).toThrow(/safe path component/) + } + }) + + it("rejects symlinked runs roots and run directories", () => { + const root = mkdtempSync(join(tmpdir(), "vernier-ledger-root-")) + const outside = mkdtempSync(join(tmpdir(), "vernier-ledger-outside-")) + symlinkSync(outside, join(root, "runs")) + expect(() => journalPath(root, "run-1")).toThrow(/runs root.*symlink/) + + const rootWithRun = mkdtempSync(join(tmpdir(), "vernier-ledger-root-")) + mkdirSync(join(rootWithRun, "runs")) + symlinkSync(outside, join(rootWithRun, "runs", "run-1")) + expect(() => journalPath(rootWithRun, "run-1")).toThrow(/run directory.*symlink/) + }) +}) diff --git a/test/policy.test.ts b/test/policy.test.ts index 737a952..247ad33 100644 --- a/test/policy.test.ts +++ b/test/policy.test.ts @@ -74,6 +74,13 @@ describe("decideNextStep", () => { expect(decision.notes.join("\n")).toContain("scripts/pilot1_runner.py") }) + it("escalates when effects were not observed", () => { + const decision = decideNextStep(observation({ effectsObserved: false })) + expect(decision.kind).toBe("escalate") + expect(decision.classification).toBe("failure") + expect(decision.notes.join("\n")).toContain("unknown post-step effects") + }) + it("is pure: identical observations yield identical decisions", () => { const obs = observation({ contractValid: false, contractFailedChecks: ["x"] }) expect(decideNextStep(obs)).toEqual(decideNextStep(obs)) diff --git a/test/resume.test.ts b/test/resume.test.ts index c49bdf4..e19d872 100644 --- a/test/resume.test.ts +++ b/test/resume.test.ts @@ -21,7 +21,7 @@ import { executorRegistry, scriptExecutor } from "../src/executors/script.js" import { ContractRegistry, type Contract } from "../src/kernel/contract.js" import { retryPolicy, until } from "../src/kernel/policy.js" import { noEffects, sig, type Loop } from "../src/kernel/types.js" -import { Ledger, journalPath } from "../src/ledger/ledger.js" +import { Ledger, journalPath, resumeKey } from "../src/ledger/ledger.js" function temp(): { workdir: string; ledgerRoot: string } { const root = mkdtempSync(join(tmpdir(), "vernier-resume-")) @@ -138,23 +138,19 @@ describe("resume from the ledger (linear loop)", () => { const resumed = resumeRun(loop, crashed.state.runId) expect(resumed.state).toMatchObject({ stepIndex: 0, attempt: 1, iteration: 1 }) - // ...but the tick replays the journaled result instead of executing. + // ...but the tick replays the journaled result instead of executing and + // fails closed because the crash lost the post-step effects observation. const replayedTick = await tick(resumed, d) expect(counts.double).toBe(1) // NOT re-run - expect(replayedTick.decision.kind).toBe("continue") - expect(resumed.state.values.n).toBe(6) + expect(replayedTick.decision.kind).toBe("escalate") + expect(replayedTick.decision.notes.join("\n")).toContain("unknown post-step effects") + expect(replayedTick.state.status).toBe("needs_human") - const outcome = await driveRun(resumed, d) - expect(outcome.state.status).toBe("done") - expect(counts).toEqual({ double: 1, doubleAgain: 1 }) - expect(finalOutput(loop, resumed.state, outcome.decision)).toEqual({ n: 12, doubledTwice: true }) - - // The replayed tick re-appended the missing pieces: exactly one - // step_result for `double`, and its recomputed contract entry. const entries = Ledger.load(journal) expect(entries.filter((e) => e.type === "step_result" && e.stepId === "double")).toHaveLength(1) expect(entries.filter((e) => e.type === "contract" && e.stepId === "double")).toHaveLength(1) - expect(summarizeJournal(entries).status).toBe("done") + expect(entries.filter((e) => e.type === "effects" && e.stepId === "double")).toHaveLength(1) + expect(summarizeJournal(entries).status).toBe("needs_human") }) it("a terminal run resumes as terminal; ticking it refuses", async () => { @@ -305,12 +301,10 @@ describe("resume from the ledger (ITERATING loop)", () => { const replayed = await tick(resumed, d) expect(ex.counts.answer).toBe(2) // replayed from the ledger, NOT re-executed - expect(replayed.decision.kind).toBe("continue") + expect(replayed.decision.kind).toBe("escalate") + expect(replayed.decision.notes.join("\n")).toContain("unknown post-step effects") + expect(replayed.state.status).toBe("needs_human") expect(resumed.state.values.answer).toBe("revised answer") // iteration 2's output — the v2 key did not collapse passes - - const outcome = await driveRun(resumed, d) - expect(outcome.state.status).toBe("done") - expect(ex.counts).toEqual({ answer: 2, grade: 2 }) }) }) @@ -375,6 +369,38 @@ describe("resume never double-applies side effects", () => { expect(ex.counts).toEqual({ remember: 1, confirm: 1 }) }) + it("started-only crash records an interrupted slot and escalates before re-entering it", async () => { + const { workdir, ledgerRoot } = temp() + const store = join(workdir, "..", "rules.txt") + const ex = rememberExecutors(store) + const d = deps([ex.remember, ex.confirm], workdir) + const loop = rememberLoop(ledgerRoot) + + const crashed = startRun(loop, { rule: "always verify" }, d) + const key = resumeKey("remember", { rule: "always verify" }, 1, 1) + crashed.ledger.append({ + type: "step_started", + key, + stepId: "remember", + attempt: 1, + iteration: 1, + executorId: "script:remember", + at: "2026-06-10T00:00:01.000Z", + }) + + const resumed = resumeRun(loop, crashed.state.runId) + const recovered = await tick(resumed, d) + + expect(ex.counts.remember).toBe(0) + expect(storeLines(store)).toEqual([]) + expect(recovered.decision.kind).toBe("escalate") + expect(recovered.decision.notes.join("\n")).toContain("unknown post-step effects") + expect(recovered.state).toMatchObject({ stepIndex: 0, attempt: 1, status: "needs_human" }) + const entries = Ledger.load(journalPath(ledgerRoot, crashed.state.runId)) + expect(entries.filter((e) => e.type === "step_result" && e.stepId === "remember")).toHaveLength(1) + expect(entries.filter((e) => e.type === "effects" && e.stepId === "remember")).toHaveLength(1) + }) + it("torn tick after the remember step's result: the slot is replayed from the ledger — still exactly one append", async () => { const { workdir, ledgerRoot } = temp() const store = join(workdir, "..", "rules.txt") // OUTSIDE the observed workdir, like the real Memory store @@ -389,10 +415,10 @@ describe("resume never double-applies side effects", () => { const resumed = resumeRun(loop, crashed.state.runId) expect(resumed.state.stepIndex).toBe(0) // fold lands back on `remember`... - const outcome = await driveRun(resumed, d) - expect(outcome.state.status).toBe("done") + const replayed = await tick(resumed, d) + expect(replayed.state.status).toBe("needs_human") + expect(replayed.decision.notes.join("\n")).toContain("unknown post-step effects") expect(storeLines(store)).toEqual(["always verify"]) // ...but replay, not re-execution: ONE append - expect(ex.counts).toEqual({ remember: 1, confirm: 1 }) - expect(finalOutput(loop, resumed.state, outcome.decision)).toEqual({ stored: true, confirmed: true }) + expect(ex.counts).toEqual({ remember: 1, confirm: 0 }) }) })