Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/engine/resume.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export function resumeRun(loop: Loop, runId: string): Run {
const replayed =
meta.keyVersion === KEY_VERSION
? view
: { ...view, completed: new Map<string, StepResultEntry>() }
: { ...view, started: new Map(), terminal: new Map(), completed: new Map<string, StepResultEntry>() }
return { loop, ledger: new Ledger(path), state, replayed }
}

Expand Down
62 changes: 54 additions & 8 deletions src/engine/tick.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { hashObserver, type EffectObservation, type EffectsObserver } from "../k
import type { Decision, Observation } from "../kernel/policy.js"
import type { Executor, Loop, MemoryStore, RunContext, Step, StepResult, StepSkill, StepSpec } from "../kernel/types.js"
import { derivedOutputSchema, zeroUsage } from "../kernel/types.js"
import { journalPath, Ledger, resolveLedgerRoot, resumeKey, KEY_VERSION, type Replay, type StepResultEntry } from "../ledger/ledger.js"
import { assertSafePathComponent, journalPath, Ledger, resolveLedgerRoot, resumeKey, KEY_VERSION, type Replay, type StepResultEntry, type StepStartedEntry } from "../ledger/ledger.js"
import { embedSkillsInPrompt, nativeSkillsDirective, skillBody, snapshotSkills } from "../skills/skills.js"

export type RunStatus = "running" | "done" | "needs_human" | "stopped"
Expand Down Expand Up @@ -221,6 +221,7 @@ function applyFinalOutputGuard(

/** Fresh run id: loopId + timestamp + entropy. Exposed so a driver can lease the run dir BEFORE the first journal write. */
export function newRunId(loop: Loop): string {
assertSafePathComponent(loop.id, "loop id")
return `${loop.id}-${now().replace(/[-:T]/g, "").slice(0, 14)}-${randomBytes(3).toString("hex")}`
}

Expand Down Expand Up @@ -271,13 +272,15 @@ export async function tick(run: Run, deps: EngineDeps): Promise<TickOutcome> {
const inputs = step.signature.input.parse(state.values) as Record<string, unknown>
const key = resumeKey(step.id, inputs, state.iteration, state.attempt)

// RESUME IS REPLAY: when the ledger already holds a completed result for
// RESUME IS REPLAY: when the ledger already holds a terminal result for
// this exact (stepId, iteration, attempt, inputs) slot — a crash landed
// after the step_result but before its decision — the slot is replayed
// from the ledger, never re-executed. LLM steps are non-deterministic and
// side-effecting steps (codex writes, `remember`) must not double-apply.
const journaled = run.replayed?.completed.get(key)
const journaled = run.replayed?.terminal.get(key)
if (journaled) return replayTick(run, deps, step, key, journaled)
const started = run.replayed?.started.get(key)
if (started) return recoverStartedTick(run, deps, step, key, started)

const executor = deps.executors.get(step.executor)
if (!executor) throw new Error(`Unknown executor id \`${step.executor}\` for step \`${step.id}\`.`)
Expand Down Expand Up @@ -373,7 +376,18 @@ export async function tick(run: Run, deps: EngineDeps): Promise<TickOutcome> {
workdir: deps.workdir,
...(deps.memory ? { memory: deps.memory } : {}),
})
const effects: EffectObservation = await observer.assess(deps.workdir, before, step.effects)
let effects: EffectObservation
try {
effects = await observer.assess(deps.workdir, before, step.effects)
} catch (error) {
effects = {
changed: [],
allowed: false,
unexpected: ["<effects unknown: observer failed after executor>"],
observed: false,
reason: `effect observer failed after executor: ${error instanceof Error ? error.message : String(error)}`,
}
}

// 5. Project engine-observed fields over the executor's output (e.g. an
// artifact path from effect attribution — the projection wins on
Expand Down Expand Up @@ -417,6 +431,7 @@ export async function tick(run: Run, deps: EngineDeps): Promise<TickOutcome> {
contractFailedChecks: contractResult ? failedCheckMessages(contractResult) : [],
effectsAllowed: effects.allowed,
unexpectedChanges: effects.unexpected,
effectsObserved: effects.observed !== false,
output: validatedOutput,
}
const decision = applyFinalOutputGuard(loop, state, loop.policy(observation), validatedOutput)
Expand All @@ -435,14 +450,14 @@ export async function tick(run: Run, deps: EngineDeps): Promise<TickOutcome> {
* (signature parse; the contract, when its entry is missing) are recomputed;
* non-deterministic and side-effecting pieces are never re-run.
*/
function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journaled: StepResultEntry): TickOutcome {
function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journaled: StepResultEntry, recoveredEffects?: EffectObservation): TickOutcome {
const { loop, ledger, state } = run
// Tolerant lookup: replaying must not require the executor to be wired.
const executorId = deps.executors.get(step.executor)?.id ?? step.executor

const output = journaled.output
const outputParse = step.signature.output.safeParse(output)
const outputValid = outputParse.success
const outputValid = journaled.status === "completed" && journaled.outputValid && outputParse.success

let contractResult: ContractResult | null = null
if (step.contract) {
Expand All @@ -465,9 +480,13 @@ function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journal

// Effects: replay the ledgered observation. If the crash landed before the
// effects entry was written, the before-snapshot is gone and observation is
// impossible — assume a clean scope rather than re-executing the step.
// impossible. Fail loud instead of assuming a clean scope.
const ledgeredEffects = run.replayed?.effects.get(key)?.observation
const effects: EffectObservation =
run.replayed?.effects.get(key)?.observation ?? { changed: [], allowed: true, unexpected: [] }
ledgeredEffects ?? recoveredEffects ?? { changed: [], allowed: false, unexpected: ["<effects unknown: crash before observation>"], observed: false, reason: "crash before effects journal entry" }
if (!ledgeredEffects) {
ledger.append({ type: "effects", key, stepId: step.id, attempt: state.attempt, iteration: state.iteration, observation: effects, at: now() })
}

const validatedOutput = outputValid ? (outputParse.data as Record<string, unknown>) : null
const observation: Observation = {
Expand All @@ -488,6 +507,7 @@ function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journal
contractFailedChecks: contractResult ? failedCheckMessages(contractResult) : [],
effectsAllowed: effects.allowed,
unexpectedChanges: effects.unexpected,
effectsObserved: effects.observed !== false,
output: validatedOutput,
}
const decision = applyFinalOutputGuard(loop, state, loop.policy(observation), validatedOutput)
Expand All @@ -496,6 +516,32 @@ function replayTick(run: Run, deps: EngineDeps, step: Step, key: string, journal
return { state: run.state, decision }
}

function recoverStartedTick(run: Run, deps: EngineDeps, step: Step, key: string, started: StepStartedEntry): TickOutcome {
const { ledger } = run
const journaled: StepResultEntry = {
type: "step_result",
key,
stepId: step.id,
attempt: started.attempt,
iteration: started.iteration,
status: "interrupted",
output: { error: "crash after step_started before step_result" },
outputValid: false,
evidence: [],
usage: zeroUsage(),
at: now(),
}
const effects: EffectObservation = {
changed: [],
allowed: false,
unexpected: ["<effects unknown: crash after step_started before step_result>"],
observed: false,
reason: "crash after step_started before step_result",
}
ledger.append(journaled)
return replayTick(run, deps, step, key, journaled, effects)
}

/** The journal-to-state projection. Exported for the resume fold (engine/resume.ts); not a public API. */
export function nextState(loop: Loop, state: RunState, decision: Decision, output: Record<string, unknown>): RunState {
const values = { ...state.values, ...output }
Expand Down
46 changes: 40 additions & 6 deletions src/kernel/contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
// ContractResult / registry). A `path`-valued output field whose file
// content gets validated remains the common case (run-trace.v1 below).

import { existsSync, readFileSync } from "node:fs"
import { isAbsolute, join } from "node:path"
import { closeSync, constants, fstatSync, openSync, readSync, realpathSync } from "node:fs"
import { isAbsolute, relative, resolve } from "node:path"

export interface ContractCheck {
readonly label: string
Expand Down Expand Up @@ -60,6 +60,40 @@ export class ContractRegistry {

export const RUN_TRACE_V1 = "run-trace.v1"

const MAX_TRACE_BYTES = 1_000_000

function safeReadWorkdirRelativeTrace(tracePath: string, workdir: string): { exists: boolean; text: string; detail: string } {
if (!tracePath) return { exists: false, text: "", detail: "expected a readable trace file at `<missing trace output field>`" }
if (isAbsolute(tracePath)) return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to be relative to the workdir` }
const root = resolve(workdir)
const absolute = resolve(root, tracePath)
const rel = relative(root, absolute)
if (rel === "" || rel.startsWith("..") || rel.includes(":") || rel.startsWith("/")) {
return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to stay inside the workdir` }
}
try {
const realRoot = realpathSync(root)
const realAbsolute = realpathSync(absolute)
const realRel = relative(realRoot, realAbsolute)
if (realRel === "" || realRel.startsWith("..") || realRel.includes(":") || realRel.startsWith("/")) {
return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to stay inside the real workdir` }
}
const fd = openSync(absolute, constants.O_RDONLY | constants.O_NOFOLLOW)
try {
const stat = fstatSync(fd)
if (!stat.isFile()) return { exists: false, text: "", detail: `expected trace path \`${tracePath}\` to be a regular file` }
if (stat.size > MAX_TRACE_BYTES) return { exists: false, text: "", detail: `expected trace file \`${tracePath}\` to be <= ${MAX_TRACE_BYTES} bytes` }
const buffer = Buffer.alloc(Math.min(stat.size, MAX_TRACE_BYTES))
const bytes = readSync(fd, buffer, 0, buffer.length, 0)
return { exists: true, text: buffer.subarray(0, bytes).toString("utf8"), detail: `expected a readable trace file at \`${tracePath}\`` }
} finally {
closeSync(fd)
}
} catch {
return { exists: false, text: "", detail: `expected a readable trace file at \`${tracePath}\`` }
}
}

function metadataLineContains(text: string, field: string, value: string): boolean {
const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
return new RegExp(`^.*\`${escape(field)}\`.*${escape(value)}.*$`, "m").test(text)
Expand All @@ -69,12 +103,12 @@ export const runTraceV1: Contract = {
id: RUN_TRACE_V1,
validate(output, ctx) {
const tracePath = typeof output.trace === "string" ? output.trace : ""
const absolute = tracePath && !isAbsolute(tracePath) ? join(ctx.workdir, tracePath) : tracePath
const exists = Boolean(absolute) && existsSync(absolute)
const text = exists ? readFileSync(absolute, "utf8") : ""
const trace = safeReadWorkdirRelativeTrace(tracePath, ctx.workdir)
const exists = trace.exists
const text = trace.text

const checks: ContractCheck[] = [
{ label: "trace file exists", passed: exists, detail: `expected a readable trace file at \`${tracePath || "<missing trace output field>"}\`` },
{ label: "trace file exists", passed: exists, detail: trace.detail },
{ label: "trace heading recorded", passed: text.trimStart().startsWith("# Trace:"), detail: "expected a `# Trace:` markdown heading" },
{ label: "trace id recorded", passed: text.includes(ctx.traceId), detail: `expected trace id \`${ctx.traceId}\`` },
{ label: "loop id recorded", passed: metadataLineContains(text, "loop_id", ctx.loopId), detail: `expected loop id \`${ctx.loopId}\` in trace metadata` },
Expand Down
53 changes: 51 additions & 2 deletions src/kernel/effects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
// worktree.ts only answers "did anything change?" — this attributes.

import { createHash } from "node:crypto"
import { readdirSync, readFileSync } from "node:fs"
import { lstatSync, readdirSync, readFileSync, readlinkSync, realpathSync, statSync } from "node:fs"
import { join, relative } from "node:path"
import type { EffectScope, OutputProjection } from "./types.js"

Expand All @@ -19,17 +19,63 @@ export function snapshotDir(root: string): Snapshot {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (SKIP_DIRS.has(entry.name)) continue
const full = join(dir, entry.name)
const rel = relative(root, full).split("\\").join("/")
if (entry.isDirectory()) walk(full)
else if (entry.isSymbolicLink()) out.set(rel, symlinkDigest(full))
else if (entry.isFile()) {
const rel = relative(root, full).split("\\").join("/")
out.set(rel, createHash("sha256").update(readFileSync(full)).digest("hex"))
} else {
out.set(rel, `type:${entryType(full)}`)
}
}
}
walk(root)
return out
}

function symlinkDigest(path: string, seen = new Set<string>()): string {
const target = readlinkSync(path)
try {
const stat = statSync(path)
if (stat.isFile()) return `symlink:${target}:file:${createHash("sha256").update(readFileSync(path)).digest("hex")}`
if (stat.isDirectory()) return `symlink:${target}:directory:${directoryDigest(path, seen)}`
Comment thread
futuretyler marked this conversation as resolved.
Resolved
return `symlink:${target}:target:${entryType(path, false)}`
} catch (error) {
const reason = error instanceof Error ? ((error as NodeJS.ErrnoException).code ?? error.message) : String(error)
return `symlink:${target}:unreadable:${reason}`
}
}

function directoryDigest(path: string, seen: Set<string>): string {
const real = realpathSync(path)
if (seen.has(real)) return "cycle"
seen.add(real)
const entries = readdirSync(path, { withFileTypes: true })
.filter((entry) => !SKIP_DIRS.has(entry.name))
.sort((a, b) => a.name.localeCompare(b.name))
const parts = entries.map((entry) => {
const full = join(path, entry.name)
if (entry.isDirectory()) return `${entry.name}/:${directoryDigest(full, seen)}`
if (entry.isSymbolicLink()) return `${entry.name}:${symlinkDigest(full, seen)}`
if (entry.isFile()) return `${entry.name}:file:${createHash("sha256").update(readFileSync(full)).digest("hex")}`
return `${entry.name}:type:${entryType(full)}`
})
seen.delete(real)
return createHash("sha256").update(JSON.stringify(parts)).digest("hex")
}

function entryType(path: string, noFollow = true): string {
const stat = noFollow ? lstatSync(path) : statSync(path)
if (stat.isFile()) return "file"
if (stat.isDirectory()) return "directory"
if (stat.isSymbolicLink()) return "symlink"
if (stat.isFIFO()) return "fifo"
if (stat.isSocket()) return "socket"
if (stat.isBlockDevice()) return "block-device"
if (stat.isCharacterDevice()) return "character-device"
return "unknown"
}

export function changedFiles(before: Snapshot, after: Snapshot): string[] {
const changed = new Set<string>()
for (const [path, digest] of after) if (before.get(path) !== digest) changed.add(path)
Expand All @@ -46,6 +92,9 @@ export interface EffectObservation {
readonly changed: readonly string[]
readonly allowed: boolean
readonly unexpected: readonly string[]
/** False only when a crash made post-step effect observation unknowable. Missing means observed=true for old journals. */
readonly observed?: boolean
readonly reason?: string
}

export function assessChanges(before: Snapshot, after: Snapshot, scope: EffectScope): EffectObservation {
Expand Down
14 changes: 14 additions & 0 deletions src/kernel/policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ export interface Observation {
readonly contractValid: boolean
readonly contractFailedChecks: readonly string[]
readonly effectsAllowed: boolean
/** False when post-step effects are unknowable because the run crashed or observation failed. */
readonly effectsObserved?: boolean
readonly unexpectedChanges: readonly string[]
/** The signature-validated output value of this step; null when outputValid is false. */
readonly output: Readonly<Record<string, unknown>> | null
Expand Down Expand Up @@ -80,6 +82,17 @@ export function decideNextStep(obs: Observation): Decision {
}
}

if (obs.effectsObserved === false) {
const reason = `step \`${obs.stepId}\` has unknown post-step effects.`
return {
kind: "escalate",
classification: "failure",
summary: `${reason} Human review is required before continuing.`,
notes: [reason],
improvement: "Recover or rerun in a clean worktree so effect attribution is explicit before promotion.",
}
}

if (obs.stepStatus !== "completed") {
const reason = `executor \`${obs.executorId}\` ${obs.stepStatus === "interrupted" ? "was interrupted" : "failed"} on step \`${obs.stepId}\`.`
return {
Expand All @@ -106,6 +119,7 @@ export function decideNextStep(obs: Observation): Decision {
}
}


if (!obs.effectsAllowed || obs.unexpectedChanges.length > 0) {
const notes: string[] = []
if (!obs.effectsAllowed) notes.push("At least one attributed change was outside the allowed effect scope.")
Expand Down
4 changes: 2 additions & 2 deletions src/ledger/evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ export function projectRunEvidence(input: ProjectRunEvidenceInput): RunEvidenceP
const contractStatus = contractEvidence(contract)
const effectsStatus = effectEvidence(effects)
const unexpectedEffects = effects?.observation.unexpected.map(String) ?? []
const observedEffects = effects ? true : null
const observedEffects = effects ? effects.observation.observed !== false : null

steps.push({
key: result.key,
Expand Down Expand Up @@ -232,7 +232,7 @@ function contractEvidence(contract: { readonly result: { readonly valid: boolean
}

function effectEvidence(effects: EffectsEntry | undefined): EffectEvidenceStatus {
if (!effects) return "unknown"
if (!effects || effects.observation.observed === false) return "unknown"
return effects.observation.allowed ? "passed" : "failed"
}

Expand Down
Loading
Loading