Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -3297,15 +3297,12 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
"podName": {
"type": "string",
},
"version": {
"type": "string",
},
},
"required": [
"instanceId",
],
"type": "object",
},
"workerVersion": {
"type": "string",
},
},
"required": [
"_id",
Expand Down
54 changes: 53 additions & 1 deletion apps/api/src/routes/requests/archive.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -781,7 +781,7 @@ describe("run import/export — round-trip (export → import)", () => {
startedAt: new Date("2026-05-11T23:04:37.172Z"),
finishedAt: new Date("2026-05-11T23:04:47.446Z"),
durationMs: 10274, // finishedAt − startedAt; computed on write at import time
workerVersion: "copilot-unknown-unknown-unknown",
worker: { instanceId: "instance-rich-1", version: "copilot-unknown-unknown-unknown" },
aiCallCount: 2,
os: { platform: "linux", release: "6.6.114.1", arch: "x64" },
tokenUsage: { promptTokens: 16, completionTokens: 397, totalTokens: 413 },
Expand Down Expand Up @@ -953,6 +953,58 @@ describe("run import/export — round-trip (export → import)", () => {
});
});

describe("run import — legacy run.workerVersion (#855)", () => {
/** Minimal terminal run.yaml; `runExtra` lines are appended under `run:`. */
function makeYaml(id: string, runExtra: string[]) {
return [
`_id: ${id}`,
`projectId: ${TEST_PROJECT_ID}`,
`scenario:`,
` task: "legacy worker version"`,
` criteria: []`,
`workerType: coder-acp-copilot`,
`createdAt: 2026-01-01T00:00:00Z`,
`run:`,
` _id: ${id}`,
` attemptNumber: 1`,
` status: done`,
...runExtra,
].join("\n") + "\n";
}

/** Upload a single-run archive and return the inserted request document. */
async function importYaml(id: string, yaml: string) {
const reqCollection = makeRequestCollection();
const app = buildApp(reqCollection, makeTaskPromptStore());
const archive = await buildTarGz({ [`${id}/run.yaml`]: yaml });

const res = await request(app)
.post(`/api/v1/runs/upload?projectId=${TEST_PROJECT_ID}`)
.attach("archive", archive, "archive.tar.gz");

expect(res.status).toBe(201);
return reqCollection.docs.get(id);
}

it("moves a legacy run.workerVersion into run.worker.version", async () => {
const inserted = await importYaml("legacy-wv-1", makeYaml("legacy-wv-1", [` workerVersion: copilot-1.0-20260101T000000Z-abc`]));

expect(inserted.run.worker).toEqual({ version: "copilot-1.0-20260101T000000Z-abc" });
expect(inserted.run).not.toHaveProperty("workerVersion");
});

it("keeps existing worker identity alongside the folded version", async () => {
const inserted = await importYaml("legacy-wv-2", makeYaml("legacy-wv-2", [
` workerVersion: copilot-1.0-x-y`,
` worker:`,
` instanceId: inst-1`,
` podName: pod-a`,
]));

expect(inserted.run.worker).toEqual({ instanceId: "inst-1", podName: "pod-a", version: "copilot-1.0-x-y" });
});
});

describe("run import — task-prompt entity creation (#832)", () => {
// Task-prompt entities back features, report triggers, group-by-task,
// the runs-list task filter and the criteria/MDP analysis. The submission
Expand Down
10 changes: 9 additions & 1 deletion apps/api/src/routes/requests/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
RunFacetsResponseSchema,
RunStateSchema,
decodeCursor,
foldLegacyWorkerVersion,
encodeCursor,
runDurationMs,
parseExtensionSpec,
Expand Down Expand Up @@ -2478,6 +2479,13 @@ async function finalizePendingRun(
throw new ImportError(400, `Failed to parse run.yaml: ${parseErr}`);
}

// Archives exported before issue #855 carry `run.workerVersion`; move it to
// `run.worker.version` so validation doesn't strip it as an unknown key.
if (runDocRaw && typeof runDocRaw === "object" && "run" in runDocRaw) {
const raw = runDocRaw as { run: unknown };
raw.run = foldLegacyWorkerVersion(raw.run);
}

const ImportSchema = RequestResponseSchema.extend({ run: RunStateSchema });
const parsed = ImportSchema.safeParse(runDocRaw);
if (!parsed.success) {
Expand Down Expand Up @@ -2589,7 +2597,7 @@ async function finalizePendingRun(
// stripped any unknown fields, so what's in `runDoc` is exactly the
// optional surface we care to preserve (taskPromptId, model, agentVersion,
// mcpServers, skillRevisions, extensions, profileId, profileVersionId,
// run.os, run.workerVersion, run.aiCallCount, run.startedAt,
// run.os, run.worker.version, run.aiCallCount, run.startedAt,
// run.finishedAt, …). The previous allowlist construction silently
// dropped all of these on round-trip.
const docToInsert: RequestDocument = {
Expand Down
5 changes: 5 additions & 0 deletions apps/cli/src/run-get-action.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ export async function runGetAction(options: RunGetOptions): Promise<void> {
{ key: 'workerType', label: 'Worker' },
{ key: 'model', label: 'Model' },
{ key: 'profileId', label: 'Profile', formatter: (r: any) => r.profileId ?? '' },
{ key: 'agentVersion', label: 'Agent Version' },
{ key: 'workerVersion', label: 'Worker Version' },
{ key: 'status', label: 'Status' },
{ key: 'maxIterations', label: 'Max Iterations' },
{ key: 'gatesCount', label: 'Gates' },
Expand All @@ -74,6 +76,7 @@ export async function runGetAction(options: RunGetOptions): Promise<void> {
status: rs?.status,
outcome: rs?.outcome,
error: rs?.error,
workerVersion: rs?.worker?.version,
turnsCount: rs?.turns?.length ?? 0,
gatesCount: run.gates?.length ?? run.gateSummaries?.length ?? 0,
gateSummaries: (run.gateSummaries ?? []).map((g) => `${g.gate}:${g.status}`).join(', '),
Expand All @@ -91,6 +94,8 @@ export async function runGetAction(options: RunGetOptions): Promise<void> {
if (run.model) console.log(`${label('Model:')} ${value(run.model)}`);
if (run.profileId) console.log(`${label('Profile:')} ${value(run.profileId)}`);
if (run.profileVersionId) console.log(`${label('Profile Ver:')} ${dimTimestamp(run.profileVersionId)}`);
if (run.agentVersion) console.log(`${label('Agent Ver:')} ${value(run.agentVersion)}`);
if (rs?.worker?.version) console.log(`${label('Worker Ver:')} ${dimTimestamp(rs.worker.version)}`);

const statusColor = rs?.outcome === 'succeeded' ? successText
: (rs?.outcome === 'failed' || rs?.outcome === 'finished') ? errorText
Expand Down
24 changes: 24 additions & 0 deletions apps/cli/src/run-get.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,30 @@ describe('run get', () => {
});
});

describe('worker version (#855)', () => {
it('shows run.worker.version in table and JSON output', async () => {
const run = { ...fullRun, run: { ...fullRun.run, worker: { instanceId: 'i-1', version: 'claude-1.0-20260101T000000Z-abc' } } };
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: () => Promise.resolve(run) }));
const { runGetAction } = await import('./run-get-action.js');

const table = captureConsole();
try {
await runGetAction({ id: 'run-002', url: 'http://localhost:3100', output: 'table' });
} finally {
table.restore();
}
expect(table.lines.join('\n')).toMatch(/Worker Ver:.*claude-1\.0-20260101T000000Z-abc/);

const json = captureConsole();
try {
await runGetAction({ id: 'run-002', url: 'http://localhost:3100', output: 'json' });
} finally {
json.restore();
}
expect(JSON.parse(json.lines.join('\n'))[0].workerVersion).toBe('claude-1.0-20260101T000000Z-abc');
});
});

describe('error handling', () => {
it('exits with error on 404', async () => {
const mockFetch = vi.fn().mockResolvedValue({
Expand Down
10 changes: 6 additions & 4 deletions apps/portal/src/components/StatusBadge.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,12 @@ export function StatusBadge({
<div className="space-y-0.5 text-xs">
{worker && (
<>
<div>
<span className="text-muted-foreground">Worker: </span>
<span className="font-mono">{worker.instanceId}</span>
</div>
{worker.instanceId && (
<div>
<span className="text-muted-foreground">Worker: </span>
<span className="font-mono">{worker.instanceId}</span>
</div>
)}
{worker.podName && (
<div>
<span className="text-muted-foreground">Pod: </span>
Expand Down
8 changes: 4 additions & 4 deletions apps/portal/src/pages/RunDetail.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -692,7 +692,7 @@ export function RunDetail() {
{run.agentVersion && (
<MetaItem
label="Version"
title={activeRun?.workerVersion ? `Worker: ${activeRun?.workerVersion}` : undefined}
title={activeRun?.worker?.version ? `Worker: ${activeRun.worker.version}` : undefined}
value={<span className="font-mono">{run.agentVersion}</span>}
/>
)}
Expand Down Expand Up @@ -1363,7 +1363,7 @@ export function RunDetail() {
)}

{/* Version Info card */}
{(run.agentVersion || activeRun?.workerVersion || activeRun?.os) && (
{(run.agentVersion || activeRun?.worker?.version || activeRun?.os) && (
<Card>
<CardHeader>
<CardTitle className="text-lg">Version Info</CardTitle>
Expand All @@ -1375,10 +1375,10 @@ export function RunDetail() {
<span className="font-mono font-medium">{run.agentVersion}</span>
</div>
)}
{activeRun?.workerVersion && (
{activeRun?.worker?.version && (
<div>
<span className="text-muted-foreground">Worker Version:</span>{" "}
<span className="font-mono font-medium">{activeRun?.workerVersion}</span>
<span className="font-mono font-medium">{activeRun.worker.version}</span>
</div>
)}
{activeRun?.os && (
Expand Down
6 changes: 3 additions & 3 deletions apps/portal/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,12 @@ export interface RunState {
error?: string;
logsUrl?: string;
turns?: ConversationTurn[];
workerVersion?: string;
os?: OsInfo;
/** Wall-clock time the owning worker last extended visibility for this run's queue message. */
lastHeartbeatAt?: string;
/** Identity of the worker process currently processing the run. */
worker?: { instanceId: string; podName?: string };
/** Identity and build of the worker process that handled the run (issue #855).
* `instanceId` is absent on legacy runs where only the version was recorded. */
worker?: { instanceId?: string; podName?: string; version?: string };
harUrl?: string;
videoUrls?: string[];
setupVideoUrls?: string[];
Expand Down
21 changes: 18 additions & 3 deletions apps/scheduler/src/stuck-run-reaper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ describe("StuckRunReaper", () => {
expect(errSpy).toHaveBeenCalled();
});

it("uses {run.worker:{$exists:false}} filter when worker identity is absent", async () => {
it("uses {run.worker.instanceId:{$exists:false}} filter when worker identity is absent", async () => {
const docs = [{ _id: "req-1", run: { _id: "run-1", startedAt: OLD() } }];
const collection = mockCollection(docs);
const store = new InMemoryHeartbeatStore();
Expand All @@ -228,8 +228,23 @@ describe("StuckRunReaper", () => {
await reaper.sweep();
await reaper.sweep();
const [filter] = collection.findOneAndUpdate.mock.calls[0];
expect(filter).toMatchObject({ "run.worker": { $exists: false } });
expect(filter).not.toHaveProperty("run.worker.instanceId");
expect(filter).toMatchObject({ "run.worker.instanceId": { $exists: false } });
expect(filter).not.toHaveProperty("run.worker");
});

it("treats a legacy version-only worker sub-document as having no owner (#855)", async () => {
// Migration 031 moves a pre-#846 run's version into run.worker.version
// without an instanceId — the claim must still match that document.
const docs = [{ _id: "req-1", run: { _id: "run-1", startedAt: OLD(), worker: { version: "v-legacy" } } }];
const collection = mockCollection(docs);
const store = new InMemoryHeartbeatStore();
const reaper = makeReaper(collection, store);

await reaper.sweep();
await reaper.sweep();
const [filter, update] = collection.findOneAndUpdate.mock.calls[0];
expect(filter).toMatchObject({ "run.worker.instanceId": { $exists: false } });
expect(update.$set["run.error"]).toMatch(/instance=unknown/);
});

it("does not delete the heartbeat when the claim no-ops (ownership changed)", async () => {
Expand Down
4 changes: 2 additions & 2 deletions apps/scheduler/src/stuck-run-reaper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ export class StuckRunReaper {
if (startedAt && startedAt.getTime() > cutoff) continue;
const worker = run.worker;
const workerDesc = worker
? `instance=${worker.instanceId}${worker.podName ? ` pod=${worker.podName}` : ""}`
? `instance=${worker.instanceId ?? "unknown"}${worker.podName ? ` pod=${worker.podName}` : ""}`
: "unknown worker";
out.push({
_id: d._id as any,
Expand Down Expand Up @@ -335,7 +335,7 @@ export class StuckRunReaper {
"run.status": "processing",
...(c.ownerInstanceId
? { "run.worker.instanceId": c.ownerInstanceId }
: { "run.worker": { $exists: false } }),
: { "run.worker.instanceId": { $exists: false } }),
} as any,
{
$set: {
Expand Down
2 changes: 2 additions & 0 deletions docs/architecture/db-migrations.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ Defaults match the `docker-compose.yml` local dev environment, so no configurati
### Common patterns

- **RU-paced backfill** — use `batchUpdate(col, filter, update, label)` from `../batch-update.js` (collects `_id`s, then `updateMany({_id:{$in:batch}}, update)` in small batches with 429 retry). Static update only.
- **Per-document value copy** — when each document's new value comes from another of its own fields (e.g. 031 moving `run.workerVersion` → `run.worker.version`), iterate a projected cursor and write each doc with `updateOneWithRetry(col, filter, update, label)` from `../batch-update.js`, pacing with `sleep(INTER_BATCH_DELAY_MS)`. Avoid aggregation-pipeline updates (`$set: "$field"`) for Cosmos compatibility.
- **Per-document backfill referencing `_id`** — when the new value derives from each doc's own `_id` (e.g. `keyId = _id`), `batchUpdate` can't express it. Use a paced `bulkWrite` loop of `updateOne` ops instead (mirror `025-create-projects`), reusing `BATCH_SIZE`/`sleep`/`getRetryAfterMs` for pacing and 429 handling.
- **Idempotent index create** — wrap `createIndex` in try/catch and tolerate "already exists" codes (`85`/`86`/`68`); rethrow real failures so validation surfaces them.
- **Unique-index swap** — before creating a composite **unique** index on a non-empty collection, `$group`/`$match` to assert no duplicate keys exist first (a pre-existing collision fails index creation on Cosmos). Drop the old index tolerating "not found" (`26`/`27`). `down()` for index ops is log-only by convention.
Expand Down Expand Up @@ -114,6 +115,7 @@ When you run a command:
| `027-uuid-keys-mcp-profileversions` | Opaque UUID `_id` + reference key for `mcp-servers` (`slug`) and `profile-versions` (`ref`) with `{projectId,slug}` / `{projectId,ref}` indexes; drops dead `prompt-feature-extractions` (see [db.md](db.md#per-project-entity-keying-migration-027)) |
| `028-isolate-mcp-secrets-per-project` | Reconciles the token-manager `mcp-secrets` unique index — drops the legacy global-unique `{mcpId,name}` and (re)creates the per-project `{projectId,mcpId,name}` (see [token-manager.md](token-manager.md#mcp-secrets)) |
| `029-create-users-collection` | Creates the database-enforced unique identity index for authentication; uses Cosmos collection-creation extensions for continuous-backup accounts and preserves native MongoDB index creation (see below) |
| `031-move-worker-version` | Moves the worker build into the worker sub-document: `requests.run.workerVersion` → `run.worker.version` and `runs.workerVersion` → `worker.version` (issue #855); never overwrites an existing `worker.version`; unsets the legacy field. Pre-#846 runs get a version-only `worker` (no `instanceId`). `down()` reverses the move |

### Migration 029: users identity uniqueness

Expand Down
2 changes: 1 addition & 1 deletion docs/architecture/overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ walkthrough, failures, cache isolation, and implemented/deferred boundary.
### Benchmark execution

1. **Submit** — A user submits a task via CLI or Portal, selecting a worker, model, criteria, and optionally an agent version. The API validates the selection (model must be in `supportedModels`, version must be active, at least one criterion required), resolves the agent version's queue, creates a run record in CosmosDB, and enqueues a message.
2. **Execute** — KEDA scales the target worker pod from 0→N. The worker dequeues the message, spins up the coding agent, and executes the task. The worker stamps `workerVersion` (exact build identity) on the run.
2. **Execute** — KEDA scales the target worker pod from 0→N. The worker dequeues the message, spins up the coding agent, and executes the task. The worker stamps `run.worker = { instanceId, podName?, version }` (process identity plus exact build) on the run.
3. **Stream** — Workers publish real-time log events to Redis Pub/Sub. The API relays these as SSE streams to the CLI/Portal.
4. **Judge** — After the agent completes, the worker invokes the Judge to evaluate output against criteria. Results (pass/fail per criterion, scores) are persisted to CosmosDB.
5. **Snapshot** — Each iteration's workspace is snapshotted to Blob Storage for later inspection.
Expand Down
Loading