From bdb3b5b46fbaecd87b8a46a7242b2415f1f28a94 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 27 Oct 2026 23:47:56 -0700 Subject: [PATCH 1/2] feat(cli): republish submission archive workflow upstream Carry the existing CLI feature from #1563 onto fresh microsoft/scope main without Taxonome, Observation, or Experiment ancestry. Source commits: 2f7ecb99a1aca44c1cbc5c8ab59a15c28982caf3, d5e75c49bdcebf915ac00b8696424fa009872259, 7a7db97735e73d0fe32f1e49aa443286ea838296, 072a8307d5299ad0cae8ead735f4b07de1a18764. Preserve CLI code, tests, and snapshot exactly. Correct standalone documentation to avoid advertising absent Experiment commands or API filters. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- apps/cli/README.md | 19 ++ .../__snapshots__/cli-commands.test.ts.snap | 2 + .../src/commands/run-download-batch.test.ts | 265 ++++++++++++++++++ apps/cli/src/commands/run-request-ids.test.ts | 155 ++++++++++ apps/cli/src/commands/run-request-ids.ts | 192 +++++++++++++ apps/cli/src/commands/run.test.ts | 13 + apps/cli/src/commands/run.ts | 86 +++--- apps/cli/src/utils/archive-download.test.ts | 153 ++++++++++ apps/cli/src/utils/archive-download.ts | 205 ++++++++++++++ docs/architecture/app-design.md | 8 +- docs/architecture/cli-distribution.md | 11 + .../docs/guides/submitting-requests-cli.md | 16 ++ 12 files changed, 1090 insertions(+), 35 deletions(-) create mode 100644 apps/cli/src/commands/run-download-batch.test.ts create mode 100644 apps/cli/src/commands/run-request-ids.test.ts create mode 100644 apps/cli/src/commands/run-request-ids.ts create mode 100644 apps/cli/src/utils/archive-download.test.ts create mode 100644 apps/cli/src/utils/archive-download.ts diff --git a/apps/cli/README.md b/apps/cli/README.md index b40bc2b5d..00f7ba767 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -73,6 +73,7 @@ Never print or persist tokens in logs. See [the auth contract](../../docs/archit ```bash # List all benchmark runs scope run list +scope run list --submission-id # Submit a run scope run submit -s config/scenarios/hello-world.yaml -w coder-acp-copilot @@ -83,6 +84,11 @@ scope run get -i # Stream logs scope run logs -i +# Download a batch by explicit IDs, submission, or experiment +scope run download-batch --ids +scope run download-batch --submission-id +scope run download-batch --experiment-id + # List criteria scope criteria list @@ -91,6 +97,19 @@ scope --help scope run --help ``` +`run download-batch` accepts exactly one of `--ids`, `--submission-id`, or +`--experiment-id`. Submission/experiment selectors require a project and must +resolve to at most 100 runs, matching the batch archive endpoint limit. +The experiment selector requires an API deployment that supports the +`experimentId` request filter; the standalone CLI change does not add that API +capability or an `experiment` command group. + +Run archives download to an exclusive same-directory temporary file while +stderr reports compressed-byte progress. Scope validates a declared +`Content-Length` and the complete gzip stream/trailer before atomically +renaming the file into place. Interrupted or corrupt downloads remove the +temporary file and never overwrite an existing archive. + ### Provider secrets `scope secret` manages provider credentials (GitHub tokens, Anthropic keys, diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index 66f3e414c..36f31a637 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -758,6 +758,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "download-batch": { "options": [ "--dir", + "--experiment-id", "--extract", "--ids", "--output", @@ -779,6 +780,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--created-after", "--created-before", "--criteria", + "--experiment-id", "--include-deleted", "--max-iterations", "--model", diff --git a/apps/cli/src/commands/run-download-batch.test.ts b/apps/cli/src/commands/run-download-batch.test.ts new file mode 100644 index 000000000..14e48ef9a --- /dev/null +++ b/apps/cli/src/commands/run-download-batch.test.ts @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Command } from "commander"; +import { registerRunCommands } from "./run.js"; + +function makeProgram(): Command { + const program = new Command(); + registerRunCommands(program); + return program; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + delete process.env.SCOPE_MT_DOWNLOAD_OUTPUT_DIR; +}); + +describe("run download-batch", () => { + it("preserves explicit --ids behavior and archive request order", async () => { + const directory = await mkdtemp(join(tmpdir(), "scope-batch-")); + process.env.SCOPE_MT_DOWNLOAD_OUTPUT_DIR = directory; + let archiveBody: unknown; + const archive = gzipSync(Buffer.from("explicit IDs archive")); + vi.stubGlobal("fetch", vi.fn(async (request: Request) => { + archiveBody = JSON.parse(await request.text()) as unknown; + return new Response(archive, { + status: 200, + headers: { "Content-Length": String(archive.length) }, + }); + + describe("run download", () => { + it("uses the validated atomic downloader for a single run", async () => { + const directory = await mkdtemp(join(tmpdir(), "scope-single-download-")); + const outputPath = join(directory, "run-1.tar.gz"); + const archive = gzipSync(Buffer.from("single run archive")); + let requestCount = 0; + vi.stubGlobal("fetch", vi.fn(async () => { + requestCount++; + if (requestCount === 1) { + return new Response(JSON.stringify({ + id: "run-1", + workerType: "coder-acp-copilot", + run: { status: "done", turns: [{ iteration: 1 }] }, + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(archive, { + status: 200, + headers: { "Content-Length": String(archive.length) }, + }); + })); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + try { + await makeProgram().parseAsync([ + "run", + "download", + "--id", + "run-1", + "--output", + outputPath, + "-u", + "http://localhost:3100", + ], { from: "user" }); + + expect(requestCount).toBe(2); + expect(await readFile(outputPath)).toEqual(archive); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + }); + })); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + try { + await makeProgram().parseAsync([ + "run", + "download-batch", + "--ids", + "run-2", + "run-1", + "run-2", + "-u", + "http://localhost:3100", + ], { from: "user" }); + + expect(archiveBody).toEqual({ ids: ["run-2", "run-1"] }); + const files = await import("node:fs/promises").then(fs => + fs.readdir(directory) + ); + expect(files).toHaveLength(1); + expect(await readFile(join(directory, files[0]))).toEqual( + archive, + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("resolves a submission through cursor pages before posting exact stable IDs", async () => { + const directory = await mkdtemp(join(tmpdir(), "scope-submission-batch-")); + process.env.SCOPE_MT_DOWNLOAD_OUTPUT_DIR = directory; + const submissionId = "01234567-89ab-4cde-8fab-0123456789ab"; + const archive = gzipSync(Buffer.from("submission archive")); + const requests: Array<{ url: URL; method: string; body: string }> = []; + vi.stubGlobal("fetch", vi.fn(async (request: Request) => { + const captured = { + url: new URL(request.url), + method: request.method, + body: await request.text(), + }; + requests.push(captured); + + if (captured.url.pathname.endsWith("/requests/archive")) { + return new Response(archive, { + status: 200, + headers: { "Content-Length": String(archive.length) }, + }); + } + if (!captured.url.searchParams.has("after")) { + return new Response(JSON.stringify({ + data: [ + { id: "run-exact-1", submissionId }, + { id: "run-prefix-only", submissionId: `${submissionId}-suffix` }, + ], + cursors: { next: "cursor / next?" }, + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ + data: [ + { id: "run-exact-1", submissionId }, + { id: "run-exact-2", submissionId }, + ], + cursors: { next: null }, + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + })); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + try { + await makeProgram().parseAsync([ + "run", + "download-batch", + "--submission-id", + submissionId, + "--project", + "project-1", + "-u", + "http://localhost:3100", + ], { from: "user" }); + + expect(requests).toHaveLength(3); + for (const request of requests.slice(0, 2)) { + expect(request.method).toBe("GET"); + expect(request.url.searchParams.get("submissionId")) + .toBe(submissionId); + expect(request.url.searchParams.get("projectId")).toBe("project-1"); + expect(request.url.searchParams.get("limit")).toBe("100"); + } + expect(requests[0].url.searchParams.get("after")).toBeNull(); + expect(requests[1].url.searchParams.get("after")) + .toBe("cursor / next?"); + expect(requests[2].url.pathname).toMatch(/\/requests\/archive$/); + expect(requests[2].method).toBe("POST"); + expect(JSON.parse(requests[2].body)).toEqual({ + ids: ["run-exact-1", "run-exact-2"], + }); + + const files = await import("node:fs/promises").then(fs => + fs.readdir(directory) + ); + expect(files).toHaveLength(1); + expect(await readFile(join(directory, files[0]))).toEqual( + archive, + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("rejects multiple ID sources before network I/O", async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process, "exit").mockImplementation( + (code?: string | number | null): never => { + throw new Error(`process.exit(${String(code)})`); + }, + ); + + await expect(makeProgram().parseAsync([ + "run", + "download-batch", + "--ids", + "run-1", + "--submission-id", + "submission-1", + "-u", + "http://localhost:3100", + ], { from: "user" })).rejects.toThrow("process.exit(1)"); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("does not publish a truncated gzip archive", async () => { + const directory = await mkdtemp(join(tmpdir(), "scope-truncated-batch-")); + const outputPath = join(directory, "batch.tar.gz"); + await import("node:fs/promises").then(fs => + fs.writeFile(outputPath, "existing-valid-archive") + ); + const truncated = gzipSync(Buffer.from("large archive contents")) + .subarray(0, -4); + vi.stubGlobal("fetch", vi.fn(async () => + new Response(truncated, { + status: 200, + headers: { "Content-Length": String(truncated.length) }, + }) + )); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + vi.spyOn(process, "exit").mockImplementation( + (code?: string | number | null): never => { + throw new Error(`process.exit(${String(code)})`); + }, + ); + + try { + await expect(makeProgram().parseAsync([ + "run", + "download-batch", + "--ids", + "run-1", + "--output", + outputPath, + "-u", + "http://localhost:3100", + ], { from: "user" })).rejects.toThrow("process.exit(1)"); + expect(await readFile(outputPath, "utf8")) + .toBe("existing-valid-archive"); + const files = await import("node:fs/promises").then(fs => + fs.readdir(directory) + ); + expect(files.some(name => name.includes(".part-"))).toBe(false); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/cli/src/commands/run-request-ids.test.ts b/apps/cli/src/commands/run-request-ids.test.ts new file mode 100644 index 000000000..09f0f2d24 --- /dev/null +++ b/apps/cli/src/commands/run-request-ids.test.ts @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it, vi } from "vitest"; +import { + assertBatchArchiveLimit, + assertResolvedRequestIds, + isFullUuid, + resolvePaginatedRequestIds, + selectBatchIdSource, +} from "./run-request-ids.js"; + +function response(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +describe("run download-batch ID resolution", () => { + it("requires exactly one source and preserves explicit IDs", () => { + expect(() => selectBatchIdSource({})) + .toThrow("exactly one of --ids, --experiment-id, or --submission-id"); + expect(() => selectBatchIdSource({ + ids: ["run-1"], + submissionId: "submission-1", + })).toThrow("exactly one"); + expect(selectBatchIdSource({ ids: ["run-1", "run-2"] })).toEqual({ + kind: "ids", + ids: ["run-1", "run-2"], + }); + }); + + it("walks cursors with encoded filters and stable deduplication", async () => { + const fetcher = vi.fn() + .mockResolvedValueOnce(response({ + data: [{ id: "run-1" }, { _id: "run-2" }], + cursors: { next: "cursor / two?" }, + })) + .mockResolvedValueOnce(response({ + data: [{ id: "run-2" }, { id: "run-3" }], + cursors: { next: null }, + })); + + const ids = await resolvePaginatedRequestIds({ + baseUrl: "http://localhost:3100", + projectId: "project-1", + filter: "experimentId", + value: "experiment / value?", + fetcher, + }); + + expect(ids).toEqual(["run-1", "run-2", "run-3"]); + expect(fetcher).toHaveBeenCalledTimes(2); + const firstPath = fetcher.mock.calls[0][1] as string; + const secondPath = fetcher.mock.calls[1][1] as string; + expect(new URL(`http://scope${firstPath}`).searchParams.get("experimentId")) + .toBe("experiment / value?"); + expect(new URL(`http://scope${firstPath}`).searchParams.get("limit")) + .toBe("100"); + expect(new URL(`http://scope${secondPath}`).searchParams.get("after")) + .toBe("cursor / two?"); + expect(fetcher.mock.calls[0][2]).toEqual({ projectId: "project-1" }); + }); + + it("filters a full UUID submission selector to exact matches", async () => { + const submissionId = "01234567-89ab-4cde-8fab-0123456789ab"; + expect(isFullUuid(submissionId)).toBe(true); + const fetcher = vi.fn().mockResolvedValue(response({ + data: [ + { id: "exact-1", submissionId }, + { id: "prefix-only", submissionId: `${submissionId}-suffix` }, + { id: "other", submissionId: "01234567-89ab-4cde-8fab-000000000000" }, + { id: "exact-2", submissionId }, + ], + cursors: { next: null }, + })); + + await expect(resolvePaginatedRequestIds({ + baseUrl: "http://localhost:3100", + projectId: "project-1", + filter: "submissionId", + value: submissionId, + fetcher, + })).resolves.toEqual(["exact-1", "exact-2"]); + const path = fetcher.mock.calls[0][1] as string; + expect(new URL(`http://scope${path}`).searchParams.get("submissionId")) + .toBe(submissionId); + expect(fetcher.mock.calls[0][2]).toEqual({ projectId: "project-1" }); + }); + + it("rejects empty exact results and selections above the archive limit", async () => { + const submissionId = "01234567-89ab-4cde-8fab-0123456789ab"; + const empty = vi.fn().mockResolvedValue(response({ + data: [{ id: "prefix", submissionId: `${submissionId}-suffix` }], + cursors: { next: null }, + })); + await expect(resolvePaginatedRequestIds({ + baseUrl: "http://localhost:3100", + projectId: "project-1", + filter: "submissionId", + value: submissionId, + fetcher: empty, + })).resolves.toEqual([]); + expect(() => assertResolvedRequestIds([], "submission")) + .toThrow("No exact runs found for this submission"); + + expect(() => assertBatchArchiveLimit( + Array.from({ length: 101 }, (_, index) => `run-${index}`), + )).toThrow("at most 100 run IDs"); + }); + + it("rejects a paginated selector as soon as it exceeds 100 IDs", async () => { + const fetcher = vi.fn() + .mockResolvedValueOnce(response({ + data: Array.from({ length: 100 }, (_, index) => ({ + id: `run-${index}`, + })), + cursors: { next: "next" }, + })) + .mockResolvedValueOnce(response({ + data: [{ id: "run-100" }], + cursors: { next: null }, + })); + + await expect(resolvePaginatedRequestIds({ + baseUrl: "http://localhost:3100", + projectId: "project-1", + filter: "experimentId", + value: "experiment-1", + fetcher, + })).rejects.toThrow("resolved 101"); + expect(fetcher).toHaveBeenCalledTimes(2); + }); + + it("rejects repeated pagination cursors", async () => { + const fetcher = vi.fn() + .mockResolvedValueOnce(response({ + data: [{ id: "run-1" }], + cursors: { next: "same-cursor" }, + })) + .mockResolvedValueOnce(response({ + data: [{ id: "run-2" }], + cursors: { next: "same-cursor" }, + })); + + await expect(resolvePaginatedRequestIds({ + baseUrl: "http://localhost:3100", + projectId: "project-1", + filter: "experimentId", + value: "experiment-1", + fetcher, + })).rejects.toThrow("repeated cursor"); + }); +}); diff --git a/apps/cli/src/commands/run-request-ids.ts b/apps/cli/src/commands/run-request-ids.ts new file mode 100644 index 000000000..ef6483c24 --- /dev/null +++ b/apps/cli/src/commands/run-request-ids.ts @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { apiFetch } from "../utils/api-client.js"; + +export const MAX_BATCH_ARCHIVE_IDS = 100; + +/** One supported source of run IDs for `run download-batch`. */ +export type BatchIdSource = + | { kind: "ids"; ids: string[] } + | { kind: "experiment"; value: string } + | { kind: "submission"; value: string }; + +/** Minimal request-list item needed for batch resolution. */ +interface RequestListItem { + id: string; + experimentId?: string; + submissionId?: string; +} + +/** Cursor-paged request-list response. */ +interface RequestListPage { + data: RequestListItem[]; + next: string | null; +} + +type RequestFetcher = typeof apiFetch; + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; +} + +/** Narrows the request-list JSON boundary immediately after parsing. */ +function parseRequestListPage(value: unknown): RequestListPage { + const page = record(value); + const rawData = Array.isArray(value) + ? value + : Array.isArray(page?.data) + ? page.data + : undefined; + if (!rawData) { + throw new Error("Request list response is missing a data array"); + } + + const data = rawData.map((raw, index) => { + const item = record(raw); + const id = typeof item?.id === "string" + ? item.id + : typeof item?._id === "string" + ? item._id + : undefined; + if (!id) { + throw new Error(`Request list item ${index} is missing an id`); + } + return { + id, + ...(typeof item?.experimentId === "string" + ? { experimentId: item.experimentId } + : {}), + ...(typeof item?.submissionId === "string" + ? { submissionId: item.submissionId } + : {}), + }; + }); + + const cursors = record(page?.cursors); + const next = typeof cursors?.next === "string" && cursors.next + ? cursors.next + : null; + return { data, next }; +} + +/** Returns true only for a complete canonical UUID selector. */ +export function isFullUuid(value: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + .test(value); +} + +/** Validates the exactly-one ID source contract. */ +export function selectBatchIdSource(options: { + ids?: string[]; + experimentId?: string; + submissionId?: string; +}): BatchIdSource { + const sources = [ + options.ids?.length ? "ids" : undefined, + options.experimentId ? "experiment" : undefined, + options.submissionId ? "submission" : undefined, + ].filter((source): source is string => source !== undefined); + + if (sources.length !== 1) { + throw new Error( + "Provide exactly one of --ids, --experiment-id, or --submission-id", + ); + } + if (sources[0] === "ids") { + return { kind: "ids", ids: options.ids! }; + } + if (sources[0] === "experiment") { + return { kind: "experiment", value: options.experimentId! }; + } + return { kind: "submission", value: options.submissionId! }; +} + +/** Enforces the request archive endpoint's documented ID limit. */ +export function assertBatchArchiveLimit(ids: readonly string[]): void { + if (ids.length > MAX_BATCH_ARCHIVE_IDS) { + throw new Error( + `Batch archive supports at most ${MAX_BATCH_ARCHIVE_IDS} run IDs; ` + + `the selector resolved ${ids.length}. Narrow the selection or download smaller batches.`, + ); + } +} + +/** Fails when a resolved experiment/submission contains no exact runs. */ +export function assertResolvedRequestIds( + ids: readonly string[], + source: "experiment" | "submission", +): void { + if (ids.length === 0) { + throw new Error(`No exact runs found for this ${source}`); + } +} + +/** + * Resolves request IDs from a cursor-paged API filter. + * + * Submission UUIDs are filtered exactly because the API filter is prefix-based. + */ +export async function resolvePaginatedRequestIds(options: { + baseUrl: string | undefined; + projectId: string; + filter: "experimentId" | "submissionId"; + value: string; + fetcher?: RequestFetcher; +}): Promise { + const fetcher = options.fetcher ?? apiFetch; + const ids: string[] = []; + const seenIds = new Set(); + const seenCursors = new Set(); + let after: string | undefined; + const requireExactSubmission = + options.filter === "submissionId" && isFullUuid(options.value); + + do { + const params = new URLSearchParams({ + [options.filter]: options.value, + limit: "100", + }); + if (after) params.set("after", after); + + const response = await fetcher( + options.baseUrl, + `/requests?${params.toString()}`, + { projectId: options.projectId }, + ); + if (!response.ok) { + const body: unknown = await response.json().catch(() => undefined); + const message = typeof record(body)?.error === "string" + ? String(record(body)?.error) + : `HTTP ${response.status}`; + throw new Error(`Failed to resolve runs: ${message}`); + } + + const page = parseRequestListPage(await response.json() as unknown); + for (const request of page.data) { + if ( + requireExactSubmission && + request.submissionId !== options.value + ) { + continue; + } + if (!seenIds.has(request.id)) { + seenIds.add(request.id); + ids.push(request.id); + assertBatchArchiveLimit(ids); + } + } + + after = page.next ?? undefined; + if (after) { + if (seenCursors.has(after)) { + throw new Error("Request pagination returned a repeated cursor"); + } + seenCursors.add(after); + } + } while (after); + + return ids; +} diff --git a/apps/cli/src/commands/run.test.ts b/apps/cli/src/commands/run.test.ts index b26334bab..eaffd1339 100644 --- a/apps/cli/src/commands/run.test.ts +++ b/apps/cli/src/commands/run.test.ts @@ -175,6 +175,19 @@ describe("run list", () => { expect(output).toContain("status: done"); expect(output).not.toContain("Found 1 request(s):"); }); + + it("forwards an encoded submission filter with project scoping", async () => { + mockFetchWith({ data: [], cursors: { next: null, prev: null } }); + + await runListAndCaptureOutput([ + "--submission-id", + "submission / value?", + ]); + + const url = new URL(lastRequest!.url); + expect(url.searchParams.get("submissionId")).toBe("submission / value?"); + expect(url.searchParams.get("projectId")).toBe("proj-test"); + }); }); describe("run submit", () => { diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index 4c2727699..af77c5e7c 100644 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -4,11 +4,9 @@ import { Command, Option } from "commander"; import EventSource from "eventsource"; import { execSync } from "child_process"; -import { mkdtempSync, mkdirSync, createWriteStream, rmSync, readFileSync, readdirSync, existsSync, statSync } from "fs"; +import { mkdtempSync, mkdirSync, rmSync, readFileSync, readdirSync, existsSync, statSync } from "fs"; import { tmpdir } from "os"; import { join, resolve, dirname, basename } from "path"; -import { pipeline } from "stream/promises"; -import { Readable } from "stream"; import { resolveScenarioAndPersona } from "../config-loader.js"; import { configureHelp } from "../utils/helpFormatter.js"; import { colorLevel, dimTimestamp, errorText, successText, label, value, banner, warnBanner, criterionIcon, styleText } from "../utils/style.js"; @@ -20,6 +18,16 @@ import { requireProjectId } from "../utils/config.js"; import { apiFetch, getApiBasePath } from "../utils/api-client.js"; import { parseGatesOption } from "../utils/gates.js"; import { buildResourceBindingSpecs, collectRepeatable } from "../utils/resources.js"; +import { + createArchiveProgressReporter, + downloadArchiveAtomic, +} from "../utils/archive-download.js"; +import { + assertBatchArchiveLimit, + assertResolvedRequestIds, + resolvePaginatedRequestIds, + selectBatchIdSource, +} from "./run-request-ids.js"; /** * Resolve a CLI option that may be either a literal string or a `@path` @@ -504,6 +512,7 @@ run .option("--search ", "Free-text search over id, task, model, and worker") .option("--created-after ", "Only runs created at/after this ISO-8601 datetime") .option("--created-before ", "Only runs created at/before this ISO-8601 datetime") + .option("--experiment-id ", "Filter by experiment ID") .option("--submission-id ", "Filter by submission ID") .option("--turns ", "Filter by actual turns (e.g. '>=5', '<=10', '=3')") .option("--max-iterations ", "Filter by configured maxIterations (e.g. '>=5', '<=10', '=3')") @@ -545,6 +554,9 @@ run if (options.createdBefore) { params.set("createdBefore", options.createdBefore); } + if (options.experimentId) { + params.set("experimentId", options.experimentId); + } if (options.submissionId) { params.set("submissionId", options.submissionId); } @@ -749,9 +761,11 @@ run } const outputPath = resolve(outputFile); - mkdirSync(dirname(outputPath), { recursive: true }); - const fileStream = createWriteStream(outputPath); - await pipeline(Readable.fromWeb(archiveResp.body as any), fileStream); + await downloadArchiveAtomic({ + response: archiveResp, + outputPath, + onProgress: createArchiveProgressReporter(), + }); console.log(`${successText('Archive:')} ${value(outputPath)}`); // Step 3: Optionally extract @@ -787,44 +801,48 @@ run .command("download-batch") .description("Download multiple runs at once into a single batch archive") .option("-i, --ids ", "One or more run IDs") - .option("--submission-id ", "Download all runs for a submission") + .option("--experiment-id ", "Download all runs for an experiment") + .option("--submission-id ", "Download all exact runs for a submission") .option("-o, --output ", "Output file path (default: batch-.tar.gz)") .option("-e, --extract", "Extract the archive after downloading") .option("-d, --dir ", "Extraction directory (implies --extract)") .option("-u, --url ", "API base URL", getDefaultApiUrl()) - .option("--project ", "Project ID for scoped operations (required with --submission-id)") + .option("--project ", "Project ID for scoped operations (required with --experiment-id or --submission-id)") .action(async (options) => { const { url } = options; const shouldExtract = options.extract || !!options.dir; try { - // Resolve IDs: either from --ids or by querying --submission-id - let ids: string[] = options.ids || []; + const source = selectBatchIdSource({ + ids: options.ids, + experimentId: options.experimentId, + submissionId: options.submissionId, + }); + let ids: string[]; - if (options.submissionId) { - console.log(`${label('Fetching runs for submission')} ${value(options.submissionId)}...`); + if (source.kind === "ids") { + ids = [...new Set(source.ids)]; + assertBatchArchiveLimit(ids); + } else { + const sourceLabel = source.kind === "experiment" + ? "experiment" + : "submission"; + console.log( + `${label(`Fetching runs for ${sourceLabel}`)} ${value(source.value)}...`, + ); const projectId = requireProjectId(options.project); - const listResp = await apiFetch(url, `/requests?submissionId=${encodeURIComponent(options.submissionId)}&limit=1000`, { projectId }); - if (!listResp.ok) { - const error = await listResp.json(); - console.error(errorText("Error fetching runs:"), error); - process.exit(1); - } - const listData = await listResp.json(); - const runs = listData.data || listData; - if (!Array.isArray(runs) || runs.length === 0) { - console.error(errorText("No runs found for this submission")); - process.exit(1); - } - ids = runs.map((r: { _id: string }) => r._id); + ids = await resolvePaginatedRequestIds({ + baseUrl: url, + projectId, + filter: source.kind === "experiment" + ? "experimentId" + : "submissionId", + value: source.value, + }); + assertResolvedRequestIds(ids, sourceLabel); console.log(`${label('Found')} ${value(String(ids.length))} runs`); } - if (ids.length === 0) { - console.error(errorText("Error: provide --ids or --submission-id")); - process.exit(1); - } - // Determine output file const downloadDir = process.env.SCOPE_MT_DOWNLOAD_OUTPUT_DIR; const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); @@ -846,9 +864,11 @@ run } const outputPath = resolve(outputFile); - mkdirSync(dirname(outputPath), { recursive: true }); - const fileStream = createWriteStream(outputPath); - await pipeline(Readable.fromWeb(archiveResp.body as any), fileStream); + await downloadArchiveAtomic({ + response: archiveResp, + outputPath, + onProgress: createArchiveProgressReporter(), + }); console.log(`${successText('Archive:')} ${value(outputPath)}`); // Optionally extract diff --git a/apps/cli/src/utils/archive-download.test.ts b/apps/cli/src/utils/archive-download.test.ts new file mode 100644 index 000000000..595d462a3 --- /dev/null +++ b/apps/cli/src/utils/archive-download.test.ts @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createArchiveProgressReporter, + downloadArchiveAtomic, + formatDownloadBytes, + type ArchiveDownloadProgress, +} from "./archive-download.js"; + +const directories: string[] = []; + +afterEach(async () => { + await Promise.all(directories.splice(0).map(directory => + rm(directory, { recursive: true, force: true }) + )); +}); + +async function directory(): Promise { + const value = await mkdtemp(join(tmpdir(), "scope-archive-download-")); + directories.push(value); + return value; +} + +function archiveResponse( + bytes: Uint8Array, + contentLength = bytes.length, +): Response { + const body = bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ) as ArrayBuffer; + return new Response(body, { + status: 200, + headers: { + "Content-Type": "application/gzip", + "Content-Length": String(contentLength), + }, + }); +} + +describe("atomic archive download", () => { + it("reports progress, validates gzip, and atomically replaces the destination", async () => { + const root = await directory(); + const outputPath = join(root, "batch.tar.gz"); + await writeFile(outputPath, "old archive"); + const bytes = gzipSync(Buffer.from("validated archive contents")); + const progress: ArchiveDownloadProgress[] = []; + + await expect(downloadArchiveAtomic({ + response: archiveResponse(bytes), + outputPath, + progressIntervalMs: 0, + onProgress: event => progress.push(event), + })).resolves.toEqual({ + path: outputPath, + bytes: bytes.length, + }); + + expect(await readFile(outputPath)).toEqual(bytes); + expect(progress.at(-1)).toEqual({ + downloadedBytes: bytes.length, + totalBytes: bytes.length, + complete: true, + }); + expect((await readdir(root)).filter(name => name.includes(".part-"))) + .toEqual([]); + }); + + it("preserves an existing destination and removes temp files on length mismatch", async () => { + const root = await directory(); + const outputPath = join(root, "batch.tar.gz"); + await writeFile(outputPath, "known-good"); + const bytes = gzipSync(Buffer.from("truncated response")); + + await expect(downloadArchiveAtomic({ + response: archiveResponse(bytes, bytes.length + 100), + outputPath, + })).rejects.toThrow( + `received ${bytes.length} of ${bytes.length + 100} bytes`, + ); + expect(await readFile(outputPath, "utf8")).toBe("known-good"); + expect((await readdir(root)).filter(name => name.includes(".part-"))) + .toEqual([]); + }); + + it.each([ + ["non-gzip", Buffer.from("not gzip"), /not gzip data/], + [ + "truncated gzip", + gzipSync(Buffer.from("integrity checked")).subarray(0, -4), + /gzip integrity validation/, + ], + ])("rejects %s without publishing it", async (_name, bytes, expected) => { + const root = await directory(); + const outputPath = join(root, "batch.tar.gz"); + + await expect(downloadArchiveAtomic({ + response: archiveResponse(bytes), + outputPath, + })).rejects.toThrow(expected); + await expect(readFile(outputPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect((await readdir(root)).filter(name => name.includes(".part-"))) + .toEqual([]); + }); + + it("formats byte progress for redirected and TTY streams", () => { + expect(formatDownloadBytes(1024 ** 3 * 6.64)) + .toBe("6.64 GiB"); + + let redirected = ""; + const redirectedStream = { + isTTY: false, + write: (text: string) => { + redirected += text; + return true; + }, + } as NodeJS.WriteStream; + createArchiveProgressReporter(redirectedStream)({ + downloadedBytes: 512, + totalBytes: 1024, + complete: true, + }); + expect(redirected).toContain("Downloading: 512 B / 1.00 KiB (50.0%)\n"); + + let tty = ""; + const ttyStream = { + isTTY: true, + write: (text: string) => { + tty += text; + return true; + }, + } as NodeJS.WriteStream; + const report = createArchiveProgressReporter(ttyStream); + report({ downloadedBytes: 1, complete: false }); + report({ downloadedBytes: 2, complete: true }); + expect(tty).toContain("\rDownloading:"); + expect(tty.endsWith("\n")).toBe(true); + }); +}); diff --git a/apps/cli/src/utils/archive-download.ts b/apps/cli/src/utils/archive-download.ts new file mode 100644 index 000000000..ff22c00dd --- /dev/null +++ b/apps/cli/src/utils/archive-download.ts @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + createReadStream, + createWriteStream, +} from "node:fs"; +import { + mkdir, + open, + rename, + rm, +} from "node:fs/promises"; +import { dirname } from "node:path"; +import { randomUUID } from "node:crypto"; +import { Readable, Transform, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; +import type { ReadableStream as NodeReadableStream } from "node:stream/web"; + +export interface ArchiveDownloadProgress { + /** Bytes durably streamed into the temporary file so far. */ + downloadedBytes: number; + /** Declared response length, when available and valid. */ + totalBytes?: number; + /** True for the final progress event after the response stream ends. */ + complete: boolean; +} + +export interface ArchiveDownloadResult { + /** Final atomic destination path. */ + path: string; + /** Exact compressed archive bytes downloaded. */ + bytes: number; +} + +interface DownloadArchiveOptions { + response: Response; + outputPath: string; + onProgress?: (progress: ArchiveDownloadProgress) => void; + progressIntervalMs?: number; +} + +function parseContentLength(response: Response): number | undefined { + const raw = response.headers.get("content-length"); + if (raw === null) return undefined; + const value = Number(raw); + return Number.isSafeInteger(value) && value >= 0 ? value : undefined; +} + +/** Validates gzip framing and checksum without materializing decompressed data. */ +async function verifyGzip(path: string): Promise { + const file = await open(path, "r"); + const prefix = Buffer.alloc(2); + let bytesRead: number; + try { + ({ bytesRead } = await file.read(prefix, 0, prefix.length, 0)); + } finally { + await file.close(); + } + if (bytesRead < 2 || prefix[0] !== 0x1f || prefix[1] !== 0x8b) { + throw new Error("Downloaded archive is not gzip data"); + } + await pipeline( + createReadStream(path), + createGunzip(), + new Writable({ + write(_chunk, _encoding, callback) { + callback(); + }, + }), + ); +} + +/** + * Downloads one gzip archive atomically with progress and integrity checks. + * + * Bytes are streamed to an exclusive same-directory temporary file. The final + * destination is replaced only after Content-Length (when present) and the + * gzip trailer/checksum both validate. + */ +export async function downloadArchiveAtomic( + options: DownloadArchiveOptions, +): Promise { + if (!options.response.body) { + throw new Error("Archive response has no body"); + } + + const outputDirectory = dirname(options.outputPath); + await mkdir(outputDirectory, { recursive: true }); + const temporaryPath = `${options.outputPath}.part-${process.pid}-${randomUUID()}`; + const totalBytes = parseContentLength(options.response); + const intervalMs = options.progressIntervalMs ?? 250; + let downloadedBytes = 0; + let lastProgressAt = 0; + + const report = (force = false) => { + const now = Date.now(); + if ( + options.onProgress && + (force || now - lastProgressAt >= intervalMs) + ) { + lastProgressAt = now; + options.onProgress({ downloadedBytes, totalBytes, complete: force }); + } + }; + + try { + const counter = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + downloadedBytes += chunk.length; + report(); + callback(null, chunk); + }, + }); + await pipeline( + Readable.fromWeb( + options.response.body as unknown as NodeReadableStream, + ), + counter, + createWriteStream(temporaryPath, { flags: "wx", mode: 0o600 }), + ); + const temporaryFile = await open(temporaryPath, "r"); + try { + await temporaryFile.sync(); + } finally { + await temporaryFile.close(); + } + report(true); + + if ( + totalBytes !== undefined && + downloadedBytes !== totalBytes + ) { + throw new Error( + `Archive download is incomplete: received ${downloadedBytes} of ${totalBytes} bytes`, + ); + } + + try { + await verifyGzip(temporaryPath); + } catch (error) { + throw new Error( + `Downloaded archive failed gzip integrity validation: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + await rename(temporaryPath, options.outputPath); + return { path: options.outputPath, bytes: downloadedBytes }; + } catch (error) { + await rm(temporaryPath, { force: true }).catch(() => {}); + throw error; + } +} + +/** Formats an integer byte count using binary units. */ +export function formatDownloadBytes(bytes: number): string { + const units = ["B", "KiB", "MiB", "GiB", "TiB"]; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + const precision = unit === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2; + return `${value.toFixed(precision)} ${units[unit]}`; +} + +/** Creates one stderr progress reporter for interactive and redirected use. */ +export function createArchiveProgressReporter( + stream: NodeJS.WriteStream = process.stderr, +): (progress: ArchiveDownloadProgress) => void { + let lastLineLength = 0; + let lastRedirectedBytes = 0; + return ({ downloadedBytes, totalBytes, complete }) => { + if ( + !stream.isTTY && + !complete && + downloadedBytes - lastRedirectedBytes < 64 * 1024 * 1024 + ) { + return; + } + lastRedirectedBytes = downloadedBytes; + const downloaded = formatDownloadBytes(downloadedBytes); + const line = totalBytes === undefined + ? `Downloading: ${downloaded}` + : `Downloading: ${downloaded} / ${formatDownloadBytes(totalBytes)} (${ + totalBytes === 0 + ? "100.0" + : ((downloadedBytes / totalBytes) * 100).toFixed(1) + }%)`; + if (stream.isTTY) { + stream.write(`\r${line.padEnd(lastLineLength)}`); + lastLineLength = Math.max(lastLineLength, line.length); + if (complete) { + stream.write("\n"); + lastLineLength = 0; + } + } else { + stream.write(`${line}\n`); + } + }; +} diff --git a/docs/architecture/app-design.md b/docs/architecture/app-design.md index f1a243262..bd8e8fded 100644 --- a/docs/architecture/app-design.md +++ b/docs/architecture/app-design.md @@ -598,8 +598,12 @@ applying them. `scope run list` exposes every Portal filter and the sort controls, forwarding them to the same API params: `--worker` (multi), `--status`, `--outcome`, `--task`, `--profile`, `--criteria`, `--model`, `--os`, `--priority`, `--agent-version`, `--search`, -`--created-after`, `--created-before`, `--sort-by`, `--sort-dir`. (The CLI list stays flat; -grouping is a Portal view.) +`--created-after`, `--created-before`, `--submission-id`, `--experiment-id`, +`--sort-by`, `--sort-dir`. (The CLI list stays flat; grouping is a Portal view.) + +The CLI also forwards `--experiment-id` for compatibility with Experiment-capable +API deployments. The standalone request-list API described here does not yet +implement that filter; the CLI option does not introduce an Experiment dependency. ## Real-Time Log Streaming diff --git a/docs/architecture/cli-distribution.md b/docs/architecture/cli-distribution.md index b2906b3ed..5d4a1cbed 100644 --- a/docs/architecture/cli-distribution.md +++ b/docs/architecture/cli-distribution.md @@ -107,6 +107,17 @@ modules. Coverage includes the extensionless installed `scope` executable, mock API commands, missing-URL failures, explicit URL precedence, offline help/version, and update checks; no release is published by these tests. +### Archive download integrity + +`scope run download` and `scope run download-batch` share one atomic archive +downloader. It streams compressed bytes to an exclusive temporary file in the +destination directory, reports progress on stderr, verifies response +`Content-Length` when present, and reads the full gzip stream through Node's +gunzip implementation to validate framing and checksum. Only a validated +temporary file is atomically renamed to the final destination. Failures remove +the temporary file and preserve any existing destination; extraction starts +only after validation succeeds. + ## Versioning The release workflow sorts valid `cli/v*` tags in `microsoft/scope` semantically diff --git a/website/src/content/docs/guides/submitting-requests-cli.md b/website/src/content/docs/guides/submitting-requests-cli.md index a5e6175bb..df1f6013d 100644 --- a/website/src/content/docs/guides/submitting-requests-cli.md +++ b/website/src/content/docs/guides/submitting-requests-cli.md @@ -104,6 +104,7 @@ scope run status -i scope run list scope run list -w coder-acp-copilot # filter by worker scope run list --submission-id # filter by submission +scope run list --experiment-id # filter by experiment scope run list -o json # machine-readable ``` @@ -139,9 +140,24 @@ scope run download -i -e -d ./output # extract For bulk downloads: ```bash +scope run download-batch --experiment-id -e -d ./batch scope run download-batch --submission-id -e -d ./batch ``` +`download-batch` accepts exactly one source: explicit `--ids`, +`--submission-id`, or `--experiment-id`. Submission and experiment selectors +require a project and may resolve at most 100 runs, matching the batch archive +endpoint limit. +The experiment selector requires an API deployment with `experimentId` request +filtering; installing this standalone CLI change does not enable Experiment API +support. + +Archive downloads report compressed-byte progress and are published +atomically. The CLI first writes a temporary file beside the destination, +checks `Content-Length` when supplied, validates the complete gzip stream, and +only then renames it to the requested output. A truncated or corrupt transfer +leaves no invalid destination file and preserves any existing archive. + ## Retry a failed run ```bash From 168208921d01a516072b40ce3c57dcd45b29abaf Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Tue, 27 Oct 2026 23:51:04 -0700 Subject: [PATCH 2/2] fix(website): separate canonical license headers from descriptions C CI requires the exact canonical MIT header block. Split the existing license and descriptive comments in two inherited main CSS files without changing styles or the extracted CLI feature. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- website/src/styles/contributions.css | 4 +++- website/src/styles/contributors.css | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/website/src/styles/contributions.css b/website/src/styles/contributions.css index 6aa468603..98f3bd382 100644 --- a/website/src/styles/contributions.css +++ b/website/src/styles/contributions.css @@ -1,7 +1,9 @@ /* * Copyright (c) Microsoft Corporation. * Licensed under the MIT License. - * + */ + +/* * Shared presentation for website contribution data: status messages, * count pills, cards, and development-only sample labels. */ diff --git a/website/src/styles/contributors.css b/website/src/styles/contributors.css index 5e75ab0ed..03ef9015a 100644 --- a/website/src/styles/contributors.css +++ b/website/src/styles/contributors.css @@ -1,7 +1,9 @@ /* * Copyright (c) Microsoft Corporation. * Licensed under the MIT License. - * + */ + +/* * Contributors (). An avatar wall on the * landing page and a card grid on the Community > Contribute page. Shares * the card, message and sample-tag styles in contributions.css.