Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,10 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = `
},
"type": "array",
},
"workspaceSizeBytes": {
"minimum": 0,
"type": "integer",
},
},
"required": [
"iteration",
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/test-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ export function createMockBlobStorage(): BlobStorage {
getLogsBlobUrl: vi.fn().mockImplementation((blobName: string) => `https://mockaccount.blob.core.windows.net/logs/${blobName}`),
appendLogEvent: vi.fn().mockResolvedValue(undefined),
ensureContainer: vi.fn().mockResolvedValue(undefined),
uploadWorkspaceSnapshot: vi.fn().mockResolvedValue("https://blob/snapshot"),
uploadWorkspaceSnapshot: vi.fn().mockResolvedValue({ url: "https://blob/snapshot", sizeBytes: 2048 }),
downloadSnapshot: vi.fn().mockResolvedValue(undefined),
listSnapshots: vi.fn().mockResolvedValue([]),
} as unknown as BlobStorage;
Expand Down
16 changes: 14 additions & 2 deletions apps/cli/src/run-get-action.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/
import { colorLevel, dimTimestamp, errorText, successText, label, value, banner, warnBanner, criterionIcon } from "./utils/style.js";
import { GATE_METADATA, GATE_ORDER, type ConversationTurn, type GateId, type GateRunSummary, type RequestDocument } from "shared/types";
import { formatData, isMachineReadable } from "./utils/formatters.js";
import { formatBytes, formatData, isMachineReadable, summarizeWorkspaceSizes } from "./utils/formatters.js";
import type { OutputFormat, DisplayField } from "./utils/types.js";
import { apiFetch } from "./utils/api-client.js";

Expand Down Expand Up @@ -49,6 +49,7 @@ export async function runGetAction(options: RunGetOptions): Promise<void> {
const run = await response.json() as RequestDocument & { id?: string; completedAt?: string | Date };
// Per-attempt state is nested under run.run (RunState).
const rs = run.run;
const workspaceSizes = summarizeWorkspaceSizes(rs?.turns ?? []);

// Machine-readable output
if (isMachineReadable(format)) {
Expand All @@ -62,6 +63,7 @@ export async function runGetAction(options: RunGetOptions): Promise<void> {
{ key: 'gatesCount', label: 'Gates' },
{ key: 'gateSummaries', label: 'Gate Status' },
{ key: 'turnsCount', label: 'Turns' },
{ key: 'workspaceSizeBytes', label: 'Workspace Snapshot Bytes' },
{ key: 'passed', label: 'Passed' },
{ key: 'task', label: 'Task' },
{ key: 'criteriaCount', label: 'Criteria' },
Expand All @@ -75,6 +77,8 @@ export async function runGetAction(options: RunGetOptions): Promise<void> {
outcome: rs?.outcome,
error: rs?.error,
turnsCount: rs?.turns?.length ?? 0,
// Total compressed snapshot bytes; empty (N/A) when no turn recorded a size.
workspaceSizeBytes: workspaceSizes.measured > 0 ? workspaceSizes.totalBytes : undefined,
gatesCount: run.gates?.length ?? run.gateSummaries?.length ?? 0,
gateSummaries: (run.gateSummaries ?? []).map((g) => `${g.gate}:${g.status}`).join(', '),
passed: rs?.outcome === 'succeeded' ? 'yes' : rs?.outcome === 'failed' || rs?.outcome === 'finished' ? 'no' : '-',
Expand Down Expand Up @@ -165,9 +169,17 @@ export async function runGetAction(options: RunGetOptions): Promise<void> {
const criteriaStr = turn.criteriaResults?.length
? ` — ${turn.criteriaResults.filter((cr: { passed: boolean }) => cr.passed).length}/${turn.criteriaResults.length} criteria passed`
: '';
console.log(` ${label(`Iteration ${turn.iteration}:`)} ${passIcon}${criteriaStr}`);
const sizeStr = turn.workspaceSizeBytes != null ? ` — snapshot ${formatBytes(turn.workspaceSizeBytes)}` : '';
console.log(` ${label(`Iteration ${turn.iteration}:`)} ${passIcon}${criteriaStr}${sizeStr}`);
});
}

// Total workspace snapshot size; legacy/imported runs never recorded sizes.
const { totalBytes, measured, missing } = workspaceSizes;
const sizeSummary = measured === 0
? dimTimestamp('unavailable')
: `${value(formatBytes(totalBytes))}${missing > 0 ? dimTimestamp(` (${measured}/${rs.turns.length} iterations measured)`) : ''}`;
console.log(` ${label('Workspace snapshots:')} ${sizeSummary}`);
}

// Error
Expand Down
57 changes: 57 additions & 0 deletions apps/cli/src/run-get.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,63 @@ describe('run get', () => {
});
});

describe('workspace snapshot sizes (#1564)', () => {
/** Run `run get` against a copy of `fullRun` whose turns carry the given sizes. */
async function runWithSizes(sizes: Array<number | undefined>, output: string) {
const run = {
...fullRun,
run: {
...fullRun.run,
turns: fullRun.run.turns.map((turn, i) => ({ ...turn, workspaceSizeBytes: sizes[i] })),
},
};
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: () => Promise.resolve(run) }));

const { lines, restore } = captureConsole();
try {
const { runGetAction } = await import('./run-get-action.js');
await runGetAction({ id: 'run-002', url: 'http://localhost:3100', output });
} finally {
restore();
}
return lines.join('\n');
}

it('shows per-iteration sizes and the total', async () => {
const output = await runWithSizes([1024, 2048], 'table');

expect(output).toContain('snapshot 1.0 KB');
expect(output).toContain('snapshot 2.0 KB');
expect(output).toMatch(/Workspace snapshots:.*3\.0 KB/);
expect(output).not.toContain('measured');
});

it('notes partial coverage when some iterations lack a size', async () => {
const output = await runWithSizes([1024, undefined], 'table');

expect(output).toMatch(/Workspace snapshots:.*1\.0 KB.*1\/2 iterations measured/);
});

it('shows unavailable for legacy runs without recorded sizes', async () => {
const output = await runWithSizes([undefined, undefined], 'table');

expect(output).toMatch(/Workspace snapshots:.*unavailable/);
expect(output).not.toContain('0 B');
});

it('includes the total bytes in JSON output', async () => {
const output = await runWithSizes([1024, 2048], 'json');

expect(JSON.parse(output)[0].workspaceSizeBytes).toBe('3072');
});

it('reports N/A in JSON output when no size was recorded', async () => {
const output = await runWithSizes([undefined, undefined], 'json');

expect(JSON.parse(output)[0].workspaceSizeBytes).toBe('N/A');
});
});

describe('error handling', () => {
it('exits with error on 404', async () => {
const mockFetch = vi.fn().mockResolvedValue({
Expand Down
23 changes: 23 additions & 0 deletions apps/cli/src/utils/formatters.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import {
getValueOrNA,
formatDate,
maxColumnWidth,
formatBytes,
summarizeWorkspaceSizes,
} from './formatters.js';
import type { DisplayField, TableColumn } from './types.js';

Expand Down Expand Up @@ -356,3 +358,24 @@ describe('isMachineReadable', () => {
expect(isMachineReadable('table')).toBe(false);
});
});

describe('formatBytes', () => {
it('keeps small values in bytes and scales through GB', () => {
expect(formatBytes(0)).toBe('0 B');
expect(formatBytes(1023)).toBe('1023 B');
expect(formatBytes(1536)).toBe('1.5 KB');
expect(formatBytes(5 * 1024 * 1024)).toBe('5.0 MB');
expect(formatBytes(2.25 * 1024 ** 3)).toBe('2.3 GB');
});
});

describe('summarizeWorkspaceSizes', () => {
it('sums recorded sizes and counts turns without one as missing', () => {
expect(summarizeWorkspaceSizes([{ workspaceSizeBytes: 100 }, {}, { workspaceSizeBytes: 0 }]))
.toEqual({ totalBytes: 100, measured: 2, missing: 1 });
});

it('reports nothing measured for legacy turns', () => {
expect(summarizeWorkspaceSizes([{}, {}])).toEqual({ totalBytes: 0, measured: 0, missing: 2 });
});
});
47 changes: 47 additions & 0 deletions apps/cli/src/utils/formatters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,53 @@ export function formatAsTSV<T extends ListItem>(
return output.trimEnd();
}

// ── Workspace snapshot sizes ─────────────────────────────────────────────────

const BYTE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB'] as const;

/**
* Format a byte count using binary (1024-based) units up to TB.
*
* @param bytes - Non-negative byte count.
* @returns e.g. `"512 B"`, `"1.5 KB"`, `"2.3 GB"`.
*/
export function formatBytes(bytes: number): string {
let size = bytes;
let unitIndex = 0;

// Step up a unit while the value is at least 1024 and a larger unit exists.
while (size >= 1024 && unitIndex < BYTE_UNITS.length - 1) {
size /= 1024;
unitIndex++;
}

return unitIndex === 0 ? `${size} B` : `${size.toFixed(1)} ${BYTE_UNITS[unitIndex]}`;
}

/**
* Sum the recorded workspace snapshot sizes (`workspaceSizeBytes`) of a run's turns.
* Legacy/imported turns have no recorded size and are counted as `missing`
* rather than as zero bytes.
*
* @param turns - Turns of a single run attempt.
* @returns `totalBytes` over the `measured` turns, plus the `missing` count.
*/
export function summarizeWorkspaceSizes(
turns: ReadonlyArray<{ workspaceSizeBytes?: number }>,
): { totalBytes: number; measured: number; missing: number } {
let totalBytes = 0;
let measured = 0;

for (const turn of turns) {
if (typeof turn.workspaceSizeBytes === 'number') {
totalBytes += turn.workspaceSizeBytes;
measured++;
}
}

return { totalBytes, measured, missing: turns.length - measured };
}

// ── Format: JSON ─────────────────────────────────────────────────────────────

/**
Expand Down
54 changes: 54 additions & 0 deletions apps/portal/src/components/TurnTimeline.stories.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import type { Meta, StoryObj } from "@storybook/react-vite";
import { expect } from "storybook/test";

import { TurnTimeline } from "./TurnTimeline";
import type { ConversationTurn } from "@/types";

/** Build a minimal completed turn for the timeline. */
function makeTurn(iteration: number, overrides: Partial<ConversationTurn> = {}): ConversationTurn {
return {
iteration,
judgeFeedback: `Feedback for iteration ${iteration}`,
snapshotUrl: `https://blob/snapshot-${iteration}.tar.gz`,
passed: false,
timestamp: "2026-01-01T10:00:00.000Z",
durationMs: 65_000,
...overrides,
};
}

const meta = {
component: TurnTimeline,
tags: ["ai-generated"],
args: {
runId: "run-123",
turns: [
makeTurn(1, { workspaceSizeBytes: 3 * 1024 * 1024 }),
makeTurn(2, { workspaceSizeBytes: 1.2 * 1024 ** 3, passed: true }),
],
},
} satisfies Meta<typeof TurnTimeline>;

export default meta;
type Story = StoryObj<typeof meta>;

/** Each iteration shows its compressed workspace snapshot size (#1564). */
export const WithSnapshotSizes: Story = {
play: async ({ canvas }) => {
await expect(canvas.getByText("3.0 MB")).toBeInTheDocument();
await expect(canvas.getByText("1.2 GB")).toBeInTheDocument();
},
};

/** Legacy/imported runs have no recorded size — shown as unavailable, never 0 B. */
export const LegacyWithoutSizes: Story = {
args: {
turns: [makeTurn(1), makeTurn(2, { passed: true })],
},
play: async ({ canvas }) => {
await expect(canvas.getAllByText("size unavailable")).toHaveLength(2);
},
};
51 changes: 51 additions & 0 deletions apps/portal/src/components/TurnTimeline.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

// @vitest-environment happy-dom
import { afterEach, describe, expect, it } from "vitest";
import { cleanup, render, screen } from "@testing-library/react";

import { TurnTimeline } from "./TurnTimeline";
import type { ConversationTurn } from "@/types";

/** Build a minimal completed turn for the timeline. */
function makeTurn(iteration: number, overrides: Partial<ConversationTurn> = {}): ConversationTurn {
return {
iteration,
judgeFeedback: "",
snapshotUrl: `https://blob/snapshot-${iteration}.tar.gz`,
passed: false,
timestamp: "2026-01-01T10:00:00.000Z",
...overrides,
};
}

describe("TurnTimeline — workspace snapshot size (#1564)", () => {
afterEach(() => cleanup());

it("shows the formatted size with the exact byte count as a tooltip", () => {
render(<TurnTimeline runId="run-1" turns={[makeTurn(1, { workspaceSizeBytes: 1536 })]} />);

const size = screen.getByText("1.5 KB");
expect(size.getAttribute("title")).toContain("1,536 bytes");
});

it("shows 'size unavailable' for a turn without a recorded size", () => {
render(<TurnTimeline runId="run-1" turns={[makeTurn(1)]} />);

expect(screen.getByText("size unavailable")).toBeTruthy();
expect(screen.queryByText("0 B")).toBeNull();
});

it("shows a recorded size of zero bytes as 0 B", () => {
render(<TurnTimeline runId="run-1" turns={[makeTurn(1, { workspaceSizeBytes: 0 })]} />);

expect(screen.getByText("0 B")).toBeTruthy();
});

it("omits the size when the turn has no snapshot", () => {
render(<TurnTimeline runId="run-1" turns={[makeTurn(1, { snapshotUrl: "" })]} />);

expect(screen.queryByText("size unavailable")).toBeNull();
});
});
12 changes: 12 additions & 0 deletions apps/portal/src/components/TurnTimeline.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { MarkdownRenderer } from "@/components/MarkdownRenderer";
import { api } from "@/lib/api";
import { cn } from "@/lib/utils";
import { formatDuration } from "@/lib/utils";
import { formatSnapshotBytes } from "@/lib/workspace-size";
import type { ConversationTurn } from "@/types";
import { useState } from "react";

Expand Down Expand Up @@ -88,6 +89,17 @@ export function TurnTimeline({ turns, runId, attemptRunId }: TurnTimelineProps)
{turn.aiCallCount} LLM calls
</span>
)}
{/* Workspace snapshot size — legacy/imported turns have none recorded */}
{turn.snapshotUrl && (
<span
className="font-mono"
title={turn.workspaceSizeBytes != null
? `${turn.workspaceSizeBytes.toLocaleString()} bytes (compressed workspace snapshot)`
: "Workspace snapshot size was not recorded for this iteration"}
>
{turn.workspaceSizeBytes != null ? formatSnapshotBytes(turn.workspaceSizeBytes) : "size unavailable"}
</span>
)}
{turn.snapshotUrl && (
<Button
variant="ghost"
Expand Down
Loading