From 0858d0d50b72c010cd9238a061a70430070e44c1 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Mon, 1 Jun 2026 23:01:10 -0700 Subject: [PATCH 01/54] feat(shared): typed prompts, size-based blob storage, experiment fields Add AGENTS.md support and GEPA experiment grouping to the shared layer: - TaskPromptDocument gains optional `type` ('task' | 'agents.md') and `contentBlobUrl`; `text` becomes optional. computeTaskPromptId is type-aware and backward-compatible (task/undefined keep identical _ids). - TaskPromptStore: size-based storage (inline <= PROMPT_INLINE_MAX_BYTES, else blob at prompts/{id}.txt), findOrCreate({type}), getAll({type}), resolvePromptText(doc), 256 KB hard cap. BlobStorage.uploadText added. - RequestDocument + schemas gain experimentId, agentsMdPromptId, agentsMdParentIds (multi-parent lineage). GroupByKey adds 'experiment'. - Tests: 35 passing covering type namespacing, backward compat, blob offload on create-miss, resolvePromptText, max-size reject, getAll filter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/shared/src/schemas/request.ts | 17 ++- packages/shared/src/schemas/task-prompt.ts | 12 +- packages/shared/src/storage/blob-storage.ts | 22 +++ .../shared/src/task-prompts/task-prompt-id.ts | 23 +++- .../task-prompts/task-prompt-store.test.ts | 126 +++++++++++++++++- .../src/task-prompts/task-prompt-store.ts | 121 +++++++++++++++-- packages/shared/src/types/types.ts | 48 ++++++- 7 files changed, 342 insertions(+), 27 deletions(-) diff --git a/packages/shared/src/schemas/request.ts b/packages/shared/src/schemas/request.ts index bdcf6cc88..7892f7bdc 100644 --- a/packages/shared/src/schemas/request.ts +++ b/packages/shared/src/schemas/request.ts @@ -101,6 +101,17 @@ export const CreateRequestInputSchema = z profileId: z.string().optional(), profileVariations: z.array(z.string()).optional(), priority: z.number().int().optional(), + // Optional grouping id tying together all requests of one experiment + // (e.g. a GEPA optimization run: the seed request and every candidate). + experimentId: z.string().optional(), + // Raw AGENTS.md body to deliver to the worker workspace. When present, the + // API findOrCreates an `agents.md`-typed prompt and stores its id on the + // request (see `agentsMdPromptId`). + agentsMd: z.string().optional(), + // Lineage edges (parent AGENTS.md prompt ids) when this candidate was + // derived from earlier ones: [] / omitted = root, [p] = mutation, + // [i, j] = merge of two parents. + agentsMdParentIds: z.array(z.string()).optional(), }) .openapi("CreateRequestInput"); @@ -118,6 +129,9 @@ export const RequestResponseSchema = z persona: PersonaSchema.optional(), deletedAt: z.coerce.date().optional(), taskPromptId: z.string().optional(), + experimentId: z.string().optional(), + agentsMdPromptId: z.string().optional(), + agentsMdParentIds: z.array(z.string()).optional(), mcpServers: z.array(z.string()).optional(), skillRevisions: z.array(z.string()).optional(), extensions: z.array(z.string()).optional(), @@ -200,9 +214,10 @@ export const ListRequestsQuerySchema = z criteria: z.string().optional(), submissionId: z.string().optional(), profileId: z.string().optional(), + experimentId: z.string().optional(), status: RequestStatusSchema.optional(), outcome: RequestOutcomeSchema.optional(), - groupBy: z.enum(["task", "submissionId", "profile"]).optional(), + groupBy: z.enum(["task", "submissionId", "profile", "experiment"]).optional(), limit: z.coerce.number().int().min(1).max(100).optional(), after: z.string().optional(), before: z.string().optional(), diff --git a/packages/shared/src/schemas/task-prompt.ts b/packages/shared/src/schemas/task-prompt.ts index bf66e3e50..5adbd4556 100644 --- a/packages/shared/src/schemas/task-prompt.ts +++ b/packages/shared/src/schemas/task-prompt.ts @@ -7,16 +7,26 @@ import { PromptFeatureResultSchema } from "./prompt-feature.js"; extendZodWithOpenApi(z); +export const TaskPromptTypeSchema = z + .enum(["task", "agents.md"]) + .openapi("TaskPromptType"); + export const CreateTaskPromptInputSchema = z .object({ text: z.string(), + type: TaskPromptTypeSchema.optional(), }) .openapi("CreateTaskPromptInput"); export const TaskPromptResponseSchema = z .object({ _id: z.string(), - text: z.string(), + // Present for inline-stored bodies; absent when the body lives in blob + // storage (see `contentBlobUrl`). Use the `/content` endpoint to always + // get the resolved text. + text: z.string().optional(), + contentBlobUrl: z.string().optional(), + type: TaskPromptTypeSchema.optional(), features: z.array(PromptFeatureResultSchema).optional(), featuresExtractedAt: z.coerce.date().optional(), createdAt: z.coerce.date(), diff --git a/packages/shared/src/storage/blob-storage.ts b/packages/shared/src/storage/blob-storage.ts index 19a7e3db2..c9b50038b 100644 --- a/packages/shared/src/storage/blob-storage.ts +++ b/packages/shared/src/storage/blob-storage.ts @@ -438,6 +438,28 @@ export class BlobStorage { return blockBlobClient.url; } + /** + * Uploads an in-memory UTF-8 text body to the snapshots container as a single + * block blob. Overwrites if the blob already exists. Returns the blob URL. + * + * Use this for plain-text/markdown bodies (e.g. large prompt or AGENTS.md + * content) where the JSON content-type from `uploadJson` would be misleading. + */ + async uploadText( + blobName: string, + text: string, + contentType: string = "text/plain; charset=utf-8", + ): Promise { + await this.ensureContainer(); + + const buf = Buffer.from(text, "utf-8"); + const blockBlobClient = this.containerClient.getBlockBlobClient(blobName); + await blockBlobClient.uploadData(buf, { + blobHTTPHeaders: { blobContentType: contentType }, + }); + return blockBlobClient.url; + } + /** * Downloads a snapshot from blob storage and extracts it to the target directory. */ diff --git a/packages/shared/src/task-prompts/task-prompt-id.ts b/packages/shared/src/task-prompts/task-prompt-id.ts index 7a7e51656..5f120d6d3 100644 --- a/packages/shared/src/task-prompts/task-prompt-id.ts +++ b/packages/shared/src/task-prompts/task-prompt-id.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { v5 as uuidv5 } from 'uuid'; +import type { TaskPromptType } from '../types/types.js'; /** * Namespace UUID derived from 'task.scope-mt.dev' using the RFC 4122 DNS namespace. @@ -10,13 +11,21 @@ import { v5 as uuidv5 } from 'uuid'; export const TASK_PROMPT_NAMESPACE = uuidv5('task.scope-mt.dev', uuidv5.DNS); /** - * Compute a deterministic UUIDv5 for a task prompt from its text content. + * Compute a deterministic UUIDv5 for a task prompt from its text content (and type). * - * The ID is derived from `uuidv5(text.trim(), TASK_PROMPT_NAMESPACE)`, making it: - * - **Content-addressed**: same text always produces the same ID - * - **Trim-insensitive**: leading/trailing whitespace is ignored - * - **Standard UUID format**: 36-char hyphenated UUID, consistent with the rest of the codebase + * The ID is content-addressed and **backward-compatible**: + * - **`task` / undefined type**: derived from `uuidv5(text.trim(), NS)` — identical + * to the original scheme, so every existing task prompt keeps its exact `_id`. + * - **Non-task types** (e.g. `agents.md`): the type is folded into the hash as + * `uuidv5(type + '\n' + text.trim(), NS)`, namespacing it so a prompt of one type + * never collides with an identical-text prompt of another type. + * + * Properties: content-addressed (same text+type → same ID), trim-insensitive, + * standard 36-char hyphenated UUID. */ -export function computeTaskPromptId(text: string): string { - return uuidv5(text.trim(), TASK_PROMPT_NAMESPACE); +export function computeTaskPromptId(text: string, type?: TaskPromptType): string { + const trimmed = text.trim(); + const namespaced = + !type || type === 'task' ? trimmed : `${type}\n${trimmed}`; + return uuidv5(namespaced, TASK_PROMPT_NAMESPACE); } diff --git a/packages/shared/src/task-prompts/task-prompt-store.test.ts b/packages/shared/src/task-prompts/task-prompt-store.test.ts index 930846978..b7ca6b22f 100644 --- a/packages/shared/src/task-prompts/task-prompt-store.test.ts +++ b/packages/shared/src/task-prompts/task-prompt-store.test.ts @@ -63,9 +63,10 @@ function createMockCollection() { let count = 0; for (const doc of docs.values()) { if (filter.deletedAt?.$exists === false && doc.deletedAt) continue; + if (!matchesType(filter, doc)) continue; if (filter.text?.$regex) { const regex = new RegExp(filter.text.$regex, filter.text.$options); - if (!regex.test(doc.text)) continue; + if (!regex.test(doc.text ?? "")) continue; } count++; } @@ -76,9 +77,10 @@ function createMockCollection() { const results: TaskPromptDocument[] = []; for (const doc of docs.values()) { if (filter.deletedAt?.$exists === false && doc.deletedAt) continue; + if (!matchesType(filter, doc)) continue; if (filter.text?.$regex) { const regex = new RegExp(filter.text.$regex, filter.text.$options); - if (!regex.test(doc.text)) continue; + if (!regex.test(doc.text ?? "")) continue; } results.push({ ...doc }); } @@ -87,6 +89,18 @@ function createMockCollection() { }; } +/** Replicates the store's `$or`/`type` filter against an in-memory doc. */ +function matchesType(filter: any, doc: TaskPromptDocument): boolean { + if (filter.$or) { + return filter.$or.some((clause: any) => { + if (clause.type?.$exists === false) return doc.type === undefined; + return doc.type === clause.type; + }); + } + if (filter.type !== undefined) return doc.type === filter.type; + return true; +} + // ── Tests ──────────────────────────────────────────────────────────────────── describe("TaskPromptStore", () => { @@ -131,6 +145,98 @@ describe("TaskPromptStore", () => { expect(revived._id).toBe(doc._id); expect(revived.deletedAt).toBeUndefined(); }); + + it("namespaces non-task types into a distinct id", async () => { + const task = await store.findOrCreate("same text"); + const agents = await store.findOrCreate("same text", { type: "agents.md" }); + expect(agents._id).not.toBe(task._id); + expect(agents._id).toBe(computeTaskPromptId("same text", "agents.md")); + expect(agents.type).toBe("agents.md"); + }); + + it("keeps task ids backward-compatible (unchanged by type arg)", async () => { + const a = await store.findOrCreate("hello"); + const b = await store.findOrCreate("hello", { type: "task" }); + expect(a._id).toBe(b._id); + expect(a._id).toBe(computeTaskPromptId("hello")); + }); + }); + + // -- size-based storage --------------------------------------------------- + + describe("size-based storage", () => { + function createMockBlob() { + const blobs = new Map(); + return { + store: blobs, + uploadText: vi.fn(async (name: string, text: string) => { + blobs.set(name, text); + return `https://acct.blob/snapshots/${name}`; + }), + downloadBlobToBuffer: vi.fn(async (name: string) => { + const v = blobs.get(name); + if (v === undefined) throw new Error(`blob ${name} not found`); + return Buffer.from(v, "utf-8"); + }), + }; + } + + it("stores small bodies inline and never touches blob", async () => { + const blob = createMockBlob(); + const s = new TaskPromptStore(col as any, blob as any, 1024); + const doc = await s.findOrCreate("tiny"); + expect(doc.text).toBe("tiny"); + expect(doc.contentBlobUrl).toBeUndefined(); + expect(blob.uploadText).not.toHaveBeenCalled(); + }); + + it("uploads over-threshold bodies to blob on a create miss", async () => { + const blob = createMockBlob(); + const s = new TaskPromptStore(col as any, blob as any, 16); + const big = "x".repeat(64); + const doc = await s.findOrCreate(big, { type: "agents.md" }); + expect(doc.text).toBeUndefined(); + expect(doc.contentBlobUrl).toContain(`prompts/${doc._id}.txt`); + expect(blob.uploadText).toHaveBeenCalledTimes(1); + }); + + it("skips the upload on a hit (idempotent)", async () => { + const blob = createMockBlob(); + const s = new TaskPromptStore(col as any, blob as any, 16); + const big = "y".repeat(64); + await s.findOrCreate(big, { type: "agents.md" }); + await s.findOrCreate(big, { type: "agents.md" }); + expect(blob.uploadText).toHaveBeenCalledTimes(1); + }); + + it("resolvePromptText returns inline text without blob access", async () => { + const blob = createMockBlob(); + const s = new TaskPromptStore(col as any, blob as any, 1024); + const doc = await s.findOrCreate("inline body"); + expect(await s.resolvePromptText(doc)).toBe("inline body"); + expect(blob.downloadBlobToBuffer).not.toHaveBeenCalled(); + }); + + it("resolvePromptText downloads blob-backed bodies", async () => { + const blob = createMockBlob(); + const s = new TaskPromptStore(col as any, blob as any, 16); + const big = "z".repeat(64); + const doc = await s.findOrCreate(big, { type: "agents.md" }); + expect(await s.resolvePromptText(doc)).toBe(big); + expect(blob.downloadBlobToBuffer).toHaveBeenCalledTimes(1); + }); + + it("rejects bodies over the hard max size", async () => { + const blob = createMockBlob(); + const s = new TaskPromptStore(col as any, blob as any, 16); + const huge = "a".repeat(256 * 1024 + 1); + await expect(s.findOrCreate(huge)).rejects.toThrow(/maximum/); + }); + + it("throws when an over-threshold body has no blob storage configured", async () => { + const s = new TaskPromptStore(col as any, undefined, 16); + await expect(s.findOrCreate("b".repeat(64))).rejects.toThrow(/BlobStorage/); + }); }); // -- get ------------------------------------------------------------------ @@ -222,6 +328,22 @@ describe("TaskPromptStore", () => { expect(total).toBe(1); expect(items[0].text).toBe("Azure deployment"); }); + + it("defaults to task-typed prompts (and legacy untyped docs)", async () => { + await store.findOrCreate("a task prompt"); + await store.findOrCreate("an agents file", { type: "agents.md" }); + const { items, total } = await store.getAll(); + expect(total).toBe(1); + expect(items[0].text).toBe("a task prompt"); + }); + + it("filters by agents.md type", async () => { + await store.findOrCreate("a task prompt"); + await store.findOrCreate("an agents file", { type: "agents.md" }); + const { items, total } = await store.getAll({ type: "agents.md" }); + expect(total).toBe(1); + expect(items[0].text).toBe("an agents file"); + }); }); // -- attachFeatures ------------------------------------------------------- diff --git a/packages/shared/src/task-prompts/task-prompt-store.ts b/packages/shared/src/task-prompts/task-prompt-store.ts index fa2907c1f..5e4c232ef 100644 --- a/packages/shared/src/task-prompts/task-prompt-store.ts +++ b/packages/shared/src/task-prompts/task-prompt-store.ts @@ -2,38 +2,97 @@ // Licensed under the MIT License. import { Collection } from 'mongodb'; -import { TaskPromptDocument, PromptFeatureResult } from '../types/types.js'; +import { TaskPromptDocument, PromptFeatureResult, TaskPromptType } from '../types/types.js'; import { computeTaskPromptId } from './task-prompt-id.js'; +import type { BlobStorage } from '../storage/blob-storage.js'; + +/** Default inline-vs-blob threshold (UTF-8 bytes). Bodies larger than this are + * stored in blob storage; smaller bodies stay inline in Mongo. */ +export const DEFAULT_PROMPT_INLINE_MAX_BYTES = 16 * 1024; + +/** Hard guard cap on prompt body size (UTF-8 bytes). Bodies larger than this + * are rejected outright, regardless of storage location. */ +export const PROMPT_MAX_BYTES = 256 * 1024; + +/** Deterministic blob path for a prompt body that lives in blob storage. */ +export function promptBlobName(id: string): string { + return `prompts/${id}.txt`; +} /** * MongoDB-backed store for task prompt entities. * * Task prompts are **immutable and content-addressed**: the `_id` is a UUIDv5 - * derived from the trimmed prompt text. The same text always resolves to the - * same document — `findOrCreate` is idempotent. + * derived from the trimmed prompt text (and, for non-`task` types, the type). + * The same text+type always resolves to the same document — `findOrCreate` is + * idempotent. + * + * The body is stored **inline** (`text`) when small, or in **blob storage** + * (`contentBlobUrl`) when it exceeds the configured inline threshold. Storage + * location is decided purely by size, independent of the prompt `type`. * * Documents are soft-deleted (deletedAt) rather than removed. */ export class TaskPromptStore { - constructor(private collection: Collection) {} + private readonly inlineMaxBytes: number; + + constructor( + private collection: Collection, + private blobStorage?: BlobStorage, + inlineMaxBytes?: number, + ) { + this.inlineMaxBytes = + inlineMaxBytes ?? + (process.env.PROMPT_INLINE_MAX_BYTES + ? parseInt(process.env.PROMPT_INLINE_MAX_BYTES, 10) + : DEFAULT_PROMPT_INLINE_MAX_BYTES); + } /** Get a single task prompt by ID (non-deleted) */ async get(id: string): Promise { return this.collection.findOne({ _id: id, deletedAt: { $exists: false } }); } - /** Get a task prompt by its text content (non-deleted) */ - async getByText(text: string): Promise { - return this.get(computeTaskPromptId(text)); + /** Get a task prompt by its text content (and type) (non-deleted) */ + async getByText(text: string, type?: TaskPromptType): Promise { + return this.get(computeTaskPromptId(text, type)); } /** - * Find an existing task prompt by text, or create a new one. - * Idempotent — same text always returns the same document. + * Resolve a prompt document's body to plain text, regardless of storage + * location: returns inline `text` when present, otherwise downloads the body + * from blob storage. Throws if the document has neither (corrupt) or if blob + * storage is needed but not configured. */ - async findOrCreate(text: string): Promise { + async resolvePromptText(doc: TaskPromptDocument): Promise { + if (doc.text != null) return doc.text; + if (doc.contentBlobUrl) { + if (!this.blobStorage) { + throw new Error( + `Task prompt '${doc._id}' body is in blob storage but no BlobStorage is configured`, + ); + } + const buf = await this.blobStorage.downloadBlobToBuffer(promptBlobName(doc._id)); + return buf.toString('utf-8'); + } + throw new Error(`Task prompt '${doc._id}' has neither inline text nor a blob reference`); + } + + /** + * Find an existing task prompt by text (and type), or create a new one. + * Idempotent — same text+type always returns the same document. + * + * Bodies over the inline threshold are uploaded to blob storage on a create + * **miss** (and the doc stores only a `contentBlobUrl`); smaller bodies are + * stored inline. The dedup/lookup is by content hash and never touches blob. + */ + async findOrCreate( + text: string, + opts?: { type?: TaskPromptType }, + ): Promise { const trimmed = text.trim(); - const id = computeTaskPromptId(trimmed); + const type = opts?.type; + const id = computeTaskPromptId(trimmed, type); // Try to find existing (including soft-deleted — revive if needed) const existing = await this.collection.findOne({ _id: id }); @@ -49,12 +108,35 @@ export class TaskPromptStore { return existing; } - // Create new document + // Enforce the hard size cap before storing anywhere. + const byteLength = Buffer.byteLength(trimmed, 'utf-8'); + if (byteLength > PROMPT_MAX_BYTES) { + throw new Error( + `Prompt body is ${byteLength} bytes, exceeding the maximum of ${PROMPT_MAX_BYTES} bytes`, + ); + } + const doc: TaskPromptDocument = { _id: id, - text: trimmed, createdAt: new Date(), }; + if (type) doc.type = type; + + // Size-based storage: large bodies go to blob, small bodies stay inline. + if (byteLength > this.inlineMaxBytes) { + if (!this.blobStorage) { + throw new Error( + `Prompt body is ${byteLength} bytes (over the ${this.inlineMaxBytes}-byte inline ` + + `threshold) but no BlobStorage is configured to store it`, + ); + } + doc.contentBlobUrl = await this.blobStorage.uploadText( + promptBlobName(id), + trimmed, + ); + } else { + doc.text = trimmed; + } await this.collection.insertOne(doc as any); return doc; @@ -62,15 +144,26 @@ export class TaskPromptStore { /** * List active (non-deleted) task prompts. - * Supports pagination and optional substring search on text. + * Supports pagination, optional substring search on text, and type scoping. + * + * `type` defaults to `task`: when omitted or `'task'`, both explicitly-typed + * task prompts and legacy docs without a `type` field are returned (so absent + * type behaves as `task`). Pass `'agents.md'` to list AGENTS.md prompts. */ async getAll(opts?: { limit?: number; offset?: number; search?: string; + type?: TaskPromptType; }): Promise<{ items: TaskPromptDocument[]; total: number }> { const filter: Record = { deletedAt: { $exists: false } }; + if (!opts?.type || opts.type === 'task') { + filter.$or = [{ type: 'task' }, { type: { $exists: false } }]; + } else { + filter.type = opts.type; + } + if (opts?.search) { filter.text = { $regex: opts.search, $options: 'i' }; } diff --git a/packages/shared/src/types/types.ts b/packages/shared/src/types/types.ts index c23444417..5aacba693 100644 --- a/packages/shared/src/types/types.ts +++ b/packages/shared/src/types/types.ts @@ -165,6 +165,26 @@ export interface RequestDocument { deletedAt?: Date; // Soft-delete timestamp (null/absent = active) taskPromptId?: string; // Materialized UUIDv5 of scenario.task (FK → TaskPromptDocument._id) promptFeatureExtractionId?: string; // @deprecated — use TaskPromptDocument.features via taskPromptId instead + /** + * Optional grouping id tying together all requests of one experiment + * (e.g. a single GEPA optimization run: the seed request plus every + * candidate-evaluation request). Free-form, supplied by the client. + */ + experimentId?: string; + /** + * FK → TaskPromptDocument._id of an AGENTS.md-typed prompt to deliver into + * the agent's workspace for this run. When set, the worker writes the + * resolved content to `/AGENTS.md` before the run starts. + */ + agentsMdPromptId?: string; + /** + * Lineage edges for the AGENTS.md candidate: the parent AGENTS.md prompt ids + * this candidate was derived from. Empty/absent = root (seed); one entry = + * reflective mutation; two entries = merge (GEPA's lineage is a multi-parent + * DAG). Recorded on the request so the experiment's full DAG can be + * reconstructed by querying its requests. + */ + agentsMdParentIds?: string[]; mcpServers?: string[]; // MCP server slugs selected for this run skillRevisions?: string[]; // Skill revision refs (e.g. "vercel-labs/agent-skills/my-skill@a1b2c3d") extensions?: string[]; // VS Code extension IDs selected for this run (e.g. "ms-python.python") @@ -406,7 +426,7 @@ export interface QueueProcessorConfig extends BaseQueueProcessorConfig { // --- Runs grouping types (shared between API and portal) --- -export type GroupByKey = "none" | "task" | "submissionId" | "profile"; +export type GroupByKey = "none" | "task" | "submissionId" | "profile" | "experiment"; export interface AggregateStats { min: number; @@ -610,10 +630,28 @@ export interface InsightDocument { // Task prompts are immutable, content-addressed entities identified by UUIDv5(text, namespace). // Runs reference task prompts via a materialized taskPromptId derived from scenario.task. +/** Prompt type. `task` (default) = a task/scenario prompt; `agents.md` = an + * AGENTS.md system-style prompt delivered into the agent workspace. */ +export type TaskPromptType = 'task' | 'agents.md'; + /** Task prompt document stored in MongoDB. Immutable — text cannot be changed after creation. */ export interface TaskPromptDocument { _id: string; // UUIDv5 of text.trim() (content-addressed) - text: string; // Full task prompt text + /** + * Inline prompt body. Present when the body is small enough to store in + * Mongo (≤ PROMPT_INLINE_MAX_BYTES). Mutually exclusive with + * `contentBlobUrl` — exactly one is set. Optional so large bodies can live + * in blob storage instead. + */ + text?: string; + /** + * Blob reference to the prompt body when it exceeds the inline size + * threshold. Mutually exclusive with `text`. The body is fetched via + * `resolvePromptText` server-side. + */ + contentBlobUrl?: string; + /** Prompt type. Absent ⇒ `'task'` (backward compatible). */ + type?: TaskPromptType; features?: PromptFeatureResult[]; // Detected prompt features featuresExtractedAt?: Date; // When features were last extracted createdAt: Date; @@ -628,6 +666,12 @@ export interface TaskPromptDocument { export interface PromptFeatureConfig { id: string; prompt: string; + /** + * Which prompt type this feature applies to. Absent ⇒ `'task'` (backward + * compatible). Feature extraction only considers features whose `type` + * matches the prompt being extracted. + */ + type?: TaskPromptType; } /** Prompt feature document stored in MongoDB (extends PromptFeatureConfig with DB metadata) */ From 71c891a56c2bf0df16374bf5644d4240193ef1ae Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Mon, 1 Jun 2026 23:01:20 -0700 Subject: [PATCH 02/54] feat(api): experiment grouping + AGENTS.md on requests, typed task prompts Wire the shared typed-prompt and experiment capabilities into the API: - POST /api/v1/requests accepts experimentId, agentsMd, agentsMdParentIds; agentsMd is resolved via findOrCreate(type: 'agents.md') and stored as agentsMdPromptId across all construction paths (variation, count>1, single); retry/resubmit/import preserve lineage. - GET /api/v1/requests: ?experimentId= filter + groupBy=experiment. - Task prompts: ?type= list filter, create with type, new GET /:id/content (resolvePromptText), extract-features scoped by prompt type. TaskPromptStore constructed with blobStorage. - grouping.ts buildGroupingPipeline supports 'experiment'. - Tests: typed task prompts, /:id/content, experiment/AGENTS.md request creation + lineage, experiment filter/group. OpenAPI snapshot updated. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openapi-snapshot.test.ts.snap | 107 +++++++++- apps/api/src/endpoints.test.ts | 182 ++++++++++++++++++ apps/api/src/grouping.ts | 6 +- apps/api/src/index.ts | 9 +- apps/api/src/routes/requests/index.ts | 52 ++++- apps/api/src/routes/task-prompts.ts | 59 +++++- apps/api/src/test-helpers.ts | 1 + 7 files changed, 399 insertions(+), 17 deletions(-) diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index 7912955e3..07c23708b 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -664,6 +664,18 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "CreateRequestInput": { "properties": { + "agentsMd": { + "type": "string", + }, + "agentsMdParentIds": { + "items": { + "type": "string", + }, + "type": "array", + }, + "experimentId": { + "type": "string", + }, "extensions": { "items": { "type": "string", @@ -755,6 +767,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "text": { "type": "string", }, + "type": { + "$ref": "#/components/schemas/TaskPromptType", + }, }, "required": [ "text", @@ -2087,6 +2102,15 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "agentVersion": { "type": "string", }, + "agentsMdParentIds": { + "items": { + "type": "string", + }, + "type": "array", + }, + "agentsMdPromptId": { + "type": "string", + }, "createdAt": { "format": "date-time", "type": [ @@ -2101,6 +2125,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "null", ], }, + "experimentId": { + "type": "string", + }, "extensions": { "items": { "type": "string", @@ -2622,6 +2649,9 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "_id": { "type": "string", }, + "contentBlobUrl": { + "type": "string", + }, "createdAt": { "format": "date-time", "type": [ @@ -2652,14 +2682,23 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "text": { "type": "string", }, + "type": { + "$ref": "#/components/schemas/TaskPromptType", + }, }, "required": [ "_id", - "text", "createdAt", ], "type": "object", }, + "TaskPromptType": { + "enum": [ + "task", + "agents.md", + ], + "type": "string", + }, "TokenUsage": { "properties": { "completionTokens": { @@ -6588,6 +6627,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "experimentId", + "required": false, + "schema": { + "type": "string", + }, + }, { "in": "query", "name": "status", @@ -6625,6 +6672,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "task", "submissionId", "profile", + "experiment", ], "type": "string", }, @@ -8541,6 +8589,18 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "type", + "required": false, + "schema": { + "enum": [ + "task", + "agents.md", + ], + "type": "string", + }, + }, ], "responses": { "200": { @@ -8736,6 +8796,51 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], }, }, + "/api/v1/task-prompts/{id}/content": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "id": { + "type": "string", + }, + "text": { + "type": "string", + }, + }, + "required": [ + "id", + "text", + ], + "type": "object", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Task prompt not found", + }, + }, + "summary": "Get resolved task prompt content", + "tags": [ + "Task Prompts", + ], + }, + }, "/api/v1/task-prompts/{id}/extract-features": { "post": { "parameters": [ diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 9a8422d9d..ad2cd5dce 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -502,6 +502,33 @@ describe("API Endpoints", () => { expect(res.body.data[0]).toHaveProperty("key", "sub-1"); }); + it("returns grouped results when groupBy=experiment", async () => { + const groupedDocs = [ + { key: "exp-1", label: "exp-1", aggregates: { count: 4, turns: null, duration: null, promptTokens: null, completionTokens: null }, uniform: {} }, + ]; + (mocks.collection.aggregate as any) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([{ _id: "exp-1" }]) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue(groupedDocs) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }) + .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); + + const res = await request(app).get("/api/v1/requests?groupBy=experiment"); + expect(res.status).toBe(200); + expect(res.body.data[0]).toHaveProperty("key", "exp-1"); + }); + + it("filters by experimentId", async () => { + (mocks.collection.find as any).mockReturnValue({ + sort: vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ toArray: vi.fn().mockResolvedValue([]) }), + }), + }); + + await request(app).get("/api/v1/requests?experimentId=exp-42"); + const filter = (mocks.collection.find as any).mock.calls[0][0]; + expect(filter).toHaveProperty("experimentId", "exp-42"); + }); + it("calls aggregate pipeline when groupBy is provided", async () => { (mocks.collection.aggregate as any) .mockReturnValueOnce({ toArray: vi.fn().mockResolvedValue([]) }); // keys (empty) @@ -974,6 +1001,62 @@ describe("API Endpoints", () => { }); }); + // =================================================================== + // Typed task prompts (task | agents.md) + content resolution + // =================================================================== + + describe("Typed task prompts", () => { + it("passes ?type=agents.md through to the store getAll filter", async () => { + (mocks.taskPromptStore as any).getAll.mockResolvedValue({ items: [], total: 0 }); + + const res = await request(app).get("/api/v1/task-prompts?type=agents.md"); + expect(res.status).toBe(200); + const arg = (mocks.taskPromptStore.getAll as any).mock.calls[0][0]; + expect(arg).toHaveProperty("type", "agents.md"); + }); + + it("creates a prompt with type=agents.md", async () => { + (mocks.taskPromptStore as any).findOrCreate.mockResolvedValue({ + _id: "agents-1", + type: "agents.md", + text: "# AGENTS\nBe concise.", + createdAt: new Date(), + }); + + const res = await request(app) + .post("/api/v1/task-prompts") + .send({ text: "# AGENTS\nBe concise.", type: "agents.md" }); + + expect(res.status).toBe(201); + expect(res.body).toHaveProperty("type", "agents.md"); + const arg = (mocks.taskPromptStore.findOrCreate as any).mock.calls[0]; + expect(arg[0]).toBe("# AGENTS\nBe concise."); + expect(arg[1]).toHaveProperty("type", "agents.md"); + }); + + it("GET /:id/content resolves the prompt body via resolvePromptText", async () => { + (mocks.taskPromptStore as any).get.mockResolvedValue({ + _id: "agents-1", + type: "agents.md", + contentBlobUrl: "https://blob/prompts/agents-1.txt", + createdAt: new Date(), + }); + (mocks.taskPromptStore as any).resolvePromptText.mockResolvedValue("resolved body"); + + const res = await request(app).get("/api/v1/task-prompts/agents-1/content"); + expect(res.status).toBe(200); + expect(res.body).toEqual({ id: "agents-1", text: "resolved body" }); + expect(mocks.taskPromptStore.resolvePromptText).toHaveBeenCalled(); + }); + + it("GET /:id/content returns 404 when the prompt does not exist", async () => { + (mocks.taskPromptStore as any).get.mockResolvedValue(null); + + const res = await request(app).get("/api/v1/task-prompts/missing/content"); + expect(res.status).toBe(404); + }); + }); + // =================================================================== // Submit with profile — server-side field resolution // =================================================================== @@ -1181,6 +1264,105 @@ describe("API Endpoints", () => { }); }); + // =================================================================== + // Experiment grouping + AGENTS.md on request creation + // =================================================================== + + describe("POST /api/v1/requests?worker=... (experiment + AGENTS.md)", () => { + const setupCopilotAgent = () => { + (mocks.agentCollection.findOne as any).mockResolvedValue({ + _id: "coder-acp-copilot", + versions: [{ agentVersion: "v1", status: "active", queueName: "queue-coder-acp-copilot", createdAt: new Date() }], + supportedModels: ["claude-haiku-4.5"], + }); + }; + + it("persists experimentId on the request document", async () => { + setupCopilotAgent(); + + const res = await request(app) + .post("/api/v1/requests?worker=coder-acp-copilot") + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + model: "claude-haiku-4.5", + experimentId: "exp-123", + }); + + expect(res.status).toBe(201); + const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; + expect(doc).toHaveProperty("experimentId", "exp-123"); + }); + + it("resolves agentsMd via findOrCreate(agents.md) and stores agentsMdPromptId", async () => { + setupCopilotAgent(); + (mocks.taskPromptStore as any).findOrCreate.mockResolvedValue({ + _id: "agents-xyz", + type: "agents.md", + text: "# AGENTS\nBe terse.", + createdAt: new Date(), + }); + + const res = await request(app) + .post("/api/v1/requests?worker=coder-acp-copilot") + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + model: "claude-haiku-4.5", + experimentId: "exp-123", + agentsMd: "# AGENTS\nBe terse.", + }); + + expect(res.status).toBe(201); + const foc = (mocks.taskPromptStore.findOrCreate as any).mock.calls.find( + (c: any[]) => c[1] && c[1].type === "agents.md" + ); + expect(foc).toBeDefined(); + expect(foc[0]).toBe("# AGENTS\nBe terse."); + const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; + expect(doc).toHaveProperty("agentsMdPromptId", "agents-xyz"); + }); + + it("persists agentsMdParentIds lineage on the request", async () => { + setupCopilotAgent(); + (mocks.taskPromptStore as any).findOrCreate.mockResolvedValue({ + _id: "agents-child", + type: "agents.md", + text: "child", + createdAt: new Date(), + }); + + const res = await request(app) + .post("/api/v1/requests?worker=coder-acp-copilot") + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + model: "claude-haiku-4.5", + experimentId: "exp-123", + agentsMd: "child", + agentsMdParentIds: ["agents-p1", "agents-p2"], + }); + + expect(res.status).toBe(201); + const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; + expect(doc).toHaveProperty("agentsMdParentIds", ["agents-p1", "agents-p2"]); + }); + + it("omits experiment fields when not provided", async () => { + setupCopilotAgent(); + + const res = await request(app) + .post("/api/v1/requests?worker=coder-acp-copilot") + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + model: "claude-haiku-4.5", + }); + + expect(res.status).toBe(201); + const doc = (mocks.collection.insertOne as any).mock.calls[0][0]; + expect(doc.experimentId).toBeUndefined(); + expect(doc.agentsMdPromptId).toBeUndefined(); + expect(doc.agentsMdParentIds).toBeUndefined(); + }); + }); + // =================================================================== // Bulk resubmit // =================================================================== diff --git a/apps/api/src/grouping.ts b/apps/api/src/grouping.ts index 11e7c4a36..430e9e302 100644 --- a/apps/api/src/grouping.ts +++ b/apps/api/src/grouping.ts @@ -12,12 +12,12 @@ import type { GroupByKey, RunGroup, AggregateStats, GroupUniformValues } from "s * The pipeline is appended after a $match stage with the caller's filter. */ export function buildGroupingPipeline( - groupBy: "task" | "submissionId" | "profile", + groupBy: "task" | "submissionId" | "profile" | "experiment", ): Document[] { const groupField = - groupBy === "task" ? "$taskPromptId" : groupBy === "profile" ? "$profileId" : "$submissionId"; + groupBy === "task" ? "$taskPromptId" : groupBy === "profile" ? "$profileId" : groupBy === "experiment" ? "$experimentId" : "$submissionId"; const fallbackGroupField = - groupBy === "task" ? "$scenario.task" : groupBy === "profile" ? { $literal: "no-profile" } : { $literal: "no-submission" }; + groupBy === "task" ? "$scenario.task" : groupBy === "profile" ? { $literal: "no-profile" } : groupBy === "experiment" ? { $literal: "no-experiment" } : { $literal: "no-submission" }; return [ // 1. Compute per-run derived values diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 6c0877b70..7daf0ab61 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -124,7 +124,10 @@ async function initializeClients(): Promise { mcpServerCollection = db.collection("mcp-servers"); insightsCollection = db.collection("insights"); taskPromptCollection = db.collection("task-prompts"); - taskPromptStore = new TaskPromptStore(taskPromptCollection); + // Blob storage is also (re)assigned below for logs/snapshots; construct an + // instance here so the task-prompt store can offload over-threshold bodies. + blobStorage = new BlobStorage({ storageAccountName, storageConnectionString }); + taskPromptStore = new TaskPromptStore(taskPromptCollection, blobStorage); featureFlagCollection = db.collection("feature-flags"); reportTemplateCollection = db.collection("report-templates"); skillCollection = db.collection("skills"); @@ -201,7 +204,9 @@ async function initializeClients(): Promise { console.log(`Initialized Queue clients for workers: ${Array.from(queueClients.keys()).join(", ")}, report`); - // Initialize blob storage (used for log persistence and snapshots) + // Initialize blob storage (used for log persistence and snapshots). + // Already constructed above for the task-prompt store; reassign to keep the + // original initialization order/comment intact. blobStorage = new BlobStorage({ storageAccountName, storageConnectionString }); // Initialize Redis-backed heartbeat store. Workers write per-run diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index cd3fa56f1..6a119bf1c 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -74,9 +74,38 @@ apiRoute(ctx.app, ctx.registry, { response: z.union([RequestResponseSchema, z.array(RequestResponseSchema)]), successStatus: 201, handler: async (req, res) => { - const { scenario: scenarioObj, persona: personaObj, maxIterations, personaInstructions, count = 1, promptFeatureExtractionId, model: requestedModel, reasoningEffort: requestedReasoningEffort, mcpServers: mcpServerSlugs, skills: skillSlugs, extensions: extensionIds, agentVersion: requestedAgentVersion, profileId: requestedProfileSpec, profileVariations, priority: requestedPriority } = req.body; + const { scenario: scenarioObj, persona: personaObj, maxIterations, personaInstructions, count = 1, promptFeatureExtractionId, model: requestedModel, reasoningEffort: requestedReasoningEffort, mcpServers: mcpServerSlugs, skills: skillSlugs, extensions: extensionIds, agentVersion: requestedAgentVersion, profileId: requestedProfileSpec, profileVariations, priority: requestedPriority, experimentId: requestedExperimentId, agentsMd: requestedAgentsMd, agentsMdParentIds: requestedAgentsMdParentIds } = req.body; let worker = req.query.worker as string | undefined; + // Experiment grouping + AGENTS.md lineage (used by the GEPA optimizer and + // any caller that wants to group requests / attach an AGENTS.md body). + const experimentId = + typeof requestedExperimentId === "string" && requestedExperimentId.trim() + ? requestedExperimentId + : undefined; + const agentsMdText = + typeof requestedAgentsMd === "string" && requestedAgentsMd.trim() + ? requestedAgentsMd + : undefined; + const agentsMdParentIds = Array.isArray(requestedAgentsMdParentIds) + ? requestedAgentsMdParentIds.filter((x: unknown): x is string => typeof x === "string") + : undefined; + + // Resolve the AGENTS.md body to an `agents.md`-typed prompt id (idempotent, + // content-addressed; large bodies are offloaded to blob by the store). + const resolveAgentsMdPromptId = async (): Promise => { + if (!agentsMdText) return undefined; + const p = await ctx.taskPromptStore.findOrCreate(agentsMdText, { type: "agents.md" }); + return p._id; + }; + + // Common request fields for experiment grouping / AGENTS.md lineage. + const buildExperimentFields = (agentsMdPromptId?: string) => ({ + ...(experimentId ? { experimentId } : {}), + ...(agentsMdPromptId ? { agentsMdPromptId } : {}), + ...(agentsMdParentIds && agentsMdParentIds.length ? { agentsMdParentIds } : {}), + }); + type VariationInput = { profileId: string; profileVersion?: number; @@ -343,6 +372,8 @@ apiRoute(ctx.app, ctx.registry, { const allNewIds: string[] = []; const newDocs: RequestDocument[] = []; const variationResults: Array<{ profileId: string; label?: string; ids: string[] }> = []; + const agentsMdPromptId = await resolveAgentsMdPromptId(); + const experimentFields = buildExperimentFields(agentsMdPromptId); for (const r of resolved) { const newIds: string[] = []; @@ -374,6 +405,7 @@ apiRoute(ctx.app, ctx.registry, { profileId: r.profile._id, profileVersionId: r.profileVersion._id, submissionId, + ...experimentFields, run: { _id: runId, attemptNumber: 1, status: "pending", logsUrl: ctx.blobStorage.getLogsBlobUrl(`${requestId}/runs/${runId}/run.jsonl`) }, }; newDocs.push(requestDoc); @@ -719,6 +751,10 @@ apiRoute(ctx.app, ctx.registry, { const taskPrompt = await ctx.taskPromptStore.findOrCreate(scenario.task); const taskPromptId = taskPrompt._id; + // Resolve AGENTS.md lineage once for this submission (shared across count>1). + const agentsMdPromptId = await resolveAgentsMdPromptId(); + const experimentFields = buildExperimentFields(agentsMdPromptId); + // Generate a submission ID to group all runs from this request const submissionId = uuidv4(); @@ -752,6 +788,7 @@ apiRoute(ctx.app, ctx.registry, { ...(profileId ? { profileId } : {}), ...(profileVersionId ? { profileVersionId } : {}), submissionId, + ...experimentFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries // never overwrite a previous attempt's blobs. @@ -810,6 +847,7 @@ apiRoute(ctx.app, ctx.registry, { ...(profileId ? { profileId } : {}), ...(profileVersionId ? { profileVersionId } : {}), submissionId, + ...experimentFields, // Mint a distinct run id for the first attempt. Blob artifacts // are scoped under `{requestId}/runs/{runId}/...` so retries // never overwrite a previous attempt's blobs. @@ -887,6 +925,7 @@ apiRoute(ctx.app, ctx.registry, { const criteriaFilter = req.query.criteria as string; const submissionIdFilter = req.query.submissionId as string; const profileIdFilter = req.query.profileId as string; + const experimentIdFilter = req.query.experimentId as string; const statusFilter = req.query.status as string; const outcomeFilter = req.query.outcome as string; const turnsFilterRaw = req.query.turns as string | undefined; @@ -894,7 +933,7 @@ apiRoute(ctx.app, ctx.registry, { const maxIterationsFilterRaw = req.query.maxIterations as string | undefined; const maxIterationsOpFilter = (req.query.maxIterationsOp as string | undefined) ?? "eq"; const includeDeleted = req.query.includeDeleted === "true"; - const groupByParam = req.query.groupBy as "task" | "submissionId" | "profile" | undefined; + const groupByParam = req.query.groupBy as "task" | "submissionId" | "profile" | "experiment" | undefined; const limit = Math.min(Math.max(Number(req.query.limit) || 10, 1), 100); const afterParam = req.query.after as string | undefined; const beforeParam = req.query.before as string | undefined; @@ -926,6 +965,9 @@ apiRoute(ctx.app, ctx.registry, { if (profileIdFilter) { filter.profileId = profileIdFilter; } + if (experimentIdFilter) { + filter.experimentId = experimentIdFilter; + } if (submissionIdFilter) { // Prefix-based matching: allow filtering by partial submission ID filter.submissionId = { $regex: `^${submissionIdFilter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}` }; @@ -982,7 +1024,7 @@ apiRoute(ctx.app, ctx.registry, { // Grouped mode: paginated RunGroup[] via two-phase aggregation if (groupByParam) { - const groupByField = groupByParam === "task" ? "taskPromptId" : groupByParam === "profile" ? "profileId" : "submissionId"; + const groupByField = groupByParam === "task" ? "taskPromptId" : groupByParam === "profile" ? "profileId" : groupByParam === "experiment" ? "experimentId" : "submissionId"; const groupByAggField = `$${groupByField}`; // Decode group cursor @@ -1445,6 +1487,10 @@ apiRoute(ctx.app, ctx.registry, { ...(original.taskPromptId ? { taskPromptId: original.taskPromptId } : {}), ...(effectiveProfileId ? { profileId: effectiveProfileId } : {}), ...(effectiveProfileVersionId ? { profileVersionId: effectiveProfileVersionId } : {}), + // Preserve experiment grouping + AGENTS.md lineage across re-submits. + ...(original.experimentId ? { experimentId: original.experimentId } : {}), + ...(original.agentsMdPromptId ? { agentsMdPromptId: original.agentsMdPromptId } : {}), + ...(original.agentsMdParentIds && original.agentsMdParentIds.length > 0 ? { agentsMdParentIds: original.agentsMdParentIds } : {}), submissionId, // Bulk re-submit creates a brand-new request — mint a distinct // run id for the first attempt so blob artifacts live under diff --git a/apps/api/src/routes/task-prompts.ts b/apps/api/src/routes/task-prompts.ts index ec5988a50..e64feaded 100644 --- a/apps/api/src/routes/task-prompts.ts +++ b/apps/api/src/routes/task-prompts.ts @@ -46,7 +46,10 @@ apiRoute(ctx.app, ctx.registry, { // Fetch recent task prompts as context (avoid duplicates) const recentPrompts = await ctx.taskPromptCollection - .find({ deletedAt: { $exists: false } }) + .find({ + deletedAt: { $exists: false }, + $or: [{ type: "task" }, { type: { $exists: false } }], + }) .sort({ createdAt: -1 }) .limit(20) .project({ text: 1, _id: 0 }) @@ -75,6 +78,7 @@ apiRoute(ctx.app, ctx.registry, { limit: z.coerce.number().optional(), offset: z.coerce.number().optional(), search: z.string().optional(), + type: z.enum(["task", "agents.md"]).optional(), }), response: z.object({ items: z.array(TaskPromptResponseSchema), @@ -86,8 +90,9 @@ apiRoute(ctx.app, ctx.registry, { const limit = req.query.limit ?? 50; const offset = req.query.offset ?? 0; const search = req.query.search; + const type = req.query.type; - const { items, total } = await ctx.taskPromptStore.getAll({ limit, offset, search }); + const { items, total } = await ctx.taskPromptStore.getAll({ limit, offset, search, type }); res.json({ items, total, limit, offset }); }, }); @@ -114,7 +119,33 @@ apiRoute(ctx.app, ctx.registry, { }, }); -// POST /api/v1/task-prompts — create (or find existing) task prompt. Idempotent. +// GET /api/v1/task-prompts/:id/content — get the resolved plain-text body +// (inline or downloaded from blob). Used by workers to fetch AGENTS.md / task text. +apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/task-prompts/:id/content", + tags: ["Task Prompts"], + summary: "Get resolved task prompt content", + params: z.object({ id: z.string() }), + response: z.object({ id: z.string(), text: z.string() }), + errorResponses: { + 404: { description: "Task prompt not found" }, + }, + handler: async (req, res, next) => { + try { + const { id } = req.params; + const taskPrompt = await ctx.taskPromptStore.get(id); + if (!taskPrompt) { + res.status(404).json({ error: "Task prompt not found" }); + return; + } + const text = await ctx.taskPromptStore.resolvePromptText(taskPrompt); + res.json({ id, text }); + } catch (err) { + next(err); + } + }, +}); apiRoute(ctx.app, ctx.registry, { method: "post", path: "/api/v1/task-prompts", @@ -126,14 +157,18 @@ apiRoute(ctx.app, ctx.registry, { 400: { description: "Empty text string" }, }, handler: async (req, res, next) => { - const { text } = req.body; + const { text, type } = req.body; if (!text || typeof text !== "string" || !text.trim()) { res.status(400).json({ error: "Body must contain a non-empty 'text' string" }); return; } - const taskPrompt = await ctx.taskPromptStore.findOrCreate(text); - res.status(201).json(taskPrompt); + try { + const taskPrompt = await ctx.taskPromptStore.findOrCreate(text, { type }); + res.status(201).json(taskPrompt); + } catch (err) { + next(err); + } }, }); @@ -211,13 +246,21 @@ apiRoute(ctx.app, ctx.registry, { return; } + // Scope candidate features to the prompt's own type: task prompts get task + // features; AGENTS.md prompts get agents.md features. Absent type ⇒ task. + const promptType = taskPrompt.type ?? "task"; + const typeFilter = + promptType === "task" + ? { $or: [{ type: "task" }, { type: { $exists: false } }] } + : { type: promptType }; const allFeatures = await ctx.promptFeatureCollection - .find({ deletedAt: { $exists: false } }) + .find({ deletedAt: { $exists: false }, ...typeFilter }) .toArray(); const featureConfigs = allFeatures.map(f => ({ id: f.id, prompt: f.prompt })); try { - const { results, suggestedFeatures } = await extractPromptFeatures(taskPrompt.text, featureConfigs, model); + const promptText = await ctx.taskPromptStore.resolvePromptText(taskPrompt); + const { results, suggestedFeatures } = await extractPromptFeatures(promptText, featureConfigs, model); // Store features on the task prompt entity const updated = await ctx.taskPromptStore.attachFeatures(id, results); diff --git a/apps/api/src/test-helpers.ts b/apps/api/src/test-helpers.ts index 6230d6d02..a35282d41 100644 --- a/apps/api/src/test-helpers.ts +++ b/apps/api/src/test-helpers.ts @@ -119,6 +119,7 @@ export function createMockTaskPromptStore(): Record { createdAt: new Date(), }), getAll: vi.fn().mockResolvedValue({ items: [], total: 0 }), + resolvePromptText: vi.fn().mockResolvedValue("mock task"), delete: vi.fn().mockResolvedValue(undefined), attachFeatures: vi.fn().mockResolvedValue({ _id: "tp-mock-id", From c6fadeb3e43b3b6ef929e07ca681b2ec1a60e4a5 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Mon, 1 Jun 2026 23:06:23 -0700 Subject: [PATCH 03/54] feat(prompt-features): type prompt features (task | agents.md) Features are now scoped by prompt type so a feature only applies to prompts of its own type: - PromptFeatureStore: getAll({type}), create({type}), seed carries type. - Schemas: CreatePromptFeatureInput + PromptFeatureResponse add optional type. - API: GET /prompt-features?type= filter (task includes legacy untyped), create persists type, seed $setOnInsert carries type, extract-from-text scopes candidate features by ?type=. task-prompts extract-features already scopes by the prompt's own type. - FS provider loads optional `type` from feature YAML. - Tests: 6 store tests + API filter/create tests. OpenAPI snapshot updated. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openapi-snapshot.test.ts.snap | 33 ++++++ apps/api/src/endpoints.test.ts | 38 ++++++ apps/api/src/routes/prompt-features.ts | 34 +++++- apps/api/src/routes/task-prompts.ts | 2 +- .../prompt-feature-provider-fs.ts | 1 + .../prompt-feature-store.test.ts | 112 ++++++++++++++++++ .../prompt-features/prompt-feature-store.ts | 23 +++- packages/shared/src/schemas/prompt-feature.ts | 2 + 8 files changed, 233 insertions(+), 12 deletions(-) create mode 100644 packages/shared/src/prompt-features/prompt-feature-store.test.ts diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index 07c23708b..f07f14183 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -606,6 +606,13 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "prompt": { "type": "string", }, + "type": { + "enum": [ + "task", + "agents.md", + ], + "type": "string", + }, }, "required": [ "id", @@ -1731,6 +1738,13 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "prompt": { "type": "string", }, + "type": { + "enum": [ + "task", + "agents.md", + ], + "type": "string", + }, "updatedAt": { "format": "date-time", "type": [ @@ -5555,6 +5569,18 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "type", + "required": false, + "schema": { + "enum": [ + "task", + "agents.md", + ], + "type": "string", + }, + }, ], "responses": { "200": { @@ -5623,6 +5649,13 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "text": { "type": "string", }, + "type": { + "enum": [ + "task", + "agents.md", + ], + "type": "string", + }, }, "required": [ "text", diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index ad2cd5dce..cbf9b440a 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -914,6 +914,44 @@ describe("API Endpoints", () => { expect(res.status).toBe(400); }); + + it("persists type=agents.md when creating a feature", async () => { + (mocks.promptFeatureCollection.findOne as any).mockResolvedValue(null); + + const res = await request(app) + .post("/api/v1/prompt-features") + .send({ id: "agents_feat", prompt: "Does AGENTS.md mention tests?", type: "agents.md" }); + + expect(res.status).toBe(201); + expect(res.body).toHaveProperty("type", "agents.md"); + const doc = (mocks.promptFeatureCollection.insertOne as any).mock.calls[0][0]; + expect(doc).toHaveProperty("type", "agents.md"); + }); + }); + + describe("GET /api/v1/prompt-features?type=", () => { + it("scopes the query to agents.md features", async () => { + (mocks.promptFeatureCollection.find as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }); + + const res = await request(app).get("/api/v1/prompt-features?type=agents.md"); + expect(res.status).toBe(200); + const filter = (mocks.promptFeatureCollection.find as any).mock.calls[0][0]; + expect(JSON.stringify(filter)).toContain("agents.md"); + }); + + it("treats task type as including legacy untyped features", async () => { + (mocks.promptFeatureCollection.find as any).mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }); + + await request(app).get("/api/v1/prompt-features?type=task"); + const filter = (mocks.promptFeatureCollection.find as any).mock.calls[0][0]; + const json = JSON.stringify(filter); + expect(json).toContain("$exists"); + expect(json).toContain("task"); + }); }); describe("GET /api/v1/prompt-features/:id", () => { diff --git a/apps/api/src/routes/prompt-features.ts b/apps/api/src/routes/prompt-features.ts index 6a462d23c..3fb51f743 100644 --- a/apps/api/src/routes/prompt-features.ts +++ b/apps/api/src/routes/prompt-features.ts @@ -110,6 +110,7 @@ apiRoute(ctx.app, ctx.registry, { $setOnInsert: { id: trimmedId, prompt: config.prompt.trim(), + ...(config.type ? { type: config.type } : {}), createdAt: new Date(), }, }, @@ -134,6 +135,7 @@ apiRoute(ctx.app, ctx.registry, { body: z.object({ text: z.string(), model: z.string().optional(), + type: z.enum(["task", "agents.md"]).optional(), }), response: z.object({ features: z.array(PromptFeatureResultSchema), @@ -145,7 +147,7 @@ apiRoute(ctx.app, ctx.registry, { 503: { description: "LLM not configured" }, }, handler: async (req, res, next) => { - const { text, model } = req.body; + const { text, model, type } = req.body; if (!text || typeof text !== "string" || !text.trim()) { res.status(400).json({ error: "Body must contain a non-empty 'text' string" }); return; @@ -156,8 +158,15 @@ apiRoute(ctx.app, ctx.registry, { return; } + // Scope candidate features to the requested type (default = task, + // including legacy features with no `type`). + const promptType = type ?? "task"; + const typeFilter: Record = + promptType === "task" + ? { $or: [{ type: "task" }, { type: { $exists: false } }] } + : { type: promptType }; const allFeatures = await ctx.promptFeatureCollection - .find({ deletedAt: { $exists: false } }) + .find({ deletedAt: { $exists: false }, ...typeFilter }) .toArray(); const featureConfigs = allFeatures.map(f => ({ id: f.id, prompt: f.prompt })); @@ -185,16 +194,30 @@ apiRoute(ctx.app, ctx.registry, { path: "/api/v1/prompt-features", tags: ["Prompt Features"], summary: "List features", - query: z.object({ q: z.string().optional() }), + query: z.object({ q: z.string().optional(), type: z.enum(["task", "agents.md"]).optional() }), response: z.array(PromptFeatureResponseSchema), handler: async (req, res) => { const q = req.query.q; + const type = req.query.type; const filter: Record = { deletedAt: { $exists: false } }; + if (type) { + // Absent `type` is treated as 'task' for backward compatibility. + filter.$and = [ + type === "task" + ? { $or: [{ type: "task" }, { type: { $exists: false } }] } + : { type }, + ]; + } if (q) { - filter.$or = [ + const search = [ { id: { $regex: q, $options: "i" } }, { prompt: { $regex: q, $options: "i" } }, ]; + if (filter.$and) { + (filter.$and as unknown[]).push({ $or: search }); + } else { + filter.$or = search; + } } const features = await ctx.promptFeatureCollection.find(filter).toArray(); features.sort((a, b) => a.id.localeCompare(b.id)); @@ -239,7 +262,7 @@ apiRoute(ctx.app, ctx.registry, { 409: { description: "Feature already exists" }, }, handler: async (req, res) => { - const { id, prompt } = req.body; + const { id, prompt, type } = req.body; if (!id || typeof id !== "string") { res.status(400).json({ error: "id is required and must be a string" }); @@ -263,6 +286,7 @@ apiRoute(ctx.app, ctx.registry, { const doc: PromptFeatureDocument = { id, prompt: prompt.trim(), + ...(type ? { type } : {}), createdAt: new Date(), }; diff --git a/apps/api/src/routes/task-prompts.ts b/apps/api/src/routes/task-prompts.ts index e64feaded..ce5716fb7 100644 --- a/apps/api/src/routes/task-prompts.ts +++ b/apps/api/src/routes/task-prompts.ts @@ -249,7 +249,7 @@ apiRoute(ctx.app, ctx.registry, { // Scope candidate features to the prompt's own type: task prompts get task // features; AGENTS.md prompts get agents.md features. Absent type ⇒ task. const promptType = taskPrompt.type ?? "task"; - const typeFilter = + const typeFilter: Record = promptType === "task" ? { $or: [{ type: "task" }, { type: { $exists: false } }] } : { type: promptType }; diff --git a/packages/shared/src/prompt-features/prompt-feature-provider-fs.ts b/packages/shared/src/prompt-features/prompt-feature-provider-fs.ts index c76574ace..bf23767ff 100644 --- a/packages/shared/src/prompt-features/prompt-feature-provider-fs.ts +++ b/packages/shared/src/prompt-features/prompt-feature-provider-fs.ts @@ -50,6 +50,7 @@ export class FileSystemPromptFeatureProvider implements PromptFeatureProvider { const feature: PromptFeatureConfig = { id: data.id.trim(), prompt: data.prompt.trim(), + ...(data.type === "task" || data.type === "agents.md" ? { type: data.type } : {}), }; if (this.registry.has(feature.id)) { diff --git a/packages/shared/src/prompt-features/prompt-feature-store.test.ts b/packages/shared/src/prompt-features/prompt-feature-store.test.ts new file mode 100644 index 000000000..61010b8dd --- /dev/null +++ b/packages/shared/src/prompt-features/prompt-feature-store.test.ts @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, beforeEach } from "vitest"; +import { PromptFeatureStore } from "./prompt-feature-store.js"; +import type { PromptFeatureDocument } from "../types/types.js"; + +// ── Minimal in-memory collection ───────────────────────────────────────────── +// Supports the subset of query operators the store uses: `id`, `deletedAt` +// existence, and the `$or`/`type` existence filters for type scoping. + +function matchesType(doc: PromptFeatureDocument, clause: any): boolean { + if (clause.type !== undefined) { + if (typeof clause.type === "object" && clause.type.$exists === false) { + return doc.type === undefined; + } + return doc.type === clause.type; + } + return true; +} + +function createMockCollection() { + const docs: PromptFeatureDocument[] = []; + + return { + _docs: docs, + find(filter: any) { + let results = docs.filter((d) => { + if (filter.deletedAt && filter.deletedAt.$exists === false && d.deletedAt) return false; + if (filter.$or) { + if (!filter.$or.some((clause: any) => matchesType(d, clause))) return false; + } + return true; + }); + return { + sort() { return this; }, + toArray() { return Promise.resolve(results.map((d) => ({ ...d }))); }, + }; + }, + findOne(filter: any) { + const doc = docs.find((d) => { + if (filter.id && d.id !== filter.id) return false; + if (filter.deletedAt && filter.deletedAt.$exists === false && d.deletedAt) return false; + return true; + }); + return Promise.resolve(doc ? { ...doc } : null); + }, + insertOne(doc: PromptFeatureDocument) { + docs.push({ ...doc }); + return Promise.resolve({ insertedId: doc.id }); + }, + updateOne() { return Promise.resolve({ matchedCount: 1, modifiedCount: 1 }); }, + }; +} + +describe("PromptFeatureStore type support", () => { + let store: PromptFeatureStore; + let collection: ReturnType; + + beforeEach(() => { + collection = createMockCollection(); + store = new PromptFeatureStore(collection as any); + }); + + it("create persists the type field when provided", async () => { + const doc = await store.create({ id: "agents_terse", prompt: "Be terse", type: "agents.md" }); + expect(doc.type).toBe("agents.md"); + expect(collection._docs[0].type).toBe("agents.md"); + }); + + it("create omits type when not provided (backward compatible)", async () => { + const doc = await store.create({ id: "task_feat", prompt: "A task feature" }); + expect(doc.type).toBeUndefined(); + expect("type" in collection._docs[0]).toBe(false); + }); + + it("getAll({type: 'agents.md'}) returns only agents.md features", async () => { + await store.create({ id: "task_a", prompt: "p", type: "task" }); + await store.create({ id: "legacy", prompt: "p" }); // no type → task + await store.create({ id: "agents_b", prompt: "p", type: "agents.md" }); + + const result = await store.getAll({ type: "agents.md" }); + expect(result.map((f) => f.id)).toEqual(["agents_b"]); + }); + + it("getAll({type: 'task'}) includes legacy (untyped) features", async () => { + await store.create({ id: "task_a", prompt: "p", type: "task" }); + await store.create({ id: "legacy", prompt: "p" }); + await store.create({ id: "agents_b", prompt: "p", type: "agents.md" }); + + const result = await store.getAll({ type: "task" }); + expect(result.map((f) => f.id).sort()).toEqual(["legacy", "task_a"]); + }); + + it("getAll() with no filter returns all features", async () => { + await store.create({ id: "task_a", prompt: "p" }); + await store.create({ id: "agents_b", prompt: "p", type: "agents.md" }); + + const result = await store.getAll(); + expect(result).toHaveLength(2); + }); + + it("seed carries the type field for new features", async () => { + const inserted = await store.seed([ + { id: "agents_seed", prompt: "p", type: "agents.md" }, + { id: "task_seed", prompt: "p" }, + ]); + expect(inserted).toBe(2); + expect(collection._docs.find((d) => d.id === "agents_seed")?.type).toBe("agents.md"); + expect("type" in (collection._docs.find((d) => d.id === "task_seed") as any)).toBe(false); + }); +}); diff --git a/packages/shared/src/prompt-features/prompt-feature-store.ts b/packages/shared/src/prompt-features/prompt-feature-store.ts index 7fda78964..5965dddd8 100644 --- a/packages/shared/src/prompt-features/prompt-feature-store.ts +++ b/packages/shared/src/prompt-features/prompt-feature-store.ts @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { Collection } from 'mongodb'; -import { PromptFeatureConfig, PromptFeatureDocument } from '../types/types.js'; +import { Collection, Filter } from 'mongodb'; +import { PromptFeatureConfig, PromptFeatureDocument, TaskPromptType } from '../types/types.js'; /** * MongoDB-backed prompt feature store for CRUD operations on prompt feature definitions. @@ -13,10 +13,18 @@ import { PromptFeatureConfig, PromptFeatureDocument } from '../types/types.js'; export class PromptFeatureStore { constructor(private collection: Collection) {} - /** List all active (non-deleted) prompt features */ - async getAll(): Promise { + /** List all active (non-deleted) prompt features, optionally filtered by type */ + async getAll(opts?: { type?: TaskPromptType }): Promise { + const filter: Filter = { deletedAt: { $exists: false } }; + if (opts?.type) { + // Absent `type` is treated as 'task' for backward compatibility. + filter.$or = + opts.type === "task" + ? [{ type: "task" }, { type: { $exists: false } }] + : [{ type: opts.type }]; + } return this.collection - .find({ deletedAt: { $exists: false } }) + .find(filter) .sort({ id: 1 }) .toArray(); } @@ -30,8 +38,9 @@ export class PromptFeatureStore { async create(input: { id: string; prompt: string; + type?: TaskPromptType; }): Promise { - const { id, prompt } = input; + const { id, prompt, type } = input; // Validate ID format if (!/^[a-z0-9_-]+$/.test(id)) { @@ -49,6 +58,7 @@ export class PromptFeatureStore { const doc: PromptFeatureDocument = { id, prompt: prompt.trim(), + ...(type ? { type } : {}), createdAt: new Date(), }; @@ -102,6 +112,7 @@ export class PromptFeatureStore { await this.collection.insertOne({ id: config.id, prompt: config.prompt, + ...(config.type ? { type: config.type } : {}), createdAt: new Date(), } as any); inserted++; diff --git a/packages/shared/src/schemas/prompt-feature.ts b/packages/shared/src/schemas/prompt-feature.ts index bf4d96ec3..ec2e294dd 100644 --- a/packages/shared/src/schemas/prompt-feature.ts +++ b/packages/shared/src/schemas/prompt-feature.ts @@ -10,6 +10,7 @@ export const CreatePromptFeatureInputSchema = z .object({ id: z.string().regex(/^[a-z][a-z0-9_]*$/), prompt: z.string(), + type: z.enum(["task", "agents.md"]).optional(), }) .openapi("CreatePromptFeatureInput"); @@ -23,6 +24,7 @@ export const PromptFeatureResponseSchema = z .object({ id: z.string(), prompt: z.string(), + type: z.enum(["task", "agents.md"]).optional(), createdAt: z.coerce.date(), updatedAt: z.coerce.date().optional(), deletedAt: z.coerce.date().optional(), From 97b56d117a7c1e55f513372f6ed144c39866f6c5 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Mon, 1 Jun 2026 23:10:31 -0700 Subject: [PATCH 04/54] feat(worker): deliver AGENTS.md into workspace before run Add AGENTS.md delivery to the shared queue processor so all workers honor a request's optional agentsMdPromptId: - New PromptClient.getText(id) REST client resolves a prompt's plain-text body via GET /api/v1/task-prompts/:id/content (server downloads blob if the doc is blob-backed), with clear errors on 404 / non-OK / missing text. - writeAgentsMd() resolves the AGENTS.md text once before the run and writes /AGENTS.md (constant for the whole run). No-ops when the request has no agentsMdPromptId; throws (fails the run loudly via the base processor's catch) when set but apiBaseUrl is missing or resolution fails. Tests: PromptClient success/404/non-OK/missing-text; writeAgentsMd no-op, missing-apiBaseUrl throw, and writes resolved body into a temp workspace. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../shared/src/queue/queue-processor.test.ts | 59 ++++++++++++++++++- packages/shared/src/queue/queue-processor.ts | 45 ++++++++++++++ packages/shared/src/task-prompts/index.ts | 1 + .../src/task-prompts/prompt-client.test.ts | 45 ++++++++++++++ .../shared/src/task-prompts/prompt-client.ts | 42 +++++++++++++ 5 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 packages/shared/src/task-prompts/prompt-client.test.ts create mode 100644 packages/shared/src/task-prompts/prompt-client.ts diff --git a/packages/shared/src/queue/queue-processor.test.ts b/packages/shared/src/queue/queue-processor.test.ts index fb2a1dc95..ba670abdf 100644 --- a/packages/shared/src/queue/queue-processor.test.ts +++ b/packages/shared/src/queue/queue-processor.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import os from "node:os"; import { CodingAgentQueueProcessor } from "./queue-processor.js"; import type { QueueProcessorConfig, WorkerProcessor, WorkerResult } from "../types/types.js"; @@ -82,6 +82,63 @@ describe("CodingAgentQueueProcessor.getVersionFields", () => { }); }); +// ─── AGENTS.md workspace delivery ──────────────────────────────────────────── +describe("CodingAgentQueueProcessor.writeAgentsMd", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("no-ops when the request has no agentsMdPromptId", async () => { + const qp = new CodingAgentQueueProcessor(testConfig, stubProcessor); + const log = vi.fn().mockResolvedValue(undefined); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + await (qp as any).writeAgentsMd({ _id: "r1" }, "/tmp/does-not-matter", log); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("throws when agentsMdPromptId is set but apiBaseUrl is missing", async () => { + // testConfig has no apiBaseUrl. + const qp = new CodingAgentQueueProcessor(testConfig, stubProcessor); + const log = vi.fn().mockResolvedValue(undefined); + + await expect( + (qp as any).writeAgentsMd({ _id: "r1", agentsMdPromptId: "agents-1" }, "/tmp", log), + ).rejects.toThrow(/apiBaseUrl/); + }); + + it("writes the resolved AGENTS.md body into the workspace root", async () => { + const { mkdtemp, readFile, rm } = await import("node:fs/promises"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const workspace = await mkdtemp(join(tmpdir(), "agents-md-test-")); + + try { + const qp = new CodingAgentQueueProcessor( + { ...testConfig, apiBaseUrl: "https://api.example.com" }, + stubProcessor, + ); + const log = vi.fn().mockResolvedValue(undefined); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ id: "agents-1", text: "# AGENTS\nBe terse." }), + }), + ); + + await (qp as any).writeAgentsMd({ _id: "r1", agentsMdPromptId: "agents-1" }, workspace, log); + + const written = await readFile(join(workspace, "AGENTS.md"), "utf-8"); + expect(written).toBe("# AGENTS\nBe terse."); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); +}); + // ─── Redelivery fail-fast (handleRequest pre-checks) ───────────────────────── describe("CodingAgentQueueProcessor.handleRequest redelivery handling", () => { function makeHarness(runStatus: string, runOverrides: Record = {}) { diff --git a/packages/shared/src/queue/queue-processor.ts b/packages/shared/src/queue/queue-processor.ts index f663284f7..de32a4e2f 100644 --- a/packages/shared/src/queue/queue-processor.ts +++ b/packages/shared/src/queue/queue-processor.ts @@ -30,6 +30,9 @@ import { McpSecretClient } from "../mcp/mcp-secret-client.js"; import { SkillClient } from "../skills/skill-client.js"; import { ExtensionClient } from "../extensions/extension-client.js"; import { extractSkillsToWorkspace } from "../skills/skill-extractor.js"; +import { PromptClient } from "../task-prompts/prompt-client.js"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; /** * Queue processor for coding agent workers. @@ -388,6 +391,44 @@ export class CodingAgentQueueProcessor extends BaseQueueProcessor/AGENTS.md` before the first turn. The text is resolved via + * the API (`GET /api/v1/task-prompts/:id/content`), which downloads from blob + * storage when the prompt is blob-backed — the worker never touches blob. + * + * Fails loudly (throws) when `agentsMdPromptId` is set but `apiBaseUrl` is + * missing or the fetch/write fails, so the run is marked failed rather than + * silently evaluating the baseline worker (which would corrupt GEPA results). + */ + private async writeAgentsMd( + requestDoc: RequestDocument, + workspacePath: string, + log: (level: LogEvent["level"], msg: string, data?: Record) => Promise, + ): Promise { + const agentsMdPromptId = requestDoc.agentsMdPromptId; + if (!agentsMdPromptId) return; + + const apiBaseUrl = (this.config as QueueProcessorConfig).apiBaseUrl; + if (!apiBaseUrl) { + throw new Error( + `Request has agentsMdPromptId '${agentsMdPromptId}' but no apiBaseUrl is configured; cannot resolve AGENTS.md`, + ); + } + + const promptClient = new PromptClient(apiBaseUrl); + const text = await promptClient.getText(agentsMdPromptId); + const agentsMdPath = join(workspacePath, "AGENTS.md"); + await writeFile(agentsMdPath, text, "utf-8"); + await log("info", `Wrote AGENTS.md to workspace`, { + agentsMdPromptId, + path: agentsMdPath, + bytes: Buffer.byteLength(text, "utf-8"), + }); + } + /** * Multi-turn processing with judge loop. */ @@ -511,6 +552,10 @@ export class CodingAgentQueueProcessor extends BaseQueueProcessor { + vi.restoreAllMocks(); +}); + +describe("PromptClient.getText", () => { + it("fetches resolved text from the content endpoint", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ id: "p1", text: "# AGENTS\nBe concise." }), + }); + vi.stubGlobal("fetch", fetchMock); + + const client = new PromptClient("https://api.example.com/"); + const text = await client.getText("p1"); + + expect(text).toBe("# AGENTS\nBe concise."); + // Trailing slash on base URL is trimmed; id is path-encoded. + expect(fetchMock).toHaveBeenCalledWith("https://api.example.com/api/v1/task-prompts/p1/content"); + }); + + it("throws a clear error on 404", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 404, statusText: "Not Found" })); + const client = new PromptClient("https://api.example.com"); + await expect(client.getText("missing")).rejects.toThrow(/not found/i); + }); + + it("throws on non-OK HTTP status", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 500, statusText: "Server Error" })); + const client = new PromptClient("https://api.example.com"); + await expect(client.getText("p1")).rejects.toThrow(/500/); + }); + + it("throws when the response lacks a text field", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ id: "p1" }) })); + const client = new PromptClient("https://api.example.com"); + await expect(client.getText("p1")).rejects.toThrow(/no text/i); + }); +}); diff --git a/packages/shared/src/task-prompts/prompt-client.ts b/packages/shared/src/task-prompts/prompt-client.ts new file mode 100644 index 000000000..ade51cc2f --- /dev/null +++ b/packages/shared/src/task-prompts/prompt-client.ts @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Client for resolving prompt text (task or AGENTS.md) via the Scope REST API. + * + * Used by queue processors at message-processing time to fetch the resolved + * plain-text body of a prompt (inline or blob-backed) without needing direct + * blob storage access. The API endpoint `GET /api/v1/task-prompts/:id/content` + * runs `resolvePromptText` server-side and returns `{ id, text }`. + */ +export class PromptClient { + private readonly apiUrl: string; + + constructor(apiUrl: string) { + this.apiUrl = apiUrl.replace(/\/+$/, ""); + } + + /** + * Fetch the resolved plain-text body of a prompt by id. + * + * @param id - Prompt id (content hash) of a task or agents.md prompt. + * @throws Error if the prompt cannot be resolved (404 or HTTP error). + */ + async getText(id: string): Promise { + const url = `${this.apiUrl}/api/v1/task-prompts/${encodeURIComponent(id)}/content`; + const res = await fetch(url); + + if (res.status === 404) { + throw new Error(`Prompt '${id}' not found via API`); + } + if (!res.ok) { + throw new Error(`[PromptClient] GET ${url} failed: ${res.status} ${res.statusText}`); + } + + const data = (await res.json()) as { id: string; text: string }; + if (typeof data.text !== "string") { + throw new Error(`[PromptClient] GET ${url} returned no text for prompt '${id}'`); + } + return data.text; + } +} From b9dd1a242d3006c181882cbd842e273323b0ddb9 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Mon, 1 Jun 2026 23:12:11 -0700 Subject: [PATCH 05/54] feat(db): add sparse index migration for requests.experimentId GEPA experiments group a seed request and all follow-up requests under a shared experimentId. Both experiment grouping (groupBy=experiment) and AGENTS.md lineage reconstruction filter requests by this field, so add a sparse single-field index to keep those queries efficient. - 018-add-experiment-id-index.ts: try/catch-wrapped createIndex, log-only non-destructive down(). - Registered in required-migrations.ts. - Unit tests cover index creation, idempotent already-exists, and no-op down. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/018-add-experiment-id-index.test.ts | 51 +++++++++++++++++++ .../migrations/018-add-experiment-id-index.ts | 41 +++++++++++++++ .../db-migrations/src/required-migrations.ts | 1 + 3 files changed, 93 insertions(+) create mode 100644 packages/db-migrations/src/018-add-experiment-id-index.test.ts create mode 100644 packages/db-migrations/src/migrations/018-add-experiment-id-index.ts diff --git a/packages/db-migrations/src/018-add-experiment-id-index.test.ts b/packages/db-migrations/src/018-add-experiment-id-index.test.ts new file mode 100644 index 000000000..095d4539e --- /dev/null +++ b/packages/db-migrations/src/018-add-experiment-id-index.test.ts @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, vi } from "vitest"; +import type { Db } from "mongodb"; + +function makeMockDb(createIndexImpl?: (...args: any[]) => any) { + const createIndex = vi.fn(createIndexImpl ?? (async () => "experimentId_1")); + const requests = { createIndex }; + const db = { + collection: vi.fn((name: string) => { + if (name === "requests") return requests; + throw new Error(`unexpected collection ${name}`); + }), + _requests: requests, + } as any; + return db as Db & { _requests: { createIndex: ReturnType } }; +} + +const { AddExperimentIdIndex } = await import("./migrations/018-add-experiment-id-index.js"); + +describe("migration 018: AddExperimentIdIndex", () => { + describe("up()", () => { + it("creates a sparse single-field index on requests.experimentId", async () => { + const db = makeMockDb(); + await new AddExperimentIdIndex().up(db); + + expect(db.collection).toHaveBeenCalledWith("requests"); + expect(db._requests.createIndex).toHaveBeenCalledTimes(1); + expect(db._requests.createIndex).toHaveBeenCalledWith( + { experimentId: 1 }, + { sparse: true }, + ); + }); + + it("swallows errors when the index already exists", async () => { + const db = makeMockDb(async () => { + throw new Error("index already exists"); + }); + await expect(new AddExperimentIdIndex().up(db)).resolves.toBeUndefined(); + }); + }); + + describe("down()", () => { + it("does not drop the index (no-op)", async () => { + const db = makeMockDb(); + await expect(new AddExperimentIdIndex().down(db)).resolves.toBeUndefined(); + expect(db._requests.createIndex).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/db-migrations/src/migrations/018-add-experiment-id-index.ts b/packages/db-migrations/src/migrations/018-add-experiment-id-index.ts new file mode 100644 index 000000000..020756556 --- /dev/null +++ b/packages/db-migrations/src/migrations/018-add-experiment-id-index.ts @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Migration: Create a sparse index on experimentId in the requests collection. + * + * GEPA optimizer runs group a seed request and all follow-up (mutation/merge) + * requests under a shared `experimentId`. Both experiment grouping + * (`groupBy=experiment`) and AGENTS.md lineage reconstruction filter requests + * by this field, so a single-field index keeps those queries efficient. + * + * The index is sparse so it only indexes documents that actually carry an + * `experimentId` (most requests don't), avoiding write-RU cost on the rest. + * No other indexes are needed: the new `type` fields on task-prompts / + * prompt-features are tiny, human-curated collections where `?type=` scans are + * cheap, and request lineage is stored inline on the request + * (`agentsMdParentIds`) and queried via this same experimentId index. + */ + +import type { Db } from "mongodb"; +import type { MigrationInterface } from "mongo-migrate-ts"; + +export class AddExperimentIdIndex implements MigrationInterface { + async up(db: Db): Promise { + const col = db.collection("requests"); + try { + await col.createIndex({ experimentId: 1 }, { sparse: true }); + console.log(" Created sparse index on requests.experimentId"); + } catch (err: any) { + console.log( + ` Index on requests.experimentId already exists or couldn't be created: ${err.message ?? err}`, + ); + } + } + + async down(db: Db): Promise { + console.log( + " Skipping index drop — indexes should be dropped manually if needed", + ); + } +} diff --git a/packages/db-migrations/src/required-migrations.ts b/packages/db-migrations/src/required-migrations.ts index 120a54d0d..08ebf54b4 100644 --- a/packages/db-migrations/src/required-migrations.ts +++ b/packages/db-migrations/src/required-migrations.ts @@ -30,4 +30,5 @@ export const REQUIRED_MIGRATIONS: readonly string[] = [ "015-add-priority-and-scheduler-index.ts", "016-fix-scheduler-sort-index.ts", "017-add-post-processor-dispatch-index.ts", + "018-add-experiment-id-index.ts", ]; From b2ee59ddd47f3858d05f8ff631066d335e880201 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Mon, 1 Jun 2026 23:21:08 -0700 Subject: [PATCH 06/54] feat(portal): AGENTS.md submit field, experiment grouping, typed prompt features - SubmitRun: optional AGENTS.md textarea + experimentId input - RunDetail: render AGENTS.md as Markdown with lineage/experiment badges - RunsList: add "Experiment" group-by option (client-side grouping by experimentId) - PromptFeatureWizard/List: typed features (task | agents.md) with filter + column - api.ts/types.ts: new request fields, TaskPromptType, getTaskPromptContent Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/components/PromptFeatureWizard.tsx | 24 ++++++++++- apps/portal/src/lib/api.ts | 30 +++++++++---- apps/portal/src/pages/PromptFeatureList.tsx | 41 +++++++++++++++--- apps/portal/src/pages/RunDetail.tsx | 42 +++++++++++++++++++ apps/portal/src/pages/RunsList.tsx | 14 ++++--- apps/portal/src/pages/SubmitRun.tsx | 29 +++++++++++++ apps/portal/src/types.ts | 17 +++++++- 7 files changed, 174 insertions(+), 23 deletions(-) diff --git a/apps/portal/src/components/PromptFeatureWizard.tsx b/apps/portal/src/components/PromptFeatureWizard.tsx index fde815683..90dd5538d 100644 --- a/apps/portal/src/components/PromptFeatureWizard.tsx +++ b/apps/portal/src/components/PromptFeatureWizard.tsx @@ -10,8 +10,9 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; import { Badge } from "@/components/ui/badge"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Stepper } from "@/components/Stepper"; -import type { PromptFeatureDocument } from "@/types"; +import type { PromptFeatureDocument, TaskPromptType } from "@/types"; import { ArrowLeft, ArrowRight, @@ -35,6 +36,8 @@ export interface PromptFeatureWizardProps { initialId?: string; /** Pre-fill detection prompt (skips to step 2 when provided) */ initialPrompt?: string; + /** Pre-select the prompt type this feature applies to (default "task") */ + initialType?: TaskPromptType; /** Called after successful feature creation */ onCreated?: (feature: PromptFeatureDocument) => void; /** Called when user clicks Cancel (step 1) or Back at step 1 */ @@ -45,6 +48,7 @@ export function PromptFeatureWizard({ initialBehavior = "", initialId = "", initialPrompt = "", + initialType = "task", onCreated, onCancel, }: PromptFeatureWizardProps) { @@ -58,6 +62,7 @@ export function PromptFeatureWizard({ const [id, setId] = useState(initialId); const [idManuallyEdited, setIdManuallyEdited] = useState(!!initialId); const [idEditMode, setIdEditMode] = useState(false); + const [type, setType] = useState(initialType); // Step 2 fields const [prompt, setPrompt] = useState(initialPrompt); @@ -117,6 +122,7 @@ export function PromptFeatureWizard({ createMutation.mutate({ id: id.trim(), prompt: prompt.trim(), + ...(type !== "task" ? { type } : {}), }); }; @@ -170,6 +176,22 @@ export function PromptFeatureWizard({

+
+ + +

+ Features are only extracted against prompts of their own type. +

+
+