From c2eab7f6e98ed38460673ac40542957b78c1b252 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Wed, 21 Oct 2026 00:45:08 -0700 Subject: [PATCH 1/2] feat(portal): specialize inline observation authoring for New Run Adapt the shared wizard slice from 6378ec19 onto the taxonomy layer without Experiment dependencies. Preserve generic Criteria defaults and existing observation subject/result semantics. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d0c36bff-9146-4622-aaf0-858de5876813 --- .../CreateCriterionDialog.stories.tsx | 86 +++++++++++ .../components/CreateCriterionDialog.test.tsx | 139 ++++++++++++++++++ .../src/components/CreateCriterionDialog.tsx | 36 ++++- .../src/components/CriteriaPicker.stories.tsx | 57 +++++++ .../src/components/CriteriaPicker.test.tsx | 23 ++- apps/portal/src/components/CriteriaPicker.tsx | 26 +++- .../src/components/CriteriaWizardStep1.tsx | 73 +++++---- .../components/CriteriaWizardStep2.test.tsx | 26 +++- .../src/components/CriteriaWizardStep2.tsx | 69 ++++++--- .../src/hooks/useCriteriaWizard.test.tsx | 86 +++++++++++ apps/portal/src/hooks/useCriteriaWizard.ts | 10 +- apps/portal/src/pages/SubmitRun.stories.tsx | 43 ++++++ apps/portal/src/pages/SubmitRun.test.tsx | 115 +++++++++++++++ apps/portal/src/pages/SubmitRun.tsx | 4 +- docs/architecture/app-design.md | 16 ++ 15 files changed, 738 insertions(+), 71 deletions(-) create mode 100644 apps/portal/src/components/CreateCriterionDialog.stories.tsx create mode 100644 apps/portal/src/components/CreateCriterionDialog.test.tsx create mode 100644 apps/portal/src/components/CriteriaPicker.stories.tsx create mode 100644 apps/portal/src/pages/SubmitRun.stories.tsx create mode 100644 apps/portal/src/pages/SubmitRun.test.tsx diff --git a/apps/portal/src/components/CreateCriterionDialog.stories.tsx b/apps/portal/src/components/CreateCriterionDialog.stories.tsx new file mode 100644 index 000000000..ddaab78bc --- /dev/null +++ b/apps/portal/src/components/CreateCriterionDialog.stories.tsx @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState } from "react"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, fn, userEvent, waitFor, within } from "storybook/test"; +import { http, HttpResponse } from "msw"; +import { setSelectedProjectIdHolder } from "@/lib/project-scope"; +import { CreateCriterionDialog } from "./CreateCriterionDialog"; + +const meta = { + title: "Components/CreateCriterionDialog", + component: CreateCriterionDialog, + tags: ["ai-generated", "needs-work"], + args: { open: true, onOpenChange: fn(), onCreated: fn() }, + decorators: [(Story) => { + setSelectedProjectIdHolder("demo-project"); + return ; + }], + render: (args) => { + const [open, setOpen] = useState(args.open); + return ; + }, + parameters: { + msw: { + handlers: [ + http.get("*/api/v1/criteria", () => HttpResponse.json([])), + http.post("*/api/v1/criteria/generate-prompt", () => HttpResponse.json({ + prompt: "Record whether the codebase contains unit tests.", + suggestedId: "has_unit_tests", + suggestedParents: [], + suggestedChildren: [], + suggestedSubject: "run", + })), + http.post("*/api/v1/criteria", () => HttpResponse.json({ + id: "has_unit_tests", prompt: "Record whether the codebase contains unit tests.", createdAt: "", + })), + ], + }, + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const GenericCriteria: Story = { + play: async ({ canvasElement, args }) => { + const page = within(canvasElement.ownerDocument.body); + await waitFor(() => expect(page.getByRole("heading", { name: "New Criteria" })).toBeVisible()); + await expect(page.getByLabelText("Criteria kind")).toBeVisible(); + await userEvent.type(page.getByLabelText(/What behavior do you want to track/), "Has unit tests"); + await userEvent.click(page.getByRole("button", { name: "Continue" })); + await expect(await page.findByText("Criteria Prompt (AI Generated)")).toBeVisible(); + await expect(page.getByText("Gate compatibility")).toBeVisible(); + await userEvent.click(page.getByRole("button", { name: "Create" })); + await waitFor(() => expect(args.onCreated).toHaveBeenCalledWith("has_unit_tests")); + }, +}; + +export const ObservationCreation: Story = { + args: { mode: "observation" }, + play: async ({ canvasElement, args }) => { + const page = within(canvasElement.ownerDocument.body); + await waitFor(() => expect(page.getByRole("heading", { name: "New Observation" })).toBeVisible()); + await expect(page.queryByLabelText("Criteria kind")).not.toBeInTheDocument(); + await expect(page.getByLabelText("Evaluation subject")).toHaveTextContent("Whole run"); + await expect(page.getByText("Observation dimension")).toBeVisible(); + await userEvent.type(page.getByLabelText(/What behavior do you want to observe/), "Has unit tests"); + await userEvent.click(page.getByRole("button", { name: "Continue" })); + await expect(await page.findByText("Observation Prompt (AI Generated)")).toBeVisible(); + await expect(page.getAllByPlaceholderText("Search observations…")).toHaveLength(2); + await expect(page.queryByText(/No gate-compatible criteria/)).not.toBeInTheDocument(); + await userEvent.click(page.getByRole("button", { name: "Create Observation" })); + await waitFor(() => expect(args.onCreated).toHaveBeenCalledWith("has_unit_tests")); + }, +}; + +export const LegacyObservationDefault: Story = { + args: { defaultKind: "observation" }, + play: async ({ canvasElement }) => { + const page = within(canvasElement.ownerDocument.body); + await waitFor(() => expect(page.getByRole("heading", { name: "New Criteria" })).toBeVisible()); + await expect(page.getByLabelText("Criteria kind")).toHaveTextContent("Observation"); + await expect(page.getByLabelText("Evaluation subject")).toBeVisible(); + }, +}; diff --git a/apps/portal/src/components/CreateCriterionDialog.test.tsx b/apps/portal/src/components/CreateCriterionDialog.test.tsx new file mode 100644 index 000000000..0e478935d --- /dev/null +++ b/apps/portal/src/components/CreateCriterionDialog.test.tsx @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ComponentProps } from "react"; +import { CreateCriterionDialog } from "./CreateCriterionDialog"; +import { api } from "@/lib/api"; + +vi.mock("@/lib/api", () => ({ + api: { + listCriteria: vi.fn(), + generateCriteriaPrompt: vi.fn(), + createCriterion: vi.fn(), + }, +})); + +beforeEach(() => { + vi.mocked(api.listCriteria).mockResolvedValue([]); + vi.mocked(api.generateCriteriaPrompt).mockResolvedValue({ + prompt: "Record whether the codebase contains unit tests.", + suggestedId: "ai_suggested_id", + suggestedParents: [], + suggestedChildren: [], + }); + vi.mocked(api.createCriterion).mockImplementation(async (input) => ({ + ...input, + createdAt: "", + })); +}); + +afterEach(() => { + cleanup(); + vi.resetAllMocks(); +}); + +function renderDialog(props: Partial> = {}) { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }); + const onCreated = vi.fn(); + const onOpenChange = vi.fn(); + + render( + + + , + ); + + return { onCreated, onOpenChange }; +} + +describe("CreateCriterionDialog modes", () => { + it("keeps generic Criteria creation and locked gates unchanged by default", async () => { + renderDialog({ defaultGates: ["build"], lockedGates: ["build"] }); + + expect(screen.getByRole("heading", { name: "New Criteria" })).toBeTruthy(); + expect(screen.getByLabelText("Criteria kind")).toBeTruthy(); + expect(screen.getByText(/Gate criteria steer the agent/)).toBeTruthy(); + expect(screen.getByText(/Criteria ID/)).toBeTruthy(); + + fireEvent.change(screen.getByLabelText(/What behavior do you want to track/), { + target: { value: "Has unit tests" }, + }); + await waitFor(() => expect(screen.getByRole("button", { name: /Continue/ }).hasAttribute("disabled")).toBe(false)); + fireEvent.click(screen.getByRole("button", { name: /Continue/ })); + await screen.findByText("Criteria Prompt (AI Generated)"); + fireEvent.click(screen.getByRole("button", { name: "Create" })); + + await waitFor(() => expect(vi.mocked(api.createCriterion).mock.calls[0]?.[0]).toEqual({ + id: "has_unit_tests", + prompt: "Record whether the codebase contains unit tests.", + dependsOn: undefined, + kind: "gate", + gates: ["build"], + })); + }); + + it("retains defaultKind without opting into specialized Observation copy", () => { + renderDialog({ defaultKind: "observation" }); + + expect(screen.getByRole("heading", { name: "New Criteria" })).toBeTruthy(); + expect(screen.getByLabelText("Criteria kind").textContent).toContain("Observation"); + expect(screen.getByLabelText("Evaluation subject")).toBeTruthy(); + }); + + it("fixes the Observation kind and creates a run-subject observation without gates", async () => { + const { onCreated, onOpenChange } = renderDialog({ + mode: "observation", + defaultKind: "gate", + defaultGates: ["build"], + }); + + expect(screen.getByRole("heading", { name: "New Observation" })).toBeTruthy(); + expect(screen.getByText(/Observation ID/)).toBeTruthy(); + expect(screen.getByText("Observation dimension")).toBeTruthy(); + expect(screen.getByLabelText("Evaluation subject").textContent).toContain("Whole run"); + expect(screen.queryByLabelText("Criteria kind")).toBeNull(); + expect(screen.queryByText(/Gate criteria steer the agent/)).toBeNull(); + expect(screen.queryByText("Gate compatibility")).toBeNull(); + + fireEvent.change(screen.getByLabelText(/What behavior do you want to observe/), { + target: { value: "Has unit tests" }, + }); + await waitFor(() => expect(screen.getByRole("button", { name: /Continue/ }).hasAttribute("disabled")).toBe(false)); + fireEvent.click(screen.getByRole("button", { name: /Continue/ })); + + await screen.findByText("Observation Prompt (AI Generated)"); + expect(screen.getByRole("heading", { name: "Review & Create Observation" })).toBeTruthy(); + expect(screen.getAllByPlaceholderText("Search observations…")).toHaveLength(2); + expect(screen.queryByText(/Criteria Prompt/)).toBeNull(); + expect(screen.queryByText(/observation criteria/)).toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "Create Observation" })); + + await waitFor(() => { + expect(vi.mocked(api.createCriterion).mock.calls[0]?.[0]).toEqual({ + dependsOn: undefined, + id: "has_unit_tests", + prompt: "Record whether the codebase contains unit tests.", + kind: "observation", + subject: "run", + }); + expect(onCreated).toHaveBeenCalledWith("has_unit_tests"); + expect(onOpenChange).toHaveBeenCalledWith(false); + }); + expect(api.generateCriteriaPrompt).toHaveBeenCalledWith("Has unit tests", undefined, undefined); + }); +}); diff --git a/apps/portal/src/components/CreateCriterionDialog.tsx b/apps/portal/src/components/CreateCriterionDialog.tsx index be51969f3..7e550bfe3 100644 --- a/apps/portal/src/components/CreateCriterionDialog.tsx +++ b/apps/portal/src/components/CreateCriterionDialog.tsx @@ -28,13 +28,25 @@ interface CreateCriterionDialogProps { lockedGates?: GateId[]; /** Pre-populated criteria kind (defaults to gate) */ defaultKind?: CriterionKind; + /** Specialized creation experience for inline observation entry points. */ + mode?: "criteria" | "observation"; } -export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGates, lockedGates, defaultKind }: CreateCriterionDialogProps) { +export function CreateCriterionDialog({ + open, + onOpenChange, + onCreated, + defaultGates, + lockedGates, + defaultKind, + mode = "criteria", +}: CreateCriterionDialogProps) { + const observationMode = mode === "observation"; const wizard = useCriteriaWizard({ initialGates: defaultGates, lockedGates, - initialKind: defaultKind, + initialKind: observationMode ? "observation" : defaultKind, + entityLabel: observationMode ? "Observation" : "Criterion", onSuccess: (id) => { onCreated(id); handleOpenChange(false); @@ -65,14 +77,20 @@ export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGa {wizard.step === 1 ? ( <> - New Criteria + {observationMode ? "New Observation" : "New Criteria"} - Describe the behavior to evaluate + {observationMode + ? "Describe the behavior to observe" + : "Describe the behavior to evaluate"}
- +
@@ -90,14 +108,16 @@ export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGa ) : ( <> - Review & Create + + {observationMode ? "Review & Create Observation" : "Review & Create"} + {wizard.behavior}
- +
@@ -120,7 +140,7 @@ export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGa ) : ( )} - Create + {observationMode ? "Create Observation" : "Create"} diff --git a/apps/portal/src/components/CriteriaPicker.stories.tsx b/apps/portal/src/components/CriteriaPicker.stories.tsx new file mode 100644 index 000000000..1972ab668 --- /dev/null +++ b/apps/portal/src/components/CriteriaPicker.stories.tsx @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useState } from "react"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, userEvent, within } from "storybook/test"; +import { http, HttpResponse } from "msw"; +import { setSelectedProjectIdHolder } from "@/lib/project-scope"; +import { CriteriaPicker } from "./CriteriaPicker"; + +const meta = { + title: "Components/CriteriaPicker", + component: CriteriaPicker, + tags: ["ai-generated", "needs-work"], + args: { selected: [], onChange: () => undefined }, + decorators: [(Story) => { + setSelectedProjectIdHolder("demo-project"); + return
; + }], + render: (args) => { + const [selected, setSelected] = useState([]); + return ; + }, + parameters: { + msw: { + handlers: [http.get("*/api/v1/criteria", () => HttpResponse.json([ + { id: "gate_requirement", prompt: "The project builds.", kind: "gate", createdAt: "" }, + { id: "has_tests", prompt: "Record whether tests exist.", kind: "observation", createdAt: "" }, + ]))], + }, + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const GenericCriteria: Story = { + play: async ({ canvas, canvasElement }) => { + const input = await canvas.findByPlaceholderText("Type to search criteria…"); + await userEvent.type(input, "missing"); + await expect(await within(canvasElement.ownerDocument.body).findByText("No matching criteria")).toBeVisible(); + }, +}; + +export const ObservationDependencies: Story = { + args: { kind: "observation", entityLabel: "observations" }, + play: async ({ canvas, canvasElement }) => { + const page = within(canvasElement.ownerDocument.body); + const input = await canvas.findByPlaceholderText("Search observations…"); + await userEvent.click(input); + await expect(page.queryByRole("button", { name: /gate_requirement/ })).not.toBeInTheDocument(); + await userEvent.click(await page.findByRole("button", { name: /has_tests/ })); + await expect(canvas.getByText("has_tests")).toBeVisible(); + await userEvent.type(input, "missing"); + await expect(await page.findByText("No matching observations")).toBeVisible(); + }, +}; diff --git a/apps/portal/src/components/CriteriaPicker.test.tsx b/apps/portal/src/components/CriteriaPicker.test.tsx index 7c488c1f8..2f4bea56e 100644 --- a/apps/portal/src/components/CriteriaPicker.test.tsx +++ b/apps/portal/src/components/CriteriaPicker.test.tsx @@ -9,8 +9,9 @@ import { CriteriaPicker } from "./CriteriaPicker"; import type { CriteriaDocument } from "@/types"; const CRITERIA: CriteriaDocument[] = [ - { id: "crit_a", prompt: "a" } as CriteriaDocument, - { id: "crit_b", prompt: "b" } as CriteriaDocument, + { id: "crit_a", prompt: "a", createdAt: "" }, + { id: "crit_b", prompt: "b", createdAt: "" }, + { id: "observation_a", prompt: "record a", kind: "observation", createdAt: "" }, ]; vi.mock("@/lib/api", () => ({ @@ -60,4 +61,22 @@ describe("CriteriaPicker dropdown", () => { fireEvent.click(option); expect(selected).toEqual(["crit_a"]); }); + + it("uses specialized copy without admitting gate criteria to observation suggestions", async () => { + const onChange = vi.fn(); + const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( + + + , + ); + const input = await screen.findByPlaceholderText("Search observations…"); + fireEvent.focus(input); + expect(screen.queryByRole("button", { name: /crit_a/ })).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: /observation_a/ })); + expect(onChange).toHaveBeenCalledWith(["observation_a"]); + + fireEvent.change(input, { target: { value: "missing" } }); + expect(screen.getByText("No matching observations")).toBeTruthy(); + }); }); diff --git a/apps/portal/src/components/CriteriaPicker.tsx b/apps/portal/src/components/CriteriaPicker.tsx index 19ed40767..25b68fb4f 100644 --- a/apps/portal/src/components/CriteriaPicker.tsx +++ b/apps/portal/src/components/CriteriaPicker.tsx @@ -25,6 +25,8 @@ interface CriteriaPickerProps { gate?: GateId; /** Restrict suggestions by criterion kind. Missing kind is treated as "gate". */ kind?: CriterionKind; + /** User-facing noun used by specialized creation flows. */ + entityLabel?: "criteria" | "observations"; /** * Additional predicate restricting which criteria are offered as suggestions. * Used to enforce gate-compatibility invariants (e.g. only show criteria that @@ -33,7 +35,17 @@ interface CriteriaPickerProps { filter?: (criterion: CriteriaDocument) => boolean; } -export function CriteriaPicker({ selected, onChange, aiSuggested = [], inputId, trailingAction, gate, kind, filter }: CriteriaPickerProps) { +export function CriteriaPicker({ + selected, + onChange, + aiSuggested = [], + inputId, + trailingAction, + gate, + kind, + entityLabel, + filter, +}: CriteriaPickerProps) { const [query, setQuery] = useState(""); const [open, setOpen] = useState(false); const [highlightIdx, setHighlightIdx] = useState(0); @@ -215,7 +227,15 @@ export function CriteriaPicker({ selected, onChange, aiSuggested = [], inputId, }} onFocus={() => setOpen(true)} onKeyDown={handleKeyDown} - placeholder={kind === "observation" ? "Search observation criteria…" : gate ? `Search ${GATE_METADATA[gate].label} criteria…` : "Type to search criteria…"} + placeholder={ + entityLabel + ? `Search ${entityLabel}…` + : kind === "observation" + ? "Search observation criteria…" + : gate + ? `Search ${GATE_METADATA[gate].label} criteria…` + : "Type to search criteria…" + } className="h-9 font-mono text-sm" /> {trailingAction} @@ -249,7 +269,7 @@ export function CriteriaPicker({ selected, onChange, aiSuggested = [], inputId, {open && query && suggestions.length === 0 && createPortal(
- No matching criteria + No matching {entityLabel ?? "criteria"}
, document.body, )} diff --git a/apps/portal/src/components/CriteriaWizardStep1.tsx b/apps/portal/src/components/CriteriaWizardStep1.tsx index 3cda0d8e3..09b545c51 100644 --- a/apps/portal/src/components/CriteriaWizardStep1.tsx +++ b/apps/portal/src/components/CriteriaWizardStep1.tsx @@ -13,9 +13,14 @@ interface CriteriaWizardStep1Props { wizard: CriteriaWizardState; /** HTML id prefix to avoid collisions when rendered in different contexts */ idPrefix?: string; + observationMode?: boolean; } -export function CriteriaWizardStep1({ wizard, idPrefix = "" }: CriteriaWizardStep1Props) { +export function CriteriaWizardStep1({ + wizard, + idPrefix = "", + observationMode = false, +}: CriteriaWizardStep1Props) { const { behavior, handleBehaviorChange, @@ -41,27 +46,34 @@ export function CriteriaWizardStep1({ wizard, idPrefix = "" }: CriteriaWizardSte {/* Behavior description */}