From c5d24b5aff760431d74d6f4e0163fb4df24ba70b Mon Sep 17 00:00:00 2001 From: Will Leeney Date: Mon, 14 Sep 2026 08:38:46 +0100 Subject: [PATCH 1/3] test(examples): execute the real example files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tests/examples/` reimplemented each example's logic against `../../src` and never imported `examples/*.ts`. An example could rot or break outright and CI stayed green — and four of the eight had no test at all. Examples do their work at import time via top-level await, so importing one runs it. `tests/examples/run-example.ts` makes that safe under vitest: - `STACKONE_BASE_URL` points the SDK at the MSW mock (toolsets.ts:630 reads it), so an example constructing `new StackOneToolSet()` with no explicit baseUrl still hits the mock rather than production. No example needed changing. - `process.exit` becomes a throw that `runExample` catches and reports as `exitCode`, so an example bailing on missing config is an assertable outcome instead of a torn-down vitest worker. Seven of the eight examples now execute for real. The eighth, claude-agent-sdk-integration, still cannot: `query()` spawns a claude-code subprocess that MSW cannot intercept. Its existing setup-level test is kept and the exclusion is now stated in the file rather than implied. Mock gaps that blocked execution, all of which made the examples exercise nothing: - The three `workday_*` actions every example filters on were absent from the MCP catalog, so the filters matched zero tools. - No Anthropic handler existed at all (mocks/handlers.anthropic.ts). - The OpenAI handlers keyed off prompts none of the examples send; the examples all prompt "List the first 5 employees". - Semantic search (`/actions/search`) was unmocked, so search-tools hit the real network. Added scoped to that test via `server.use` — a global handler pre-empts src/semantic-search.test.ts, which asserts against an unmocked endpoint. Tool-count assertions are exact rather than `> 0`. A `> 0` check cannot fail: `fetchTools` appends tool_feedback *after* filtering, so a filter matching nothing still returns one tool. Verified by mutation — pointing an example at a nonexistent action now fails with `expected 1 to be 4`, where before it passed. defender-config keeps all seven original wire-payload tests, which assert the exact RPC body per defender mode; the four execution tests are added alongside. fetch-tools and tanstack-ai-integration describe examples that do not exist. Relabelled in place rather than moved — the `examples` vitest project globs only `tests/examples/**` and `root` globs only `src/`+`scripts/`, so relocating them would have dropped them from both projects silently. 348 passing, up from 329. The one failure is pre-existing and environmental (`spawn tsc ENOENT`); it fails identically on origin/main. --- mocks/handlers.anthropic.ts | 50 +++++++++++ mocks/handlers.openai.ts | 55 ++++++++++++ mocks/handlers.ts | 2 + mocks/mcp-server.ts | 36 ++++++++ tests/examples/ai-sdk-integration.test.ts | 36 ++++++++ tests/examples/anthropic-integration.test.ts | 38 +++++++++ tests/examples/auth-management.test.ts | 51 +++++++++++ .../claude-agent-sdk-integration.test.ts | 9 +- tests/examples/defender-config.test.ts | 45 +++++++++- tests/examples/fetch-tools.test.ts | 7 +- tests/examples/openai-integration.test.ts | 78 ++++++----------- .../openai-responses-integration.test.ts | 66 +++++---------- tests/examples/run-example.ts | 84 +++++++++++++++++++ tests/examples/search-tools.test.ts | 72 ++++++++++++++++ .../examples/tanstack-ai-integration.test.ts | 7 +- 15 files changed, 524 insertions(+), 112 deletions(-) create mode 100644 mocks/handlers.anthropic.ts create mode 100644 tests/examples/ai-sdk-integration.test.ts create mode 100644 tests/examples/anthropic-integration.test.ts create mode 100644 tests/examples/auth-management.test.ts create mode 100644 tests/examples/run-example.ts create mode 100644 tests/examples/search-tools.test.ts diff --git a/mocks/handlers.anthropic.ts b/mocks/handlers.anthropic.ts new file mode 100644 index 00000000..59846406 --- /dev/null +++ b/mocks/handlers.anthropic.ts @@ -0,0 +1,50 @@ +import { http, HttpResponse } from 'msw'; + +/** + * Anthropic Messages API, for anthropic-integration.ts. + */ +export const anthropicHandlers = [ + http.post('https://api.anthropic.com/v1/messages', async ({ request }) => { + const body = (await request.json()) as { + messages?: Array<{ content?: unknown; role?: string }>; + tools?: Array<{ name?: string }>; + }; + + const userMessage = body.messages + ?.filter((message) => message.role === 'user') + .map((message) => + typeof message.content === 'string' ? message.content : JSON.stringify(message.content), + ) + .join(' '); + const hasTools = Boolean(body.tools?.length); + + if (hasTools && userMessage?.includes('List the first 5 employees')) { + return HttpResponse.json({ + id: 'msg_mock_list', + type: 'message', + role: 'assistant', + model: 'claude-haiku-4-5-20241022', + content: [ + { + type: 'tool_use', + id: 'toolu_mock_list', + name: 'workday_list_workers', + input: { query: { limit: 5 } }, + }, + ], + stop_reason: 'tool_use', + usage: { input_tokens: 100, output_tokens: 50 }, + }); + } + + return HttpResponse.json({ + id: 'msg_default', + type: 'message', + role: 'assistant', + model: 'claude-haiku-4-5-20241022', + content: [{ type: 'text', text: 'Mock response' }], + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 10 }, + }); + }), +]; diff --git a/mocks/handlers.openai.ts b/mocks/handlers.openai.ts index 6f004c2e..d6214a64 100644 --- a/mocks/handlers.openai.ts +++ b/mocks/handlers.openai.ts @@ -65,6 +65,29 @@ export const openaiHandlers = [ }); } + // For openai-responses-integration.ts and search-tools.ts, which prompt with + // the same sentence and expect a tool call back. + if (hasTools && userMessage.includes('List the first 5 employees')) { + return HttpResponse.json({ + id: 'resp_mock_list', + object: 'response', + created_at: Date.now(), + model: 'gpt-5', + status: 'completed', + output: [ + { + type: 'function_call', + id: 'call_mock_list', + call_id: 'call_mock_list', + name: 'workday_list_workers', + arguments: JSON.stringify({ query: { limit: 5 } }), + status: 'completed', + }, + ], + usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, + }); + } + // For human-in-the-loop.ts if (hasTools && userMessage.includes('Create a new employee')) { return HttpResponse.json({ @@ -158,6 +181,38 @@ export const openaiHandlers = [ }); } + // For openai-integration.ts, which filters to workday actions and prompts + // with this sentence. + if (hasTools && userMessage.includes('List the first 5 employees')) { + return HttpResponse.json({ + id: 'chatcmpl-mock-list', + object: 'chat.completion', + created: Date.now(), + model: 'gpt-5', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_mock_list', + type: 'function', + function: { + name: 'workday_list_workers', + arguments: JSON.stringify({ query: { limit: 5 } }), + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, + }); + } + // For human-in-the-loop.ts - returns tool call for bamboohr_create_employee if (hasTools && userMessage.includes('Create a new employee')) { return HttpResponse.json({ diff --git a/mocks/handlers.ts b/mocks/handlers.ts index d1fc784c..20033fef 100644 --- a/mocks/handlers.ts +++ b/mocks/handlers.ts @@ -1,3 +1,4 @@ +import { anthropicHandlers } from './handlers.anthropic'; import { exampleApiHandlers } from './handlers.example-api'; import { mcpHandlers } from './handlers.mcp'; import { openaiHandlers } from './handlers.openai'; @@ -5,6 +6,7 @@ import { stackoneAiHandlers } from './handlers.stackone-ai'; import { stackoneRpcHandlers } from './handlers.stackone-rpc'; export const handlers = [ + ...anthropicHandlers, ...openaiHandlers, ...stackoneRpcHandlers, ...stackoneAiHandlers, diff --git a/mocks/mcp-server.ts b/mocks/mcp-server.ts index 24a53258..4187d363 100644 --- a/mocks/mcp-server.ts +++ b/mocks/mcp-server.ts @@ -227,6 +227,42 @@ export const exampleBamboohrTools = [ required: ['name'], }, }, + // The examples filter on these three action names. Without them the examples + // execute against an empty toolset and assert nothing. + { + name: 'workday_list_workers', + description: 'List workers from Workday', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'object', + properties: { + limit: { type: 'number', description: 'Limit the number of results' }, + }, + }, + }, + }, + }, + { + name: 'workday_get_worker', + description: 'Get a single worker by ID from Workday', + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'The worker ID' }, + }, + required: ['id'], + }, + }, + { + name: 'workday_get_current_user', + description: 'Get the currently authenticated Workday user', + inputSchema: { + type: 'object', + properties: {}, + }, + }, ] as const satisfies McpToolDefinition[]; export const mixedProviderTools = [ diff --git a/tests/examples/ai-sdk-integration.test.ts b/tests/examples/ai-sdk-integration.test.ts new file mode 100644 index 00000000..bb08825f --- /dev/null +++ b/tests/examples/ai-sdk-integration.test.ts @@ -0,0 +1,36 @@ +/** + * Executes examples/ai-sdk-integration.ts. + */ + +import { runExample, stubExampleEnv } from './run-example'; + +describe('ai-sdk-integration example', () => { + beforeEach(() => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('loads tools and drives a Vercel AI SDK generation', async () => { + const { stdout, exitCode } = await runExample('../../examples/ai-sdk-integration.ts'); + + expect(exitCode).toBeUndefined(); + + // 4: 3 workday actions the example filters to, plus the auto-appended tool_feedback. + // A bare > 0 check cannot fail here — tool_feedback is appended after + // filtering, so even a filter matching nothing still yields one tool. + const loaded = stdout.match(/Loaded (\d+) tools/); + expect(loaded).not.toBeNull(); + expect(Number(loaded?.[1])).toBe(4); + }); + + it('skips cleanly when OPENAI_API_KEY is absent', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); + + const { stdout, exitCode } = await runExample('../../examples/ai-sdk-integration.ts'); + + expect(exitCode).toBe(0); + }); +}); diff --git a/tests/examples/anthropic-integration.test.ts b/tests/examples/anthropic-integration.test.ts new file mode 100644 index 00000000..affc1c33 --- /dev/null +++ b/tests/examples/anthropic-integration.test.ts @@ -0,0 +1,38 @@ +/** + * Executes examples/anthropic-integration.ts. + */ + +import { runExample, stubExampleEnv } from './run-example'; + +describe('anthropic-integration example', () => { + beforeEach(() => { + stubExampleEnv({ ANTHROPIC_API_KEY: 'test-anthropic-key' }); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('loads tools, calls Anthropic, and reports the tool use', async () => { + const { stdout, exitCode } = await runExample('../../examples/anthropic-integration.ts'); + + expect(exitCode).toBeUndefined(); + + // 3: 2 tools matching the example's filters, plus the auto-appended tool_feedback. + // A bare > 0 check cannot fail here — tool_feedback is appended after + // filtering, so even a filter matching nothing still yields one tool. + const loaded = stdout.match(/Loaded (\d+) tools/); + expect(loaded).not.toBeNull(); + expect(Number(loaded?.[1])).toBe(3); + + expect(stdout).toContain('workday_list_workers'); + }); + + it('skips cleanly when ANTHROPIC_API_KEY is absent', async () => { + vi.stubEnv('ANTHROPIC_API_KEY', ''); + + const { stdout, exitCode } = await runExample('../../examples/anthropic-integration.ts'); + + expect(exitCode).toBe(0); + }); +}); diff --git a/tests/examples/auth-management.test.ts b/tests/examples/auth-management.test.ts new file mode 100644 index 00000000..f8d3d2b5 --- /dev/null +++ b/tests/examples/auth-management.test.ts @@ -0,0 +1,51 @@ +/** + * Executes examples/auth-management.ts. + */ + +import { runExample, stubExampleEnv } from './run-example'; + +describe('auth-management example', () => { + beforeEach(() => { + stubExampleEnv(); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('runs every auth pattern to completion', async () => { + const { stdout, exitCode } = await runExample('../../examples/auth-management.ts'); + + expect(exitCode).toBeUndefined(); + expect(stdout).toContain('Done — all auth patterns demonstrated.'); + }); + + it('loads tools through each configuration path', async () => { + const { stdout } = await runExample('../../examples/auth-management.ts'); + + for (const section of [ + '1a. API Key from environment', + '1b. Explicit API key', + '2. Account ID from environment', + '3. Account ID in constructor', + '4. setAccounts() — global account list', + '5. Per-tool account override', + ]) { + expect(stdout).toContain(section); + } + + // Every fetchTools() call must return a non-empty toolset; "Loaded 0 tools" + // would mean the example ran but silently exercised nothing. + const loaded = [...stdout.matchAll(/Loaded (\d+) tools/g)].map((match) => Number(match[1])); + expect(loaded.length).toBeGreaterThan(0); + expect(loaded.every((count) => count > 0)).toBe(true); + }); + + it('exits non-zero when STACKONE_API_KEY is absent', async () => { + vi.stubEnv('STACKONE_API_KEY', ''); + + const { exitCode } = await runExample('../../examples/auth-management.ts'); + + expect(exitCode).toBe(1); + }); +}); diff --git a/tests/examples/claude-agent-sdk-integration.test.ts b/tests/examples/claude-agent-sdk-integration.test.ts index 4bd5a38f..05da426e 100644 --- a/tests/examples/claude-agent-sdk-integration.test.ts +++ b/tests/examples/claude-agent-sdk-integration.test.ts @@ -3,10 +3,11 @@ * * Tests the setup of StackOne tools with Claude Agent SDK. * - * Note: The Claude Agent SDK spawns a subprocess to run claude-code, which - * requires the ANTHROPIC_API_KEY environment variable and a running claude-code - * installation. This test validates the tool setup and MCP server creation, - * but does not test the actual query execution. + * This is the one example NOT executed directly (see tests/examples/run-example.ts + * for how the others are). `query()` spawns a claude-code subprocess, which MSW + * cannot intercept and which needs a real ANTHROPIC_API_KEY and installation. + * So this reimplements the example's setup — tool fetch, tool wrapper, MCP server + * creation and handler execution — and stops short of the query call. */ import { tool, createSdkMcpServer } from '@anthropic-ai/claude-agent-sdk'; diff --git a/tests/examples/defender-config.test.ts b/tests/examples/defender-config.test.ts index 13368652..70337046 100644 --- a/tests/examples/defender-config.test.ts +++ b/tests/examples/defender-config.test.ts @@ -8,9 +8,10 @@ */ import { TEST_BASE_URL } from '../../mocks/constants'; +import { runExample, stubExampleEnv } from './run-example'; import { DEFAULT_DEFENDER_CONFIG, StackOneToolSet, ToolSetConfigError } from '../../src'; -describe('defender-config example e2e', () => { +describe('defender-config wire payloads', () => { beforeEach(() => { vi.stubEnv('STACKONE_API_KEY', 'test-key'); // Silence override warnings so they don't pollute test output. @@ -153,3 +154,45 @@ describe('defender-config example e2e', () => { }); }); }); + +describe('defender-config example', () => { + beforeEach(() => { + // The example defaults section 8 to gmail_list_messages, which the MCP mock + // does not serve; TOOL_NAME is the example's own knob for pointing it elsewhere. + stubExampleEnv({ TOOL_NAME: 'workday_list_workers' }); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('demonstrates every defender pattern to completion', async () => { + const { stdout, exitCode } = await runExample('../../examples/defender-config.ts'); + + expect(exitCode).toBeUndefined(); + expect(stdout).toContain('Done — defender patterns demonstrated.'); + }); + + it('shows both the omitted and the all-false payload shapes', async () => { + const { stdout } = await runExample('../../examples/defender-config.ts'); + + expect(stdout).toContain('SDK adds no defender_config to the RPC payload.'); + expect(stdout).toContain('SDK sends defender_config with all fields false.'); + }); + + it('reaches the live section and gets defender metadata back', async () => { + const { stdout } = await runExample('../../examples/defender-config.ts'); + + // With STACKONE_API_KEY stubbed the example runs section 8 rather than + // printing its skip notice. + expect(stdout).not.toContain('Skipping — set STACKONE_API_KEY to run this section.'); + expect(stdout).toContain('defenderMetadata:'); + }); + + it('still rejects an invalid defender combination', async () => { + const { stdout } = await runExample('../../examples/defender-config.ts'); + + // The example prints this only if the invalid combo failed to throw. + expect(stdout).not.toContain('(no throw — unexpected!)'); + }); +}); diff --git a/tests/examples/fetch-tools.test.ts b/tests/examples/fetch-tools.test.ts index 37bb6f76..1860949d 100644 --- a/tests/examples/fetch-tools.test.ts +++ b/tests/examples/fetch-tools.test.ts @@ -1,7 +1,8 @@ /** - * E2E test for fetch-tools.ts example + * E2E coverage for fetching and filtering tools via MCP. * - * Tests the complete flow of fetching and filtering tools via MCP. + * There is no `examples/fetch-tools.ts`; this exercises the discovery flow the + * other examples build on, against the MCP mock. */ import { http, HttpResponse } from 'msw'; @@ -9,7 +10,7 @@ import { server } from '../../mocks/node'; import { TEST_BASE_URL } from '../../mocks/constants'; import { StackOneToolSet } from '../../src'; -describe('fetch-tools example e2e', () => { +describe('fetch-tools e2e', () => { beforeEach(() => { vi.stubEnv('STACKONE_API_KEY', 'test-key'); }); diff --git a/tests/examples/openai-integration.test.ts b/tests/examples/openai-integration.test.ts index da5793fd..d1919ca4 100644 --- a/tests/examples/openai-integration.test.ts +++ b/tests/examples/openai-integration.test.ts @@ -1,73 +1,43 @@ /** - * E2E test for openai-integration.ts example - * - * Tests the complete flow of using StackOne tools with OpenAI Chat Completions API. + * Executes examples/openai-integration.ts. */ -import OpenAI from 'openai'; -import { TEST_BASE_URL } from '../../mocks/constants'; -import { StackOneToolSet } from '../../src'; +import { runExample, stubExampleEnv } from './run-example'; -describe('openai-integration example e2e', () => { +describe('openai-integration example', () => { beforeEach(() => { - vi.stubEnv('STACKONE_API_KEY', 'test-key'); - vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); }); afterEach(() => { vi.unstubAllEnvs(); }); - it('should fetch tools, convert to OpenAI format, and create chat completion with tool calls', async () => { - const toolset = new StackOneToolSet({ - accountId: 'your-bamboohr-account-id', - baseUrl: TEST_BASE_URL, - }); + it('loads tools, calls OpenAI, and reports the tool calls', async () => { + const { stdout, exitCode } = await runExample('../../examples/openai-integration.ts'); - // Fetch all tools for this account via MCP - const tools = await toolset.fetchTools(); - const openAITools = tools.toOpenAI(); + expect(exitCode).toBeUndefined(); - // Verify tools are in OpenAI format - expect(Array.isArray(openAITools)).toBe(true); - expect(openAITools.length).toBeGreaterThan(0); - expect(openAITools[0]).toHaveProperty('type', 'function'); - expect(openAITools[0]).toHaveProperty('function'); + // A zero here would mean the action filter matched nothing and the example + // sent OpenAI an empty tool list. + // 4: 3 workday actions the example filters to, plus the auto-appended tool_feedback. + // A bare > 0 check cannot fail here — tool_feedback is appended after + // filtering, so even a filter matching nothing still yields one tool. + const loaded = stdout.match(/Loaded (\d+) tools for OpenAI/); + expect(loaded).not.toBeNull(); + expect(Number(loaded?.[1])).toBe(4); - // Initialize OpenAI client - const openai = new OpenAI(); - - // Create a chat completion with tool calls - const response = await openai.chat.completions.create({ - model: 'gpt-5', - messages: [ - { - role: 'system', - content: 'You are a helpful assistant that can access BambooHR information.', - }, - { - role: 'user', - content: - 'What is the employee with id: c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA phone number?', - }, - ], - tools: openAITools, - }); - - // Verify the response contains tool calls - expect(response.choices.length).toBeGreaterThan(0); + expect(stdout).toContain('Model returned 1 choice(s)'); + expect(stdout).toContain('Tool calls made: 1'); + expect(stdout).toContain('Tool: workday_list_workers'); + }); - const choice = response.choices[0]; - expect(choice.message.tool_calls).toBeDefined(); - expect(choice.message.tool_calls!.length).toBeGreaterThan(0); + it('skips cleanly when OPENAI_API_KEY is absent', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); - const toolCall = choice.message.tool_calls![0]; - assert(toolCall.type === 'function'); - expect(toolCall.function.name).toBe('bamboohr_get_employee'); + const { stdout, exitCode } = await runExample('../../examples/openai-integration.ts'); - // Parse the arguments to verify they contain the expected fields - const args: unknown = JSON.parse(toolCall.function.arguments); - assert(typeof args === 'object' && args !== null && 'id' in args); - expect(args.id).toBe('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA'); + expect(exitCode).toBe(0); + expect(stdout).toContain('Skipping: OPENAI_API_KEY is not set'); }); }); diff --git a/tests/examples/openai-responses-integration.test.ts b/tests/examples/openai-responses-integration.test.ts index bc2845c4..98697f25 100644 --- a/tests/examples/openai-responses-integration.test.ts +++ b/tests/examples/openai-responses-integration.test.ts @@ -1,66 +1,38 @@ /** - * E2E test for openai-responses-integration.ts example - * - * Tests the complete flow of using StackOne tools with OpenAI Responses API. + * Executes examples/openai-responses-integration.ts. */ -import OpenAI from 'openai'; -import { TEST_BASE_URL } from '../../mocks/constants'; -import { StackOneToolSet } from '../../src'; +import { runExample, stubExampleEnv } from './run-example'; -describe('openai-responses-integration example e2e', () => { +describe('openai-responses-integration example', () => { beforeEach(() => { - vi.stubEnv('STACKONE_API_KEY', 'test-key'); - vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); }); afterEach(() => { vi.unstubAllEnvs(); }); - it('should fetch tools, convert to OpenAI Responses format, and create response with tool calls', async () => { - const toolset = new StackOneToolSet({ - accountId: 'your-stackone-account-id', - baseUrl: TEST_BASE_URL, - }); + it('loads tools, calls the Responses API, and reports the tool calls', async () => { + const { stdout, exitCode } = await runExample('../../examples/openai-responses-integration.ts'); - // Fetch tools via MCP with action filter - const tools = await toolset.fetchTools({ - actions: ['*_list_*'], - }); - const openAIResponsesTools = tools.toOpenAIResponses(); + expect(exitCode).toBeUndefined(); - // Verify tools are in OpenAI Responses format - expect(Array.isArray(openAIResponsesTools)).toBe(true); - expect(openAIResponsesTools.length).toBeGreaterThan(0); + // 3: 2 tools matching the example's '*_list_*' filter, plus the auto-appended tool_feedback. + // A bare > 0 check cannot fail here — tool_feedback is appended after + // filtering, so even a filter matching nothing still yields one tool. + const loaded = stdout.match(/Loaded (\d+) tools/); + expect(loaded).not.toBeNull(); + expect(Number(loaded?.[1])).toBe(3); - // Initialize OpenAI client - const openai = new OpenAI(); - - // Create a response with tool calls using the Responses API - const response = await openai.responses.create({ - model: 'gpt-5', - instructions: 'You are a helpful assistant that can access various tools.', - input: 'What is the employee with id: c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA phone number?', - tools: openAIResponsesTools, - }); - - // Verify the response contains expected data - expect(response.id).toBeDefined(); - expect(response.model).toBeDefined(); - - // Check if the model made any tool calls - const toolCalls = response.output.filter( - (item): item is OpenAI.Responses.ResponseFunctionToolCall => item.type === 'function_call', - ); + expect(stdout).toContain('workday_list_workers'); + }); - expect(toolCalls.length).toBeGreaterThan(0); + it('skips cleanly when OPENAI_API_KEY is absent', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); - const toolCall = toolCalls[0]; - expect(toolCall.name).toBe('bamboohr_get_employee'); + const { stdout, exitCode } = await runExample('../../examples/openai-responses-integration.ts'); - // Parse the arguments to verify they contain the expected fields - const args = JSON.parse(toolCall.arguments); - expect(args.id).toBe('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA'); + expect(exitCode).toBe(0); }); }); diff --git a/tests/examples/run-example.ts b/tests/examples/run-example.ts new file mode 100644 index 00000000..83a3fa0b --- /dev/null +++ b/tests/examples/run-example.ts @@ -0,0 +1,84 @@ +/** + * Executes a real file from `examples/` under MSW. + * + * Examples run their work at import time via top-level await, so importing one + * *is* running it. Two things make that safe under vitest: + * + * - `STACKONE_BASE_URL` points the SDK at the mock (toolsets.ts reads it), so + * examples that construct `new StackOneToolSet()` with no explicit baseUrl + * still hit MSW rather than production. + * - `process.exit` is swapped for a throw. An example that bails on missing + * config would otherwise tear down the whole vitest worker, turning a real + * failure into a confusing crash. + */ + +import process from 'node:process'; +import { TEST_BASE_URL } from '../../mocks/constants'; + +/** Account id the MCP mock serves example tools for (mocks/handlers.mcp.ts). */ +export const EXAMPLE_ACCOUNT_ID = 'your-bamboohr-account-id'; + +export interface ExampleRun { + stdout: string; + exitCode: number | undefined; +} + +export class ExampleExitError extends Error { + constructor(readonly code: number | undefined) { + super(`example called process.exit(${code})`); + this.name = 'ExampleExitError'; + } +} + +/** + * Import and execute an example, capturing its console output. + * + * @param specifier module specifier relative to this file, e.g. `../../examples/auth-management.ts` + */ +export async function runExample(specifier: string): Promise { + const lines: string[] = []; + const record = + (stream: 'log' | 'error') => + (...args: unknown[]): void => { + lines.push(`${stream === 'error' ? '[stderr] ' : ''}${args.map(String).join(' ')}`); + }; + + const logSpy = vi.spyOn(console, 'log').mockImplementation(record('log')); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(record('error')); + + let exitCode: number | undefined; + const realExit = process.exit; + // biome-ignore lint/suspicious/noExplicitAny: process.exit's never-returning signature + process.exit = ((code?: number): never => { + exitCode = code; + throw new ExampleExitError(code); + }) as any; + + try { + // Query param busts vite's module cache so each test executes the example + // afresh rather than replaying the first import's side effects. + await import(/* @vite-ignore */ `${specifier}?run=${crypto.randomUUID()}`); + } catch (error) { + // An example bailing via process.exit is a normal outcome to assert on, not + // a harness failure — surface it as `exitCode` with its output intact. + if (!(error instanceof ExampleExitError)) { + throw error; + } + } finally { + process.exit = realExit; + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + return { stdout: lines.join('\n'), exitCode }; +} + +/** Env every example needs to reach the mock instead of production. */ +export function stubExampleEnv(extra: Record = {}): void { + vi.stubEnv('STACKONE_API_KEY', 'test-key'); + vi.stubEnv('STACKONE_ACCOUNT_ID', EXAMPLE_ACCOUNT_ID); + vi.stubEnv('STACKONE_BASE_URL', TEST_BASE_URL); + for (const [key, value] of Object.entries(extra)) { + vi.stubEnv(key, value); + } +} diff --git a/tests/examples/search-tools.test.ts b/tests/examples/search-tools.test.ts new file mode 100644 index 00000000..8f8a727b --- /dev/null +++ b/tests/examples/search-tools.test.ts @@ -0,0 +1,72 @@ +/** + * Executes examples/search-tools.ts. + */ + +import { http, HttpResponse } from 'msw'; +import { TEST_BASE_URL } from '../../mocks/constants'; +import { server } from '../../mocks/node'; +import { runExample, stubExampleEnv } from './run-example'; + +describe('search-tools example', () => { + beforeEach(() => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + + // Scoped rather than a global handler: src/semantic-search.test.ts asserts + // against an unmocked /actions/search and a default would pre-empt it. + server.use( + http.post(`${TEST_BASE_URL}/actions/search`, async ({ request }) => { + const body = (await request.json()) as { query?: string; top_k?: number }; + const candidates = [ + 'workday_list_workers', + 'workday_get_worker', + 'workday_get_current_user', + 'bamboohr_list_employees', + ]; + const results = candidates + .slice(0, body.top_k ?? candidates.length) + .map((id, index) => ({ id, similarity_score: 1 - index * 0.1 })); + + return HttpResponse.json({ + results, + total_count: results.length, + query: body.query ?? '', + }); + }), + ); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('runs every discovery mode to completion', async () => { + const { stdout, exitCode } = await runExample('../../examples/search-tools.ts'); + + expect(exitCode).toBeUndefined(); + + for (const section of [ + '1. Direct Fetch (action filters)', + '2. Semantic Search', + '3. Local Search (BM25 + TF-IDF)', + '4. Auto Search + getSearchTool() Callable', + ]) { + expect(stdout).toContain(section); + } + }); + + it('fetches tools matching the workday glob', async () => { + const { stdout } = await runExample('../../examples/search-tools.ts'); + + const fetched = stdout.match(/Fetched (\d+) tools matching "workday_\*"/); + expect(fetched).not.toBeNull(); + expect(Number(fetched?.[1])).toBeGreaterThan(0); + }); + + it('exits non-zero without STACKONE_ACCOUNT_ID', async () => { + vi.stubEnv('STACKONE_ACCOUNT_ID', ''); + + const { stdout, exitCode } = await runExample('../../examples/search-tools.ts'); + + expect(exitCode).toBe(1); + }); +}); diff --git a/tests/examples/tanstack-ai-integration.test.ts b/tests/examples/tanstack-ai-integration.test.ts index 7c13e59d..1fb2e1af 100644 --- a/tests/examples/tanstack-ai-integration.test.ts +++ b/tests/examples/tanstack-ai-integration.test.ts @@ -1,7 +1,8 @@ /** - * E2E test for tanstack-ai-integration.ts example + * E2E coverage for using StackOne tools with TanStack AI. * - * Tests the complete flow of using StackOne tools with TanStack AI. + * There is no `examples/tanstack-ai-integration.ts`; this covers the TanStack + * conversion path directly. * * Note: TanStack AI requires Zod schemas for tool input validation. * This test validates tool setup and schema conversion, but the actual @@ -12,7 +13,7 @@ import { TEST_BASE_URL } from '../../mocks/constants'; import { StackOneToolSet } from '../../src'; -describe('tanstack-ai-integration example e2e', () => { +describe('tanstack-ai-integration e2e', () => { beforeEach(() => { vi.stubEnv('STACKONE_API_KEY', 'test-key'); vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); From 6914e41a51dddae80b5aef334b2ffaeee8219b1b Mon Sep 17 00:00:00 2001 From: S2D2 Date: Fri, 13 Nov 2026 11:58:07 +0000 Subject: [PATCH 2/3] test(examples): run each example file against the mocks Builds on the original approach (run the real files, not a copy of their logic) for the examples on main today: - tests/examples/run-example.ts runs examples/.ts, captures what it prints and turns process.exit into an exit code; recordToolCalls() serves the mock catalog and records each tools/call - auth-management, openai, openai-responses, ai-sdk and anthropic each run for real, asserting their exact output and, where the example executes a tool, the tools/call that reached the mock; each skip path is covered - the openai and openai-responses tests no longer reimplement the example - search-and-execute uses the shared helpers - mock models answer the prompt the examples send, return a tool call, then answer once the tool result is back; adds the Anthropic handler - vitest.setup.ts fails any unmocked request to a remote host, so an example can no longer reach a real API from the test suite (it did: the Anthropic example called api.anthropic.com) Co-Authored-By: Will Leeney Co-Authored-By: Claude Opus 5.5 Co-authored-by: Guillaume Lebedel <10488548+glebedel@users.noreply.github.com> --- mocks/handlers.anthropic.ts | 70 ++-- mocks/handlers.openai.ts | 318 ++++-------------- mocks/handlers.ts | 8 +- mocks/handlers.utils.ts | 36 +- mocks/mcp-server.ts | 3 +- tests/examples/ai-sdk-integration.test.ts | 46 +-- tests/examples/anthropic-integration.test.ts | 40 ++- tests/examples/auth-management.test.ts | 59 ++-- tests/examples/openai-integration.test.ts | 105 ++---- .../openai-responses-integration.test.ts | 80 ++--- tests/examples/run-example.ts | 119 +++---- tests/examples/search-and-execute.test.ts | 27 +- vitest.setup.ts | 12 +- 13 files changed, 344 insertions(+), 579 deletions(-) diff --git a/mocks/handlers.anthropic.ts b/mocks/handlers.anthropic.ts index 59846406..91afc882 100644 --- a/mocks/handlers.anthropic.ts +++ b/mocks/handlers.anthropic.ts @@ -1,50 +1,40 @@ import { http, HttpResponse } from 'msw'; +import { + EXAMPLE_ANSWER, + EXAMPLE_PROMPT, + EXAMPLE_TOOL_CALL, + bodyField, + carriesToolResult, + isRecord, + offersTools, +} from './handlers.utils'; -/** - * Anthropic Messages API, for anthropic-integration.ts. - */ +/** Whether any user message is the example prompt. */ +const asksExamplePrompt = (messages: unknown): boolean => { + const items: unknown[] = Array.isArray(messages) ? messages : []; + return items.some( + (item) => isRecord(item) && item.role === 'user' && item.content === EXAMPLE_PROMPT, + ); +}; + +/** Anthropic Messages, answering the prompt anthropic-integration.ts sends. */ export const anthropicHandlers = [ http.post('https://api.anthropic.com/v1/messages', async ({ request }) => { - const body = (await request.json()) as { - messages?: Array<{ content?: unknown; role?: string }>; - tools?: Array<{ name?: string }>; - }; - - const userMessage = body.messages - ?.filter((message) => message.role === 'user') - .map((message) => - typeof message.content === 'string' ? message.content : JSON.stringify(message.content), - ) - .join(' '); - const hasTools = Boolean(body.tools?.length); - - if (hasTools && userMessage?.includes('List the first 5 employees')) { - return HttpResponse.json({ - id: 'msg_mock_list', - type: 'message', - role: 'assistant', - model: 'claude-haiku-4-5-20241022', - content: [ - { - type: 'tool_use', - id: 'toolu_mock_list', - name: 'workday_list_workers', - input: { query: { limit: 5 } }, - }, - ], - stop_reason: 'tool_use', - usage: { input_tokens: 100, output_tokens: 50 }, - }); - } - + const body = await request.json(); + const callsTool = + !carriesToolResult(body) && + offersTools(body) && + asksExamplePrompt(bodyField(body, 'messages')); return HttpResponse.json({ - id: 'msg_default', + id: 'msg_mock', type: 'message', role: 'assistant', - model: 'claude-haiku-4-5-20241022', - content: [{ type: 'text', text: 'Mock response' }], - stop_reason: 'end_turn', - usage: { input_tokens: 10, output_tokens: 10 }, + model: 'claude-haiku-4-5', + content: callsTool + ? [{ type: 'tool_use', id: 'toolu_mock_list', ...EXAMPLE_TOOL_CALL }] + : [{ type: 'text', text: carriesToolResult(body) ? EXAMPLE_ANSWER : 'Mock response' }], + stop_reason: callsTool ? 'tool_use' : 'end_turn', + usage: { input_tokens: 100, output_tokens: 50 }, }); }), ]; diff --git a/mocks/handlers.openai.ts b/mocks/handlers.openai.ts index ab7756f6..26c72d40 100644 --- a/mocks/handlers.openai.ts +++ b/mocks/handlers.openai.ts @@ -1,277 +1,97 @@ import { http, HttpResponse } from 'msw'; -import { bodyField, extractTextFromInput, isRecord, offersTools } from './handlers.utils'; +import { + EXAMPLE_ANSWER, + EXAMPLE_PROMPT, + EXAMPLE_TOOL_CALL, + bodyField, + carriesToolResult, + extractTextFromInput, + offersTools, +} from './handlers.utils'; -/** The first user message of a chat completion that mentions an employee, or `''`. */ -const employeeQuestion = (messages: unknown): string => { - if (!Array.isArray(messages)) { - return ''; +/** The model's next turn: the answer once a tool result is back, else a tool call for the prompt. */ +const nextTurn = (body: unknown, userText: string): 'answer' | 'tool_call' | 'default' => { + if (carriesToolResult(body)) { + return 'answer'; } - const candidates: unknown[] = messages; - for (const message of candidates) { - if (isRecord(message) && message.role === 'user' && typeof message.content === 'string') { - if (message.content.includes('employee')) { - return message.content; - } - } - } - return ''; + return offersTools(body) && userText.includes(EXAMPLE_PROMPT) ? 'tool_call' : 'default'; }; +const usage = { input_tokens: 100, output_tokens: 50, total_tokens: 150 }; + /** - * OpenAI API endpoints for AI SDK and OpenAI integration examples + * OpenAI Responses (used by the AI SDK and openai-responses-integration.ts) and Chat Completions + * (openai-integration.ts), answering the prompt the examples send. */ export const openaiHandlers = [ - // OpenAI Responses API (used by AI SDK) http.post('https://api.openai.com/v1/responses', async ({ request }) => { const body = await request.json(); - const userMessage = extractTextFromInput(bodyField(body, 'input')); - const hasTools = offersTools(body); - - // For ai-sdk-integration.ts - if (hasTools && userMessage.includes('Get all details')) { - return HttpResponse.json({ - id: 'resp_mock_ai_sdk', - object: 'response', - created_at: Date.now(), - model: 'gpt-5', - status: 'completed', - output: [ - { - type: 'message', - id: 'msg_mock_ai_sdk', - role: 'assistant', - status: 'completed', - content: [ - { - type: 'output_text', - text: 'The employee Michael Scott has the following details: ID c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA, phone +1-555-0100.', - annotations: [], - }, - ], - }, - ], - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }); - } - - // For openai-responses-integration.ts - if (hasTools && userMessage.includes('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA')) { - return HttpResponse.json({ - id: 'resp_mock_responses', - object: 'response', - created_at: Date.now(), - model: 'gpt-5', - status: 'completed', - output: [ - { - type: 'function_call', - id: 'call_mock_get', - call_id: 'call_mock_get', - name: 'bamboohr_get_employee', - arguments: JSON.stringify({ - id: 'c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA', - }), - status: 'completed', - }, - ], - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }); - } - - // For openai-responses-integration.ts and search-tools.ts, which prompt with - // the same sentence and expect a tool call back. - if (hasTools && userMessage.includes('List the first 5 employees')) { - return HttpResponse.json({ - id: 'resp_mock_list', - object: 'response', - created_at: Date.now(), - model: 'gpt-5', - status: 'completed', - output: [ - { - type: 'function_call', - id: 'call_mock_list', - call_id: 'call_mock_list', - name: 'workday_list_workers', - arguments: JSON.stringify({ query: { limit: 5 } }), - status: 'completed', - }, - ], - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }); - } - - // For human-in-the-loop.ts - if (hasTools && userMessage.includes('Create a new employee')) { - return HttpResponse.json({ - id: 'resp_mock_hitl', - object: 'response', - created_at: Date.now(), - model: 'gpt-5', - status: 'completed', - output: [ - { - type: 'function_call', - id: 'call_mock_create', - call_id: 'call_mock_create', - name: 'bamboohr_create_employee', - arguments: JSON.stringify({ - name: 'John Doe', - personal_email: 'john.doe@example.com', - department: 'Engineering', - start_date: '2025-01-01', - hire_date: '2025-01-01', - }), - status: 'completed', - }, - ], - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }); - } - - // Default response + const turn = nextTurn(body, extractTextFromInput(bodyField(body, 'input'))); + const output = + turn === 'tool_call' + ? [ + { + type: 'function_call', + id: 'call_mock_list', + call_id: 'call_mock_list', + name: EXAMPLE_TOOL_CALL.name, + arguments: JSON.stringify(EXAMPLE_TOOL_CALL.input), + status: 'completed', + }, + ] + : [ + { + type: 'message', + id: 'msg_mock', + role: 'assistant', + status: 'completed', + content: [ + { + type: 'output_text', + text: turn === 'answer' ? EXAMPLE_ANSWER : 'Mock response', + annotations: [], + }, + ], + }, + ]; return HttpResponse.json({ - id: 'resp_default', + id: `resp_mock_${turn}`, object: 'response', created_at: Date.now(), model: 'gpt-5', status: 'completed', - output: [ - { - type: 'message', - id: 'msg_default', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'Mock response', annotations: [] }], - }, - ], - usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 }, + output, + usage, }); }), - // OpenAI Chat Completions API (used by OpenAI SDK directly) http.post('https://api.openai.com/v1/chat/completions', async ({ request }) => { const body = await request.json(); - const userMessage = employeeQuestion(bodyField(body, 'messages')); - const hasTools = offersTools(body); - - // For openai-integration.ts - returns tool call for bamboohr_get_employee - if (hasTools && userMessage.includes('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA')) { - return HttpResponse.json({ - id: 'chatcmpl-mock', - object: 'chat.completion', - created: Date.now(), - model: 'gpt-5', - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: null, - tool_calls: [ - { - id: 'call_mock', - type: 'function', - function: { - name: 'bamboohr_get_employee', - arguments: JSON.stringify({ - id: 'c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA', - fields: 'phone_number', - }), - }, - }, - ], - }, - finish_reason: 'tool_calls', - }, - ], - usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, - }); - } - - // For openai-integration.ts, which filters to workday actions and prompts - // with this sentence. - if (hasTools && userMessage.includes('List the first 5 employees')) { - return HttpResponse.json({ - id: 'chatcmpl-mock-list', - object: 'chat.completion', - created: Date.now(), - model: 'gpt-5', - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: null, - tool_calls: [ - { - id: 'call_mock_list', - type: 'function', - function: { - name: 'workday_list_workers', - arguments: JSON.stringify({ query: { limit: 5 } }), - }, - }, - ], - }, - finish_reason: 'tool_calls', - }, - ], - usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, - }); - } - - // For human-in-the-loop.ts - returns tool call for bamboohr_create_employee - if (hasTools && userMessage.includes('Create a new employee')) { - return HttpResponse.json({ - id: 'chatcmpl-mock-hitl', - object: 'chat.completion', - created: Date.now(), - model: 'gpt-5', - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: null, - tool_calls: [ - { - id: 'call_mock_create', - type: 'function', - function: { - name: 'bamboohr_create_employee', - arguments: JSON.stringify({ - name: 'John Doe', - personal_email: 'john.doe@example.com', - department: 'Engineering', - start_date: '2025-01-01', - hire_date: '2025-01-01', - }), - }, + const turn = nextTurn(body, extractTextFromInput(bodyField(body, 'messages'))); + const message = + turn === 'tool_call' + ? { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_mock_list', + type: 'function', + function: { + name: EXAMPLE_TOOL_CALL.name, + arguments: JSON.stringify(EXAMPLE_TOOL_CALL.input), }, - ], - }, - finish_reason: 'tool_calls', - }, - ], - usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, - }); - } - - // Default response + }, + ], + } + : { role: 'assistant', content: turn === 'answer' ? EXAMPLE_ANSWER : 'Mock response' }; return HttpResponse.json({ - id: 'chatcmpl-default', + id: `chatcmpl-mock-${turn}`, object: 'chat.completion', created: Date.now(), model: 'gpt-5', - choices: [ - { - index: 0, - message: { role: 'assistant', content: 'Mock response' }, - finish_reason: 'stop', - }, - ], - usage: { prompt_tokens: 10, completion_tokens: 10, total_tokens: 20 }, + choices: [{ index: 0, message, finish_reason: turn === 'tool_call' ? 'tool_calls' : 'stop' }], + usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, }); }), ]; diff --git a/mocks/handlers.ts b/mocks/handlers.ts index 43b6b4c0..978f89a8 100644 --- a/mocks/handlers.ts +++ b/mocks/handlers.ts @@ -1,5 +1,11 @@ +import { anthropicHandlers } from './handlers.anthropic'; import { mcpHandlers } from './handlers.mcp'; import { openaiHandlers } from './handlers.openai'; import { stackoneAccountsHandlers } from './handlers.stackone-accounts'; -export const handlers = [...openaiHandlers, ...stackoneAccountsHandlers, ...mcpHandlers]; +export const handlers = [ + ...anthropicHandlers, + ...openaiHandlers, + ...stackoneAccountsHandlers, + ...mcpHandlers, +]; diff --git a/mocks/handlers.utils.ts b/mocks/handlers.utils.ts index c8225dfa..d7eb661c 100644 --- a/mocks/handlers.utils.ts +++ b/mocks/handlers.utils.ts @@ -6,7 +6,7 @@ export const isRecord = (value: unknown): value is Record => export const bodyField = (body: unknown, key: string): unknown => isRecord(body) ? body[key] : undefined; -/** Whether a parsed OpenAI request body offers the model at least one tool. */ +/** Whether a parsed OpenAI or Anthropic request body offers the model at least one tool. */ export const offersTools = (body: unknown): boolean => { const tools = bodyField(body, 'tools'); return Array.isArray(tools) && tools.length > 0; @@ -35,9 +35,7 @@ const userMessageText = (item: unknown): string | undefined => { return typeof inputText.text === 'string' ? inputText.text : ''; }; -/** - * Helper to extract text content from OpenAI responses API input - */ +/** The first user message text of a Responses `input` (or the string itself) or a chat `messages`. */ export const extractTextFromInput = (input: unknown): string => { // Handle string input directly (OpenAI Responses API can accept plain strings) if (typeof input === 'string') { @@ -55,3 +53,33 @@ export const extractTextFromInput = (input: unknown): string => { } return ''; }; + +/** The prompt every LLM example sends. */ +export const EXAMPLE_PROMPT = 'List the first 5 employees'; + +/** The tool call the mock models make for {@link EXAMPLE_PROMPT}. */ +export const EXAMPLE_TOOL_CALL = { name: 'workday_list_workers', input: { query: { limit: 5 } } }; + +/** The mock models' reply once the tool result is back. */ +export const EXAMPLE_ANSWER = 'Here are the first 5 employees.'; + +/** + * Whether a request already carries a tool result: a chat `tool` message, a Responses + * `function_call_output` item, or an Anthropic `tool_result` block. + */ +export const carriesToolResult = (body: unknown): boolean => { + const items: unknown[] = [bodyField(body, 'messages'), bodyField(body, 'input')].flatMap( + (list: unknown) => (Array.isArray(list) ? list : []), + ); + return items.some((item) => { + if (!isRecord(item)) { + return false; + } + const parts: unknown[] = Array.isArray(item.content) ? item.content : []; + return ( + item.role === 'tool' || + item.type === 'function_call_output' || + parts.some((part) => isRecord(part) && part.type === 'tool_result') + ); + }); +}; diff --git a/mocks/mcp-server.ts b/mocks/mcp-server.ts index f61acc80..ab73c6b9 100644 --- a/mocks/mcp-server.ts +++ b/mocks/mcp-server.ts @@ -733,8 +733,7 @@ export const exampleBamboohrTools = [ required: ['name'], }, }, - // The examples filter on these three action names. Without them the examples - // execute against an empty toolset and assert nothing. + // The LLM examples filter on these three actions. { name: 'workday_list_workers', description: 'List workers from Workday', diff --git a/tests/examples/ai-sdk-integration.test.ts b/tests/examples/ai-sdk-integration.test.ts index bb08825f..9738736f 100644 --- a/tests/examples/ai-sdk-integration.test.ts +++ b/tests/examples/ai-sdk-integration.test.ts @@ -1,36 +1,42 @@ /** - * Executes examples/ai-sdk-integration.ts. + * Runs examples/ai-sdk-integration.ts against the mock StackOne and OpenAI APIs. */ -import { runExample, stubExampleEnv } from './run-example'; +import { EXAMPLE_ANSWER } from '../../mocks/handlers.utils'; +import { recordToolCalls, runExample, stubExampleEnv } from './run-example'; describe('ai-sdk-integration example', () => { - beforeEach(() => { - stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); - }); - afterEach(() => { vi.unstubAllEnvs(); }); - it('loads tools and drives a Vercel AI SDK generation', async () => { - const { stdout, exitCode } = await runExample('../../examples/ai-sdk-integration.ts'); + it('lets the AI SDK execute the tool, then prints the answer', async () => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + const calls = recordToolCalls(); - expect(exitCode).toBeUndefined(); + const { output, exitCode } = await runExample('ai-sdk-integration'); - // 4: 3 workday actions the example filters to, plus the auto-appended tool_feedback. - // A bare > 0 check cannot fail here — tool_feedback is appended after - // filtering, so even a filter matching nothing still yields one tool. - const loaded = stdout.match(/Loaded (\d+) tools/); - expect(loaded).not.toBeNull(); - expect(Number(loaded?.[1])).toBe(4); + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 3 tools for AI SDK', + `AI response: ${EXAMPLE_ANSWER}`, + 'Steps taken: 2', + ' Tool call: workday_list_workers', + ' Arguments: {"query":{"limit":5}}', + ].join('\n'), + ); + expect(calls.map(({ name, arguments: args }) => [name, args])).toEqual([ + ['workday_list_workers', { query: { limit: 5 } }], + ]); }); - it('skips cleanly when OPENAI_API_KEY is absent', async () => { - vi.stubEnv('OPENAI_API_KEY', ''); - - const { stdout, exitCode } = await runExample('../../examples/ai-sdk-integration.ts'); + it('skips without OPENAI_API_KEY', async () => { + stubExampleEnv({ OPENAI_API_KEY: '' }); - expect(exitCode).toBe(0); + expect(await runExample('ai-sdk-integration')).toEqual({ + output: 'Skipping: OPENAI_API_KEY is not set', + exitCode: 0, + }); }); }); diff --git a/tests/examples/anthropic-integration.test.ts b/tests/examples/anthropic-integration.test.ts index affc1c33..f1b2480e 100644 --- a/tests/examples/anthropic-integration.test.ts +++ b/tests/examples/anthropic-integration.test.ts @@ -1,38 +1,36 @@ /** - * Executes examples/anthropic-integration.ts. + * Runs examples/anthropic-integration.ts against the mock StackOne and Anthropic APIs. */ import { runExample, stubExampleEnv } from './run-example'; describe('anthropic-integration example', () => { - beforeEach(() => { - stubExampleEnv({ ANTHROPIC_API_KEY: 'test-anthropic-key' }); - }); - afterEach(() => { vi.unstubAllEnvs(); }); - it('loads tools, calls Anthropic, and reports the tool use', async () => { - const { stdout, exitCode } = await runExample('../../examples/anthropic-integration.ts'); - - expect(exitCode).toBeUndefined(); + it('sends the list and search tools and prints the tool use the model returns', async () => { + stubExampleEnv({ ANTHROPIC_API_KEY: 'test-anthropic-key' }); - // 3: 2 tools matching the example's filters, plus the auto-appended tool_feedback. - // A bare > 0 check cannot fail here — tool_feedback is appended after - // filtering, so even a filter matching nothing still yields one tool. - const loaded = stdout.match(/Loaded (\d+) tools/); - expect(loaded).not.toBeNull(); - expect(Number(loaded?.[1])).toBe(3); + const { output, exitCode } = await runExample('anthropic-integration'); - expect(stdout).toContain('workday_list_workers'); + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 2 tools for Anthropic', + 'Response content blocks: 1', + ' Tool use: workday_list_workers', + ' Input: {"query":{"limit":5}}', + ].join('\n'), + ); }); - it('skips cleanly when ANTHROPIC_API_KEY is absent', async () => { - vi.stubEnv('ANTHROPIC_API_KEY', ''); - - const { stdout, exitCode } = await runExample('../../examples/anthropic-integration.ts'); + it('skips without ANTHROPIC_API_KEY', async () => { + stubExampleEnv({ ANTHROPIC_API_KEY: '' }); - expect(exitCode).toBe(0); + expect(await runExample('anthropic-integration')).toEqual({ + output: 'Skipping: ANTHROPIC_API_KEY is not set', + exitCode: 0, + }); }); }); diff --git a/tests/examples/auth-management.test.ts b/tests/examples/auth-management.test.ts index f8d3d2b5..f26e5e02 100644 --- a/tests/examples/auth-management.test.ts +++ b/tests/examples/auth-management.test.ts @@ -1,51 +1,42 @@ /** - * Executes examples/auth-management.ts. + * Runs examples/auth-management.ts against the mock StackOne API. */ import { runExample, stubExampleEnv } from './run-example'; describe('auth-management example', () => { - beforeEach(() => { - stubExampleEnv(); - }); - afterEach(() => { vi.unstubAllEnvs(); }); - it('runs every auth pattern to completion', async () => { - const { stdout, exitCode } = await runExample('../../examples/auth-management.ts'); - - expect(exitCode).toBeUndefined(); - expect(stdout).toContain('Done — all auth patterns demonstrated.'); - }); - - it('loads tools through each configuration path', async () => { - const { stdout } = await runExample('../../examples/auth-management.ts'); + it('loads tools through every way of passing the key and the account', async () => { + stubExampleEnv(); - for (const section of [ - '1a. API Key from environment', - '1b. Explicit API key', - '2. Account ID from environment', - '3. Account ID in constructor', - '4. setAccounts() — global account list', - '5. Per-tool account override', - ]) { - expect(stdout).toContain(section); - } + const { output, exitCode } = await runExample('auth-management'); - // Every fetchTools() call must return a non-empty toolset; "Loaded 0 tools" - // would mean the example ran but silently exercised nothing. - const loaded = [...stdout.matchAll(/Loaded (\d+) tools/g)].map((match) => Number(match[1])); - expect(loaded.length).toBeGreaterThan(0); - expect(loaded.every((count) => count > 0)).toBe(true); + expect(exitCode).toBeUndefined(); + expect(output.split('\n').filter((line) => line.startsWith(' '))).toEqual([ + ' Loaded 2 tools using env API key', + ' Loaded 2 tools using explicit API key', + ' 2 linked account(s), 1 active', + ' Loaded 2 tools across them', + ' STACKONE_ACCOUNT_ID is set', + ' Loaded 6 tools for account from env', + ' Loaded 6 tools for configured account', + ' Called setAccounts() with configured account', + ' Loaded 6 tools after setAccounts()', + // The discovered mock account serves no workday tools. + ' (workday_list_workers not available — skipping single-tool demo)', + ]); + expect(output).toMatch(/Done — all auth patterns demonstrated\.$/); }); - it('exits non-zero when STACKONE_API_KEY is absent', async () => { - vi.stubEnv('STACKONE_API_KEY', ''); - - const { exitCode } = await runExample('../../examples/auth-management.ts'); + it('exits 1 without STACKONE_API_KEY', async () => { + stubExampleEnv({ STACKONE_API_KEY: '' }); - expect(exitCode).toBe(1); + expect(await runExample('auth-management')).toEqual({ + output: '[error] Set STACKONE_API_KEY to run this example.', + exitCode: 1, + }); }); }); diff --git a/tests/examples/openai-integration.test.ts b/tests/examples/openai-integration.test.ts index 3ccf586e..d784aaa6 100644 --- a/tests/examples/openai-integration.test.ts +++ b/tests/examples/openai-integration.test.ts @@ -1,89 +1,42 @@ /** - * E2E test for openai-integration.ts example - * - * Tests the complete flow of using StackOne tools with OpenAI Chat Completions API. + * Runs examples/openai-integration.ts against the mock StackOne and OpenAI APIs. */ -import OpenAI from 'openai'; -import { TEST_BASE_URL } from '../../mocks/constants'; -import { StackOneToolSet } from '../../src'; - -describe('openai-integration example e2e', () => { - beforeEach(() => { - vi.stubEnv('STACKONE_API_KEY', 'test-key'); - vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); - }); +import { EXAMPLE_ANSWER } from '../../mocks/handlers.utils'; +import { recordToolCalls, runExample, stubExampleEnv } from './run-example'; +describe('openai-integration example', () => { afterEach(() => { vi.unstubAllEnvs(); }); - it('should fetch tools, convert to OpenAI format, and create chat completion with tool calls', async () => { - const toolset = new StackOneToolSet({ - accountId: 'your-bamboohr-account-id', - baseUrl: TEST_BASE_URL, - }); - - // Fetch all tools for this account via MCP - const tools = await toolset.fetchTools(); - const openAITools = tools.toOpenAI(); - - // Verify tools are in OpenAI format - expect(Array.isArray(openAITools)).toBe(true); - expect(openAITools.length).toBeGreaterThan(0); - expect(openAITools[0]).toHaveProperty('type', 'function'); - expect(openAITools[0]).toHaveProperty('function'); - - // Initialize OpenAI client - const openai = new OpenAI(); - - // Create a chat completion with tool calls - const response = await openai.chat.completions.create({ - model: 'gpt-5', - messages: [ - { - role: 'system', - content: 'You are a helpful assistant that can access BambooHR information.', - }, - { - role: 'user', - content: - 'What is the employee with id: c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA phone number?', - }, - ], - tools: openAITools, - }); - - // Verify the response contains tool calls - expect(response.choices.length).toBeGreaterThan(0); - - const choice = response.choices[0]; - expect(choice.message.tool_calls).toBeDefined(); - expect(choice.message.tool_calls!.length).toBeGreaterThan(0); - - const toolCall = choice.message.tool_calls![0]; - assert(toolCall.type === 'function'); - expect(toolCall.function.name).toBe('bamboohr_get_employee'); + it('executes the tool the model calls and prints its answer', async () => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + const calls = recordToolCalls(); + + const { output, exitCode } = await runExample('openai-integration'); + + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 3 tools for OpenAI', + 'Tool calls made: 1', + ' Tool: workday_list_workers', + ' Arguments: {"query":{"limit":5}}', + `Answer: ${EXAMPLE_ANSWER}`, + ].join('\n'), + ); + expect(calls.map(({ name, arguments: args }) => [name, args])).toEqual([ + ['workday_list_workers', { query: { limit: 5 } }], + ]); + }); - // Parse the arguments to verify they contain the expected fields - const args: unknown = JSON.parse(toolCall.function.arguments); - assert(typeof args === 'object' && args !== null && 'id' in args); - expect(args.id).toBe('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA'); + it('skips without OPENAI_API_KEY', async () => { + stubExampleEnv({ OPENAI_API_KEY: '' }); - // Run the tool calls and turn them into the messages to send back - const toolMessages = await tools.executeOpenAIToolCalls(choice.message.tool_calls); - expect(toolMessages).toHaveLength(1); - expect(toolMessages[0]).toMatchObject({ role: 'tool', tool_call_id: 'call_mock' }); - const content = toolMessages[0]?.content; - assert(typeof content === 'string', 'the tool message carries no text'); - expect(JSON.parse(content)).toMatchObject({ - isError: false, - result: { - data: { - action: 'bamboohr_get_employee', - arguments: { id: 'c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA' }, - }, - }, + expect(await runExample('openai-integration')).toEqual({ + output: 'Skipping: OPENAI_API_KEY is not set', + exitCode: 0, }); }); }); diff --git a/tests/examples/openai-responses-integration.test.ts b/tests/examples/openai-responses-integration.test.ts index 97c1db24..ea40822d 100644 --- a/tests/examples/openai-responses-integration.test.ts +++ b/tests/examples/openai-responses-integration.test.ts @@ -1,68 +1,38 @@ /** - * E2E test for openai-responses-integration.ts example - * - * Tests the complete flow of using StackOne tools with OpenAI Responses API. + * Runs examples/openai-responses-integration.ts against the mock StackOne and OpenAI APIs. */ -import OpenAI from 'openai'; -import { TEST_BASE_URL } from '../../mocks/constants'; -import { StackOneToolSet } from '../../src'; - -describe('openai-responses-integration example e2e', () => { - beforeEach(() => { - vi.stubEnv('STACKONE_API_KEY', 'test-key'); - vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); - }); +import { runExample, stubExampleEnv } from './run-example'; +describe('openai-responses-integration example', () => { afterEach(() => { vi.unstubAllEnvs(); }); - it('should fetch tools, convert to OpenAI Responses format, and create response with tool calls', async () => { - const toolset = new StackOneToolSet({ - accountId: 'your-stackone-account-id', - baseUrl: TEST_BASE_URL, - }); - - // Fetch tools via MCP with action filter - const tools = await toolset.fetchTools({ - actions: ['*_list_*'], - }); - const openAIResponsesTools = tools.toOpenAIResponses(); - - // Verify tools are in OpenAI Responses format - expect(Array.isArray(openAIResponsesTools)).toBe(true); - expect(openAIResponsesTools.length).toBeGreaterThan(0); - // Strict mode makes OpenAI reject served schemas with optional fields - expect(openAIResponsesTools.map((tool) => tool.strict)).not.toContain(true); - - // Initialize OpenAI client - const openai = new OpenAI(); - - // Create a response with tool calls using the Responses API - const response = await openai.responses.create({ - model: 'gpt-5', - instructions: 'You are a helpful assistant that can access various tools.', - input: 'What is the employee with id: c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA phone number?', - tools: openAIResponsesTools, - }); - - // Verify the response contains expected data - expect(response.id).toBeDefined(); - expect(response.model).toBeDefined(); - - // Check if the model made any tool calls - const toolCalls = response.output.filter( - (item): item is OpenAI.Responses.ResponseFunctionToolCall => item.type === 'function_call', + it('sends the list tools and prints the tool call the model makes', async () => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + + const { output, exitCode } = await runExample('openai-responses-integration'); + + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 2 tools for OpenAI Responses API', + 'Response ID: resp_mock_tool_call', + 'Model: gpt-5', + 'Tool calls found: 1', + ' Tool: workday_list_workers', + ' Arguments: {"query":{"limit":5}}', + ].join('\n'), ); + }); - expect(toolCalls.length).toBeGreaterThan(0); - - const toolCall = toolCalls[0]; - expect(toolCall.name).toBe('bamboohr_get_employee'); + it('skips without OPENAI_API_KEY', async () => { + stubExampleEnv({ OPENAI_API_KEY: '' }); - // Parse the arguments to verify they contain the expected fields - const args = JSON.parse(toolCall.arguments); - expect(args.id).toBe('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA'); + expect(await runExample('openai-responses-integration')).toEqual({ + output: 'Skipping: OPENAI_API_KEY is not set', + exitCode: 0, + }); }); }); diff --git a/tests/examples/run-example.ts b/tests/examples/run-example.ts index 83a3fa0b..bd45b52a 100644 --- a/tests/examples/run-example.ts +++ b/tests/examples/run-example.ts @@ -1,84 +1,89 @@ /** - * Executes a real file from `examples/` under MSW. + * Runs a real file from `examples/` against the mocks. * - * Examples run their work at import time via top-level await, so importing one - * *is* running it. Two things make that safe under vitest: - * - * - `STACKONE_BASE_URL` points the SDK at the mock (toolsets.ts reads it), so - * examples that construct `new StackOneToolSet()` with no explicit baseUrl - * still hit MSW rather than production. - * - `process.exit` is swapped for a throw. An example that bails on missing - * config would otherwise tear down the whole vitest worker, turning a real - * failure into a confusing crash. + * An example does its work in top-level await, so importing it runs it. `STACKONE_BASE_URL` points + * the toolset at the mock API, and `process.exit` throws instead of ending the vitest worker. */ import process from 'node:process'; +import { http } from 'msw'; import { TEST_BASE_URL } from '../../mocks/constants'; +import { mockAccountTools } from '../../mocks/handlers.mcp'; +import { createMcpApp } from '../../mocks/mcp-server'; +import type { MockMcpServerOptions, RecordedToolCall } from '../../mocks/mcp-server'; +import { server } from '../../mocks/node'; -/** Account id the MCP mock serves example tools for (mocks/handlers.mcp.ts). */ -export const EXAMPLE_ACCOUNT_ID = 'your-bamboohr-account-id'; +/** An account the MCP mock serves the example tools for (mocks/handlers.mcp.ts). */ +const EXAMPLE_ACCOUNT_ID = 'your-bamboohr-account-id'; -export interface ExampleRun { - stdout: string; +interface ExampleRun { + /** Everything the example printed, one console call per line, `console.error` prefixed. */ + output: string; + /** The code the example passed to `process.exit`, or undefined when it ran to the end. */ exitCode: number | undefined; } -export class ExampleExitError extends Error { - constructor(readonly code: number | undefined) { +class ExampleExit extends Error { + readonly code: number | undefined; + + constructor(code: number | undefined) { super(`example called process.exit(${code})`); - this.name = 'ExampleExitError'; + this.code = code; } } -/** - * Import and execute an example, capturing its console output. - * - * @param specifier module specifier relative to this file, e.g. `../../examples/auth-management.ts` - */ -export async function runExample(specifier: string): Promise { - const lines: string[] = []; - const record = - (stream: 'log' | 'error') => - (...args: unknown[]): void => { - lines.push(`${stream === 'error' ? '[stderr] ' : ''}${args.map(String).join(' ')}`); - }; +/** Point every example at the mocks, with the given extra variables (an empty one unsets it). */ +export function stubExampleEnv(extra: Record = {}): void { + vi.stubEnv('STACKONE_API_KEY', 'test-key'); + vi.stubEnv('STACKONE_ACCOUNT_ID', EXAMPLE_ACCOUNT_ID); + vi.stubEnv('STACKONE_BASE_URL', TEST_BASE_URL); + for (const [key, value] of Object.entries(extra)) { + vi.stubEnv(key, value); + } +} - const logSpy = vi.spyOn(console, 'log').mockImplementation(record('log')); - const errorSpy = vi.spyOn(console, 'error').mockImplementation(record('error')); +/** Run `examples/.ts` afresh and return what it printed and how it exited. */ +export async function runExample(name: string): Promise { + const lines: string[] = []; + const spies = [ + vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(' ')); + }), + vi.spyOn(console, 'error').mockImplementation((...args: unknown[]) => { + lines.push(`[error] ${args.map(String).join(' ')}`); + }), + vi.spyOn(process, 'exit').mockImplementation((code) => { + throw new ExampleExit(code === undefined || code === null ? undefined : Number(code)); + }), + ]; let exitCode: number | undefined; - const realExit = process.exit; - // biome-ignore lint/suspicious/noExplicitAny: process.exit's never-returning signature - process.exit = ((code?: number): never => { - exitCode = code; - throw new ExampleExitError(code); - }) as any; - try { - // Query param busts vite's module cache so each test executes the example - // afresh rather than replaying the first import's side effects. - await import(/* @vite-ignore */ `${specifier}?run=${crypto.randomUUID()}`); + // A fresh query string makes each run evaluate the module again instead of reusing the cache. + await import(/* @vite-ignore */ `../../examples/${name}.ts?run=${crypto.randomUUID()}`); } catch (error) { - // An example bailing via process.exit is a normal outcome to assert on, not - // a harness failure — surface it as `exitCode` with its output intact. - if (!(error instanceof ExampleExitError)) { + if (!(error instanceof ExampleExit)) { throw error; } + exitCode = error.code; } finally { - process.exit = realExit; - logSpy.mockRestore(); - errorSpy.mockRestore(); + for (const spy of spies) { + spy.mockRestore(); + } } - - return { stdout: lines.join('\n'), exitCode }; + return { output: lines.join('\n'), exitCode }; } -/** Env every example needs to reach the mock instead of production. */ -export function stubExampleEnv(extra: Record = {}): void { - vi.stubEnv('STACKONE_API_KEY', 'test-key'); - vi.stubEnv('STACKONE_ACCOUNT_ID', EXAMPLE_ACCOUNT_ID); - vi.stubEnv('STACKONE_BASE_URL', TEST_BASE_URL); - for (const [key, value] of Object.entries(extra)) { - vi.stubEnv(key, value); - } +/** Serve the default mock catalog with these options and record every `tools/call` made. */ +export function recordToolCalls( + options: Omit = {}, +): RecordedToolCall[] { + const calls: RecordedToolCall[] = []; + const app = createMcpApp({ + ...options, + accountTools: mockAccountTools, + onToolCall: (call) => calls.push(call), + }); + server.use(http.all(`${TEST_BASE_URL}/mcp`, ({ request }) => app.fetch(request))); + return calls; } diff --git a/tests/examples/search-and-execute.test.ts b/tests/examples/search-and-execute.test.ts index 432abdad..ae060d00 100644 --- a/tests/examples/search-and-execute.test.ts +++ b/tests/examples/search-and-execute.test.ts @@ -5,13 +5,10 @@ * discover the account, search, execute the best hit and try to record feedback. */ -import { http } from 'msw'; import { TEST_BASE_URL } from '../../mocks/constants'; -import { mockAccountTools } from '../../mocks/handlers.mcp'; -import { MOCK_SEARCH_SESSION_ID, createMcpApp } from '../../mocks/mcp-server'; -import type { RecordedToolCall } from '../../mocks/mcp-server'; -import { server } from '../../mocks/node'; +import { MOCK_SEARCH_SESSION_ID } from '../../mocks/mcp-server'; import { StackOneToolSet, ToolSetLoadError } from '../../src'; +import { recordToolCalls, runExample } from './run-example'; describe('search-and-execute example e2e', () => { beforeEach(() => { @@ -21,25 +18,17 @@ describe('search-and-execute example e2e', () => { afterEach(() => { vi.unstubAllEnvs(); - vi.restoreAllMocks(); }); it('runs the example end to end', async () => { - const calls: RecordedToolCall[] = []; - const app = createMcpApp({ - accountTools: mockAccountTools, - submitFeedback: true, - onToolCall: (call) => calls.push(call), - }); - server.use(http.all(`${TEST_BASE_URL}/mcp`, ({ request }) => app.fetch(request))); - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + const calls = recordToolCalls({ submitFeedback: true }); - await import('../../examples/search-and-execute'); + const { output, exitCode } = await runExample('search-and-execute'); - const printed = log.mock.calls.map((args) => String(args[0])).join('\n'); - expect(printed).toContain('default testprovider active'); - expect(printed).toContain('0.800 mock_list_items [default]'); - expect(printed).toContain('Feedback recorded.'); + expect(exitCode).toBeUndefined(); + expect(output).toContain('default testprovider active'); + expect(output).toContain('0.800 mock_list_items [default]'); + expect(output).toContain('Feedback recorded.'); expect( calls.map((call) => [ call.name, diff --git a/vitest.setup.ts b/vitest.setup.ts index 660789ea..db2b8379 100644 --- a/vitest.setup.ts +++ b/vitest.setup.ts @@ -4,6 +4,16 @@ import { server } from './mocks/node'; // account warn. Tests that need it stub it themselves. delete process.env.STACKONE_ACCOUNT_ID; -beforeAll(() => server.listen()); +// An unmocked request to a remote API fails the test, so an example cannot reach a real one. +// Loopback stays open for the tests that start their own server. +beforeAll(() => + server.listen({ + onUnhandledRequest: (request, print) => { + if (new URL(request.url).hostname !== '127.0.0.1') { + print.error(); + } + }, + }), +); afterEach(() => server.resetHandlers()); afterAll(() => server.close()); From 045054526fbf1e914e78eb7bc3088f31ca531756 Mon Sep 17 00:00:00 2001 From: S2D2 Date: Fri, 13 Nov 2026 12:03:16 +0000 Subject: [PATCH 3/3] test(examples): cover the per-tool account path and share the mock model turn - auth-management: serve a discovered account that has workday_list_workers, so section 5 runs setAccountId/getAccountId instead of its fallback, and pin the whole output with an inline snapshot - Anthropic and OpenAI mocks share one nextTurn, and read text blocks as well as input_text parts - serveAccounts moves to mocks/node so the example and toolset tests share it - reword the stubExampleEnv doc: an empty value is set, and the examples treat it as unset Co-Authored-By: Claude Opus 5.5 Co-authored-by: Guillaume Lebedel <10488548+glebedel@users.noreply.github.com> --- mocks/handlers.anthropic.ts | 28 +++------- mocks/handlers.openai.ts | 12 +---- mocks/handlers.utils.ts | 16 ++++-- mocks/node.ts | 7 +++ src/toolsets.search-execute.test.ts | 6 +-- tests/examples/anthropic-integration.test.ts | 2 +- tests/examples/auth-management.test.ts | 54 ++++++++++++++----- .../openai-responses-integration.test.ts | 2 +- tests/examples/run-example.ts | 2 +- 9 files changed, 72 insertions(+), 57 deletions(-) diff --git a/mocks/handlers.anthropic.ts b/mocks/handlers.anthropic.ts index 91afc882..d2a3124f 100644 --- a/mocks/handlers.anthropic.ts +++ b/mocks/handlers.anthropic.ts @@ -1,39 +1,27 @@ import { http, HttpResponse } from 'msw'; import { EXAMPLE_ANSWER, - EXAMPLE_PROMPT, EXAMPLE_TOOL_CALL, bodyField, - carriesToolResult, - isRecord, - offersTools, + extractTextFromInput, + nextTurn, } from './handlers.utils'; -/** Whether any user message is the example prompt. */ -const asksExamplePrompt = (messages: unknown): boolean => { - const items: unknown[] = Array.isArray(messages) ? messages : []; - return items.some( - (item) => isRecord(item) && item.role === 'user' && item.content === EXAMPLE_PROMPT, - ); -}; - /** Anthropic Messages, answering the prompt anthropic-integration.ts sends. */ export const anthropicHandlers = [ http.post('https://api.anthropic.com/v1/messages', async ({ request }) => { const body = await request.json(); - const callsTool = - !carriesToolResult(body) && - offersTools(body) && - asksExamplePrompt(bodyField(body, 'messages')); + const turn = nextTurn(body, extractTextFromInput(bodyField(body, 'messages'))); return HttpResponse.json({ id: 'msg_mock', type: 'message', role: 'assistant', model: 'claude-haiku-4-5', - content: callsTool - ? [{ type: 'tool_use', id: 'toolu_mock_list', ...EXAMPLE_TOOL_CALL }] - : [{ type: 'text', text: carriesToolResult(body) ? EXAMPLE_ANSWER : 'Mock response' }], - stop_reason: callsTool ? 'tool_use' : 'end_turn', + content: + turn === 'tool_call' + ? [{ type: 'tool_use', id: 'toolu_mock_list', ...EXAMPLE_TOOL_CALL }] + : [{ type: 'text', text: turn === 'answer' ? EXAMPLE_ANSWER : 'Mock response' }], + stop_reason: turn === 'tool_call' ? 'tool_use' : 'end_turn', usage: { input_tokens: 100, output_tokens: 50 }, }); }), diff --git a/mocks/handlers.openai.ts b/mocks/handlers.openai.ts index 26c72d40..a27f2ee0 100644 --- a/mocks/handlers.openai.ts +++ b/mocks/handlers.openai.ts @@ -1,22 +1,12 @@ import { http, HttpResponse } from 'msw'; import { EXAMPLE_ANSWER, - EXAMPLE_PROMPT, EXAMPLE_TOOL_CALL, bodyField, - carriesToolResult, extractTextFromInput, - offersTools, + nextTurn, } from './handlers.utils'; -/** The model's next turn: the answer once a tool result is back, else a tool call for the prompt. */ -const nextTurn = (body: unknown, userText: string): 'answer' | 'tool_call' | 'default' => { - if (carriesToolResult(body)) { - return 'answer'; - } - return offersTools(body) && userText.includes(EXAMPLE_PROMPT) ? 'tool_call' : 'default'; -}; - const usage = { input_tokens: 100, output_tokens: 50, total_tokens: 150 }; /** diff --git a/mocks/handlers.utils.ts b/mocks/handlers.utils.ts index d7eb661c..62ddd834 100644 --- a/mocks/handlers.utils.ts +++ b/mocks/handlers.utils.ts @@ -12,10 +12,10 @@ export const offersTools = (body: unknown): boolean => { return Array.isArray(tools) && tools.length > 0; }; -const isInputText = (content: unknown): content is Record => - isRecord(content) && content.type === 'input_text'; +const isTextPart = (content: unknown): content is Record => + isRecord(content) && (content.type === 'input_text' || content.type === 'text'); -/** The text of a user message: its string content, or the text of its first `input_text` part. */ +/** The text of a user message: its string content, or the text of its first text part. */ const userMessageText = (item: unknown): string | undefined => { if (!isRecord(item) || item.role !== 'user') { return undefined; @@ -28,7 +28,7 @@ const userMessageText = (item: unknown): string | undefined => { return undefined; } const parts: unknown[] = item.content; - const inputText = parts.find(isInputText); + const inputText = parts.find(isTextPart); if (inputText === undefined) { return undefined; } @@ -83,3 +83,11 @@ export const carriesToolResult = (body: unknown): boolean => { ); }); }; + +/** The mock model's next turn. Any tool result in the history gets the answer: one tool round. */ +export const nextTurn = (body: unknown, userText: string): 'answer' | 'tool_call' | 'default' => { + if (carriesToolResult(body)) { + return 'answer'; + } + return offersTools(body) && userText.includes(EXAMPLE_PROMPT) ? 'tool_call' : 'default'; +}; diff --git a/mocks/node.ts b/mocks/node.ts index e52fee0a..c6242c20 100644 --- a/mocks/node.ts +++ b/mocks/node.ts @@ -1,4 +1,11 @@ +import { http, HttpResponse } from 'msw'; import { setupServer } from 'msw/node'; +import { TEST_BASE_URL } from './constants'; import { handlers } from './handlers'; export const server = setupServer(...handlers); + +/** Serve `accounts` from GET /accounts for the rest of the test. */ +export const serveAccounts = (accounts: object[]): void => { + server.use(http.get(`${TEST_BASE_URL}/accounts`, () => HttpResponse.json(accounts))); +}; diff --git a/src/toolsets.search-execute.test.ts b/src/toolsets.search-execute.test.ts index 52eefead..3a55235d 100644 --- a/src/toolsets.search-execute.test.ts +++ b/src/toolsets.search-execute.test.ts @@ -6,7 +6,7 @@ import { http, HttpResponse } from 'msw'; import { TEST_BASE_URL } from '../mocks/constants'; import { MOCK_SEARCH_SESSION_ID, createMcpApp } from '../mocks/mcp-server'; import type { RecordedToolCall } from '../mocks/mcp-server'; -import { server } from '../mocks/node'; +import { serveAccounts, server } from '../mocks/node'; import { listMcpTools } from './mcp-client'; import type { McpToolDefinition } from './mcp-client'; import { StackOneMcpTool } from './tool'; @@ -103,10 +103,6 @@ const serveMock = ( return { calls, requests, rpcRequests: () => rpcRequests }; }; -/** Serve `accounts` from GET /accounts. */ -const serveAccounts = (accounts: JsonObject[]) => - server.use(http.get(`${TEST_BASE_URL}/accounts`, () => HttpResponse.json(accounts))); - /** The endpoint of every listing so far. */ const listedEndpoints = () => listMock.mock.calls.map(([request]) => request.endpoint); diff --git a/tests/examples/anthropic-integration.test.ts b/tests/examples/anthropic-integration.test.ts index f1b2480e..61fadeec 100644 --- a/tests/examples/anthropic-integration.test.ts +++ b/tests/examples/anthropic-integration.test.ts @@ -9,7 +9,7 @@ describe('anthropic-integration example', () => { vi.unstubAllEnvs(); }); - it('sends the list and search tools and prints the tool use the model returns', async () => { + it('offers the fetched tools and prints the tool use the model returns', async () => { stubExampleEnv({ ANTHROPIC_API_KEY: 'test-anthropic-key' }); const { output, exitCode } = await runExample('anthropic-integration'); diff --git a/tests/examples/auth-management.test.ts b/tests/examples/auth-management.test.ts index f26e5e02..f4361669 100644 --- a/tests/examples/auth-management.test.ts +++ b/tests/examples/auth-management.test.ts @@ -2,6 +2,7 @@ * Runs examples/auth-management.ts against the mock StackOne API. */ +import { serveAccounts } from '../../mocks/node'; import { runExample, stubExampleEnv } from './run-example'; describe('auth-management example', () => { @@ -11,24 +12,49 @@ describe('auth-management example', () => { it('loads tools through every way of passing the key and the account', async () => { stubExampleEnv(); + // Discover the account that serves workday_list_workers, so section 5 overrides its account. + serveAccounts([ + { id: 'your-bamboohr-account-id', provider: 'bamboohr', status: 'active' }, + { id: 'dead', provider: 'brokenprovider', status: 'error' }, + ]); const { output, exitCode } = await runExample('auth-management'); expect(exitCode).toBeUndefined(); - expect(output.split('\n').filter((line) => line.startsWith(' '))).toEqual([ - ' Loaded 2 tools using env API key', - ' Loaded 2 tools using explicit API key', - ' 2 linked account(s), 1 active', - ' Loaded 2 tools across them', - ' STACKONE_ACCOUNT_ID is set', - ' Loaded 6 tools for account from env', - ' Loaded 6 tools for configured account', - ' Called setAccounts() with configured account', - ' Loaded 6 tools after setAccounts()', - // The discovered mock account serves no workday tools. - ' (workday_list_workers not available — skipping single-tool demo)', - ]); - expect(output).toMatch(/Done — all auth patterns demonstrated\.$/); + expect(output).toMatchInlineSnapshot(` + "=== 1a. API Key from environment === + + Loaded 6 tools using env API key + + === 1b. Explicit API key === + + Loaded 6 tools using explicit API key + + === 1c. API key alone === + + 2 linked account(s), 1 active + Loaded 6 tools across them + + === 2. Account ID from environment === + + STACKONE_ACCOUNT_ID is set + Loaded 6 tools for account from env + + === 3. Account ID in constructor === + + Loaded 6 tools for configured account + + === 4. setAccounts() — global account list === + + Called setAccounts() with configured account + Loaded 6 tools after setAccounts() + + === 5. Per-tool account override === + + Single tool account: "per-tool-account" + + Done — all auth patterns demonstrated." + `); }); it('exits 1 without STACKONE_API_KEY', async () => { diff --git a/tests/examples/openai-responses-integration.test.ts b/tests/examples/openai-responses-integration.test.ts index ea40822d..a3d35e5e 100644 --- a/tests/examples/openai-responses-integration.test.ts +++ b/tests/examples/openai-responses-integration.test.ts @@ -9,7 +9,7 @@ describe('openai-responses-integration example', () => { vi.unstubAllEnvs(); }); - it('sends the list tools and prints the tool call the model makes', async () => { + it('offers the fetched tools and prints the tool call the model makes', async () => { stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); const { output, exitCode } = await runExample('openai-responses-integration'); diff --git a/tests/examples/run-example.ts b/tests/examples/run-example.ts index bd45b52a..f6bcc282 100644 --- a/tests/examples/run-example.ts +++ b/tests/examples/run-example.ts @@ -32,7 +32,7 @@ class ExampleExit extends Error { } } -/** Point every example at the mocks, with the given extra variables (an empty one unsets it). */ +/** Point every example at the mocks, plus the given variables. The examples treat `''` as unset. */ export function stubExampleEnv(extra: Record = {}): void { vi.stubEnv('STACKONE_API_KEY', 'test-key'); vi.stubEnv('STACKONE_ACCOUNT_ID', EXAMPLE_ACCOUNT_ID);