diff --git a/mocks/handlers.anthropic.ts b/mocks/handlers.anthropic.ts new file mode 100644 index 00000000..d2a3124f --- /dev/null +++ b/mocks/handlers.anthropic.ts @@ -0,0 +1,28 @@ +import { http, HttpResponse } from 'msw'; +import { + EXAMPLE_ANSWER, + EXAMPLE_TOOL_CALL, + bodyField, + extractTextFromInput, + nextTurn, +} from './handlers.utils'; + +/** Anthropic Messages, answering the prompt anthropic-integration.ts sends. */ +export const anthropicHandlers = [ + http.post('https://api.anthropic.com/v1/messages', async ({ request }) => { + const body = await request.json(); + const turn = nextTurn(body, extractTextFromInput(bodyField(body, 'messages'))); + return HttpResponse.json({ + id: 'msg_mock', + type: 'message', + role: 'assistant', + model: 'claude-haiku-4-5', + content: + turn === 'tool_call' + ? [{ type: 'tool_use', id: 'toolu_mock_list', ...EXAMPLE_TOOL_CALL }] + : [{ type: 'text', text: turn === 'answer' ? EXAMPLE_ANSWER : 'Mock response' }], + stop_reason: turn === 'tool_call' ? 'tool_use' : 'end_turn', + usage: { input_tokens: 100, output_tokens: 50 }, + }); + }), +]; diff --git a/mocks/handlers.openai.ts b/mocks/handlers.openai.ts index 994d2fe2..a27f2ee0 100644 --- a/mocks/handlers.openai.ts +++ b/mocks/handlers.openai.ts @@ -1,222 +1,87 @@ import { http, HttpResponse } from 'msw'; -import { bodyField, extractTextFromInput, isRecord, offersTools } from './handlers.utils'; +import { + EXAMPLE_ANSWER, + EXAMPLE_TOOL_CALL, + bodyField, + extractTextFromInput, + nextTurn, +} from './handlers.utils'; -/** The first user message of a chat completion that mentions an employee, or `''`. */ -const employeeQuestion = (messages: unknown): string => { - if (!Array.isArray(messages)) { - return ''; - } - const candidates: unknown[] = messages; - for (const message of candidates) { - if (isRecord(message) && message.role === 'user' && typeof message.content === 'string') { - if (message.content.includes('employee')) { - return message.content; - } - } - } - return ''; -}; +const usage = { input_tokens: 100, output_tokens: 50, total_tokens: 150 }; /** - * OpenAI API endpoints for AI SDK and OpenAI integration examples + * OpenAI Responses (used by the AI SDK and openai-responses-integration.ts) and Chat Completions + * (openai-integration.ts), answering the prompt the examples send. */ export const openaiHandlers = [ - // OpenAI Responses API (used by AI SDK) http.post('https://api.openai.com/v1/responses', async ({ request }) => { const body = await request.json(); - const userMessage = extractTextFromInput(bodyField(body, 'input')); - const hasTools = offersTools(body); - - // For ai-sdk-integration.ts - if (hasTools && userMessage.includes('Get all details')) { - return HttpResponse.json({ - id: 'resp_mock_ai_sdk', - object: 'response', - created_at: Date.now(), - model: 'gpt-5', - status: 'completed', - output: [ - { - type: 'message', - id: 'msg_mock_ai_sdk', - role: 'assistant', - status: 'completed', - content: [ - { - type: 'output_text', - text: 'The employee Michael Scott has the following details: ID c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA, phone +1-555-0100.', - annotations: [], - }, - ], - }, - ], - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }); - } - - // For openai-responses-integration.ts - if (hasTools && userMessage.includes('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA')) { - return HttpResponse.json({ - id: 'resp_mock_responses', - object: 'response', - created_at: Date.now(), - model: 'gpt-5', - status: 'completed', - output: [ - { - type: 'function_call', - id: 'call_mock_get', - call_id: 'call_mock_get', - name: 'bamboohr_get_employee', - arguments: JSON.stringify({ - id: 'c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA', - }), - status: 'completed', - }, - ], - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }); - } - - // For human-in-the-loop.ts - if (hasTools && userMessage.includes('Create a new employee')) { - return HttpResponse.json({ - id: 'resp_mock_hitl', - object: 'response', - created_at: Date.now(), - model: 'gpt-5', - status: 'completed', - output: [ - { - type: 'function_call', - id: 'call_mock_create', - call_id: 'call_mock_create', - name: 'bamboohr_create_employee', - arguments: JSON.stringify({ - name: 'John Doe', - personal_email: 'john.doe@example.com', - department: 'Engineering', - start_date: '2025-01-01', - hire_date: '2025-01-01', - }), - status: 'completed', - }, - ], - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }); - } - - // Default response + const turn = nextTurn(body, extractTextFromInput(bodyField(body, 'input'))); + const output = + turn === 'tool_call' + ? [ + { + type: 'function_call', + id: 'call_mock_list', + call_id: 'call_mock_list', + name: EXAMPLE_TOOL_CALL.name, + arguments: JSON.stringify(EXAMPLE_TOOL_CALL.input), + status: 'completed', + }, + ] + : [ + { + type: 'message', + id: 'msg_mock', + role: 'assistant', + status: 'completed', + content: [ + { + type: 'output_text', + text: turn === 'answer' ? EXAMPLE_ANSWER : 'Mock response', + annotations: [], + }, + ], + }, + ]; return HttpResponse.json({ - id: 'resp_default', + id: `resp_mock_${turn}`, object: 'response', created_at: Date.now(), model: 'gpt-5', status: 'completed', - output: [ - { - type: 'message', - id: 'msg_default', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'Mock response', annotations: [] }], - }, - ], - usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 }, + output, + usage, }); }), - // OpenAI Chat Completions API (used by OpenAI SDK directly) http.post('https://api.openai.com/v1/chat/completions', async ({ request }) => { const body = await request.json(); - const userMessage = employeeQuestion(bodyField(body, 'messages')); - const hasTools = offersTools(body); - - // For openai-integration.ts - returns tool call for bamboohr_get_employee - if (hasTools && userMessage.includes('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA')) { - return HttpResponse.json({ - id: 'chatcmpl-mock', - object: 'chat.completion', - created: Date.now(), - model: 'gpt-5', - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: null, - tool_calls: [ - { - id: 'call_mock', - type: 'function', - function: { - name: 'bamboohr_get_employee', - arguments: JSON.stringify({ - id: 'c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA', - fields: 'phone_number', - }), - }, - }, - ], - }, - finish_reason: 'tool_calls', - }, - ], - usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, - }); - } - - // For human-in-the-loop.ts - returns tool call for bamboohr_create_employee - if (hasTools && userMessage.includes('Create a new employee')) { - return HttpResponse.json({ - id: 'chatcmpl-mock-hitl', - object: 'chat.completion', - created: Date.now(), - model: 'gpt-5', - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: null, - tool_calls: [ - { - id: 'call_mock_create', - type: 'function', - function: { - name: 'bamboohr_create_employee', - arguments: JSON.stringify({ - name: 'John Doe', - personal_email: 'john.doe@example.com', - department: 'Engineering', - start_date: '2025-01-01', - hire_date: '2025-01-01', - }), - }, + const turn = nextTurn(body, extractTextFromInput(bodyField(body, 'messages'))); + const message = + turn === 'tool_call' + ? { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_mock_list', + type: 'function', + function: { + name: EXAMPLE_TOOL_CALL.name, + arguments: JSON.stringify(EXAMPLE_TOOL_CALL.input), }, - ], - }, - finish_reason: 'tool_calls', - }, - ], - usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, - }); - } - - // Default response + }, + ], + } + : { role: 'assistant', content: turn === 'answer' ? EXAMPLE_ANSWER : 'Mock response' }; return HttpResponse.json({ - id: 'chatcmpl-default', + id: `chatcmpl-mock-${turn}`, object: 'chat.completion', created: Date.now(), model: 'gpt-5', - choices: [ - { - index: 0, - message: { role: 'assistant', content: 'Mock response' }, - finish_reason: 'stop', - }, - ], - usage: { prompt_tokens: 10, completion_tokens: 10, total_tokens: 20 }, + choices: [{ index: 0, message, finish_reason: turn === 'tool_call' ? 'tool_calls' : 'stop' }], + usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, }); }), ]; diff --git a/mocks/handlers.ts b/mocks/handlers.ts index 43b6b4c0..978f89a8 100644 --- a/mocks/handlers.ts +++ b/mocks/handlers.ts @@ -1,5 +1,11 @@ +import { anthropicHandlers } from './handlers.anthropic'; import { mcpHandlers } from './handlers.mcp'; import { openaiHandlers } from './handlers.openai'; import { stackoneAccountsHandlers } from './handlers.stackone-accounts'; -export const handlers = [...openaiHandlers, ...stackoneAccountsHandlers, ...mcpHandlers]; +export const handlers = [ + ...anthropicHandlers, + ...openaiHandlers, + ...stackoneAccountsHandlers, + ...mcpHandlers, +]; diff --git a/mocks/handlers.utils.ts b/mocks/handlers.utils.ts index c8225dfa..62ddd834 100644 --- a/mocks/handlers.utils.ts +++ b/mocks/handlers.utils.ts @@ -6,16 +6,16 @@ export const isRecord = (value: unknown): value is Record => export const bodyField = (body: unknown, key: string): unknown => isRecord(body) ? body[key] : undefined; -/** Whether a parsed OpenAI request body offers the model at least one tool. */ +/** Whether a parsed OpenAI or Anthropic request body offers the model at least one tool. */ export const offersTools = (body: unknown): boolean => { const tools = bodyField(body, 'tools'); return Array.isArray(tools) && tools.length > 0; }; -const isInputText = (content: unknown): content is Record => - isRecord(content) && content.type === 'input_text'; +const isTextPart = (content: unknown): content is Record => + isRecord(content) && (content.type === 'input_text' || content.type === 'text'); -/** The text of a user message: its string content, or the text of its first `input_text` part. */ +/** The text of a user message: its string content, or the text of its first text part. */ const userMessageText = (item: unknown): string | undefined => { if (!isRecord(item) || item.role !== 'user') { return undefined; @@ -28,16 +28,14 @@ const userMessageText = (item: unknown): string | undefined => { return undefined; } const parts: unknown[] = item.content; - const inputText = parts.find(isInputText); + const inputText = parts.find(isTextPart); if (inputText === undefined) { return undefined; } return typeof inputText.text === 'string' ? inputText.text : ''; }; -/** - * Helper to extract text content from OpenAI responses API input - */ +/** The first user message text of a Responses `input` (or the string itself) or a chat `messages`. */ export const extractTextFromInput = (input: unknown): string => { // Handle string input directly (OpenAI Responses API can accept plain strings) if (typeof input === 'string') { @@ -55,3 +53,41 @@ export const extractTextFromInput = (input: unknown): string => { } return ''; }; + +/** The prompt every LLM example sends. */ +export const EXAMPLE_PROMPT = 'List the first 5 employees'; + +/** The tool call the mock models make for {@link EXAMPLE_PROMPT}. */ +export const EXAMPLE_TOOL_CALL = { name: 'workday_list_workers', input: { query: { limit: 5 } } }; + +/** The mock models' reply once the tool result is back. */ +export const EXAMPLE_ANSWER = 'Here are the first 5 employees.'; + +/** + * Whether a request already carries a tool result: a chat `tool` message, a Responses + * `function_call_output` item, or an Anthropic `tool_result` block. + */ +export const carriesToolResult = (body: unknown): boolean => { + const items: unknown[] = [bodyField(body, 'messages'), bodyField(body, 'input')].flatMap( + (list: unknown) => (Array.isArray(list) ? list : []), + ); + return items.some((item) => { + if (!isRecord(item)) { + return false; + } + const parts: unknown[] = Array.isArray(item.content) ? item.content : []; + return ( + item.role === 'tool' || + item.type === 'function_call_output' || + parts.some((part) => isRecord(part) && part.type === 'tool_result') + ); + }); +}; + +/** The mock model's next turn. Any tool result in the history gets the answer: one tool round. */ +export const nextTurn = (body: unknown, userText: string): 'answer' | 'tool_call' | 'default' => { + if (carriesToolResult(body)) { + return 'answer'; + } + return offersTools(body) && userText.includes(EXAMPLE_PROMPT) ? 'tool_call' : 'default'; +}; diff --git a/mocks/mcp-server.ts b/mocks/mcp-server.ts index 703664ff..ab73c6b9 100644 --- a/mocks/mcp-server.ts +++ b/mocks/mcp-server.ts @@ -733,6 +733,41 @@ export const exampleBamboohrTools = [ required: ['name'], }, }, + // The LLM examples filter on these three actions. + { + name: 'workday_list_workers', + description: 'List workers from Workday', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'object', + properties: { + limit: { type: 'number', description: 'Limit the number of results' }, + }, + }, + }, + }, + }, + { + name: 'workday_get_worker', + description: 'Get a single worker by ID from Workday', + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'The worker ID' }, + }, + required: ['id'], + }, + }, + { + name: 'workday_get_current_user', + description: 'Get the currently authenticated Workday user', + inputSchema: { + type: 'object', + properties: {}, + }, + }, ] as const satisfies McpToolDefinition[]; /** A per-action tool whose name ends like a meta tool's, served in individual mode. */ diff --git a/mocks/node.ts b/mocks/node.ts index e52fee0a..c6242c20 100644 --- a/mocks/node.ts +++ b/mocks/node.ts @@ -1,4 +1,11 @@ +import { http, HttpResponse } from 'msw'; import { setupServer } from 'msw/node'; +import { TEST_BASE_URL } from './constants'; import { handlers } from './handlers'; export const server = setupServer(...handlers); + +/** Serve `accounts` from GET /accounts for the rest of the test. */ +export const serveAccounts = (accounts: object[]): void => { + server.use(http.get(`${TEST_BASE_URL}/accounts`, () => HttpResponse.json(accounts))); +}; diff --git a/src/toolsets.search-execute.test.ts b/src/toolsets.search-execute.test.ts index 52eefead..3a55235d 100644 --- a/src/toolsets.search-execute.test.ts +++ b/src/toolsets.search-execute.test.ts @@ -6,7 +6,7 @@ import { http, HttpResponse } from 'msw'; import { TEST_BASE_URL } from '../mocks/constants'; import { MOCK_SEARCH_SESSION_ID, createMcpApp } from '../mocks/mcp-server'; import type { RecordedToolCall } from '../mocks/mcp-server'; -import { server } from '../mocks/node'; +import { serveAccounts, server } from '../mocks/node'; import { listMcpTools } from './mcp-client'; import type { McpToolDefinition } from './mcp-client'; import { StackOneMcpTool } from './tool'; @@ -103,10 +103,6 @@ const serveMock = ( return { calls, requests, rpcRequests: () => rpcRequests }; }; -/** Serve `accounts` from GET /accounts. */ -const serveAccounts = (accounts: JsonObject[]) => - server.use(http.get(`${TEST_BASE_URL}/accounts`, () => HttpResponse.json(accounts))); - /** The endpoint of every listing so far. */ const listedEndpoints = () => listMock.mock.calls.map(([request]) => request.endpoint); diff --git a/tests/examples/ai-sdk-integration.test.ts b/tests/examples/ai-sdk-integration.test.ts new file mode 100644 index 00000000..9738736f --- /dev/null +++ b/tests/examples/ai-sdk-integration.test.ts @@ -0,0 +1,42 @@ +/** + * Runs examples/ai-sdk-integration.ts against the mock StackOne and OpenAI APIs. + */ + +import { EXAMPLE_ANSWER } from '../../mocks/handlers.utils'; +import { recordToolCalls, runExample, stubExampleEnv } from './run-example'; + +describe('ai-sdk-integration example', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('lets the AI SDK execute the tool, then prints the answer', async () => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + const calls = recordToolCalls(); + + const { output, exitCode } = await runExample('ai-sdk-integration'); + + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 3 tools for AI SDK', + `AI response: ${EXAMPLE_ANSWER}`, + 'Steps taken: 2', + ' Tool call: workday_list_workers', + ' Arguments: {"query":{"limit":5}}', + ].join('\n'), + ); + expect(calls.map(({ name, arguments: args }) => [name, args])).toEqual([ + ['workday_list_workers', { query: { limit: 5 } }], + ]); + }); + + it('skips without OPENAI_API_KEY', async () => { + stubExampleEnv({ OPENAI_API_KEY: '' }); + + expect(await runExample('ai-sdk-integration')).toEqual({ + output: 'Skipping: OPENAI_API_KEY is not set', + exitCode: 0, + }); + }); +}); diff --git a/tests/examples/anthropic-integration.test.ts b/tests/examples/anthropic-integration.test.ts new file mode 100644 index 00000000..61fadeec --- /dev/null +++ b/tests/examples/anthropic-integration.test.ts @@ -0,0 +1,36 @@ +/** + * Runs examples/anthropic-integration.ts against the mock StackOne and Anthropic APIs. + */ + +import { runExample, stubExampleEnv } from './run-example'; + +describe('anthropic-integration example', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('offers the fetched tools and prints the tool use the model returns', async () => { + stubExampleEnv({ ANTHROPIC_API_KEY: 'test-anthropic-key' }); + + const { output, exitCode } = await runExample('anthropic-integration'); + + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 2 tools for Anthropic', + 'Response content blocks: 1', + ' Tool use: workday_list_workers', + ' Input: {"query":{"limit":5}}', + ].join('\n'), + ); + }); + + it('skips without ANTHROPIC_API_KEY', async () => { + stubExampleEnv({ ANTHROPIC_API_KEY: '' }); + + expect(await runExample('anthropic-integration')).toEqual({ + output: 'Skipping: ANTHROPIC_API_KEY is not set', + exitCode: 0, + }); + }); +}); diff --git a/tests/examples/auth-management.test.ts b/tests/examples/auth-management.test.ts new file mode 100644 index 00000000..f4361669 --- /dev/null +++ b/tests/examples/auth-management.test.ts @@ -0,0 +1,68 @@ +/** + * Runs examples/auth-management.ts against the mock StackOne API. + */ + +import { serveAccounts } from '../../mocks/node'; +import { runExample, stubExampleEnv } from './run-example'; + +describe('auth-management example', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('loads tools through every way of passing the key and the account', async () => { + stubExampleEnv(); + // Discover the account that serves workday_list_workers, so section 5 overrides its account. + serveAccounts([ + { id: 'your-bamboohr-account-id', provider: 'bamboohr', status: 'active' }, + { id: 'dead', provider: 'brokenprovider', status: 'error' }, + ]); + + const { output, exitCode } = await runExample('auth-management'); + + expect(exitCode).toBeUndefined(); + expect(output).toMatchInlineSnapshot(` + "=== 1a. API Key from environment === + + Loaded 6 tools using env API key + + === 1b. Explicit API key === + + Loaded 6 tools using explicit API key + + === 1c. API key alone === + + 2 linked account(s), 1 active + Loaded 6 tools across them + + === 2. Account ID from environment === + + STACKONE_ACCOUNT_ID is set + Loaded 6 tools for account from env + + === 3. Account ID in constructor === + + Loaded 6 tools for configured account + + === 4. setAccounts() — global account list === + + Called setAccounts() with configured account + Loaded 6 tools after setAccounts() + + === 5. Per-tool account override === + + Single tool account: "per-tool-account" + + Done — all auth patterns demonstrated." + `); + }); + + it('exits 1 without STACKONE_API_KEY', async () => { + stubExampleEnv({ STACKONE_API_KEY: '' }); + + expect(await runExample('auth-management')).toEqual({ + output: '[error] Set STACKONE_API_KEY to run this example.', + exitCode: 1, + }); + }); +}); diff --git a/tests/examples/fetch-tools.test.ts b/tests/examples/fetch-tools.test.ts index 30e8f876..2ec83ae2 100644 --- a/tests/examples/fetch-tools.test.ts +++ b/tests/examples/fetch-tools.test.ts @@ -1,7 +1,8 @@ /** - * E2E test for fetch-tools.ts example + * E2E coverage for fetching and filtering tools via MCP. * - * Tests the complete flow of fetching and filtering tools via MCP. + * There is no `examples/fetch-tools.ts`; this exercises the discovery flow the + * other examples build on, against the MCP mock. */ import { http, HttpResponse } from 'msw'; @@ -10,7 +11,7 @@ import { server } from '../../mocks/node'; import { TEST_BASE_URL } from '../../mocks/constants'; import { StackOneToolSet } from '../../src'; -describe('fetch-tools example e2e', () => { +describe('fetch-tools e2e', () => { beforeEach(() => { vi.stubEnv('STACKONE_API_KEY', 'test-key'); }); diff --git a/tests/examples/openai-integration.test.ts b/tests/examples/openai-integration.test.ts index 3ccf586e..d784aaa6 100644 --- a/tests/examples/openai-integration.test.ts +++ b/tests/examples/openai-integration.test.ts @@ -1,89 +1,42 @@ /** - * E2E test for openai-integration.ts example - * - * Tests the complete flow of using StackOne tools with OpenAI Chat Completions API. + * Runs examples/openai-integration.ts against the mock StackOne and OpenAI APIs. */ -import OpenAI from 'openai'; -import { TEST_BASE_URL } from '../../mocks/constants'; -import { StackOneToolSet } from '../../src'; - -describe('openai-integration example e2e', () => { - beforeEach(() => { - vi.stubEnv('STACKONE_API_KEY', 'test-key'); - vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); - }); +import { EXAMPLE_ANSWER } from '../../mocks/handlers.utils'; +import { recordToolCalls, runExample, stubExampleEnv } from './run-example'; +describe('openai-integration example', () => { afterEach(() => { vi.unstubAllEnvs(); }); - it('should fetch tools, convert to OpenAI format, and create chat completion with tool calls', async () => { - const toolset = new StackOneToolSet({ - accountId: 'your-bamboohr-account-id', - baseUrl: TEST_BASE_URL, - }); - - // Fetch all tools for this account via MCP - const tools = await toolset.fetchTools(); - const openAITools = tools.toOpenAI(); - - // Verify tools are in OpenAI format - expect(Array.isArray(openAITools)).toBe(true); - expect(openAITools.length).toBeGreaterThan(0); - expect(openAITools[0]).toHaveProperty('type', 'function'); - expect(openAITools[0]).toHaveProperty('function'); - - // Initialize OpenAI client - const openai = new OpenAI(); - - // Create a chat completion with tool calls - const response = await openai.chat.completions.create({ - model: 'gpt-5', - messages: [ - { - role: 'system', - content: 'You are a helpful assistant that can access BambooHR information.', - }, - { - role: 'user', - content: - 'What is the employee with id: c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA phone number?', - }, - ], - tools: openAITools, - }); - - // Verify the response contains tool calls - expect(response.choices.length).toBeGreaterThan(0); - - const choice = response.choices[0]; - expect(choice.message.tool_calls).toBeDefined(); - expect(choice.message.tool_calls!.length).toBeGreaterThan(0); - - const toolCall = choice.message.tool_calls![0]; - assert(toolCall.type === 'function'); - expect(toolCall.function.name).toBe('bamboohr_get_employee'); + it('executes the tool the model calls and prints its answer', async () => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + const calls = recordToolCalls(); + + const { output, exitCode } = await runExample('openai-integration'); + + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 3 tools for OpenAI', + 'Tool calls made: 1', + ' Tool: workday_list_workers', + ' Arguments: {"query":{"limit":5}}', + `Answer: ${EXAMPLE_ANSWER}`, + ].join('\n'), + ); + expect(calls.map(({ name, arguments: args }) => [name, args])).toEqual([ + ['workday_list_workers', { query: { limit: 5 } }], + ]); + }); - // Parse the arguments to verify they contain the expected fields - const args: unknown = JSON.parse(toolCall.function.arguments); - assert(typeof args === 'object' && args !== null && 'id' in args); - expect(args.id).toBe('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA'); + it('skips without OPENAI_API_KEY', async () => { + stubExampleEnv({ OPENAI_API_KEY: '' }); - // Run the tool calls and turn them into the messages to send back - const toolMessages = await tools.executeOpenAIToolCalls(choice.message.tool_calls); - expect(toolMessages).toHaveLength(1); - expect(toolMessages[0]).toMatchObject({ role: 'tool', tool_call_id: 'call_mock' }); - const content = toolMessages[0]?.content; - assert(typeof content === 'string', 'the tool message carries no text'); - expect(JSON.parse(content)).toMatchObject({ - isError: false, - result: { - data: { - action: 'bamboohr_get_employee', - arguments: { id: 'c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA' }, - }, - }, + expect(await runExample('openai-integration')).toEqual({ + output: 'Skipping: OPENAI_API_KEY is not set', + exitCode: 0, }); }); }); diff --git a/tests/examples/openai-responses-integration.test.ts b/tests/examples/openai-responses-integration.test.ts index 97c1db24..a3d35e5e 100644 --- a/tests/examples/openai-responses-integration.test.ts +++ b/tests/examples/openai-responses-integration.test.ts @@ -1,68 +1,38 @@ /** - * E2E test for openai-responses-integration.ts example - * - * Tests the complete flow of using StackOne tools with OpenAI Responses API. + * Runs examples/openai-responses-integration.ts against the mock StackOne and OpenAI APIs. */ -import OpenAI from 'openai'; -import { TEST_BASE_URL } from '../../mocks/constants'; -import { StackOneToolSet } from '../../src'; - -describe('openai-responses-integration example e2e', () => { - beforeEach(() => { - vi.stubEnv('STACKONE_API_KEY', 'test-key'); - vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); - }); +import { runExample, stubExampleEnv } from './run-example'; +describe('openai-responses-integration example', () => { afterEach(() => { vi.unstubAllEnvs(); }); - it('should fetch tools, convert to OpenAI Responses format, and create response with tool calls', async () => { - const toolset = new StackOneToolSet({ - accountId: 'your-stackone-account-id', - baseUrl: TEST_BASE_URL, - }); - - // Fetch tools via MCP with action filter - const tools = await toolset.fetchTools({ - actions: ['*_list_*'], - }); - const openAIResponsesTools = tools.toOpenAIResponses(); - - // Verify tools are in OpenAI Responses format - expect(Array.isArray(openAIResponsesTools)).toBe(true); - expect(openAIResponsesTools.length).toBeGreaterThan(0); - // Strict mode makes OpenAI reject served schemas with optional fields - expect(openAIResponsesTools.map((tool) => tool.strict)).not.toContain(true); - - // Initialize OpenAI client - const openai = new OpenAI(); - - // Create a response with tool calls using the Responses API - const response = await openai.responses.create({ - model: 'gpt-5', - instructions: 'You are a helpful assistant that can access various tools.', - input: 'What is the employee with id: c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA phone number?', - tools: openAIResponsesTools, - }); - - // Verify the response contains expected data - expect(response.id).toBeDefined(); - expect(response.model).toBeDefined(); - - // Check if the model made any tool calls - const toolCalls = response.output.filter( - (item): item is OpenAI.Responses.ResponseFunctionToolCall => item.type === 'function_call', + it('offers the fetched tools and prints the tool call the model makes', async () => { + stubExampleEnv({ OPENAI_API_KEY: 'test-openai-key' }); + + const { output, exitCode } = await runExample('openai-responses-integration'); + + expect(exitCode).toBeUndefined(); + expect(output).toBe( + [ + 'Loaded 2 tools for OpenAI Responses API', + 'Response ID: resp_mock_tool_call', + 'Model: gpt-5', + 'Tool calls found: 1', + ' Tool: workday_list_workers', + ' Arguments: {"query":{"limit":5}}', + ].join('\n'), ); + }); - expect(toolCalls.length).toBeGreaterThan(0); - - const toolCall = toolCalls[0]; - expect(toolCall.name).toBe('bamboohr_get_employee'); + it('skips without OPENAI_API_KEY', async () => { + stubExampleEnv({ OPENAI_API_KEY: '' }); - // Parse the arguments to verify they contain the expected fields - const args = JSON.parse(toolCall.arguments); - expect(args.id).toBe('c28xIQaWQ6MzM5MzczMDA2NzMzMzkwNzIwNA'); + expect(await runExample('openai-responses-integration')).toEqual({ + output: 'Skipping: OPENAI_API_KEY is not set', + exitCode: 0, + }); }); }); diff --git a/tests/examples/run-example.ts b/tests/examples/run-example.ts new file mode 100644 index 00000000..f6bcc282 --- /dev/null +++ b/tests/examples/run-example.ts @@ -0,0 +1,89 @@ +/** + * Runs a real file from `examples/` against the mocks. + * + * An example does its work in top-level await, so importing it runs it. `STACKONE_BASE_URL` points + * the toolset at the mock API, and `process.exit` throws instead of ending the vitest worker. + */ + +import process from 'node:process'; +import { http } from 'msw'; +import { TEST_BASE_URL } from '../../mocks/constants'; +import { mockAccountTools } from '../../mocks/handlers.mcp'; +import { createMcpApp } from '../../mocks/mcp-server'; +import type { MockMcpServerOptions, RecordedToolCall } from '../../mocks/mcp-server'; +import { server } from '../../mocks/node'; + +/** An account the MCP mock serves the example tools for (mocks/handlers.mcp.ts). */ +const EXAMPLE_ACCOUNT_ID = 'your-bamboohr-account-id'; + +interface ExampleRun { + /** Everything the example printed, one console call per line, `console.error` prefixed. */ + output: string; + /** The code the example passed to `process.exit`, or undefined when it ran to the end. */ + exitCode: number | undefined; +} + +class ExampleExit extends Error { + readonly code: number | undefined; + + constructor(code: number | undefined) { + super(`example called process.exit(${code})`); + this.code = code; + } +} + +/** Point every example at the mocks, plus the given variables. The examples treat `''` as unset. */ +export function stubExampleEnv(extra: Record = {}): void { + vi.stubEnv('STACKONE_API_KEY', 'test-key'); + vi.stubEnv('STACKONE_ACCOUNT_ID', EXAMPLE_ACCOUNT_ID); + vi.stubEnv('STACKONE_BASE_URL', TEST_BASE_URL); + for (const [key, value] of Object.entries(extra)) { + vi.stubEnv(key, value); + } +} + +/** Run `examples/.ts` afresh and return what it printed and how it exited. */ +export async function runExample(name: string): Promise { + const lines: string[] = []; + const spies = [ + vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(' ')); + }), + vi.spyOn(console, 'error').mockImplementation((...args: unknown[]) => { + lines.push(`[error] ${args.map(String).join(' ')}`); + }), + vi.spyOn(process, 'exit').mockImplementation((code) => { + throw new ExampleExit(code === undefined || code === null ? undefined : Number(code)); + }), + ]; + + let exitCode: number | undefined; + try { + // A fresh query string makes each run evaluate the module again instead of reusing the cache. + await import(/* @vite-ignore */ `../../examples/${name}.ts?run=${crypto.randomUUID()}`); + } catch (error) { + if (!(error instanceof ExampleExit)) { + throw error; + } + exitCode = error.code; + } finally { + for (const spy of spies) { + spy.mockRestore(); + } + } + return { output: lines.join('\n'), exitCode }; +} + +/** Serve the default mock catalog with these options and record every `tools/call` made. */ +export function recordToolCalls( + options: Omit = {}, +): RecordedToolCall[] { + const calls: RecordedToolCall[] = []; + const app = createMcpApp({ + ...options, + accountTools: mockAccountTools, + onToolCall: (call) => calls.push(call), + }); + server.use(http.all(`${TEST_BASE_URL}/mcp`, ({ request }) => app.fetch(request))); + return calls; +} diff --git a/tests/examples/search-and-execute.test.ts b/tests/examples/search-and-execute.test.ts index 432abdad..ae060d00 100644 --- a/tests/examples/search-and-execute.test.ts +++ b/tests/examples/search-and-execute.test.ts @@ -5,13 +5,10 @@ * discover the account, search, execute the best hit and try to record feedback. */ -import { http } from 'msw'; import { TEST_BASE_URL } from '../../mocks/constants'; -import { mockAccountTools } from '../../mocks/handlers.mcp'; -import { MOCK_SEARCH_SESSION_ID, createMcpApp } from '../../mocks/mcp-server'; -import type { RecordedToolCall } from '../../mocks/mcp-server'; -import { server } from '../../mocks/node'; +import { MOCK_SEARCH_SESSION_ID } from '../../mocks/mcp-server'; import { StackOneToolSet, ToolSetLoadError } from '../../src'; +import { recordToolCalls, runExample } from './run-example'; describe('search-and-execute example e2e', () => { beforeEach(() => { @@ -21,25 +18,17 @@ describe('search-and-execute example e2e', () => { afterEach(() => { vi.unstubAllEnvs(); - vi.restoreAllMocks(); }); it('runs the example end to end', async () => { - const calls: RecordedToolCall[] = []; - const app = createMcpApp({ - accountTools: mockAccountTools, - submitFeedback: true, - onToolCall: (call) => calls.push(call), - }); - server.use(http.all(`${TEST_BASE_URL}/mcp`, ({ request }) => app.fetch(request))); - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + const calls = recordToolCalls({ submitFeedback: true }); - await import('../../examples/search-and-execute'); + const { output, exitCode } = await runExample('search-and-execute'); - const printed = log.mock.calls.map((args) => String(args[0])).join('\n'); - expect(printed).toContain('default testprovider active'); - expect(printed).toContain('0.800 mock_list_items [default]'); - expect(printed).toContain('Feedback recorded.'); + expect(exitCode).toBeUndefined(); + expect(output).toContain('default testprovider active'); + expect(output).toContain('0.800 mock_list_items [default]'); + expect(output).toContain('Feedback recorded.'); expect( calls.map((call) => [ call.name, diff --git a/tests/examples/tanstack-ai-integration.test.ts b/tests/examples/tanstack-ai-integration.test.ts index 044e2898..6a9cf079 100644 --- a/tests/examples/tanstack-ai-integration.test.ts +++ b/tests/examples/tanstack-ai-integration.test.ts @@ -1,7 +1,8 @@ /** - * E2E test for tanstack-ai-integration.ts example + * E2E coverage for using StackOne tools with TanStack AI. * - * Tests the complete flow of using StackOne tools with TanStack AI. + * There is no `examples/tanstack-ai-integration.ts`; this covers the TanStack + * conversion path directly. * * Note: TanStack AI requires Zod schemas for tool input validation. * This test validates tool setup and schema conversion, but the actual @@ -12,7 +13,7 @@ import { TEST_BASE_URL } from '../../mocks/constants'; import { StackOneToolSet } from '../../src'; -describe('tanstack-ai-integration example e2e', () => { +describe('tanstack-ai-integration e2e', () => { beforeEach(() => { vi.stubEnv('STACKONE_API_KEY', 'test-key'); vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); diff --git a/vitest.setup.ts b/vitest.setup.ts index 660789ea..db2b8379 100644 --- a/vitest.setup.ts +++ b/vitest.setup.ts @@ -4,6 +4,16 @@ import { server } from './mocks/node'; // account warn. Tests that need it stub it themselves. delete process.env.STACKONE_ACCOUNT_ID; -beforeAll(() => server.listen()); +// An unmocked request to a remote API fails the test, so an example cannot reach a real one. +// Loopback stays open for the tests that start their own server. +beforeAll(() => + server.listen({ + onUnhandledRequest: (request, print) => { + if (new URL(request.url).hostname !== '127.0.0.1') { + print.error(); + } + }, + }), +); afterEach(() => server.resetHandlers()); afterAll(() => server.close());