diff --git a/.env.example b/.env.example index a85939bd..0a781c84 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,11 @@ # Required for all examples. Account ids are discovered from the key. STACKONE_API_KEY=your-stackone-api-key -# Only examples/auth_management.py needs a specific account +# Optional: override the API host. Falls back to https://api.stackone.com. +# STACKONE_BASE_URL=https://api.stackone.com + +# Needed by the examples that pin one account: auth_management, langchain_integration, +# langgraph_integration and pydantic_ai_integration STACKONE_ACCOUNT_ID=your-account-id # Required for OpenAI-based examples (openai_integration, langchain_integration, langgraph_integration, pydantic_ai_integration) diff --git a/.github/workflows/check-title.yaml b/.github/workflows/check-title.yaml new file mode 100644 index 00000000..a322a4df --- /dev/null +++ b/.github/workflows/check-title.yaml @@ -0,0 +1,26 @@ +name: 'Check PR title' + +on: + pull_request: + types: + - opened + - reopened + - edited + - synchronize + - ready_for_review + +permissions: + pull-requests: read + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-slim + steps: + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + ignoreLabels: | + autorelease: pending + dependencies diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af9f4f7e..e38bceb8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -105,10 +105,18 @@ jobs: echo "can-skip=false" >> "$GITHUB_OUTPUT" fi + # Runs the full suite, both SDKs side by side, so a change here that the Node SDK does not + # match fails here rather than drifting. conformance: needs: check-secret if: needs.check-secret.outputs.can-skip != 'true' runs-on: ubuntu-latest + env: + # Pinned to a commit on refactor/mcp-only-toolset, not the branch: a force-push or the + # branch's deletion on merge must not silently change what this job grades. Bump + # deliberately while StackOneHQ/stackone-ai-node#383 is open; switch to a main commit + # once it merges. + NODE_SDK_REF: 2253466371a22c839524d008a635ceb9f2c0959b steps: - name: Check conformance token is configured env: @@ -128,11 +136,25 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: StackOneHQ/sdk-conformance - ref: 7e3b471ac947c9dd743ff6d47ad535afd5804a86 + ref: 55bd126b0187a0f28f187803ecf48e791d157bfc token: ${{ secrets.CONFORMANCE_REPO_TOKEN }} path: sdk-conformance persist-credentials: false + # Public, so no token. + - name: Checkout Node SDK + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: StackOneHQ/stackone-ai-node + ref: ${{ env.NODE_SDK_REF }} + path: stackone-ai-node + persist-credentials: false + + # tests/test_shared_vectors.py grades the SDK against this copy; it must be the + # pinned commit's vectors/ byte for byte. Refresh with scripts/sync_vectors.sh. + - name: Vendored vectors match the pinned sdk-conformance + run: diff -r sdk/tests/vectors sdk-conformance/vectors + - name: Setup uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: @@ -145,10 +167,21 @@ jobs: version: 10.26.0 package_json_file: sdk-conformance/package.json + # 24, not 22: the Node SDK's devEngines pins Node ^24.11.0, and the suite runs on it too. - name: Setup Node uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: 22 + node-version: 24 + + - name: Install Node SDK dependencies + working-directory: stackone-ai-node + run: pnpm install --frozen-lockfile + + # The runner imports the built dist, not src, so the harness grades the publishable + # artifact. It refuses a dist older than src. + - name: Build Node SDK + working-directory: stackone-ai-node + run: pnpm run build - name: Install conformance dependencies working-directory: sdk-conformance @@ -157,8 +190,9 @@ jobs: - name: Run conformance suite working-directory: sdk-conformance env: + NODE_SDK_DIST: ${{ github.workspace }}/stackone-ai-node/dist/index.mjs PYTHON_SDK_DIR: ${{ github.workspace }}/sdk - run: pnpm test:python -- --strict-schema + run: pnpm test -- --strict-schema # Make this the one required status check. Individually required checks go # missing when a job is renamed or a matrix entry is added, and nothing notices. diff --git a/CLAUDE.md b/CLAUDE.md index ae7dbdcd..ecb952f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,20 +15,23 @@ Requires Python >= 3.11. The package is three modules. The guiding property is that **the toolset is the served catalog** — the schema listed to a model is the schema the MCP server sent, -and the request sent to `/actions/rpc` matches it. Nothing invented, nothing lost. - -1. **`types.py`** — `ToolParameters`, `ToolMode`, the error hierarchy, filename - sanitising, shared aliases and `DEFAULT_BASE_URL`. -2. **`tools.py`** — `StackOneTool` (execution, header sanitising, framework - converters), `Tools` (container), `StackOneRpcTool` (per-action tools over - `/actions/rpc`), `StackOneMcpTool` (meta tools over MCP `tools/call`), and the - MCP client. +and a call sends the model's arguments, as given, over MCP `tools/call` to the +endpoint and account that listed the tool. Nothing invented, nothing lost. The only +non-MCP request is `GET /accounts`. + +1. **`types.py`** — `ToolParameters`, `ToolMode`, `ExecuteConfig`, the error + hierarchy, shared aliases and `DEFAULT_BASE_URL`. +2. **`tools.py`** — `StackOneTool` (schema, header sanitising, framework converters; + its `execute()` raises — a hand-built tool must override it), `Tools` (container), + `StackOneMcpTool` (every tool the toolset builds, executed over MCP `tools/call`), + and the MCP client, including `parse_tool_result`. 3. **`toolset.py`** — `StackOneToolSet`: account discovery, the cached catalog, `search()` and `execute()`. -Tools are listed from `/mcp?param-style=flat_prefixed`. An API key alone is enough: -with no account given, `GET /accounts` is called and every `active` account is used. -There is no OpenAPI parsing and no client-side search. +Tools are listed from `/mcp` (plus `?tool-mode=` when a mode is set; there is no +param-style pin, so the model sees whatever style the server serves). An API key +alone is enough: with no account given, `GET /accounts` is called and every `active` +account is used. There is no OpenAPI parsing and no client-side search. ### Two calling surfaces @@ -36,14 +39,20 @@ The same actions are reachable two ways, with **different argument shapes**: - `toolset.search()` + `toolset.execute(action_id, args)` drives the per-connector `_search_actions` / `_execute_action` meta tools (`?tool-mode=search_execute`). - Arguments are the **nested** envelope: `{"body": {"variables": {...}}}`. -- `toolset.fetch_tools()` + `tool.execute(args)` uses per-action tools over - `/actions/rpc`. Arguments are **flat-prefixed**: `body_variables`, `path_id`. + Arguments are the **nested** form: `{"body": {"variables": {...}}}`. +- `toolset.fetch_tools()` + `tool.execute(args)` calls per-action tools. Arguments + are the keys the tool's served schema names. -Both return the payload itself. The server **silently drops** arguments that do not -match the schema — a wrong key returns a normal-looking success with your filter -ignored. Flat keys passed to `toolset.execute()` fail this way; nested keys passed to -a `fetch_tools()` tool are accepted. +Both go over `tools/call` with arguments sent verbatim — the SDK never splits or +renames them. The server **silently drops** arguments that do not match the schema — +a wrong key returns a normal-looking success with your filter ignored. + +Results follow one rule for every tool (`parse_tool_result`): text parts win, else +`structuredContent`; `isError` raises `StackOneAPIError` with the payload's status. +A success is returned exactly as the server wrote it, never unwrapped: for an action +tool, `*_execute_action` and feedback that is UCA's `{"isError": false, "result": ...}` +(optionally with `defenderMetadata` / `policyMetadata`); search results are bare JSON. +Keep the result shape identical to the Node SDK's. ## Commands @@ -116,11 +125,17 @@ import, never in core. **Test doubles must model what the server demands, not what the client happens to send.** The SDK once shipped unable to list a single tool while every test passed, because the mock defaulted a missing `x-account-id` to `'default'` — inventing an -account the real API would have rejected. The mock now 400s an unscoped `/mcp` or -`/actions/rpc` request and 404s an unknown account. When adding a mock behaviour, +account the real API would have rejected. The mock now 400s an unscoped `/mcp` +request and 404s an unknown account. When adding a mock behaviour, make it refuse what the real API refuses. A fake whose signature has no failure mode cannot catch a bug. +The mock logs every MCP `tools/call` request as sent — with its account and query +string, before the +MCP SDK parses it — at `GET /__requests` (`DELETE` clears it). Assert wire shape there, not +on a handler's echo, which cannot show a stripped or null key. `MOCK_SUBMIT_FEEDBACK=off` +(the `mcp_mock_server_without_feedback` fixture) serves a project without feedback. + Verify live behaviour against a real key before claiming it works — a green suite is not evidence. Put credentials in the gitignored `.env` and run `uv run --env-file .env python ...`. @@ -134,9 +149,12 @@ pnpm install These tests **fail** rather than skip if Node dependencies are missing — a silent skip previously let them vanish while CI stayed green. +The mock lists every `inputSchema` verbatim through the MCP SDK's low-level `Server`. +Do not move it to `McpServer.registerTool`: that expects a Zod shape and lists a plain +JSON Schema as `properties: {}`, so no test sees a declared parameter. + The mock's dependencies are pinned exactly (`@modelcontextprotocol/sdk`, `zod`, -`hono`, `@hono/mcp`). Do not loosen them to caret ranges: a newer MCP SDK rejects -the raw `inputSchema` objects the mock passes. +`hono`, `@hono/mcp`). Do not loosen them to caret ranges. ## Examples @@ -193,7 +211,13 @@ via release-please after a merge to main, never from a developer machine. toolset = StackOneToolSet() # reads STACKONE_API_KEY; accounts are discovered hits = toolset.search("list recent comments", top_k=3) -toolset.execute("linear_list_comments", {"body": {"variables": {"first": 25}}}) +hit = hits[0] if hits else None +session_id = hit.get("session_id") if hit else None +account_ids = [hit["account_id"]] if hit else None +toolset.execute( + "linear_list_comments", {"body": {"variables": {"first": 25}}}, account_ids=account_ids, session_id=session_id +) +toolset.submit_feedback("positive", ["linear_list_comments"], session_id=session_id) tools = toolset.fetch_tools(providers=["linear"], actions=["*_list_*"]) ``` @@ -202,6 +226,15 @@ tools = toolset.fetch_tools(providers=["linear"], actions=["*_list_*"]) case-sensitive glob. **A leading `!` is not exclusion syntax** — it is a literal character, so `["*", "!*_delete_*"]` matches every tool. - `top_k` is per connector, and must be 1..50. +- `stackone_submit_feedback` is one global tool the server lists with every + account's catalog, in both modes, when feedback is enabled. Like every tool it is a + `StackOneMcpTool`; unlike the others it is deduped to one across accounts. It is never + invented client-side — when the server omits it, `submit_feedback()` raises. + `submit_feedback()` lists one account in `search_execute` mode and makes one + `tools/call`, through the first account (as given, else in `GET /accounts` order). +- `Tools.get_tool()` returns the first tool listed under a name, as Node's `getTool()`. +- `session_id` is copied onto each `search()` hit and forwarded by `execute()` / + `submit_feedback()` only when given. Optional wire keys are omitted, never null. ## Important Considerations @@ -209,22 +242,31 @@ tools = toolset.fetch_tools(providers=["linear"], actions=["*_list_*"]) `StackOneAPIError` and the `ToolsetError` family. Nothing outside that hierarchy should escape a public method. `str(StackOneAPIError)` leads with the server's own message. -- **File downloads**: non-JSON responses return raw bytes plus metadata. The filename - comes from an attacker-controllable header and is reduced to a safe basename. -- **Headers**: model-supplied headers are an **allowlist** driven by the served - schema — only a declared `headers_*` property passes. Match header grammar with +- **File downloads**: a file action returns a download link + (`{download_url, expires_at, file}`), unchanged; the SDK does not follow it. When no + link can be issued the server answers `isError` with status 501, which raises. +- **Headers**: header arguments — entries of a nested `headers` object, and top-level + `headers_` arguments — are an **allowlist** driven by the served schema. A nested + entry passes if `headers.properties` declares it, or if `headers` is an object with no + `properties` (open, as `*_execute_action` serves it); a flat one passes if the + `headers_` property is declared. `Authorization` / `x-account-id` / `User-Agent` + never do. Every other argument is sent unchanged. The SDK sets those three itself, + after any configured headers. Match header grammar with `fullmatch`, never `match`: `$` also matches before a trailing newline, so `match` lets `"value\n"` through. ### Modifying Tool Behaviour -- Core execution logic: `StackOneTool.execute()` in `tools.py` -- RPC envelope split: `StackOneRpcTool._split_envelope_params` -- Meta-tool execution: `StackOneMcpTool.execute()` and `StackOneToolSet.execute()` +- Execution: `StackOneMcpTool.execute()` and `call_mcp_tool()` in `tools.py` +- Result parsing: `parse_tool_result()` +- search/execute routing: `StackOneToolSet.execute()` +- Tool construction and the feedback-tool dedupe: `StackOneToolSet._create_tool` and + `_dedupe_global_tools` Schemas must reach the model intact. `to_openai_function` passes the served schema -through verbatim, stripping only the SDK's internal `nullable` marker (which becomes -the JSON Schema `required` list). `to_langchain` and `to_pydantic_ai_tool` hand over +through verbatim, stripping only the SDK's internal `nullable` marker. Its `required` +is the served root list in the served order — never rebuilt from the markers, which +re-sorted it into property order. `to_langchain` and `to_pydantic_ai_tool` hand over that same schema — never rebuild one from property types, which silently loses every nested field, enum and bound. The conformance suite's `--strict-schema` gate checks the OpenAI surface. diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..759a0758 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,433 @@ +# Migrating from 2.x to 3.0 + +3.0 is a thin client over StackOne's MCP endpoint. Every tool call goes over MCP +`tools/call`, and the only other request the SDK makes is `GET /accounts`, to find +your linked accounts. Search runs on the server. Anything MCP does not support has +been removed rather than kept on a second transport. + +Work through the sections that apply to you. Each one gives the 2.x code and its 3.0 +replacement. + +- [Installation](#installation) +- [Imports](#imports) +- [Search and execute](#search-and-execute) +- [Results](#results) +- [Tool arguments and headers](#tool-arguments-and-headers) +- [Base URL](#base-url) +- [Accounts](#accounts) +- [Feedback](#feedback) +- [Schemas given to a model](#schemas-given-to-a-model) +- [Hand-built tools and `ExecuteConfig`](#hand-built-tools-and-executeconfig) +- [LangGraph helpers](#langgraph-helpers) + +## Installation + +- **Python 3.11 or later** is required. 3.10 is no longer supported. +- **`mcp` is a core dependency.** The `stackone-ai[mcp]` extra no longer exists. +- **`langchain-core` is now optional.** `to_langchain()` and `toolset.langchain()` + need the `langchain` extra and raise an `ImportError` that says so without it. +- `bm25s` and `numpy` are no longer dependencies, because search now runs on the server. + +```bash +# 2.x +uv add 'stackone-ai[mcp]' + +# 3.0: add the extras for the frameworks you use +uv add stackone-ai +uv add 'stackone-ai[langchain]' # LangChain and LangGraph +uv add 'stackone-ai[pydantic-ai]' # Pydantic AI +``` + +## Imports + +`stackone_ai.models` is split into `stackone_ai.tools` and `stackone_ai.types`. Anything +public can also be imported from the package root. + +```python +# 2.x +from stackone_ai.models import ExecuteConfig, StackOneTool, ToolParameters, Tools +from stackone_ai.toolset import ToolsetConfigError + +# 3.0 +from stackone_ai import ExecuteConfig, StackOneTool, ToolParameters, Tools, ToolsetConfigError +``` + +`ToolsetError`, `ToolsetConfigError` and `ToolsetLoadError` now subclass `StackOneError`, +so `except StackOneError` catches everything the SDK raises. Existing +`except ToolsetError` clauses behave as before. + +Unusable tool arguments raise the new `ToolArgumentsError`, before any request: a JSON +string that does not parse or does not parse to an object, an argument that cannot be +encoded as JSON, or `tool.call()` given both positional and keyword arguments, or more +than one positional argument. Raised by `tool.execute()`, `tool.call()` and +`toolset.execute()` alike. It subclasses both `StackOneError` and `ValueError`, which is +what 2.x raised, so existing `except ValueError` clauses still catch it. + +These names have been removed and have no direct replacement: + +| Removed | Use instead | +|---|---| +| `SearchConfig`, `SearchMode`, `SearchTool`, `SemanticSearchClient`, `SemanticSearchResult`, `SemanticSearchResponse`, `SemanticSearchError` | `toolset.search()`. See [Search and execute](#search-and-execute) | +| `stackone_ai.feedback`, `create_feedback_tool` | `toolset.submit_feedback()`. See [Feedback](#feedback) | +| `stackone_ai.integrations` | LangGraph's own APIs. See [LangGraph helpers](#langgraph-helpers) | +| `ToolDefinition` | `StackOneTool` fields: `name`, `description`, `parameters` | +| `ParameterLocation`, `validate_method` | Nothing. `ExecuteConfig` no longer has a method or parameter locations | +| `StackOneTool.connector`, `Tools.get_connectors()` | `fetch_tools(providers=[...])` to filter by provider. A provider name can contain `_` (`browser_linkedin`), so do not split the tool name | + +Some names were public on `main` for a while before 3.0 but never in a release: +`StackOneRpcTool`, `MCP_PARAM_STYLE`, `is_json_content_type` and +`filename_from_content_disposition`. They are gone as well. + +## Search and execute + +Client-side search is gone: the BM25/TF-IDF index, the semantic search client, the +`search=` constructor argument, and the `tool_search`/`tool_execute` meta tools. The +server's `*_search_actions` tool now does the search, and `toolset.execute()` runs an +action by id through the server's `*_execute_action` tool. + +```python +# 2.x +toolset = StackOneToolSet(search={"method": "auto"}) +tools = toolset.search_tools("list employees", top_k=5) +result = tools[0].execute({"query_page_size": 25}) + +names = toolset.search_action_names("time off requests", top_k=5) + +# 3.0 +toolset = StackOneToolSet() +hits = toolset.search("list employees", top_k=5) # a list of action dicts, best first +hit = hits[0] +result = toolset.execute( + hit["action_id"], + {"query": {"page_size": 25}}, # the nested form in hit["input_schema"] + account_ids=[hit["account_id"]], + session_id=hit.get("session_id"), +) +``` + +`search()` returns dicts carrying `action_id` and the `account_id` that found it, plus +`description`, `similarity_score`, `input_schema`, `example_request` and `session_id` +when the server sends them. `top_k` caps the whole result, as in 2.x, after ranking +across every connector; the same action linked on two accounts is two hits. When an +action's connector is linked on more than one account, `execute()` raises +`ToolsetConfigError` unless `account_ids` picks one, such as `[hit["account_id"]]`. + +`toolset.execute()` has a new signature. It used to take a meta tool name, such as +`"tool_execute"`, with arguments as a JSON string or a dict. It now takes +`(action_id, arguments=None, *, account_ids=None, session_id=None)`, and `arguments` +must be a dict. It raises instead of returning `{"error": ...}`: `ToolsetConfigError` +before any request when `action_id`, `arguments` or `session_id` is malformed, +`ToolArgumentsError` when the arguments cannot be encoded as JSON, `ToolsetLoadError` when +no linked connector matches the action, and `StackOneAPIError` when the action fails, +including when the server rejects the arguments. + +To give a model the search and execute tools, set the tool mode on the toolset. +`openai()`, `langchain()` and `pydantic_ai()` no longer take `mode=`. + +```python +# 2.x +toolset = StackOneToolSet(search={"method": "auto"}) +openai_tools = toolset.openai(mode="search_and_execute") +# ...for each tool call the model makes: +toolset.execute(call.function.name, call.function.arguments) + +# 3.0: two tools per connector, served by the server +toolset = StackOneToolSet(tool_mode="search_execute") +tools = toolset.fetch_tools() +openai_tools = tools.to_openai() +# ...then run the model's tool calls and get the messages to send back: +messages.extend(tools.execute_openai_tool_calls(message.tool_calls)) +``` + +`get_search_tool()` and `semantic_client` are removed. Call `toolset.search()` instead. + +## Results + +**Every tool returns the server's result as the server wrote it.** For an action, +that is `{"isError": False, "result": ..., "defenderMetadata"?, "policyMetadata"?}`. +Search results are bare JSON. This applies to `tool.execute()`, `tool.call()` and +`toolset.execute()`. + +```python +tool = toolset.fetch_tools(account_ids=[account_id]).get_tool("hibob_list_employees") + +# 2.x +employees = tool.execute({})["data"] + +# 3.0 +employees = tool.execute({})["result"]["data"] +``` + +A result with `isError` set raises `StackOneAPIError`, which carries the status from the +payload in `status_code` and the body in `response_body`. + +**Integers above 2^53 keep their exact value.** This SDK parses the server's JSON +itself, so it is not limited to the integer precision of IEEE 754 doubles; the Node +SDK's result can differ for a field that size, because its JSON parser has already +rounded it by the time the SDK sees it. + +**File actions return a download link, not bytes.** The SDK does not follow the link. +When no link can be issued, the call raises `StackOneAPIError` with `status_code` 501. + +```python +# 2.x +result = download.execute({"id": "file-id"}) +with open(result["file_name"] or "download.bin", "wb") as f: + f.write(result["content"]) + +# 3.0 +from pathlib import Path + +import httpx + +link = download.execute({"id": "file-id"})["result"] +# {"download_url": ..., "expires_at": ..., "file": {"name", "content_type", "content_length"}} +name = Path(link["file"]["name"] or "download.bin").name # chosen by the provider: keep only the basename +Path(name).write_bytes(httpx.get(link["download_url"]).content) +``` + +## Tool arguments and headers + +**Arguments are sent exactly as given**, as `tools/call` arguments. The SDK no longer +splits flat `path_`/`query_`/`body_` keys into a request envelope, because the server +maps them itself. + +**`fetch_tools()` tools have the server's own argument shape.** In 2.x the SDK asked +the server for the flat, prefixed style (`?param-style=flat_prefixed`). That request is +gone, so the argument names are whatever the server serves. Read them from +`tool.parameters.properties` rather than hard-coding them: + +```python +# 2.x +tool.execute({"body_variables": {"first": 25}}) + +# 3.0: use the keys the schema names, for example +print(tool.parameters.properties) +tool.execute({"body": {"variables": {"first": 25}}}) +``` + +**Header arguments are allowlisted.** A header argument is an entry of a `headers` +object argument, or a top-level `headers_` argument. Each one is forwarded only +if the tool's schema declares it in the same form: under `headers.properties`, or as a +`headers_` property. An open `headers` object, `"type": "object"` with no +`properties` and `additionalProperties` not `false`, declares every name; without +`"type": "object"`, or with `additionalProperties: false` and no `properties`, it +declares none. `Authorization`, `x-account-id`, `User-Agent` and `x-end-user-id` are +never forwarded, even when declared, because the SDK sets them itself. A dropped header argument is +logged as a warning, except a null value, which is omitted silently. Every other +argument is sent unchanged. + +A top-level `headers` argument that isn't a plain object is dropped with a warning, +unless the schema declares `headers` itself as a non-object field, in which case it's +an ordinary argument that happens to be named `headers`: it skips header filtering and is +sent as given, subject to the same JSON-value check as every other argument. A +`headers_` argument is dropped with a warning when its value is a list or dict. + +`*_execute_action` serves an open `headers` object, so `toolset.execute()` passes your +own headers on to the action, with the exception of those four: + +```python +toolset.execute("linear_list_comments", {"headers": {"x-request-id": "abc"}}) +``` + +## Base URL + +**`STACKONE_BASE_URL` is now read.** `base_url` still takes precedence when given; +otherwise the SDK now falls back to `STACKONE_BASE_URL` before the default, +`https://api.stackone.com`. An empty `base_url` argument or an empty +`STACKONE_BASE_URL` is treated as not set, not as a literal empty host. + +## Accounts + +**With no account id, the SDK discovers your accounts.** In 2.x, calling +`fetch_tools()` with no account listed tools without an `x-account-id`. In 3.0 it asks +`GET /accounts` and lists the catalog of every active shared account (non-shared ones +too with `include_non_shared=True`). If you have many +accounts, pass `account_id=`, `account_ids=` or call `set_accounts()` so the SDK does +not fetch every catalog. + +**Discovery skips non-shared accounts.** An account with `shared: false` belongs to a +single end user, so discovery leaves it out and logs a warning naming it; when every +active account is non-shared it raises `ToolsetConfigError` instead. Pass its id, or +construct the toolset with `include_non_shared=True`, to use it. + +**A non-shared account's end user is sent for you.** The API requires every MCP request +for a non-shared account to carry the account's end-user id in `x-end-user-id`. Whenever +the SDK calls `GET /accounts` (discovery, or `fetch_accounts()`), it records the +`origin_username` of each account with `shared: false` and sends it as `x-end-user-id` on +every request for that account. With explicit account ids it makes no `GET /accounts` up +front: the first request for a non-shared account is refused, and the SDK then calls +`GET /accounts` once and retries it with the end user it records. Call +`toolset.fetch_accounts()` once first to skip that round trip. + +**`STACKONE_ACCOUNT_ID` is not read.** Pass the account id as `account_id=` or +`execute={"account_ids": [...]}`. A toolset constructed with neither while +`STACKONE_ACCOUNT_ID` is set to a non-empty value logs a warning that it is ignored. + +**An empty account id raises.** `StackOneToolSet(account_id="")` raises +`ToolsetConfigError`, where it used to be treated as no account at all. An account id +read from an environment variable that is set but empty now raises too, rather than +quietly reaching every active shared account: + +```python +# Raises when STACKONE_ACCOUNT_ID is set to "" +toolset = StackOneToolSet(account_id=os.getenv("STACKONE_ACCOUNT_ID")) + +# Falls back to discovery only when the variable is unset or empty, deliberately +toolset = StackOneToolSet(account_id=os.getenv("STACKONE_ACCOUNT_ID") or None) +``` + +An empty or non-string entry in `account_ids`, `set_accounts()` or +`execute={"account_ids": [...]}` raises `ToolsetConfigError` as well, and so does +passing a tuple (or any other non-list) there: pass a list, `list(ids)`, instead. A falsy entry +(`None`, `0`, `""`) used to produce an unscoped request with no `x-account-id`; a +truthy non-string entry used to be sent as the header's value. + +**`get_tool()` returns the first of any duplicates.** When two accounts serve the same +tool name, `Tools.get_tool()` now returns the first one listed, where 2.x returned the +last. Listings are merged in sorted account order, and a warning names the clashing +tools. `execute_openai_tool_calls()`, `to_openai()`, `to_langchain()` and +`to_pydantic_ai()` use the same one: each builds one tool per name. Pass `account_ids` +to choose the account yourself. + +**A failing account is skipped, and left out for 30 seconds.** In a multi-account scope, an +account whose listing fails with a non-429 error is skipped with a warning and left out of +the cached catalog for 30 seconds, unless every account fails: then the accounts' shared +`StackOneAPIError` is raised when they all failed with one status, and otherwise a +`ToolsetLoadError` with each account's error in `failures`. A 429 that outlasts its retries +is never skipped — it aborts the whole call, even when other accounts are healthy, since +it is the key's rate limit rather than that account's failure. A single-account scope +raises its failure rather than skipping or caching it. `execute()` does not wait the 30 +seconds for an account that could serve the action, one whose provider is the action's +connector or unknown after one `GET /accounts` to learn it: it lists it again straight +away, and refuses to run the action on another account while it still fails. + +## Feedback + +The client-side `tool_feedback` tool, `create_feedback_tool()` and the implicit +`feedback_*` execution options have been removed. `execute()` and `call()` no longer +take `options=`. + +```python +# 2.x +feedback_tool = toolset.fetch_tools(actions=["tool_*"]).get_tool("tool_feedback") +feedback_tool.call(feedback="Worked well", account_id="acc_123", tool_names=["workday_list_workers"]) +tool.execute({"id": "1"}, options={"feedback_session_id": "chat-42"}) + +# 3.0 +hit = toolset.search("list workers")[0] +toolset.execute(hit["action_id"], account_ids=[hit["account_id"]], session_id=hit.get("session_id")) +toolset.submit_feedback( + "positive", + [hit["action_id"]], + feedback="Worked well", + session_id=hit.get("session_id"), +) +``` + +When feedback is enabled for your project, the server also serves a +`stackone_submit_feedback` tool. **`fetch_tools()`, `openai()`, `langchain()` and +`pydantic_ai()` include it**, once, however many accounts are linked. If you don't want +a model to call it, filter it out: + +```python +from stackone_ai import Tools + +tools = Tools([t for t in toolset.fetch_tools() if t.name != "stackone_submit_feedback"]) +``` + +`submit_feedback()` raises `ToolsetLoadError` when feedback is not enabled. It calls the +tool once, on the lowest account id in scope, and sends `action_run_id` when given. + +## Schemas given to a model + +**`to_openai_function()` passes the served schema through.** 2.x kept only `type`, +`description`, `enum` and a shallow copy of `items`/`properties`. Now `format`, +`pattern`, `default`, bounds, `oneOf`/`anyOf` and nested `required` reach the model as +the server served them. `to_langchain()` and `to_pydantic_ai_tool()` hand over the +same schema. + +**`required` is the served list.** 2.x derived `required` from each property's +internal `nullable` marker, which made every property of a hand-built tool with no +markers required. 3.0 emits the schema's own `required` list, in the order the server +sent it, and leaves it out when that list is missing or empty. Give a hand-built tool +its `required` list directly: + +```python +# 2.x: required, because "nullable" is absent +ToolParameters(type="object", properties={"id": {"type": "string"}}) + +# 3.0 +ToolParameters(type="object", properties={"id": {"type": "string"}}, required=["id"]) +``` + +## Hand-built tools and `ExecuteConfig` + +**`StackOneTool` no longer makes HTTP requests.** Its `execute()` raises `StackOneError` +on the base class, so a hand-built tool must override it. Tools from `fetch_tools()` are +`StackOneMcpTool` instances, which execute over MCP. `_api_key` is now optional on the +base class. + +```python +# 2.x +tool = StackOneTool( + description="Get an employee", + parameters=ToolParameters(type="object", properties={"id": {"type": "string"}}), + _execute_config=ExecuteConfig( + name="get_employee", method="GET", url="https://api.example.com/employees/{id}", + parameter_locations={"id": ParameterLocation.PATH}, + ), + _api_key="...", +) + +# 3.0 +class GetEmployee(StackOneTool): + def __init__(self) -> None: + super().__init__( + description="Get an employee", + parameters=ToolParameters(type="object", properties={"id": {"type": "string"}}, required=["id"]), + _execute_config=ExecuteConfig(name="get_employee"), + ) + + def execute(self, arguments=None): + args = json.loads(arguments) if isinstance(arguments, str) else dict(arguments or {}) + return httpx.get(f"https://api.example.com/employees/{args['id']}").json() + +tool = GetEmployee() +``` + +Define `__init__` on the subclass, as above. `StackOneTool` is a pydantic model, so a +type checker gives a subclass without its own `__init__` one built from the model's +fields, which rejects `_execute_config` even though the call works at runtime. + +**`ExecuteConfig` keeps only `name`, `headers` and `timeout`**, and rejects any other +field. Passing `method`, `url`, `body_type` or `parameter_locations` raises a pydantic +`ValidationError` rather than being silently ignored. + +**`StackOneMcpTool(headers=...)` adds headers; it does not replace them.** The SDK +always sets `Authorization`, `x-account-id` and `User-Agent` itself, after your headers, +and drops any case variant of those names you pass. An `x-end-user-id` you pass is sent as +given. `headers` is optional. + +**`StackOneToolSet(headers=...)` is new.** It sends extra headers on every request the +toolset makes, `GET /accounts` included, with the same rule: `Authorization`, `x-account-id` +and `User-Agent` are ignored with a warning, and `x-end-user-id` is passed through unless +`GET /accounts` reported the account's end user. + +## LangGraph helpers + +`stackone_ai.integrations` has been removed. Pass the LangChain tools to LangGraph +directly: + +```python +# 2.x +from stackone_ai.integrations import to_tool_node +node = to_tool_node(tools) + +# 3.0 +from langgraph.prebuilt import ToolNode +node = ToolNode(tools.to_langchain()) +``` + +See [examples/langgraph_integration.py](examples/langgraph_integration.py) for a +complete agent. diff --git a/README.md b/README.md index 9c2f445a..e1f55f89 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ StackOne AI provides a unified interface for accessing various SaaS tools through AI-friendly APIs. +> **Upgrading from 2.x?** 3.0 has breaking changes. See the +> [migration guide](https://github.com/StackOneHQ/stackone-ai-python/blob/main/MIGRATION.md). + ## Features - **Search and execute**: find an action in natural language and run it, so a @@ -48,16 +51,43 @@ hits = toolset.search("list recent comments", top_k=3) # "description": "Returns a page of Linear comments as a connection object ...", # "similarity_score": 0.86, # "example_request": {"action_id": "linear_list_comments"}, -# "input_schema": {"type": "object", "properties": {"body": {...}}}}, ...] +# "input_schema": {"type": "object", "properties": {"body": {...}}}, +# "session_id": "3f9c...", +# "account_id": "acc-123"}, ...] +hit = hits[0] # 2. Run it. Build the arguments from input_schema. -result = toolset.execute("linear_list_comments", {"body": {"variables": {"first": 25}}}) -result["data"] # the provider's payload — a failed call raises instead +result = toolset.execute( + hit["action_id"], + {"body": {"variables": {"first": 25}}}, + account_ids=[hit["account_id"]], + session_id=hit.get("session_id"), +) +result["result"]["data"] # as the server wrote it: {"isError": false, "result": ...}; a failure raises + +# 3. Optionally, say how it went. The same session_id links it to the search. +toolset.submit_feedback("positive", [hit["action_id"]], session_id=hit.get("session_id")) ``` `search()` asks every linked connector and returns actions ranked by `similarity_score`, so a catalog of hundreds of tools never has to fit in a -model's context. This is the recommended way to use the SDK. +model's context. This is the recommended way to use the SDK. `top_k` (1–50, default +10) caps the whole result, after ranking across every connector. + +Each hit carries the `account_id` of the account that found it, and the same action +linked on two accounts is two hits. When an action's connector is linked on more than +one account, `execute()` raises `ToolsetConfigError` rather than pick one: pass the +hit's account as `account_ids=[hit["account_id"]]`. + +Each hit carries the `session_id` of the search that found it, when the server +issued one, so read it with `hit.get("session_id")`. Passing it to +`execute()` and `submit_feedback()` links the calls server-side; leaving it out +is fine. `submit_feedback()` makes exactly one call, on the account with the lowest id +among its `account_ids` when given, otherwise among those the toolset is configured +with or discovers, and sends `action_run_id` only when given. It raises +`ToolsetLoadError` when feedback is not enabled for your project. When it is, `fetch_tools()` also returns a single +`stackone_submit_feedback` tool — one, however many accounts are linked — that a +model can call itself. ## Integration Examples @@ -173,12 +203,13 @@ print(result["messages"][-1].content) ## Advanced Filtering `fetch_tools()` takes three filters. They combine with AND, and all of them are -applied locally to one cached listing — changing a filter never refetches. +applied locally to one cached listing — changing a filter never refetches. Call +`clear_catalog_cache()` after linking or unlinking accounts. ```python toolset = StackOneToolSet() -toolset.fetch_tools() # every tool, every active account +toolset.fetch_tools() # every tool, every active shared account toolset.fetch_tools(providers=["linear"]) # one connector toolset.fetch_tools(actions=["linear_list_*"]) # one connector's list actions toolset.fetch_tools(actions=["linear_get_issue"]) # exactly one tool @@ -188,8 +219,21 @@ toolset.fetch_tools(account_ids=["acc-123", "acc-456"]) ``` - **`account_ids`** — restrict to these accounts. Omit it and the SDK discovers - your active accounts. An empty list means "no filter", not "no accounts". A - single failing account is logged and skipped, not fatal. + your active shared accounts (non-shared ones too with `include_non_shared=True`). An empty list means "no filter", not "no accounts". A + single failing account is logged and skipped, not fatal — unless it is rate + limited (see [Rate limits](#rate-limits)) — and left out of the cached catalog for + 30 seconds, after which the next call lists it again. If every account fails with + the same HTTP status (a revoked key's 401, say), that `StackOneAPIError` is raised; + otherwise a `ToolsetLoadError` whose `failures` holds each account's error. + `execute()` does not route around a failed account that could serve the action — one + whose provider is the action's connector, or unknown even after one `GET /accounts` to + learn it: it lists it again straight away rather than after 30 seconds (unless a lookup + failed to name it in the last 30 seconds, when it neither looks it up nor lists it + again), and if it still fails it raises `ToolsetLoadError` rather than run the action + on another account. Pass the account id to use. A failed account on another provider is not listed again + early, so it never costs the call a listing timeout, though learning its provider with + explicit ids takes one `GET /accounts`; like any account, it is listed again once its + 30 seconds are up. - **`providers`** — matched **case-insensitively** as a full prefix, so `providers=["linear"]` and `["LINEAR"]` are the same, and a connector whose name contains an underscore must be spelled in full (`["browser_linkedin"]`, not @@ -205,31 +249,117 @@ toolset.set_accounts(["acc-123"]) tools = toolset.fetch_tools(providers=["linear"]) ``` +### Non-shared accounts + +A non-shared account (`shared: false` in `GET /accounts`) belongs to a single end user. +Discovery skips them, so a toolset built from an API key alone never puts one end user's +accounts in front of everyone, and logs once per discovery which it skipped. If every +active account on the key is non-shared, discovery raises `ToolsetConfigError` instead. +Pass their ids to use them, or opt in: + +```python +toolset = StackOneToolSet(include_non_shared=True) # discovery includes non-shared accounts +``` + +The API requires every MCP request for a non-shared account to carry that account's +end-user id in `x-end-user-id`. Whenever the SDK calls `GET /accounts`, during discovery +or in `fetch_accounts()`, it records the `origin_username` of each account with +`shared: false`, skipped or not, and sends it as `x-end-user-id` on every request for that +account, including from tools it built earlier. Each successful `GET /accounts` replaces +the record unless a later-started one has already been recorded; `clear_catalog_cache()` +keeps it. + +With explicit account ids the SDK does not call `GET /accounts` up front. When the API +refuses a request for an account with no recorded end user because it needs one, the SDK +calls `GET /accounts` once (joining one already in flight), and if that names the account's +end user, sends the request again with it; otherwise the API's 400 is raised, with the +lookup's error as its `__cause__` if the lookup failed (a rate-limited lookup raises its +429 `StackOneAPIError` instead). Calling +`toolset.fetch_accounts()` first saves that round trip: + +```python +toolset = StackOneToolSet(account_id="acc-123") +toolset.fetch_accounts() # records acc-123's end user if it is not shared +tools = toolset.fetch_tools() +``` + +Or pass the end-user id yourself; a value recorded from `GET /accounts` replaces it: + +```python +toolset = StackOneToolSet(account_id="acc-123", headers={"x-end-user-id": "carol"}) +``` + +### Extra headers + +`StackOneToolSet(headers={...})` sends extra headers on every request the toolset makes, +`GET /accounts` and every MCP request. `Authorization`, `x-account-id` and `User-Agent` +are always the SDK's own: if you pass them they are ignored, with a warning. + ### Two ways to call a tool -The SDK exposes the same actions through two surfaces. They take **different -argument shapes**, because each mirrors the schema the server served for it. Both -return the payload itself. +The SDK exposes the same actions through two surfaces. Both execute over MCP +`tools/call`, on the endpoint and with the account that listed the tool, and send +your arguments as given, except header arguments: each is forwarded only if the tool's +schema declares it, and `Authorization`, `x-account-id`, `User-Agent` and `x-end-user-id` +never are (see +[the migration guide](MIGRATION.md#tool-arguments-and-headers)). They take **different +argument shapes**, because each mirrors the schema the server served for it. Both return the server's result exactly +as it wrote it: `{"isError": false, "result": ..., "defenderMetadata"?, "policyMetadata"?}`. | | `search()` + `toolset.execute()` | `fetch_tools()` + `tool.execute()` | |---|---|---| | Schema to read | `input_schema` on each hit | `tool.parameters.properties` | -| Argument shape | nested — `{"body": {"variables": {...}}}` | flat, prefixed — `body_variables`, `path_id` | +| Argument shape | nested — `{"body": {"variables": {...}}}` | the keys the tool's served schema names | | Best for | agents that discover actions at run time | binding a fixed, filtered set of tools to a model | ```python -# Same action, both surfaces: +# The search/execute surface always takes the nested form: toolset.execute("linear_list_comments", {"body": {"variables": {"first": 25}}}) +# A fetch_tools() tool takes whatever its own schema declares: tool = toolset.fetch_tools(actions=["linear_list_comments"]).get_tool("linear_list_comments") -tool.execute({"body_variables": {"first": 25}}) +print(tool.parameters.properties) ``` -Mixing them fails silently in **one direction**. A `fetch_tools()` tool also -accepts the nested form. But flat keys like `body_variables` passed to -`toolset.execute()` are not an error — they are dropped, and you get the server's -defaults. +Keys `toolset.execute()` does not recognise are not an error — they are dropped, +and you get the server's defaults. + +Results keep the exact value of an integer above 2^53, unlike the Node SDK, whose +JSON parser has already rounded it by the time the SDK sees it. + +A file action's `result` is a download link, `{"download_url", "expires_at", "file": +{"name", "content_type", "content_length"}}`, rather than the file itself; the SDK does +not follow it, and `file.name` is chosen by the provider, so reduce it to a safe +basename before writing to disk. When no link can be issued the call raises `StackOneAPIError` with +`status_code` 501. + +## Rate limits + +Every request the SDK makes (`GET /accounts` and each MCP request, `tools/call` +included) retries an HTTP 429 up to three times. It waits for the response's +`Retry-After`, in seconds or as an HTTP date and capped at 30 seconds, or without one +for 1s, 2s and 4s, each scaled by a random factor between 0.5 and 1. Each retry logs a +warning. No other status is retried. A wait that would not end before the request's +`timeout` is not started: an MCP request's deadline is its whole exchange, and +`GET /accounts`'s is measured from its first attempt. + +A 429 that is still there after the fourth attempt, or that there is no time left to +wait out, raises `StackOneAPIError` with +`status_code` 429 and the server's body. That ends the whole call: when `fetch_tools()` +or `search()` spans several accounts, one rate-limited account raises rather than being +skipped, so you never get a partial catalog that looks complete. Other per-account +failures are still logged and skipped. + +## Custom base URL + +```python +from stackone_ai import StackOneToolSet + +toolset = StackOneToolSet(base_url="https://api.example-dev.com") +``` +Falls back to the `STACKONE_BASE_URL` environment variable, then to +`https://api.stackone.com`, if `base_url` is not given. ## Examples diff --git a/examples/search_and_execute.py b/examples/search_and_execute.py index f7f0e480..8d3a03cc 100644 --- a/examples/search_and_execute.py +++ b/examples/search_and_execute.py @@ -9,7 +9,8 @@ Prerequisites: STACKONE_API_KEY in .env and at least one active linked account. Expected output: the ranked actions matching the query, the JSON Schema for the -best one, and the result of executing it. +best one, the arguments taken from its example_request, the result of executing it, +and whether feedback was recorded. This is the recommended way to use the SDK: search() asks every linked connector and returns ranked actions, so a catalog of hundreds of tools never has to fit in @@ -29,7 +30,16 @@ pass from stackone_ai import StackOneToolSet -from stackone_ai.types import StackOneError, ToolsetError +from stackone_ai.types import StackOneError, ToolsetError, ToolsetLoadError + + +def _score(action: dict) -> float: + """A hit's similarity_score as a number: 0 when absent, NaN when not numeric.""" + score = action.get("similarity_score") + try: + return float(0 if score is None else score) + except (TypeError, ValueError): + return float("nan") def search_and_execute() -> None: @@ -44,19 +54,41 @@ def search_and_execute() -> None: print("\nRanked matches:") for action in actions: - print(f" {action.get('similarity_score', 0.0):.3f} {action['action_id']}") + print(f" {_score(action):.3f} {action['action_id']}") best = actions[0] - # input_schema is how you find out what an action accepts. Build the call from - # it: arguments that do not match are dropped by the server without an error, - # so a guessed parameter looks like it worked. + # input_schema is how you find out what an action accepts. Build the call from it + # and from example_request: arguments that do not match are dropped by the server + # without an error, so a guessed parameter looks like it worked. print(f"\ninput_schema for {best['action_id']}:") print(json.dumps(best.get("input_schema", {}), indent=2)[:600]) - result = toolset.execute(best["action_id"], {"body": {"variables": {"first": 2}}}) + # example_request is a call the server accepts for this action, in the nested form + # execute() takes. Start from it rather than guessing; execute() sets action_id itself. + arguments = dict(best.get("example_request") or {}) + arguments.pop("action_id", None) + print(f"\narguments: {json.dumps(arguments)}") + + # session_id links this call, and the feedback below, to the search that found + # the action. It is optional: leave it out and each call stands alone. account_id + # runs it on the account that found it: with the same provider linked on two + # accounts, execute() refuses to pick one. + session_id = best.get("session_id") + result = toolset.execute( + best["action_id"], arguments, account_ids=[best["account_id"]], session_id=session_id + ) print(f"\nresult: {json.dumps(result, default=str)[:300]}...") + try: + toolset.submit_feedback( + "positive", [best["action_id"]], feedback="Found it first try", session_id=session_id + ) + print("\nfeedback recorded") + except ToolsetLoadError as exc: + # Raised when feedback is not enabled for this project. Not worth failing over. + print(f"\nfeedback skipped: {exc}") + if __name__ == "__main__": try: diff --git a/pyproject.toml b/pyproject.toml index 68002807..f363d8e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,12 +33,12 @@ packages = ["stackone_ai"] # The sdist carried the whole dev tree — uv.lock, pnpm-lock.yaml, CLAUDE.md, .github/ # and the TypeScript mocks. A consumer building from source needs none of it. [tool.hatch.build.targets.sdist] -include = ["stackone_ai", "README.md", "LICENSE", "CHANGELOG.md", "pyproject.toml"] +include = ["stackone_ai", "README.md", "LICENSE", "CHANGELOG.md", "MIGRATION.md", "pyproject.toml"] # Per-framework adapters. Each is imported lazily inside the method that needs # it, so installing the SDK does not drag in a framework you do not use. [project.optional-dependencies] -langchain = ["langchain-core>=0.3.0"] +langchain = ["langchain-core>=0.3.36"] pydantic-ai = ["pydantic-ai-slim>=1.83.0"] examples = [ "langchain>=1.0.0", @@ -49,11 +49,10 @@ examples = [ ] dev = [ "hypothesis>=6.141.1", - "langchain-core>=0.1.0", + "langchain-core>=0.3.36", "pytest>=8.3.4", "pytest-asyncio>=0.25.3", "pytest-cov>=6.0.0", - "respx>=0.22.0", "ruff>=0.9.6", "ty>=0.0.3", ] diff --git a/scripts/sync_vectors.sh b/scripts/sync_vectors.sh new file mode 100755 index 00000000..6a864495 --- /dev/null +++ b/scripts/sync_vectors.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Vendor sdk-conformance's shared unit-test vectors into tests/vectors/. +# +# ./scripts/sync_vectors.sh ../sdk-conformance +# +# Check the checkout out at the commit CI pins (the `ref` of the conformance job in +# .github/workflows/ci.yaml) first: CI fails unless tests/vectors/ is byte-identical to +# that commit's vectors/. A checkout with uncommitted changes under vectors/ is refused, +# since no commit holds what would be copied. +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +SOURCE="$1/vectors" +TARGET="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/tests/vectors" + +if [ ! -f "$SOURCE/README.md" ]; then + echo "error: $SOURCE is not sdk-conformance's vectors/ directory" >&2 + exit 1 +fi + +if ! git -C "$1" rev-parse --git-dir >/dev/null 2>&1; then + echo "error: $1 is not a git checkout, so the copy cannot be tied to a commit" >&2 + exit 1 +fi + +if [ -n "$(git -C "$1" status --porcelain --ignored --untracked-files=all -- vectors)" ]; then + echo "error: $SOURCE has uncommitted changes" >&2 + exit 1 +fi +echo "Vendoring vectors/ from sdk-conformance@$(git -C "$1" rev-parse HEAD)" + +STAGING="$(mktemp -d "${TARGET}.tmp.XXXXXX")" +trap 'rm -rf "$STAGING"' EXIT +cp -R "$SOURCE" "$STAGING/vectors" +diff -r "$STAGING/vectors" "$SOURCE" +rm -rf "$TARGET" +mv "$STAGING/vectors" "$TARGET" +echo "Copied to $TARGET" diff --git a/stackone_ai/__init__.py b/stackone_ai/__init__.py index 487809e7..d8945e55 100644 --- a/stackone_ai/__init__.py +++ b/stackone_ai/__init__.py @@ -1,12 +1,16 @@ """StackOne AI SDK""" -from stackone_ai.tools import StackOneTool, Tools +from stackone_ai.tools import StackOneMcpTool, StackOneTool, Tools from stackone_ai.toolset import StackOneToolSet from stackone_ai.types import ( ExecuteConfig, ExecuteToolsConfig, + FeedbackCategory, + FeedbackRating, + FeedbackSource, StackOneAPIError, StackOneError, + ToolArgumentsError, ToolMode, ToolParameters, ToolsetConfigError, @@ -17,13 +21,18 @@ __all__ = [ "StackOneToolSet", "StackOneTool", + "StackOneMcpTool", "Tools", "ToolMode", "ToolParameters", "ExecuteConfig", "ExecuteToolsConfig", + "FeedbackRating", + "FeedbackCategory", + "FeedbackSource", "StackOneError", "StackOneAPIError", + "ToolArgumentsError", "ToolsetError", "ToolsetConfigError", "ToolsetLoadError", diff --git a/stackone_ai/tools.py b/stackone_ai/tools.py index 51eefc64..98421ddf 100644 --- a/stackone_ai/tools.py +++ b/stackone_ai/tools.py @@ -1,24 +1,31 @@ -"""Tools served by the StackOne MCP endpoint, and their execution over the RPC endpoint. +"""Tools served by the StackOne MCP endpoint, and their execution over MCP ``tools/call``. The guiding property of this module is that a tool is the served catalog entry: -the schema listed to a model is the schema the MCP server sent, and the request -sent to ``/actions/rpc`` matches that schema. Nothing is invented, nothing is -dropped, nothing is rebuilt. +the schema listed to a model is the schema the MCP server sent, and a call sends the +model's arguments back to the endpoint that listed it, as given. Nothing is invented, +nothing is dropped, nothing is rebuilt. """ from __future__ import annotations import asyncio import base64 +import calendar import json import logging +import math +import random import re import threading +import time from collections import Counter -from collections.abc import Coroutine, Iterable, Sequence +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Sequence +from contextlib import asynccontextmanager from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from decimal import Decimal from importlib import metadata -from typing import Any, TypeVar, cast +from typing import Any, NoReturn, TypeVar import anyio import httpx @@ -28,15 +35,13 @@ ExecuteConfig, Headers, JsonDict, - ParameterLocation, StackOneAPIError, StackOneError, + ToolArgumentsError, ToolParameters, ToolsetConfigError, ToolsetError, ToolsetLoadError, - filename_from_content_disposition, - is_json_content_type, ) logger = logging.getLogger("stackone.tools") @@ -50,30 +55,111 @@ USER_AGENT = f"stackone-ai-python/{_SDK_VERSION}" -_RPC_PARAMETER_LOCATIONS = { - "action": ParameterLocation.BODY, - "body": ParameterLocation.BODY, - "headers": ParameterLocation.BODY, - "path": ParameterLocation.BODY, - "query": ParameterLocation.BODY, +_HEADER_NAME_PATTERN = re.compile(r"[A-Za-z0-9!#$%&'*+.^_`|~-]+") +_HEADER_VALUE_PATTERN = re.compile(r"[\x20-\x7e\t\x80-\xff]*") + +# What JavaScript's String.prototype.trim() strips: its WhiteSpace and LineTerminator +# characters. str.strip() also strips \x1c-\x1f and \x85, which made "x-custom\x1f" a +# valid name here when Node refuses it as malformed, and misses the byte order mark. +_JS_WHITESPACE = ( + "\t\n\v\f\r \xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a" + "\u2028\u2029\u202f\u205f\u3000\ufeff" +) + + +def _trim_header_name(name: str) -> str: + """A header name trimmed as Node trims it.""" + return name.strip(_JS_WHITESPACE) + + +def is_sdk_owned_header(name: str) -> bool: + """Whether a caller-supplied header name is one the SDK owns and will override.""" + return _ascii_lower(_trim_header_name(name)) in _SDK_OWNED_HEADERS + + +def build_request_headers( + *, + api_key: str, + account_id: str | None = None, + end_user_id: str | None = None, + extra_headers: Headers | None = None, +) -> Headers: + """The HTTP headers for a request to StackOne: the caller's extra headers first, then + the SDK's own, so Authorization, x-account-id and User-Agent are always the SDK's — + and so is x-end-user-id when an ``end_user_id`` is given. + + Case variants of the owned names are dropped first — a server would otherwise see + both. With no ``account_id``, no x-account-id is sent at all; with no + ``end_user_id``, a caller's x-end-user-id is passed through as given. + """ + replaced = _SDK_SET_HEADER_ARGUMENTS if end_user_id else _SDK_OWNED_HEADERS + headers: Headers = { + name: value + for name, value in (extra_headers or {}).items() + if _ascii_lower(_trim_header_name(name)) not in replaced + } + headers["User-Agent"] = USER_AGENT + headers["Authorization"] = build_auth_header(api_key) + if account_id: + headers["x-account-id"] = account_id + if end_user_id: + headers[END_USER_ID_HEADER] = end_user_id + return headers + + +# Header names the SDK sets itself, after every other header. A tool call may not supply +# them even when a served schema declares them: they are the credential, the tenant +# selector and the client identity. +_SDK_OWNED_HEADERS = frozenset({"authorization", "x-account-id", "user-agent"}) + +# The end user a non-shared account belongs to, which the API requires on every MCP request +# for that account. The SDK sets it when GET /accounts recorded one for the account; a +# caller's configured headers may carry it otherwise. A model may never supply it. +END_USER_ID_HEADER = "x-end-user-id" + +# Header names a tool call's header arguments may not carry, even when declared. +_SDK_SET_HEADER_ARGUMENTS = _SDK_OWNED_HEADERS | {END_USER_ID_HEADER} + +# A top-level argument with this prefix is a header argument, as is an entry of a nested +# `headers` object argument. +_FLAT_HEADER_PREFIX = "headers_" + +# Why a header argument was dropped: each reason code, and the text its warning gives. +_HEADER_REASONS = { + "set-by-sdk": "set by the SDK", + "not-declared": "not declared by the schema", + "malformed": "malformed", + "not-an-object": "not an object", + "not-a-scalar": "not a string, number or boolean", } -# Param-style pinned on the /mcp tool-listing URL. The MCP schema and the RPC-execution unwrap -# (_split_envelope_params) must agree on this, so it is pinned rather than following the server -# default — the server default is free to change without breaking the SDK. -MCP_PARAM_STYLE = "flat_prefixed" -# Matches a flat_prefixed envelope key: `_` (e.g. `path_id`, `query_limit`). -_HEADER_NAME_PATTERN = re.compile(r"[A-Za-z0-9!#$%&'*+.^_`|~-]+") -_HEADER_VALUE_PATTERN = re.compile(r"[\x20-\x7e\t\x80-\xff]*") +def _warn_header_dropped(argument: str, header: str | None, reason: str) -> None: + """Warn that a header argument, or one entry of a ``headers`` object, was dropped.""" + if header is None: + logger.warning( + "Dropping header argument %s from a tool call: %s", _json_text(argument), _HEADER_REASONS[reason] + ) + else: + logger.warning("Dropping header %s from a tool call: %s", _json_text(header), _HEADER_REASONS[reason]) + + +# How a 429 is retried, on every request the SDK makes: up to three more attempts, each +# after the server's Retry-After (capped) or else an exponential backoff with jitter. A +# wait that would not end before the request's deadline is not started: the 429 is +# returned at once instead. +RATE_LIMIT_MAX_RETRIES = 3 +RATE_LIMIT_BACKOFF_SECONDS = (1.0, 2.0, 4.0) +RATE_LIMIT_JITTER = (0.5, 1.0) +RATE_LIMIT_MAX_DELAY_SECONDS = 30.0 -_FLAT_ENVELOPE_KEY_PATTERN = re.compile(r"^(path|query|body|headers)_(.+)$") +# Looked up at call time, so tests can replace them and not actually wait. +_sleep = time.sleep +_async_sleep = anyio.sleep +_clock = time.monotonic +_async_clock = anyio.current_time -# Added per property by the toolset when normalising a served schema; it records -# whether the field was absent from the schema's `required` list. It is an internal -# marker, not part of the served schema, so it is stripped before a schema is -# handed to a model. @dataclass class McpToolDefinition: """A tool exactly as the MCP server listed it.""" @@ -89,6 +175,12 @@ def run_async(awaitable: Coroutine[Any, Any, T]) -> T: try: asyncio.get_running_loop() except RuntimeError: + in_a_loop = False + else: + in_a_loop = True + # Run outside the except block, or every failure would carry the probe's + # "no running event loop" as its __context__. + if not in_a_loop: return asyncio.run(awaitable) result: dict[str, T] = {} @@ -110,20 +202,458 @@ def runner() -> None: return result["value"] +def _js_number(value: float) -> str: + """A float as JavaScript's ``String(number)`` writes it: ``1.0`` is "1", ``1e-7`` is "1e-7". + + Both languages print the shortest digits that round-trip, so only the layout differs: + the cut-offs for exponent notation, and the exponent's sign and padding. + """ + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Infinity" if value > 0 else "-Infinity" + if value == 0: + return "0" + sign = "-" if value < 0 else "" + parsed = Decimal(repr(abs(value))).normalize().as_tuple() + digits = "".join(map(str, parsed.digits)) + k = len(digits) + n = int(parsed.exponent) + k + if k <= n <= 21: + return sign + digits + "0" * (n - k) + if 0 < n <= 21: + return sign + digits[:n] + "." + digits[n:] + if -6 < n <= 0: + return sign + "0." + "0" * -n + digits + mantissa = digits if k == 1 else f"{digits[0]}.{digits[1:]}" + return f"{sign}{mantissa}e{'+' if n > 0 else '-'}{abs(n - 1)}" + + +def _js_json(value: Any) -> str: + """``value`` as JavaScript's ``JSON.stringify`` writes it: compact, numbers as JS prints them. + + Raises: + TypeError: If ``value`` holds something JSON cannot represent, such as a set. + """ + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return _js_number(value) if math.isfinite(value) else "null" + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, dict): + entries = ( + f"{json.dumps(key if isinstance(key, str) else _header_text(key) or 'null', ensure_ascii=False)}:" + f"{_js_json(item)}" + for key, item in value.items() + ) + return "{" + ",".join(entries) + "}" + if isinstance(value, list | tuple): + return "[" + ",".join(_js_json(item) for item in value) + "]" + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +def _json_text(value: Any) -> str: + """A value as compact JSON, the way every message quotes a name or a value. + + ``repr`` wrote Python's spelling (``'x'``, ``None``, ``True``); JSON reads the same from + either SDK. A value JSON cannot hold, such as a set, falls back to its ``repr``. + """ + try: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + except (TypeError, ValueError): + return repr(value) + + +def _json_type(value: Any) -> str: + """The JSON type of a value: object, array, string, number, boolean or null. + + A value with no JSON type, such as a set, is named by its Python type. + """ + if value is None: + return "null" + if isinstance(value, bool): + return "boolean" + if isinstance(value, int | float): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, list | tuple): + return "array" + if isinstance(value, dict): + return "object" + return type(value).__name__ + + +def _seconds_2dp(seconds: float) -> str: + """Seconds rounded half up to two decimals, then written as JavaScript writes a number.""" + return _js_number(math.floor(seconds * 100 + 0.5) / 100) + + +_ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + + +def _ascii_lower(name: str) -> str: + """A header name lower-cased in ASCII only, as HTTP compares names. + + ``casefold()`` folded "ſ" (U+017F) to "s", so "uſer-agent" read as User-Agent. It is + not a token at all, and is refused as malformed instead. + """ + return name.translate(_ASCII_LOWER) + + +def _declares_object_type(schema_type: Any) -> bool: + """Whether a JSON Schema ``type`` declares (among others) ``"object"``. + + ``type`` is either a single string or, per the JSON Schema spec, a list of strings + naming every allowed type, e.g. ``["object", "null"]``. + """ + if isinstance(schema_type, str): + return schema_type == "object" + if isinstance(schema_type, list): + return "object" in schema_type + return False + + +def _header_text(value: Any) -> str | None: + """A header argument's value as text, as Node's ``headerText`` writes it; ``None`` if null. + + Strings as they are, booleans as ``true``/``false``, numbers as JavaScript prints them + and anything else as compact JSON. ``str()`` sent Python's spelling instead: ``True``, + ``1.0``, ``['a', 'b']``. + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, bool | int): + return _js_json(value) + if isinstance(value, float): + return _js_number(value) + return _js_json(value) + + def build_auth_header(api_key: str) -> str: token = base64.b64encode(f"{api_key}:".encode()).decode() return f"Basic {token}" -def _mcp_transport(client: Any, endpoint: str, headers: dict[str, str], timeout: float) -> Any: +async def _buffer_error_body(response: httpx.Response) -> None: + """Read an error response's body while its stream is still open. + + The ``mcp`` client raises on the status after the streamed response is closed, so + reading the body afterwards got nothing and every failure said only "400 Bad + Request". The body is where the server explains itself: "Legacy accounts cannot be + used with MCP", or a 412 asking for the account to be re-linked. Node already + carries it. + """ + if response.is_error: + await response.aread() + + +# The three HTTP-date forms of RFC 9110 §5.6.7, exactly: case-sensitive, single spaces, and +# GMT only. re.ASCII, or \d would also match "٣". +_DAY = r"(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)" +_MONTH = r"(?PJan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)" +_TIME = r"(?P\d\d):(?P\d\d):(?P\d\d)" +_IMF_FIXDATE = re.compile(rf"{_DAY}, (?P\d\d) {_MONTH} (?P\d{{4}}) {_TIME} GMT", re.ASCII) +_RFC850_DATE = re.compile( + rf"(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), " + rf"(?P\d\d)-{_MONTH}-(?P\d\d) {_TIME} GMT", + re.ASCII, +) +_ASCTIME_DATE = re.compile(rf"{_DAY} {_MONTH} (?P\d\d| \d) {_TIME} (?P\d{{4}})", re.ASCII) +_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") + + +def _add_years(when: datetime, years: int) -> datetime: + """``when`` moved by whole years; 29 February lands on the 28th in a common year.""" + try: + return when.replace(year=when.year + years) + except ValueError: + return when.replace(year=when.year + years, day=28) + + +def _parse_http_date(value: str, now: datetime) -> float | None: + """An RFC 9110 HTTP-date as seconds since the epoch, or ``None`` if it is not one. + + Written out rather than left to ``email.utils.parsedate_to_datetime``, which reads + far more than an HTTP-date: trailing text, numeric zones, two-digit years in any form. + The weekday is not checked against the date, as RFC 9110 does not ask it to be. + + Seconds since the epoch, not a ``datetime``, because a leap second (second 60) on + the last representable instant, e.g. ``Fri, 31 Dec 9999 23:59:60 GMT``, rolls over + into year 10000, which ``datetime`` cannot hold. ``calendar.timegm`` takes the date + fields as plain integers and does the arithmetic without that ceiling. + """ + match = _IMF_FIXDATE.fullmatch(value) or _ASCTIME_DATE.fullmatch(value) + two_digit_year = False + if match is None: + match = _RFC850_DATE.fullmatch(value) + two_digit_year = True + if match is None: + return None + year = int(match["year"]) + month = _MONTHS.index(match["month"]) + 1 + day = int(match["day"]) + hour = int(match["hour"]) + minute = int(match["minute"]) + second = int(match["second"]) + if second > 60: + return None + if two_digit_year: + # §5.6.7: a two-digit year more than 50 years ahead is the most recent past year + # with those digits. + year += now.year - now.year % 100 + try: + # Second 60 is a leap second: validated as if it were 59, the instant before the + # next minute starts. Also catches day 32, hour 25, 30 February, year 0. + datetime(year, month, day, hour, minute, min(second, 59), tzinfo=UTC) + except ValueError: + return None + if two_digit_year: + # Bounded to a century around `now`, so no overflow risk going through datetime. + when = datetime(year, month, day, hour, minute, min(second, 59), tzinfo=UTC) + if second == 60: + when += timedelta(seconds=1) + if when > _add_years(now, 50): + when = _add_years(when, -100) + elif _add_years(when, 100) <= _add_years(now, 50): + when = _add_years(when, 100) + return calendar.timegm(when.utctimetuple()) + return calendar.timegm((year, month, day, hour, minute, second)) + + +def _retry_after_seconds(value: str | None, now: datetime | None = None) -> float | None: + """A Retry-After header as seconds from ``now``: delta-seconds or an HTTP-date. + + ``None`` when the header is absent or unreadable, so the caller falls back to its + own backoff. A date in the past means retry now. ``now`` defaults to the current time. + """ + if value is None: + return None + value = value.strip(_JS_WHITESPACE) + # ASCII digits only: str.isdigit() also accepts "²", which float() then refuses. + if re.fullmatch(r"[0-9]+", value): + return float(value) + now = now or datetime.now(UTC) + when_epoch = _parse_http_date(value, now) + if when_epoch is None: + return None + now_epoch = calendar.timegm(now.utctimetuple()) + now.microsecond / 1e6 + return max(0.0, when_epoch - now_epoch) + + +def _backoff_delay(retry: int, retry_after: float | None, random_value: float | None = None) -> float: + """How long to wait before retry number ``retry`` (1-based) of a 429. + + ``retry_after`` is the 429's Retry-After in seconds, or ``None`` if it had none readable; + then the backoff for ``retry`` is jittered by ``random_value``, uniform in [0, 1). + """ + if retry_after is None: + r = random.random() if random_value is None else random_value + low, high = RATE_LIMIT_JITTER + retry_after = RATE_LIMIT_BACKOFF_SECONDS[retry - 1] * (low + (high - low) * r) + return min(retry_after, RATE_LIMIT_MAX_DELAY_SECONDS) + + +def _rate_limit_delay(response: httpx.Response, retry: int) -> float: + """How long to wait before retry number ``retry`` (1-based) of this 429.""" + return _backoff_delay(retry, _retry_after_seconds(response.headers.get("retry-after"))) + + +def _waits_for_retry(delay: float, remaining: float) -> bool: + """Whether a wait of ``delay`` ends before a deadline ``remaining`` seconds away.""" + return delay < remaining + + +def _log_rate_limit_retry(request: httpx.Request, attempt: int, delay: float) -> None: + logger.warning( + "%s %s was rate limited (429) on attempt %d of %d; retrying in %ss", + request.method, + request.url, + attempt, + RATE_LIMIT_MAX_RETRIES + 1, + _seconds_2dp(delay), + ) + + +def _outlasts_deadline(request: httpx.Request, attempt: int, delay: float, remaining: float) -> bool: + """Whether waiting ``delay`` would not end before the deadline, logging when so. + + Such a wait is not started. Sleeping into the deadline would turn the 429 into a + timeout, which a multi-account call skips like any per-account failure, and hand back + a partial catalog that looks complete. The 429 itself ends the call. + """ + if _waits_for_retry(delay, remaining): + return False + logger.warning( + "%s %s was rate limited (429) on attempt %d of %d; not retrying, because waiting %ss " + "would pass the deadline", + request.method, + request.url, + attempt, + RATE_LIMIT_MAX_RETRIES + 1, + _seconds_2dp(delay), + ) + return True + + +class RateLimitRetryingClient(httpx.Client): + """An ``httpx.Client`` that retries a 429 before the caller ever sees it. + + Retried in ``send`` rather than in a wrapping transport: passing a custom transport + makes httpx ignore the proxy environment variables, which would quietly break the SDK + behind a corporate proxy. The last 429 is returned as it came, so the caller reports + the server's body. + + ``retry_within`` is the deadline in seconds, measured from the first attempt: a retry + whose wait would not end before it is not made, and the 429 is returned instead. A + retry that times out raises :class:`RateLimitTimeout`: the account is rate limited, and + an ordinary timeout would let a caller skip it like any other failure. + """ + + def __init__(self, *args: Any, retry_within: float | None = None, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._retry_within = retry_within + + def send(self, request: httpx.Request, **kwargs: Any) -> httpx.Response: + deadline = _clock() + self._retry_within if self._retry_within is not None else None + attempt = 1 + response = super().send(request, **kwargs) + while response.status_code == 429 and attempt <= RATE_LIMIT_MAX_RETRIES: + delay = _rate_limit_delay(response, attempt) + if deadline is not None and _outlasts_deadline(request, attempt, delay, deadline - _clock()): + break + # Read before closing, so the connection goes back to the pool. + response.read() + response.close() + _log_rate_limit_retry(request, attempt, delay) + _sleep(delay) + attempt += 1 + try: + response = super().send(request, **kwargs) + except httpx.TimeoutException as exc: + raise RateLimitTimeout(str(exc), request=request) from exc + return response + + +class RateLimitTimeout(httpx.TimeoutException): + """A request retried after a 429 timed out: the 429's doing, not an ordinary timeout.""" + + +@dataclass +class _Throttle: + """The last 429 an MCP exchange retried, while any request is still retrying one. + + Counted per request because an exchange's requests overlap: the MCP client's event-stream + GET, say, answered while a ``tools/list`` retry hangs, must not clear that retry's 429. + """ + + response: httpx.Response | None = None + retrying: int = 0 + + +class RateLimitRetryingAsyncClient(httpx.AsyncClient): + """The async counterpart of :class:`RateLimitRetryingClient`, used for every MCP request. + + The MCP client streams its responses; a discarded 429 is read and closed here, and + the final one still passes through the response hooks, so its body is buffered. + + The deadline is the enclosing cancel scope's: the ``anyio.fail_after(timeout)`` that + bounds the whole MCP exchange. A retry whose wait would not end before it is not made. + + ``throttle`` records each 429 that is retried. The exchange's timeout cancels the retry + rather than raising in it, so the caller reads this afterwards to report a timeout + after a retried 429 as that 429 (see :func:`_raise_mcp_failure`). + """ + + def __init__(self, *args: Any, throttle: _Throttle | None = None, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._throttle = throttle + + async def send(self, request: httpx.Request, **kwargs: Any) -> httpx.Response: + attempt = 1 + retried = False + response = await super().send(request, **kwargs) + while response.status_code == 429 and attempt <= RATE_LIMIT_MAX_RETRIES: + delay = _rate_limit_delay(response, attempt) + remaining = anyio.current_effective_deadline() - _async_clock() + if _outlasts_deadline(request, attempt, delay, remaining): + break + await response.aread() + await response.aclose() + if self._throttle is not None: + self._throttle.response = response + if not retried: + retried = True + self._throttle.retrying += 1 + _log_rate_limit_retry(request, attempt, delay) + await _async_sleep(delay) + attempt += 1 + response = await super().send(request, **kwargs) + if self._throttle is not None and retried and response.status_code != 429: + self._throttle.retrying -= 1 + if self._throttle.retrying == 0: + self._throttle.response = None + return response + + +def is_rate_limited(exc: BaseException) -> bool: + """Whether a failure is a 429 that outlasted every retry. + + Such a failure ends the whole call: skipping the account and carrying on would hand + back a partial catalog that looks complete. Only an HTTP 429 counts, as in Node: a + tool result whose payload says 429 was never retried, so it is that connector's + failure, skipped like any other. + """ + if not (isinstance(exc, StackOneAPIError) and exc.status_code == 429): + return False + seen: set[int] = set() + stack: list[BaseException] = [exc] + while stack: + current = stack.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, httpx.HTTPStatusError) and current.response.status_code == 429: + return True + stack.extend(getattr(current, "exceptions", None) or []) + stack.extend(e for e in (current.__cause__, current.__context__) if e is not None) + return False + + +@asynccontextmanager +async def _mcp_transport( + endpoint: str, headers: dict[str, str], timeout: float, throttle: _Throttle | None = None +) -> AsyncIterator[tuple[Any, Any, Any]]: """Open the streamable-HTTP transport with the caller's timeout on every leg. The client's own defaults are a 30s connect and a 300s SSE read, and neither was overridden — so ``StackOneToolSet(timeout=2)`` against a host that accepts and never answers hung for five minutes. The execution path honoured ``timeout``; the MCP path, which search() and execute() and every listing use, did not. + + The HTTP client is built here with the settings the ``mcp`` package's own factory + uses (redirects followed), because ``streamable_http_client`` takes a client rather + than headers and timeouts. It retries a 429, so the MCP client never sees one that + a retry got past. """ - return client(endpoint, headers=headers, timeout=timeout, sse_read_timeout=timeout) + from mcp.client.streamable_http import streamable_http_client # ty: ignore[unresolved-import] + + async with RateLimitRetryingAsyncClient( + throttle=throttle, + headers=headers, + timeout=httpx.Timeout(timeout), + follow_redirects=True, + event_hooks={"response": [_buffer_error_body]}, + ) as client: + async with streamable_http_client(endpoint, http_client=client) as streams: + yield streams def fetch_mcp_tools( @@ -136,18 +666,19 @@ def fetch_mcp_tools( try: from mcp import types as mcp_types # ty: ignore[unresolved-import] from mcp.client.session import ClientSession # ty: ignore[unresolved-import] - from mcp.client.streamable_http import streamablehttp_client # ty: ignore[unresolved-import] except ImportError as exc: # pragma: no cover - depends on optional extra raise ToolsetConfigError( "mcp is a core dependency of stackone-ai but could not be imported — reinstall the package." ) from exc + throttle = _Throttle() + async def _list() -> list[McpToolDefinition]: with anyio.fail_after(timeout): return await _list_within_deadline() async def _list_within_deadline() -> list[McpToolDefinition]: - async with _mcp_transport(streamablehttp_client, endpoint, headers, timeout) as ( + async with _mcp_transport(endpoint, headers, timeout, throttle) as ( read_stream, write_stream, _, @@ -179,8 +710,10 @@ async def _list_within_deadline() -> list[McpToolDefinition]: try: return run_async(_list()) - except BaseException as exc: - raise _describe_mcp_failure(exc, endpoint) from exc + except Exception as exc: + # Exception, not BaseException: Ctrl-C and SystemExit must stop the caller, not come + # back as a ToolsetLoadError that an agent framework hands the model as a tool result. + _raise_mcp_failure(exc, endpoint, timeout, throttle) def _response_body(response: httpx.Response) -> str: @@ -202,18 +735,17 @@ def _response_body(response: httpx.Response) -> str: return "" -def _describe_mcp_failure(exc: BaseException, endpoint: str) -> Exception: +def _describe_mcp_failure(exc: BaseException, endpoint: str, timeout: float) -> Exception: """Turn the MCP client's nested failure into something a caller can act on. The listing runs inside a TaskGroup, so any HTTP error arrives wrapped in an ExceptionGroup whose str() is "unhandled errors in a TaskGroup (1 sub-exception)". Unwrap to the underlying error and carry the status code and response body, which is where the server explains itself — a dead account, for example, - answers 412 with "re-link the account to resume". + answers 412 with "re-link the account to resume". A timeout says so, as in Node. """ seen: set[int] = set() stack: list[BaseException] = [exc] - leaf: BaseException = exc while stack: current = stack.pop() if current is None or id(current) in seen: @@ -226,44 +758,64 @@ def _describe_mcp_failure(exc: BaseException, endpoint: str) -> Exception: if isinstance(current, httpx.HTTPStatusError): body = _response_body(current.response) detail = f": {body}" if body else "" + status = f"{current.response.status_code} {current.response.reason_phrase}".rstrip() # StackOneAPIError carries the status and body as attributes, so a caller # can branch on 412 rather than pattern-matching the message. return StackOneAPIError( - f"MCP request to {endpoint} failed with " - f"{current.response.status_code} {current.response.reason_phrase}{detail}", + f"MCP request to {endpoint} failed with {status}{detail}", current.response.status_code, body or None, ) - # Track the innermost non-group exception: an ExceptionGroup's own str() is - # the "unhandled errors in a TaskGroup" boilerplate that hides the real cause. - if not getattr(current, "exceptions", None): - leaf = current stack.extend(getattr(current, "exceptions", None) or []) stack.extend(e for e in (current.__cause__, current.__context__) if e is not None) - return ToolsetLoadError(f"MCP request to {endpoint} failed: {type(leaf).__name__}: {leaf}") + # The innermost cause, through group members and __cause__ only. An ExceptionGroup's + # own str() is the "unhandled errors in a TaskGroup" boilerplate that hides the real + # cause, and a __context__ is only what was being handled when the error was raised: + # for the deadline's TimeoutError that is the cancellation, or whatever the + # cancelled task happened to be catching. + leaf = _leaf_failure(exc) + if isinstance(leaf, TimeoutError | httpx.TimeoutException): + return ToolsetLoadError(f"MCP request to {endpoint} timed out after {_js_number(timeout)}s") + return ToolsetLoadError(f"MCP request to {endpoint} failed: {type(leaf).__name__}: {leaf}") -def _describe_api_failure(exc: httpx.HTTPStatusError, body: Any) -> str: - """Lead with the server's own explanation of what went wrong. - ``str(httpx.HTTPStatusError)`` is "Client error '400 Bad Request' for url ..." plus a - link to MDN's generic status page — it never says which field was wrong, even though - the answer is already in hand. This is the error a user hits on every bad tool call, - so it is the one worth making actionable. - """ - detail: Any = None - if isinstance(body, dict): - for key in ("message", "error", "detail"): - if isinstance(body.get(key), str): - detail = body[key] - break +def _leaf_failure(exc: BaseException) -> BaseException: + """The innermost cause through group members and ``__cause__``, stopping at a timeout.""" + leaf = exc + while not isinstance(leaf, TimeoutError | httpx.TimeoutException): + members = getattr(leaf, "exceptions", None) + if members: + leaf = members[0] + elif leaf.__cause__ is not None: + leaf = leaf.__cause__ else: - detail = body - elif isinstance(body, str) and body.strip(): - detail = body.strip()[:500] + break + return leaf + - status = f"{exc.response.status_code} {exc.response.reason_phrase}".strip() - return f"{status}: {detail}" if detail else f"{status} from {exc.request.url}" +def _raise_mcp_failure(exc: Exception, endpoint: str, timeout: float, throttle: _Throttle) -> NoReturn: + """Raise an MCP exchange's failure as :func:`_describe_mcp_failure` describes it. + + Except a timeout after a retried 429, which is raised as that 429. The retry's wait + ended before the deadline, but the retried request itself could still run past it, and + as a timeout a multi-account call skipped the account like any other failure and handed + back a partial catalog. Chained through an ``HTTPStatusError``, so + :func:`is_rate_limited` reads it as the HTTP 429 it is. + """ + throttled = throttle.response + if throttled is not None and isinstance(_leaf_failure(exc), TimeoutError | httpx.TimeoutException): + status = httpx.HTTPStatusError( + f"429 {throttled.reason_phrase}", request=throttled.request, response=throttled + ) + status.__cause__ = exc + raise StackOneAPIError( + f"MCP request to {endpoint} was rate limited (429) and timed out after " + f"{_js_number(timeout)}s while retrying", + 429, + None, + ) from status + raise _describe_mcp_failure(exc, endpoint, timeout) from exc def _status_of(parsed: JsonDict) -> int: @@ -286,9 +838,55 @@ def _status_of(parsed: JsonDict) -> int: return 0 +def _reject_json_constant(constant: str) -> Any: + raise ValueError(f"{constant} is not valid JSON") + + +def _first_non_finite(value: Any) -> float | None: + """The first NaN or infinity in ``value``, depth first in key order, or ``None``. + + A container already on the path is skipped, so a cycle ends here and is reported by + ``json.dumps`` as a circular reference. + """ + path: set[int] = set() + + def walk(item: Any) -> float | None: + if isinstance(item, float): + return None if math.isfinite(item) else item + if not isinstance(item, dict | list | tuple) or id(item) in path: + return None + path.add(id(item)) + try: + for child in item.values() if isinstance(item, dict) else item: + found = walk(child) + if found is not None: + return found + return None + finally: + path.discard(id(item)) + + return walk(value) + + +def _plain_part(part: Any) -> Any: + """A content part as the JSON it arrived as: wire field names, unset fields left out.""" + if isinstance(part, BaseModel): + return part.model_dump(mode="json", by_alias=True, exclude_none=True) + return part + + def parse_tool_result(result: Any, name: str) -> JsonDict: """Turn an MCP ``CallToolResult`` into a plain dict. + Text parts are joined and parsed as JSON; a non-object is wrapped as + ``{"result": ...}``. With no text at all, ``structuredContent`` is used, and text + wins when both are present. Parts that are not text (images, embedded resources) + are kept under ``content_parts`` as plain JSON, the form they arrived in. + + The result is returned as the server wrote it. For an action tool, ``*_execute_action`` + and feedback that is ``{"isError": false, "result": ..., "defenderMetadata"?: ..., + "policyMetadata"?: ...}``; a search result is bare JSON. + Raises: StackOneAPIError: If the result carries ``isError``. A failed tool call comes back as an ordinary response with that flag set, so without this check the @@ -297,21 +895,32 @@ def parse_tool_result(result: Any, name: str) -> JsonDict: texts = [getattr(part, "text", "") for part in result.content] payload = "".join(t for t in texts if t) # Parts that are not text (images, embedded resources) have no `.text`; keep them - # rather than silently returning an empty dict. - non_text = [part for part in result.content if not getattr(part, "text", "")] + # rather than silently returning an empty dict. As plain JSON, as Node returns them: + # the mcp package's pydantic objects broke json.dumps on the result, and + # execute_openai_tool_calls() sent the model their repr. + non_text = [_plain_part(part) for part in result.content if not getattr(part, "text", "")] parsed: JsonDict = {} + structured = getattr(result, "structuredContent", None) if payload: try: - loaded = json.loads(payload) - except json.JSONDecodeError: + # NaN and Infinity are not JSON; json.loads accepts them, JSON.parse does not, + # and the Node SDK hands such a payload back as text. + loaded = json.loads(payload, parse_constant=_reject_json_constant) + except ValueError: loaded = payload parsed = loaded if isinstance(loaded, dict) else {"result": loaded} + elif isinstance(structured, dict): + # A result may carry only structuredContent. Without this it came back as `{}`, + # a success with the whole payload missing. A copy, so content_parts below does + # not write into the caller's result. + parsed = dict(structured) if getattr(result, "isError", False): # The transport succeeded, so there is no HTTP status here — but the payload # carries the real one, and a caller cannot branch on 0. - raise StackOneAPIError(f"Tool {name!r} failed: {payload or parsed}", _status_of(parsed), parsed) + detail = payload or json.dumps(parsed, ensure_ascii=False, separators=(",", ":"), default=str) + raise StackOneAPIError(f"Tool {_json_text(name)} failed: {detail}", _status_of(parsed), parsed) if non_text: parsed["content_parts"] = non_text @@ -321,21 +930,21 @@ def parse_tool_result(result: Any, name: str) -> JsonDict: def call_mcp_tool( endpoint: str, headers: dict[str, str], name: str, arguments: JsonDict, *, timeout: float = 60.0 ) -> JsonDict: - """Invoke a tool over MCP ``tools/call``. + """Invoke a tool over MCP ``tools/call``, the one way every tool is executed. - The search/execute meta tools exist only on the MCP endpoint — they have no - ``/actions/rpc`` action behind them — so they must be called this way. + ``arguments`` are sent exactly as given; the server maps them onto the action. """ from mcp import types as mcp_types # ty: ignore[unresolved-import] from mcp.client.session import ClientSession # ty: ignore[unresolved-import] - from mcp.client.streamable_http import streamablehttp_client # ty: ignore[unresolved-import] + + throttle = _Throttle() async def _call() -> JsonDict: with anyio.fail_after(timeout): return await _call_within_deadline() async def _call_within_deadline() -> JsonDict: - async with _mcp_transport(streamablehttp_client, endpoint, headers, timeout) as ( + async with _mcp_transport(endpoint, headers, timeout, throttle) as ( read_stream, write_stream, _, @@ -347,12 +956,47 @@ async def _call_within_deadline() -> JsonDict: ) async with session: await session.initialize() - return parse_tool_result(await session.call_tool(name, arguments), name) + # send_request rather than call_tool: call_tool validates structuredContent + # against the tool's output schema, and on a fresh session that means a + # tools/list first — relisting the whole catalog on every call, which the + # Node SDK never does. The result is parsed below either way. + request = mcp_types.ClientRequest( + mcp_types.CallToolRequest( + params=mcp_types.CallToolRequestParams(name=name, arguments=arguments), + ) + ) + result = await session.send_request(request, mcp_types.CallToolResult) + return parse_tool_result(result, name) try: return run_async(_call()) - except BaseException as exc: - raise _describe_mcp_failure(exc, endpoint) from exc + except Exception as exc: + # Not BaseException, as in fetch_mcp_tools: KeyboardInterrupt and SystemExit propagate. + _raise_mcp_failure(exc, endpoint, timeout, throttle) + + +def is_end_user_refusal(exc: BaseException, account_id: str) -> bool: + """Whether ``exc`` is the API's 400 for an account's request without its end user. + + The API answers an MCP request for a non-shared account whose ``x-end-user-id`` does not + name the account's end user with a 400 whose message names the account, as in Node. + """ + if not (isinstance(exc, StackOneAPIError) and exc.status_code == 400): + return False + body = exc.response_body + if isinstance(body, str): + try: + body = json.loads(body) + except ValueError: + pass + message = body.get("message") if isinstance(body, dict) else body + # The guard's fixed wording, ending with this account's id, quoted or not: not an exact + # string, so a change to how the id is written does not silently turn the lookup off. + if not isinstance(message, str) or not message.startswith( + "x-end-user-id header does not match account end user id" + ): + return False + return message.rstrip().rsplit(" ", 1)[-1].strip("\"'") == account_id def _strip_internal_keys(schema: Any) -> Any: @@ -374,13 +1018,19 @@ def _strip_internal_keys(schema: Any) -> Any: class StackOneTool(BaseModel): - """A single tool: its served schema plus the request needed to execute it.""" + """A single tool: its served schema, and how to call it. + + The base class describes a tool and converts it for agent frameworks, but cannot run + it: :meth:`execute` raises. Tools from :meth:`StackOneToolSet.fetch_tools` are + :class:`StackOneMcpTool` instances, which execute over MCP ``tools/call``. A hand-built + tool must override :meth:`execute`. + """ name: str = Field(description="Tool name") description: str = Field(description="Tool description") parameters: ToolParameters = Field(description="Tool parameters") _execute_config: ExecuteConfig = PrivateAttr() - _api_key: str = PrivateAttr() + _api_key: str | None = PrivateAttr(default=None) _account_id: str | None = PrivateAttr(default=None) def __init__( @@ -388,7 +1038,9 @@ def __init__( description: str, parameters: ToolParameters, _execute_config: ExecuteConfig, - _api_key: str, + # Optional: the base class cannot execute, so it has nothing to authenticate. Kept + # as a parameter so existing hand-built tools and overrides that read it still work. + _api_key: str | None = None, _account_id: str | None = None, ) -> None: super().__init__( @@ -400,197 +1052,164 @@ def __init__( self._api_key = _api_key self._account_id = _account_id - def _prepare_headers(self) -> Headers: - """Prepare headers for the API request""" - headers: Headers = { - "Authorization": build_auth_header(self._api_key), - "User-Agent": USER_AGENT, - } - - if self._account_id: - headers["x-account-id"] = self._account_id + def _parse_arguments(self, arguments: str | JsonDict | None) -> JsonDict: + """Arguments as a dict, from a dict, a JSON string, or nothing. - headers.update(self._execute_config.headers) - return headers - - def _prepare_request_params(self, kwargs: JsonDict) -> tuple[str, JsonDict, JsonDict]: - """Prepare URL and parameters for the API request + Raises: + ToolArgumentsError: If the string is not JSON, or the value is not a JSON object. + """ + if arguments is None: + return {} + if isinstance(arguments, str): + try: + # NaN and Infinity are not JSON: json.loads accepts them, JSON.parse does not. + parsed = json.loads(arguments, parse_constant=_reject_json_constant) + except ValueError as exc: + raise ToolArgumentsError( + f"Invalid JSON in arguments for {_json_text(self.name)}: {exc}" + ) from exc + else: + parsed = arguments + if not isinstance(parsed, dict): + raise ToolArgumentsError(f"Tool arguments for {_json_text(self.name)} must be a JSON object") + return dict(parsed) - Returns: - Tuple of (url, body_params, query_params) + def _declared_headers(self) -> tuple[set[str] | None, set[str], bool]: + """The header names this tool's served schema declares. + + Returns ``(nested, flat, ordinary_headers_field)``: the names under a nested + ``headers`` object, lower-cased in ASCII, each flat ``headers_`` property exactly as + served — a flat header is a top-level argument, so it is declared only under its + own key, as in Node — and whether the schema declares a top-level ``headers`` + property as something other than an object, an ordinary field that happens to be + named ``headers`` rather than a header container. + ``nested`` is ``None`` when the ``headers`` object is open — ``type: "object"`` with + no ``properties`` and ``additionalProperties`` not ``false``, as ``*_execute_action`` + serves it — which declares every name. Any other schema with no ``properties`` + declares none. """ - from urllib.parse import quote - - url = self._execute_config.url - body_params: JsonDict = {} - query_params: JsonDict = {} - - for key, value in kwargs.items(): - param_location = self._execute_config.parameter_locations.get(key) - - if param_location == ParameterLocation.PATH: - # Safely encode path parameters to prevent SSRF attacks - encoded_value = quote(str(value), safe="") - url = url.replace(f"{{{key}}}", encoded_value) - elif param_location == ParameterLocation.QUERY: - query_params[key] = value - elif param_location in (ParameterLocation.BODY, ParameterLocation.FILE): - body_params[key] = value - else: - if f"{{{key}}}" in url: - encoded_value = quote(str(value), safe="") - url = url.replace(f"{{{key}}}", encoded_value) - elif self._execute_config.method in {"GET", "DELETE"}: - query_params[key] = value - else: - body_params[key] = value + properties = self.parameters.properties or {} + flat = {prop for prop in properties if prop.startswith(_FLAT_HEADER_PREFIX)} + nested: set[str] | None = set() + schema = properties.get("headers") + schema_type = schema.get("type") if isinstance(schema, dict) else None + declares_object = _declares_object_type(schema_type) + # A declared type of `["string", "null"]` is a plain field that happens to be + # named `headers`; `["object", "null"]` is still a header container, same as a + # bare "object". Only a declared, non-object type makes it ordinary. + ordinary_headers_field = ( + isinstance(schema, dict) and isinstance(schema_type, str | list) and not declares_object + ) + if isinstance(schema, dict): + if "properties" not in schema: + if declares_object and schema.get("additionalProperties") is not False: + nested = None + elif isinstance(schema["properties"], dict): + nested = {_ascii_lower(str(name)) for name in schema["properties"]} + return nested, flat, ordinary_headers_field - return url, body_params, query_params + @staticmethod + def _header_refusal(name: str, text: str, declared: bool) -> str | None: + """The reason code a header argument may not be forwarded for, or ``None`` if it may.""" + # Normalise before comparing: " x-foo" and "X-FOO\t" are the same header to any + # server. ASCII only: a name that differs from an owned one outside ASCII is not a + # token, and is refused as malformed below. + if _ascii_lower(_trim_header_name(name)) in _SDK_SET_HEADER_ARGUMENTS: + return "set-by-sdk" + if not declared: + return "not-declared" + # Defence in depth on a declared header's model-supplied value. fullmatch, not + # match: `$` also matches just before a trailing newline, so `match` let "value\n" + # — the one character class this rejects — straight through. + if not (_HEADER_NAME_PATTERN.fullmatch(name) and _HEADER_VALUE_PATTERN.fullmatch(text)): + return "malformed" + return None def _sanitise_headers(self, supplied: dict[str, Any] | None) -> dict[str, str]: - """Keep only headers the served schema declared; drop everything else. + """Keep only the entries of a nested ``headers`` argument the served schema declares. An allowlist, not a denylist. Tool arguments are model-controlled, so a prompt-injected call reaches this dict directly — and a denylist has to enumerate every synonym of "credential" and "tenant selector" in every provider's vocabulary (Proxy-Authorization, x-stackone-account-id, Cookie, - X-Api-Key, ...) and is wrong the moment one is missed. The previous two-name - list let all of those through. - - The allowlist is the served schema itself, so this needs no maintenance: today - zero of the served actions declare a ``headers_*`` property, and the RPC server - ignores the envelope's ``headers`` object outright. (Note: on the MCP path for - meta tools, headers are always dropped because the target action's schema is not - fetched ahead of execution to build an allowlist.) + X-Api-Key, ...) and is wrong the moment one is missed. + + The allowlist is the served schema itself, so this needs no maintenance. An open + ``headers`` object declares every name. Authorization, x-account-id, User-Agent and + x-end-user-id are refused even when declared: the SDK sets them itself. """ - allowed = { - prop[len("headers_") :].casefold() - for prop in (self.parameters.properties or {}) - if prop.startswith("headers_") - } + declared, _, _ = self._declared_headers() clean: dict[str, str] = {} for key, value in (supplied or {}).items(): - if value is None or not isinstance(key, str): + text = _header_text(value) + if text is None or not isinstance(key, str): continue - # Normalise before comparing: " x-foo" and "X-FOO\t" are the same header to - # any server, and casefold() closes the non-ASCII folding holes lower() leaves. - name = key.strip() - if name.casefold() not in allowed: - logger.warning("Dropping header %r from a tool call: no served schema declares it", name) + name = _trim_header_name(key) + reason = self._header_refusal(name, text, declared is None or _ascii_lower(name) in declared) + if reason: + _warn_header_dropped("headers", name, reason) continue - # Defence in depth on a declared header's model-supplied value. fullmatch, - # not match: `$` also matches just before a trailing newline, so `match` let - # "value\n" — the one character class this rejects — straight through. - if not _HEADER_NAME_PATTERN.fullmatch(name) or not _HEADER_VALUE_PATTERN.fullmatch(str(value)): - logger.warning("Dropping malformed header %r from a tool call", name) - continue - clean[name] = str(value) + clean[name] = text return clean - def execute(self, arguments: str | JsonDict | None = None) -> JsonDict: - """Execute the tool with the given parameters + def _sanitise_header_arguments(self, arguments: JsonDict) -> JsonDict: + """Filter the header arguments in a call; every other argument is kept unchanged. - Returns: - For JSON responses, the parsed API response as a dict. + A header argument is an entry of a nested ``headers`` object, or a top-level + ``headers_``. Each is forwarded only if the served schema declares it in the + same form, and never if it is one the SDK sets itself. A declared ``headers_`` + keeps its value as given, as every other top-level argument does; nested entries + are written as text, as Node writes them. - For file downloads (any non-JSON Content-Type, e.g. a - ``documents_download_file`` action), a dict describing the file: - ``{"content": , "content_type": str, "status_code": int, - "headers": dict, "file_name": str | None}``. Note ``content`` holds - the raw bytes and is therefore not JSON-serializable - callers that - re-serialize tool results (e.g. for an LLM) should handle this key. + A top-level ``headers`` argument that isn't a plain object is forwarded unchanged + when the schema declares ``headers`` as something other than an object (it's an + ordinary field that happens to be named ``headers``), and dropped otherwise. A flat + ``headers_`` whose value is a list or dict is dropped too — only a string, + number or boolean can be a header value. Raises: - StackOneAPIError: If the API request fails - ValueError: If the arguments are invalid + TypeError: If a header value holds something JSON cannot represent. """ - try: - if isinstance(arguments, str): - parsed_arguments = json.loads(arguments) - else: - parsed_arguments = arguments or {} - - if not isinstance(parsed_arguments, dict): - raise ValueError("Tool arguments must be a JSON object") - - headers = self._prepare_headers() - url_used, body_params, query_params = self._prepare_request_params(parsed_arguments) - - request_kwargs: dict[str, Any] = { - "method": self._execute_config.method, - "url": url_used, - "headers": headers, - } - - if body_params: - body_type = self._execute_config.body_type or "json" - if body_type == "json": - request_kwargs["json"] = body_params - elif body_type == "form": - request_kwargs["data"] = body_params + _, declared_flat, ordinary_headers_field = self._declared_headers() + clean: JsonDict = {} + for key, value in arguments.items(): + if key == "headers": + if ordinary_headers_field: + clean[key] = value + elif isinstance(value, dict): + clean[key] = self._sanitise_headers(value) + else: + _warn_header_dropped(key, None, "not-an-object") + continue + if not key.startswith(_FLAT_HEADER_PREFIX): + clean[key] = value + continue + if isinstance(value, dict | list): + _warn_header_dropped(key, None, "not-a-scalar") + continue + text = _header_text(value) + if text is None: + continue + reason = self._header_refusal(key[len(_FLAT_HEADER_PREFIX) :], text, key in declared_flat) + if reason: + _warn_header_dropped(key, None, reason) + continue + clean[key] = value + return clean - if query_params: - request_kwargs["params"] = query_params + def execute(self, arguments: str | JsonDict | None = None) -> JsonDict: + """Execute the tool. Not implemented on the base class. - response = httpx.request(**request_kwargs, timeout=self._execute_config.timeout) - response.raise_for_status() + Tools from :meth:`StackOneToolSet.fetch_tools` execute over MCP ``tools/call``. + A hand-built ``StackOneTool`` has nothing to call, so override this to run one. - content_type = response.headers.get("content-type", "") - if response.status_code in (204, 205) or ( - not response.content and is_json_content_type(content_type) - ): - # A bodyless JSON success is not a file download. Falling through would - # return `content: b""` with a made-up octet-stream type, breaking any - # caller that re-serialises the result for a model. A zero-byte body with - # a download content type IS a download, though — an empty file, with a - # filename the caller still needs — so it must not be caught here. - return {"status_code": response.status_code} - if is_json_content_type(content_type): - try: - result = response.json() - except json.JSONDecodeError as exc: - # Not the caller's arguments — the server sent a JSON content type - # with a body that is not JSON. Saying "invalid JSON in arguments" - # here sends people to debug the wrong end of the call. - raise StackOneAPIError( - f"Server sent malformed JSON for {self.name!r}: {exc}", - response.status_code, - response.text[:500], - ) from exc - return cast(JsonDict, result) if isinstance(result, dict) else {"result": result} - - # Non-JSON bodies are file downloads (e.g. documents_download_file), which the - # API serves as raw binary with the file's own MIME type and a Content-Disposition - # header. Return the bytes plus metadata rather than forcing a JSON/UTF-8 decode. - return { - "content": response.content, - "content_type": content_type or "application/octet-stream", - "status_code": response.status_code, - "headers": dict(response.headers), - "file_name": filename_from_content_disposition(response.headers.get("content-disposition")), - } - - except json.JSONDecodeError as exc: - raise ValueError(f"Invalid JSON in arguments: {exc}") from exc - except httpx.HTTPStatusError as exc: - response_body = None - if exc.response.text: - try: - response_body = exc.response.json() - except json.JSONDecodeError: - response_body = exc.response.text - raise StackOneAPIError( - _describe_api_failure(exc, response_body), exc.response.status_code, response_body - ) from exc - except httpx.RequestError as exc: - raise StackOneError(f"Request failed: {exc}") from exc - except (UnicodeEncodeError, TypeError) as exc: - # A lone surrogate — what a model emits when a token boundary splits an emoji — - # or a value JSON cannot encode (a set, bytes, a datetime) failed deep inside - # httpx and escaped as a bare UnicodeEncodeError/TypeError, outside the SDK's - # exception contract. It is an argument problem, so report it as one. - raise ValueError(f"Arguments for {self.name!r} could not be encoded as JSON: {exc}") from exc + Raises: + StackOneError: Always, on the base class. + """ + raise StackOneError( + f"Tool {_json_text(self.name)} has no executor. Override execute() to run a hand-built tool." + ) def call(self, *args: Any, **kwargs: Any) -> JsonDict: """Call the tool with the given arguments @@ -600,11 +1219,11 @@ def call(self, *args: Any, **kwargs: Any) -> JsonDict: >>> tool.call(name="John", email="john@example.com") """ if args and kwargs: - raise ValueError("Cannot provide both positional and keyword arguments") + raise ToolArgumentsError("Cannot provide both positional and keyword arguments") if args: if len(args) > 1: - raise ValueError("Only one positional argument is allowed") + raise ToolArgumentsError("Only one positional argument is allowed") return self.execute(args[0]) return self.execute(kwargs if kwargs else None) @@ -617,32 +1236,27 @@ def to_openai_function(self) -> JsonDict: """Convert this tool to OpenAI's function format. The served schema is passed through verbatim apart from the SDK's internal - ``nullable`` marker, which is translated into the JSON Schema ``required`` - list. Constraints such as ``format``, ``pattern``, ``default``, - ``minimum``/``maximum`` and ``oneOf``/``anyOf`` reach the model intact — - without them a model cannot generate valid arguments for a constrained field. + ``nullable`` marker, which is stripped. Constraints such as ``format``, + ``pattern``, ``default``, ``minimum``/``maximum`` and ``oneOf``/``anyOf`` + reach the model intact — without them a model cannot generate valid arguments + for a constrained field. ``required`` is the served list, in the served order, + and is omitted when the server sent none or an empty one. """ properties: JsonDict = {} - required: list[str] = [] - for name, prop in self.parameters.properties.items(): - if isinstance(prop, dict): - clean_prop = _strip_internal_keys(prop) - properties[name] = clean_prop - if not prop.get("nullable", False): - required.append(name) - else: - properties[name] = {"type": "string"} - required.append(name) + properties[name] = _strip_internal_keys(prop) if isinstance(prop, dict) else {"type": "string"} parameters: JsonDict = self.parameters.model_dump() parameters["properties"] = properties - # Only set required if there are required properties, else remove it if it existed - if required: - parameters["required"] = required - else: - parameters.pop("required", None) + # Taken from the served root, not rebuilt from the per-property markers: that + # rebuild re-sorted `required` into property order, so the model saw a list the + # server never sent. A non-list `required`, or a non-string entry in one, is + # malformed and dropped rather than iterated as characters or sent on. + required = parameters.pop("required", None) + names = [name for name in required if isinstance(name, str)] if isinstance(required, list) else [] + if names: + parameters["required"] = names return { "type": "function", @@ -700,8 +1314,7 @@ def _run(self, **kwargs: Any) -> Any: # a caller opt into feeding that back and retrying. # Carry the structured fields across: without them handle_tool_error # leaves a caller with only str(exc), unable to tell 401 from 429. - # str(exc) on an httpx error is just "Client error '400 Bad Request' - # for url ..." — the field that is actually wrong is in response_body. + # The field that is actually wrong is often only in response_body. # Without it the agent retries blind, which is the whole point of # handing the error back. body = getattr(exc, "response_body", None) @@ -761,8 +1374,21 @@ def get_account_id(self) -> str | None: return self._account_id -class StackOneRpcTool(StackOneTool): - """RPC-backed tool wired to the StackOne actions RPC endpoint.""" +class StackOneMcpTool(StackOneTool): + """A tool executed over MCP ``tools/call`` on the endpoint that listed it. + + Every tool the toolset builds is one of these: per-action tools, the + ``*_search_actions`` / ``*_execute_action`` meta tools and ``stackone_submit_feedback``. + """ + + _endpoint: str = PrivateAttr() + # Set by the toolset that built the tool: the end-user id it recorded for an account. + # Looked up per request, so a tool keeps up with a later GET /accounts and with + # set_account_id(). + _end_user_id_of: Callable[[str], str | None] | None = PrivateAttr(default=None) + # Also set by the toolset: given the API's refusal of a request sent without an end + # user, looks the account's end user up, and says whether there now is one to retry with. + _look_up_end_user: Callable[[str, Exception], bool] | None = PrivateAttr(default=None) def __init__( self, @@ -771,197 +1397,110 @@ def __init__( description: str, parameters: ToolParameters, api_key: str, - base_url: str, + endpoint: str, account_id: str | None, + headers: Headers | None = None, timeout: float = 60.0, ) -> None: - execute_config = ExecuteConfig( - method="POST", - url=f"{base_url.rstrip('/')}/actions/rpc", - name=name, - headers={}, - body_type="json", - parameter_locations=dict(_RPC_PARAMETER_LOCATIONS), - timeout=timeout, - ) super().__init__( description=description, parameters=parameters, - _execute_config=execute_config, + _execute_config=ExecuteConfig(name=name, headers=dict(headers or {}), timeout=timeout), _api_key=api_key, _account_id=account_id, ) + self._endpoint = endpoint - def execute(self, arguments: str | dict[str, Any] | None = None) -> dict[str, Any]: - parsed_arguments = self._parse_arguments(arguments) - envelope = self._split_envelope_params(parsed_arguments, set(self.parameters.properties) or None) - - payload: dict[str, Any] = { - "action": self.name, - "body": envelope["body"], - "headers": self._build_action_headers(envelope["headers"] or None), - } - if envelope["path"]: - payload["path"] = envelope["path"] - if envelope["query"]: - payload["query"] = envelope["query"] - - return super().execute(payload) - - def _parse_arguments(self, arguments: str | dict[str, Any] | None) -> dict[str, Any]: - if arguments is None: - return {} - if isinstance(arguments, str): - try: - parsed = json.loads(arguments) - except json.JSONDecodeError as exc: - raise ValueError(f"Invalid JSON in arguments for {self.name!r}: {exc}") from exc - else: - parsed = arguments - if not isinstance(parsed, dict): - raise ValueError("Tool arguments must be a JSON object") - return dict(parsed) + def _prepare_headers(self) -> Headers: + """The request headers: the configured extras first, then the SDK's own. - @staticmethod - def _split_envelope_params( - params: dict[str, Any], declared: set[str] | None = None - ) -> dict[str, dict[str, Any]]: - """Split LLM-supplied tool arguments into the RPC envelope (path/query/headers/body). - - Tools are listed with ``?param-style=flat_prefixed``, so keys arrive as - ``_`` (for example ``path_id``, ``query_limit``) and the prefix - carries the parameter location. A bare dict-valued ``path``/``query``/``headers``/ - ``body`` key is still bucketed for clients holding a cached nested schema, and any - other key falls through to the body. - - ``declared`` is the served schema's property names. The prefix is stripped only from - a key the server actually declared, because the pattern alone cannot tell - ``path_id`` (a path param) from ``path_to_file`` (a body field that merely starts - with "path_"). Splitting the latter would drop the argument from the body and send - the server a path component it has no use for — silently, with the model none the - wiser. - - ``None`` or an EMPTY set both mean "no schema to consult" and trust every match: - a served schema with no usable ``properties`` must not route every ``path_*`` - key into the body and lose every path parameter. + Authorization, x-account-id and User-Agent are set last, and any case variant of + them among the extras is dropped first, so neither can replace the credential or + retarget the call at another account. So is x-end-user-id, when the toolset recorded + one for this tool's account; otherwise a configured x-end-user-id is sent as given. """ - buckets: dict[str, dict[str, Any]] = {"path": {}, "query": {}, "headers": {}, "body": {}} - reserved = ("path", "query", "headers", "body") - named = declared or set() - - # Whether to read a `_` key as located, decided ONCE from the - # schema rather than per key. Under flat_prefixed every parameter is prefixed, - # so `path_to_file` means path.to_file; under a bare schema it is a body field - # that merely starts with "path_". Asking "is THIS key declared?" got that - # wrong in both directions: it demoted undeclared `query_offset` to a body - # field silently, and it would still have mis-split a declared bare name. - # ALL, not any: under flat_prefixed every parameter is prefixed, so one bare - # name is proof the schema is not. `any` would be satisfied by the very key - # this exists to protect — a declared body field called `path_to_file`. - prefixed = not named or all(_FLAT_ENVELOPE_KEY_PATTERN.match(k) for k in named) - if named and not prefixed: - logger.warning( - "Tool schema contains bare parameter names; flat-prefix detection " - "is disabled and prefixed parameters may fall into the body." - ) - - # Two passes so precedence is deterministic rather than following the caller's - # dict order: an explicit flat_prefixed key always wins over a nested one. - nested: list[tuple[str, dict[str, Any]]] = [] - bare: list[tuple[str, Any]] = [] - for key, value in params.items(): - match = _FLAT_ENVELOPE_KEY_PATTERN.match(key) - if match and prefixed: - buckets[match.group(1)][match.group(2)] = value - continue - # A reserved word the schema declares as a property is a field, not a - # container — refusing it would reject a schema-valid call. - if key in reserved and key not in named: - # Reserved keys are containers. A scalar here is malformed input, not a - # body field — putting it in the body would smuggle `path` into the payload. - if not isinstance(value, dict): - raise ValueError( - f"{key!r} is an envelope container and must be an object, " - f"got {type(value).__name__}. Did you mean {key}_?" - ) - nested.append((key, value)) - continue - bare.append((key, value)) - - # Deferred so precedence is a property of the KIND of key, not of the caller's - # dict order: flat_prefixed beats nested beats bare, always. Assigning bare keys - # in the first pass made {"body_foo": 1, "foo": 2} and {"foo": 2, "body_foo": 1} - # produce different wire bodies — and the Node SDK a third, breaking the - # cross-language byte-equality the conformance suite asserts. - for key, value in nested: - for field, field_value in value.items(): - buckets[key].setdefault(field, field_value) - for key, value in bare: - buckets["body"].setdefault(key, value) - return buckets - - def _build_action_headers(self, additional_headers: dict[str, Any] | None) -> dict[str, str]: - headers = self._sanitise_headers(additional_headers) - - account_id = self.get_account_id() - if account_id: - headers["x-account-id"] = account_id + end_user_id = self._recorded_end_user_id() + # The constructor requires the key; only code that clears it afterwards gets here. + if not self._api_key: + raise StackOneError(f'Tool "{self.name}" has no API key to authenticate with.') + return build_request_headers( + api_key=self._api_key, + account_id=self._account_id, + end_user_id=end_user_id, + extra_headers=self._execute_config.headers, + ) - return headers + def _recorded_end_user_id(self) -> str | None: + if self._end_user_id_of is None or not self._account_id: + return None + return self._end_user_id_of(self._account_id) + def execute(self, arguments: str | JsonDict | None = None) -> JsonDict: + """Call the tool over MCP ``tools/call``. -class StackOneMcpTool(StackOneTool): - """A tool executed over MCP ``tools/call`` rather than the RPC endpoint.""" + Arguments are sent as given; the server maps them onto the action. Header arguments + — the entries of a ``headers`` object and any ``headers_`` — are the exception: + these arguments are model-controlled, so each is kept only if this tool's own schema + declares it, and never if it is Authorization, x-account-id, User-Agent or + x-end-user-id. - _endpoint: str = PrivateAttr() - _mcp_headers: Headers = PrivateAttr() + Returns: + The tool's result as the server wrote it (see :func:`parse_tool_result`): for an + action, ``{"isError": false, "result": ..., ...}``. A file action's ``result`` is a + download link, not the file. - def __init__( - self, - *, - name: str, - description: str, - parameters: ToolParameters, - api_key: str, - endpoint: str, - headers: Headers, - account_id: str | None, - timeout: float = 60.0, - ) -> None: - super().__init__( - description=description, - parameters=parameters, - _execute_config=ExecuteConfig( - method="POST", url=endpoint, name=name, headers={}, timeout=timeout - ), - _api_key=api_key, - _account_id=account_id, - ) - self._endpoint = endpoint - self._mcp_headers = headers + Raises: + StackOneAPIError: If the result carries ``isError``, with the status from its + payload, or the endpoint answers with an HTTP error. A 400 asking for the + end user of an account the toolset has none recorded for is answered first: + the toolset looks it up with ``GET /accounts`` and the call is sent once more. + ToolArgumentsError: If the arguments are not a JSON object or cannot be encoded. + A subclass of both StackOneError and ValueError. + """ + parsed = self._parse_arguments(arguments) - def execute(self, arguments: str | JsonDict | None = None) -> JsonDict: - if isinstance(arguments, str): - try: - parsed = json.loads(arguments) - except json.JSONDecodeError as exc: - raise ValueError(f"Invalid JSON in arguments for {self.name!r}: {exc}") from exc - else: - parsed = arguments or {} - if not isinstance(parsed, dict): - raise ValueError("Tool arguments must be a JSON object") - - # The meta tools take a `headers` object in the envelope, and these arguments - # are model-controlled. Without this, a prompt-injected call could put its own - # Authorization or x-account-id in the envelope the server unpacks — the guard - # the RPC path has had all along, on the path search()/execute() actually use. - supplied_headers = parsed.get("headers") - if isinstance(supplied_headers, dict): - parsed = {**parsed, "headers": self._sanitise_headers(supplied_headers)} - - return call_mcp_tool( - self._endpoint, self._mcp_headers, self.name, parsed, timeout=self._execute_config.timeout - ) + unencodable = f"Arguments for {_json_text(self.name)} could not be encoded as JSON" + try: + # Without this a prompt-injected call could put its own Authorization or + # x-account-id in a header argument the server unpacks. + parsed = self._sanitise_header_arguments(parsed) + # NaN and Infinity are not JSON; the MCP client would send them as null. + non_finite = _first_non_finite(parsed) + json.dumps(parsed, ensure_ascii=False).encode("utf-8") + except (UnicodeEncodeError, TypeError, ValueError) as exc: + # A lone surrogate, in any string or key however deep — what a model emits when a + # token boundary splits an emoji — or a value JSON cannot encode (a set, bytes, + # a cycle) would otherwise fail deep inside the MCP client and surface as a + # transport error. It is an argument problem. + raise ToolArgumentsError(f"{unencodable}: {exc}") from exc + except RecursionError as exc: + # A header value that contains itself, directly or through a cycle of nested + # dicts/lists, recurses forever as _header_text walks it. Not a different + # defect than the ones above: it's still an argument the server could never + # have accepted. + raise ToolArgumentsError(f"{unencodable}: circular reference") from exc + if non_finite is not None: + raise ToolArgumentsError(f"{unencodable}: {_js_number(non_finite)} is not a JSON number") + + def call() -> JsonDict: + timeout = self._execute_config.timeout + return call_mcp_tool(self._endpoint, self._prepare_headers(), self.name, parsed, timeout=timeout) + + account_id = self._account_id + without_end_user = account_id is not None and self._recorded_end_user_id() is None + try: + return call() + except StackOneAPIError as exc: + # Sent with no recorded end user and refused for want of one: the account is + # non-shared and its end user can be looked up. Retried once, with it. + if not ( + without_end_user + and account_id is not None + and self._look_up_end_user is not None + and self._look_up_end_user(account_id, exc) + ): + raise + return call() def _read_openai_tool_call(call: Any) -> tuple[str, str, str | JsonDict]: @@ -973,35 +1512,43 @@ def _read_openai_tool_call(call: Any) -> tuple[str, str, str | JsonDict]: return str(call.id), str(function.name), function.arguments or {} -def _json_default(value: Any) -> Any: - """Serialise what json cannot. A file download returns raw bytes, which would - otherwise crash json.dumps — base64 keeps the content intact for the model.""" - if isinstance(value, bytes): - return base64.b64encode(value).decode("ascii") - return str(value) - - class Tools: """Container for Tool instances with lookup capabilities""" def __init__(self, tools: list[StackOneTool]) -> None: self.tools = tools - self._tool_map = {tool.name: tool for tool in tools} - - # Two accounts on one provider serve identically named tools, so this dict - # silently kept the last one and get_tool() routed every call to whichever - # account happened to list last. OpenAI accepts the duplicate function names - # without complaint, so nothing downstream surfaces it either — the only - # symptom is an action running against an account the caller never chose. - if len(self._tool_map) != len(tools): - counts = Counter(tool.name for tool in tools) - clashing = sorted(name for name, count in counts.items() if count > 1) - logger.warning( - "%d tool name(s) are served by more than one account (%s). get_tool() will " - "return the last one listed — pass account_ids to choose.", - len(clashing), - ", ".join(clashing[:5]), - ) + # First listing wins, as Node's getTool() finds the first match. + self._tool_map: dict[str, StackOneTool] = {} + for tool in tools: + self._tool_map.setdefault(tool.name, tool) + + # Two accounts on one provider serve identically named tools, and get_tool() + # returns only one of them — the only symptom would otherwise be an action running + # against an account the caller never chose. + self._warn_duplicate_names() + + def _warn_duplicate_names(self) -> None: + """Warn once if more than one tool has the same name.""" + if len(self._tool_map) == len(self.tools): + return + counts = Counter(tool.name for tool in self.tools) + clashing = sorted(name for name, count in counts.items() if count > 1) + logger.warning( + "%d tool name(s) are served by more than one account (%s). The first one listed, from the " + "lowest account id, is used — pass account ids to choose.", + len(clashing), + ", ".join(clashing[:5]), + ) + + def _first_per_name(self) -> list[StackOneTool]: + """The first tool of each name, in order: the one :meth:`get_tool` returns. + + Every adapter is built from these, so a model is never offered a tool a lookup by + name would not find. OpenAI was sent two functions of one name, and LangChain and + Pydantic AI each kept whichever their own lookup happened to find. + """ + self._warn_duplicate_names() + return list(self._tool_map.values()) def __getitem__(self, index: int) -> StackOneTool: return self.tools[index] @@ -1018,7 +1565,10 @@ def to_list(self) -> list[StackOneTool]: return list(self.tools) def get_tool(self, name: str) -> StackOneTool | None: - """Get a tool by its name, or None if absent""" + """Get a tool by its name, or None if absent. + + When more than one account serves the name, this is the first one listed. + """ return self._tool_map.get(name) def set_account_id(self, account_id: str | None) -> None: @@ -1035,8 +1585,8 @@ def get_account_id(self) -> str | None: return None def to_openai(self) -> list[JsonDict]: - """Convert all tools to OpenAI function format""" - return [tool.to_openai_function() for tool in self.tools] + """Convert the tools to OpenAI function format, the first of each name only.""" + return [tool.to_openai_function() for tool in self._first_per_name()] def execute_openai_tool_calls(self, tool_calls: Iterable[Any] | None) -> list[JsonDict]: """Run a Chat Completions response's tool calls and return the ``tool`` messages. @@ -1061,27 +1611,28 @@ def execute_openai_tool_calls(self, tool_calls: Iterable[Any] | None) -> list[Js call_id, name, arguments = _read_openai_tool_call(call) tool = self.get_tool(name) if tool is None: - result: Any = {"error": f"Unknown tool {name!r}"} + result: Any = {"error": f"Unknown tool {_json_text(name)}"} else: try: result = tool.execute(arguments) except (StackOneError, ValueError) as exc: body = getattr(exc, "response_body", None) result = {"error": str(exc), **({"response_body": body} if body else {})} - content = json.dumps(result, default=_json_default) + # default=str: a hand-built tool may return something JSON cannot encode. + content = json.dumps(result, default=str) messages.append({"role": "tool", "tool_call_id": call_id, "content": content}) return messages def to_langchain(self) -> Sequence[Any]: - """Convert all tools to LangChain format. + """Convert the tools to LangChain format, the first of each name only. Requires ``stackone-ai[langchain]``. """ - return [tool.to_langchain() for tool in self.tools] + return [tool.to_langchain() for tool in self._first_per_name()] def to_pydantic_ai(self) -> list[Any]: - """Convert all tools to Pydantic AI ``Tool`` instances. + """Convert the tools to Pydantic AI ``Tool`` instances, the first of each name only. Requires ``stackone-ai[pydantic-ai]`` (installs ``pydantic-ai-slim``). """ - return [tool.to_pydantic_ai_tool() for tool in self.tools] + return [tool.to_pydantic_ai_tool() for tool in self._first_per_name()] diff --git a/stackone_ai/toolset.py b/stackone_ai/toolset.py index ce7aa66e..40bbf146 100644 --- a/stackone_ai/toolset.py +++ b/stackone_ai/toolset.py @@ -3,29 +3,42 @@ from __future__ import annotations import concurrent.futures +import contextlib import copy import fnmatch import logging import os import threading -from typing import Any +import time +from collections.abc import Callable, Collection, Sequence +from dataclasses import dataclass, field +from typing import Any, TypeVar import httpx from stackone_ai.tools import ( - MCP_PARAM_STYLE, - USER_AGENT, McpToolDefinition, + RateLimitRetryingClient, + RateLimitTimeout, StackOneMcpTool, - StackOneRpcTool, StackOneTool, Tools, - build_auth_header, + _js_number, + _json_text, + _json_type, + build_request_headers, fetch_mcp_tools, + is_end_user_refusal, + is_rate_limited, + is_sdk_owned_header, ) from stackone_ai.types import ( DEFAULT_BASE_URL, + SUBMIT_FEEDBACK_TOOL_NAME, ExecuteToolsConfig, + FeedbackCategory, + FeedbackRating, + FeedbackSource, Headers, JsonDict, StackOneAPIError, @@ -42,16 +55,103 @@ _UNSET = object() """Sentinel: `mode=None` is a real mode (individual), so it cannot mean "use the default".""" +_Listing = tuple[McpToolDefinition, str | None, str] +"""A served tool, the account that listed it, and the endpoint it was listed from.""" + +_RetryFailed = Callable[["_Catalog"], Collection[str | None]] +"""Given a cached catalog, the accounts marked failed in it to list again now.""" + # The search_actions meta tool's served schema caps top_k at 50. _MAX_TOP_K = 50 +FAILED_ACCOUNT_RETRY_SECONDS = 30.0 +"""How long an account that failed to list tools is left out of a cached catalog. + +Its healthy siblings' listings are cached all the same. A later call within this window +serves them without the failed account, and without warning about it again; the first +call after it lists the failed account again. ``execute()`` does not wait: it lists one +that could serve its action again straight away, unless a provider lookup failed to name +it within the window. +""" + +# Looked up at call time, so tests can move it on rather than wait. +_clock = time.monotonic + + +@dataclass(frozen=True) +class _Catalog: + """A cached catalog: each healthy account's listing, and when and how each failed one failed.""" + + listings: dict[str | None, list[_Listing]] + failed_at: dict[str | None, float] = field(default_factory=dict) + errors: dict[str | None, Exception] = field(default_factory=dict) + + def in_order(self, account_scope: list[str | None]) -> list[_Listing]: + return [listing for account in account_scope for listing in self.listings.get(account, [])] + + def failures(self) -> dict[str | None, Exception]: + """Each account left out of this catalog, with the error it failed with.""" + return {account: self.errors[account] for account in self.failed_at if account in self.errors} + + +_Item = TypeVar("_Item") +_Result = TypeVar("_Result") + + +def _fan_out( + fn: Callable[[_Item], _Result], items: Sequence[_Item] +) -> list[tuple[_Item, _Result | Exception]]: + """Run ``fn`` over ``items`` concurrently; each item's result or failure, in item order. + + A 429 that outlasted its retries is raised at once instead of being returned: the + caller would skip it like any other failure and hand back a partial result. Work not + yet started is cancelled; calls already running finish in the background, unread. + """ + pool = concurrent.futures.ThreadPoolExecutor(max_workers=min(len(items), 10)) + try: + futures = {pool.submit(fn, item): index for index, item in enumerate(items)} + outcomes: dict[int, _Result | Exception] = {} + for future in concurrent.futures.as_completed(futures): + try: + outcomes[futures[future]] = future.result() + except Exception as exc: + if is_rate_limited(exc): + raise + outcomes[futures[future]] = exc + finally: + pool.shutdown(wait=False, cancel_futures=True) + return [(item, outcomes[index]) for index, item in enumerate(items)] + + +def _all_accounts_failed(failures: list[tuple[str | None, Exception]]) -> Exception: + """The error for a listing in which every account failed, in account order. + + When every failure is a StackOneAPIError with the same status, the first of them: two + accounts answering 401 are a revoked key, and a caller must be able to tell that from a + 429 as it can with one account. Otherwise a ToolsetLoadError, caused by an + ExceptionGroup of the failures and keeping them in ``failures`` too. + """ + errors = [failure for _, failure in failures] + first = errors[0] + if isinstance(first, StackOneAPIError) and all( + isinstance(error, StackOneAPIError) and error.status_code == first.status_code for error in errors + ): + return first + error = ToolsetLoadError( + "Every account failed to list tools: " + + "; ".join(f"{account}: {failure}" for account, failure in failures), + failures=errors, + ) + error.__cause__ = ExceptionGroup("Every account failed to list tools", errors) + return error + class StackOneToolSet: """Main class for accessing StackOne tools. The toolset is a thin client over the served catalog: it lists tools from the - MCP endpoint and executes them against the actions RPC endpoint. It does not - rewrite, filter or invent schemas. + MCP endpoint and executes each one over MCP ``tools/call`` on the endpoint, and + with the account, that listed it. It does not rewrite, filter or invent schemas. """ def __init__( @@ -62,36 +162,67 @@ def __init__( execute: ExecuteToolsConfig | None = None, timeout: float | None = None, tool_mode: ToolMode | None = None, + headers: Headers | None = None, + include_non_shared: bool = False, ) -> None: """Initialize StackOne tools with authentication Args: api_key: Optional API key. If not provided, will try to get from STACKONE_API_KEY env var account_id: Optional account ID - base_url: Optional base URL override for API requests + base_url: Optional base URL override for API requests. Falls back to + STACKONE_BASE_URL, then to the default, https://api.stackone.com. An + empty string, whether passed here or read from the environment, is + treated as not set rather than as a literal empty host. execute: Execution configuration. Controls default account scoping for tool execution. Pass ``{"account_ids": ["acc-1"]}`` to scope tools to specific accounts. - timeout: Request timeout in seconds for tool execution HTTP calls. + timeout: Request timeout in seconds for tool listing and execution. Default: 60. Takes precedence over ``execute.timeout`` if set. Increase for slow providers (e.g. Workday). tool_mode: How the endpoint lists tools. ``"search_execute"`` returns two meta tools per connector instead of one tool per action, keeping the catalog small enough for a model's context. + headers: Extra HTTP headers sent with every request this toolset makes + (``GET /accounts`` and every MCP request). ``Authorization``, + ``x-account-id`` and ``User-Agent`` are always the SDK's own and + cannot be set here. ``x-end-user-id`` can: it is passed through as + given, unless ``GET /accounts`` reported the account's end user, + which then replaces it. + include_non_shared: Whether account discovery uses non-shared accounts + (``shared`` false in ``GET /accounts``). Each belongs to a single end + user, so by default discovery skips them, with a warning naming them; + pass their ids to use them without this. Raises: - ToolsetConfigError: If no API key is provided or found in environment + ToolsetConfigError: If no API key is provided or found in environment, or + ``account_id`` is an empty string """ + # An empty account_id is usually an unset variable, and treating it as unset would + # silently widen every call to all active shared accounts. + if account_id == "": + raise ToolsetConfigError("account_id must not be an empty string") api_key_value = api_key or os.getenv("STACKONE_API_KEY") if not api_key_value: raise ToolsetConfigError( - "API key must be provided either through api_key parameter or " - "STACKONE_API_KEY environment variable" + "An API key must be provided, either to the toolset or in the STACKONE_API_KEY " + "environment variable" + ) + ignored_headers = [name for name in (headers or {}) if is_sdk_owned_header(name)] + if ignored_headers: + logger.warning( + "Ignoring headers %s: the SDK sets them itself. Pass the API key and account ids " + "through their own options instead.", + ", ".join(_json_text(name) for name in ignored_headers), ) + self.api_key: str = api_key_value self.account_id = account_id - self.base_url = base_url or DEFAULT_BASE_URL - self._account_ids: list[str] = list(execute.get("account_ids", [])) if execute else [] + self.base_url = base_url or os.getenv("STACKONE_BASE_URL") or DEFAULT_BASE_URL + self._headers: Headers = dict(headers or {}) + self._account_ids: list[str] = self._validate_account_ids( + execute.get("account_ids") if execute else None + ) self._execute_config: ExecuteToolsConfig | None = execute execute_timeout = execute.get("timeout") if execute else None self._timeout: float = ( @@ -100,10 +231,11 @@ def __init__( # Cache the listing, not the Tools wrapper. StackOneTool objects are mutable # (Tools.set_account_id rebinds them), so handing the same instances back on a # cache hit let one caller silently rescope every later caller's tools. - self._catalog_cache: dict[ - tuple[Any, ...], list[tuple[McpToolDefinition, str | None, str, Headers]] - ] = {} + self._catalog_cache: dict[tuple[Any, ...], _Catalog] = {} self._discovered_account_ids: list[str] | None = None + # Shared while in flight, so concurrent fetch_tools() and search() calls on a + # fresh toolset make one GET /accounts between them rather than one each. + self._discovering: concurrent.futures.Future[list[str]] | None = None # Bumped by clear_catalog_cache(). A listing already in flight when the cache is # cleared captured the generation it started under, and refuses to write back if # it has moved — otherwise the stale catalog lands *after* the clear and is @@ -111,19 +243,48 @@ def __init__( # to prevent. self._cache_generation = 0 self._cache_lock = threading.Lock() + # Each non-shared account's end-user id, from the last successful GET /accounts. + # Replaced whole by the next one, and kept by clear_catalog_cache(): it describes + # the accounts, not the catalog. + self._end_user_ids: dict[str, str] = {} + # Each account's provider, from the same GET /accounts and replaced with it: what + # execute() reads to tell whether an account that failed to list could have served + # an action. + self._providers: dict[str, str] = {} + # When a provider lookup last failed to name each account, which execute() then + # neither looks up nor lists again early for FAILED_ACCOUNT_RETRY_SECONDS. + self._provider_misses: dict[str, float] = {} + # The GET /accounts in flight, if any: an account's end user is looked up by joining + # it rather than making another. + self._fetching_accounts: concurrent.futures.Future[list[JsonDict]] | None = None + self._include_non_shared = include_non_shared + # Bumped at the start of every fetch_accounts() call, so two overlapping calls — + # fetch_accounts() racing discovery, say — are ordered by when they started + # rather than when they returned: a slower, older-started response must not + # overwrite a newer one that already landed. + self._accounts_sequence = 0 + self._end_user_ids_sequence = 0 self._tool_mode: ToolMode | None = tool_mode - def set_accounts(self, account_ids: list[str]) -> StackOneToolSet: + # 2.x read STACKONE_ACCOUNT_ID; 3.x never does, and without an account it uses every + # active shared account on the key. Anyone upgrading with only the variable set would + # otherwise widen to other end users' accounts with no sign of it. An empty + # account_ids list is unset too, the same as _resolve_account_ids() treats it. + scoped = account_id is not None or bool(self._account_ids) + if os.getenv("STACKONE_ACCOUNT_ID") and not scoped: + logger.warning( + "STACKONE_ACCOUNT_ID is set, but the SDK does not read it: with no account id passed, " + "every active shared account on this API key is used. Pass an account id to scope the " + "toolset." + ) + + def set_accounts(self, account_ids: list[str] | None) -> StackOneToolSet: """Set account IDs for filtering tools Returns: This toolset instance for chaining """ - if isinstance(account_ids, str): - raise ToolsetConfigError( - f"account_ids must be a list of account ids, not a string. Did you mean [{account_ids!r}]?" - ) - self._account_ids = list(account_ids) + self._account_ids = self._validate_account_ids(account_ids) self.clear_catalog_cache() return self @@ -131,12 +292,16 @@ def clear_catalog_cache(self) -> None: """Invalidate the cached tool catalog. Call when linked accounts change outside of ``set_accounts`` or when - you need to force a fresh fetch from the StackOne MCP endpoint. + you need to force a fresh fetch from the StackOne MCP endpoint. The end-user ids + recorded from GET /accounts are kept until the next one replaces them; the accounts + a provider lookup failed to name are forgotten. """ with self._cache_lock: self._cache_generation += 1 self._catalog_cache.clear() + self._provider_misses.clear() self._discovered_account_ids = None + self._discovering = None def fetch_tools( self, @@ -160,7 +325,18 @@ def fetch_tools( Collection of tools matching the filter criteria Raises: - ToolsetLoadError: If there is an error loading the tools + ToolsetLoadError: If there is an error loading the tools. When every account + fails for differing reasons, its ``failures`` holds each account's error, + and its ``__cause__`` is an ExceptionGroup of them. + StackOneAPIError: With ``status_code`` 429 if the API is still rate limiting + after retries, even when only one of several accounts is; or with the API's + status when every account fails with that same status, so a caller can tell + a revoked key's 401 from a 429. Other per-account failures skip that account + with a warning, and leave it out for ``FAILED_ACCOUNT_RETRY_SECONDS``. + + A 429 is retried up to three times, after the server's ``Retry-After`` (capped at + 30 seconds) or else a 1s, 2s, 4s backoff with jitter. A wait that would not end + within ``timeout`` is not started, and the 429 is raised instead. Examples: tools = toolset.fetch_tools(account_ids=['123', '456']) @@ -169,39 +345,49 @@ def fetch_tools( Note: For organizations with multiple connected accounts, calling `fetch_tools()` - without `account_ids` discovers and fetches the catalog for every active account. + without `account_ids` discovers and fetches the catalog for every active shared + account (non-shared ones too with ``include_non_shared=True``). If your organization has many accounts, pass explicit `account_ids` to avoid excessive round trips and blowing model context limits. + + Discovery skips non-shared accounts unless the toolset was built with + ``include_non_shared=True``. """ - if isinstance(account_ids, str): - raise ToolsetConfigError( - f"account_ids must be a list of account ids, not a string. Did you mean [{account_ids!r}]?" - ) - try: - mode = self._tool_mode if mode is _UNSET else mode - effective_account_ids = account_ids or self._account_ids - if not effective_account_ids and self.account_id: - effective_account_ids = [self.account_id] - if not effective_account_ids: - effective_account_ids = self._discover_account_ids() + mode = self._tool_mode if mode is _UNSET else mode + tools, _ = self._fetch_tools(account_ids, providers, actions, mode, retry_failed=None) + return tools + def _fetch_tools( + self, + account_ids: list[str] | None, + providers: list[str] | None, + actions: list[str] | None, + mode: ToolMode | None, + *, + retry_failed: _RetryFailed | None, + ) -> tuple[Tools, dict[str | None, Exception]]: + """:meth:`fetch_tools`, and each account in scope it left out with the error it failed + with. ``retry_failed`` picks, from a cached catalog, the accounts marked failed to list + again now rather than once they are due. + """ + try: + # Captured before account resolution, which may discover accounts: a + # clear_catalog_cache() landing mid-discovery must stop the catalog it + # resolves to from being cached too, not just the discovered list itself. + generation = self._cache_generation account_scope: list[str | None] = sorted( - dict.fromkeys(effective_account_ids), key=lambda a: (a is None, a) + dict.fromkeys(self._resolve_account_ids(account_ids)), key=lambda a: (a is None, a) ) # Keyed on what was fetched, not on how it is filtered: providers and # actions narrow the list in memory, so they must not force a refetch. # base_url and api_key belong here — leaving them out meant reassigning # either one kept serving the old catalog, still pointed at the old host. - cache_key = self._cache_key(account_scope, mode) - with self._cache_lock: - cached = self._catalog_cache.get(cache_key) - if cached is None: - cached = self._list_catalog(account_scope, mode) + listings, failures = self._catalog(account_scope, mode, generation, retry_failed=retry_failed) all_tools = [ - self._create_tool(tool_def, account, endpoint, headers, mode) - for tool_def, account, endpoint, headers in cached + self._create_tool(tool_def, account, endpoint) + for tool_def, account, endpoint in self._dedupe_global_tools(listings) ] if providers: @@ -210,7 +396,7 @@ def fetch_tools( if actions: all_tools = [tool for tool in all_tools if self._filter_by_action(tool.name, actions)] - return Tools(all_tools) + return Tools(all_tools), failures except (ToolsetError, StackOneError): # StackOneAPIError carries the HTTP status. Re-wrapping it below would throw @@ -219,62 +405,178 @@ def fetch_tools( except Exception as exc: # pragma: no cover - unexpected runtime errors raise ToolsetLoadError(f"Error fetching tools: {exc}") from exc - def _list_catalog( - self, account_scope: list[str | None], mode: ToolMode | None - ) -> list[tuple[McpToolDefinition, str | None, str, Headers]]: - """List every scoped account's catalog, tolerating accounts that fail.""" - generation = self._cache_generation - endpoint = f"{self.base_url.rstrip('/')}/mcp?param-style={MCP_PARAM_STYLE}" + @staticmethod + def _validate_account_ids(account_ids: Any) -> list[str]: + """A copy of the account ids, refusing a bare string, a non-list, a non-string id or + an empty id. ``None`` means not given: no ids. + + An empty or ``None`` id would be sent with no ``x-account-id``, so it is rejected + rather than letting the server answer for an account nobody chose, as in Node. A + dict is refused rather than iterated, which read ``{"id": "acc-1"}`` as the account + ``"id"``. + """ + if account_ids is None: + return [] + if isinstance(account_ids, str): + raise ToolsetConfigError( + "account_ids must be a list of account ids, not a string. " + f"Did you mean [{_json_text(account_ids)}]?" + ) + if not isinstance(account_ids, list) or not all(isinstance(i, str) for i in account_ids): + raise ToolsetConfigError("account_ids must be a list of account id strings") + ids = list(account_ids) + if "" in ids: + raise ToolsetConfigError("account_ids must not contain an empty account id") + return ids + + def _resolve_account_ids(self, account_ids: list[str] | None) -> list[str]: + """The accounts a call is scoped to, in the order they were given or discovered. + + The argument, then ``set_accounts()``, then the constructor's ``account_id``, then + every active shared account ``GET /accounts`` lists (non-shared ones too with + ``include_non_shared=True``). + """ + if account_ids is not None: + account_ids = self._validate_account_ids(account_ids) + resolved = account_ids or self._account_ids + if not resolved and self.account_id: + resolved = [self.account_id] + if not resolved: + resolved = self._discover_account_ids() + return list(resolved) + + def _catalog( + self, + account_scope: list[str | None], + mode: ToolMode | None, + generation: int, + *, + retry_failed: _RetryFailed | None = None, + ) -> tuple[list[_Listing], dict[str | None, Exception]]: + """Every scoped account's catalog, from the cache where it can be. + + One unusable account must not cost the caller every other account's tools, so a + failing account is skipped with a warning. The healthy accounts' listings are cached + all the same, and the failed account is left out, silently, until + ``FAILED_ACCOUNT_RETRY_SECONDS`` have passed, when the next call lists it again. Not + caching at all made every call re-list every account, and wait out the full timeout + on one that hangs. + + A 429 that outlasts its retries is raised, not skipped: it is the key's, not the + account's, and skipping it would hand back a catalog missing whichever accounts + happened to be throttled. When every account fails, see :func:`_all_accounts_failed`. + + ``generation`` is the cache generation as of the start of the call that resolved + ``account_scope``: a clear_catalog_cache() during account discovery must stop the + catalog this scope resolves to from being cached, same as the discovered list. + + Returned with each account left out and the error it failed with. ``retry_failed`` is + given the cached catalog and picks accounts marked failed in it to list again now, + without waiting for them to be due. + """ + key = self._cache_key(account_scope, mode) + with self._cache_lock: + cached = self._catalog_cache.get(key) + now = _clock() + if cached is None: + due = account_scope + else: + retried = set(retry_failed(cached)) if retry_failed else set() + due = [ + account + for account, failed_at in cached.failed_at.items() + if account in retried or now - failed_at >= FAILED_ACCOUNT_RETRY_SECONDS + ] + if not due: + return cached.in_order(account_scope), cached.failures() + + # No param-style pin: arguments are sent verbatim and the server maps them with its + # own reverse map, so the model sees whatever style the server serves. + endpoint = f"{self.base_url.rstrip('/')}/mcp" if mode: - endpoint = f"{endpoint}&tool-mode={mode}" + endpoint = f"{endpoint}?tool-mode={mode}" def _fetch_for_account( account: str | None, - ) -> list[tuple[McpToolDefinition, str | None, str, Headers]]: - headers = self._build_mcp_headers(account) - return [ - (tool_def, account, endpoint, headers) - for tool_def in fetch_mcp_tools(endpoint, headers, timeout=self._timeout) - ] + ) -> list[_Listing]: + without_end_user = account is not None and self._end_user_id(account) is None + try: + listed = fetch_mcp_tools(endpoint, self._build_mcp_headers(account), timeout=self._timeout) + except StackOneAPIError as exc: + # Refused for want of an end user it was sent without: look it up, and retry + # once with it. + if not (without_end_user and account is not None and self._look_up_end_user(account, exc)): + raise + listed = fetch_mcp_tools(endpoint, self._build_mcp_headers(account), timeout=self._timeout) + return [(tool_def, account, endpoint) for tool_def in listed] + + def _store(catalog: _Catalog) -> None: + # Merged with whatever is cached now, not overwritten: a concurrent call for the + # same key may have stored a healthier catalog since this call read it, and a + # wholesale overwrite would hide its successes behind this call's failures for + # FAILED_ACCOUNT_RETRY_SECONDS. A listing wins over a failure for the same account. + with self._cache_lock: + if generation != self._cache_generation: + return + existing = self._catalog_cache.get(key) + if existing is not None: + merged_listings = {**existing.listings, **catalog.listings} + merged_failed_at = {**existing.failed_at, **catalog.failed_at} + merged_errors = {**existing.errors, **catalog.errors} + for account in catalog.listings: + merged_failed_at.pop(account, None) + merged_errors.pop(account, None) + catalog = _Catalog(merged_listings, merged_failed_at, merged_errors) + self._catalog_cache[key] = catalog - listings: list[tuple[McpToolDefinition, str | None, str, Headers]] = [] if len(account_scope) == 1: - listings.extend(_fetch_for_account(account_scope[0])) - self._store_listing(account_scope, mode, listings, generation) - return listings - - failures: list[str] = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(account_scope), 10)) as pool: - futures = {pool.submit(_fetch_for_account, acc): acc for acc in account_scope} - for future, account in futures.items(): - try: - listings.extend(future.result()) - except Exception as exc: - # One unusable account must not cost the caller every other - # account's tools; report which one, keep the rest. - failures.append(f"{account}: {exc}") - if failures and not listings: - raise ToolsetLoadError("No account returned tools. " + " | ".join(failures)) - for failure in failures: - logger.warning("Skipping account that failed to list tools — %s", failure) - - # A degraded catalog must not be cached: the warning fires once, and every later - # call would then serve the short list silently, for the life of the process. - if not failures: - self._store_listing(account_scope, mode, listings, generation) - return listings + catalog = _Catalog({account_scope[0]: _fetch_for_account(account_scope[0])}) + _store(catalog) + return catalog.in_order(account_scope), {} + + listings = dict(cached.listings) if cached else {} + failed_at = dict(cached.failed_at) if cached else {} + errors = dict(cached.errors) if cached else {} + failures: list[tuple[str | None, Exception]] = [] + for account, outcome in _fan_out(_fetch_for_account, due): + if isinstance(outcome, Exception): + failures.append((account, outcome)) + else: + listings[account] = outcome + failed_at.pop(account, None) + errors.pop(account, None) + if not listings: + raise _all_accounts_failed(failures) + failed_now = _clock() + for account, failure in failures: + logger.warning("Skipping account that failed to list tools — %s: %s", account, failure) + failed_at[account] = failed_now + errors[account] = failure + + catalog = _Catalog(listings, failed_at, errors) + _store(catalog) + return catalog.in_order(account_scope), catalog.failures() - def _store_listing( - self, - account_scope: list[str | None], - mode: ToolMode | None, - listings: list[tuple[McpToolDefinition, str | None, str, Headers]], - generation: int, - ) -> None: - """Cache a listing, unless the cache was cleared while it was being fetched.""" - with self._cache_lock: - if generation == self._cache_generation: - self._catalog_cache[self._cache_key(account_scope, mode)] = listings + @staticmethod + def _dedupe_global_tools( + listings: list[_Listing], + ) -> list[_Listing]: + """Keep only the first listing of the feedback tool. + + It is global rather than account-scoped, so every account's listing carries an + identical copy. Left in, N accounts meant N same-named tools and a duplicate-name + warning about a clash that cannot misroute anything — noise that teaches callers + to ignore the warning that does matter. + """ + seen_feedback = False + kept: list[_Listing] = [] + for listing in listings: + if listing[0].name == SUBMIT_FEEDBACK_TOOL_NAME: + if seen_feedback: + continue + seen_feedback = True + kept.append(listing) + return kept def _cache_key(self, account_scope: list[str | None], mode: ToolMode | None) -> tuple[Any, ...]: return ( @@ -292,8 +594,16 @@ def _meta_tools(self, suffix: str, account_ids: list[str] | None = None) -> list the flipped mode partway through and cache search_execute meta tools under the individual-mode key, permanently, for every later caller. """ - tools = self.fetch_tools(account_ids=account_ids, mode="search_execute") - return [tool for tool in tools if tool.name.endswith(suffix)] + tools, _ = self._meta_tools_and_failures(suffix, account_ids, retry_failed=None) + return tools + + def _meta_tools_and_failures( + self, suffix: str, account_ids: list[str] | None, *, retry_failed: _RetryFailed | None + ) -> tuple[list[StackOneTool], dict[str | None, Exception]]: + tools, failures = self._fetch_tools( + account_ids, None, None, "search_execute", retry_failed=retry_failed + ) + return [tool for tool in tools if tool.name.endswith(suffix)], failures def search( self, @@ -309,17 +619,31 @@ def search( Args: query: What you want to do, e.g. "list recent comments". - top_k: Maximum results per connector. - account_ids: Restrict to these accounts. Defaults to all active ones. + top_k: Maximum results, across every connector searched. + account_ids: Restrict to these accounts. Defaults to every active shared one + (non-shared ones too with ``include_non_shared=True``). Returns: - Action dicts carrying at least ``action_id`` and ``description``. + At most ``top_k`` action dicts, best first across every connector, each carrying + at least ``action_id`` and ``description``, the ``account_id`` of the account + whose connector found it, and the ``session_id`` of the search when the server + issued one. The same action linked on two accounts is two hits. Pass + ``session_id`` to :meth:`execute` and :meth:`submit_feedback` to link the calls, + and ``account_id`` in ``account_ids`` to run the action on that account. + + Raises: + StackOneAPIError: With ``status_code`` 429 if the API is still rate limiting + after retries, for any account or connector. A 429 is retried as in + :meth:`fetch_tools`; other per-connector failures are skipped with a + warning. """ # The server rejects anything outside 1..50, but only after a round trip per # connector — and reports it as a load failure, which reads like an outage # rather than a typo. Fail here instead, where the caller can see why. if not isinstance(top_k, int) or isinstance(top_k, bool) or not 1 <= top_k <= _MAX_TOP_K: - raise ToolsetConfigError(f"top_k must be an integer between 1 and {_MAX_TOP_K}, got {top_k!r}") + raise ToolsetConfigError( + f"top_k must be an integer between 1 and {_MAX_TOP_K}, got {_json_text(top_k)}" + ) tools = self._meta_tools("_search_actions", account_ids) if not tools: @@ -327,23 +651,33 @@ def search( def _search_one(tool: StackOneTool) -> list[JsonDict]: found = tool.execute({"query": query, "top_k": top_k}) - return list(found.get("actions", [])) + actions = list(found.get("actions", [])) + # The server returns session_id once per search, beside the actions. Results + # from every connector are merged and re-ranked below, so this is the last + # point at which a hit can still be traced to the search, and the account, + # that produced it. + session_id = found.get("session_id") + traced: JsonDict = {} + if isinstance(session_id, str) and session_id: + traced["session_id"] = session_id + account_id = tool.get_account_id() + if account_id: + traced["account_id"] = account_id + return [{**action, **traced} for action in actions] results: list[JsonDict] = [] failures: list[str] = [] # Fan out the way fetch_tools() does. Serially, a customer with a dozen # connectors pays the sum of every connector's latency on the headline call. - with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(tools), 10)) as pool: - futures = {pool.submit(_search_one, tool): tool for tool in tools} - for future, tool in futures.items(): - try: - results.extend(future.result()) - except Exception as exc: - # Catch everything, as fetch_tools() does. Catching only StackOneError - # let a transport failure — which _describe_mcp_failure reports as a - # ToolsetLoadError — abort the whole search, which is precisely the - # flakiness this guard exists to absorb. - failures.append(f"{tool.name}: {exc}") + for tool, outcome in _fan_out(_search_one, tools): + if isinstance(outcome, Exception): + # Skip everything but a rate limit, as fetch_tools() does. Catching only + # StackOneError let a transport failure — which _describe_mcp_failure + # reports as a ToolsetLoadError — abort the whole search, which is + # precisely the flakiness this guard exists to absorb. + failures.append(f"{tool.name}: {outcome}") + else: + results.extend(outcome) if failures and not results: raise ToolsetLoadError("No connector returned results. " + " | ".join(failures)) @@ -360,8 +694,10 @@ def _score(action: JsonDict) -> float: return 0.0 return float(raw) + # Then cut to top_k: each connector was asked for top_k, so the merged list can + # hold many more. results.sort(key=_score, reverse=True) - return results + return results[:top_k] def execute( self, @@ -369,25 +705,101 @@ def execute( arguments: JsonDict | None = None, *, account_ids: list[str] | None = None, + session_id: str | None = None, ) -> JsonDict: """Execute an action by id, as returned by :meth:`search`. Always runs through the connector's ``_execute_action`` meta tool, so - ``arguments`` is the nested envelope every action's ``example_request`` - shows — ``{"query": {...}, "path": {...}, "body": {...}}``. The flat, - prefixed form belongs to ``fetch_tools()`` tools, whose own served schema - names the keys; routing by whether an id happened to be in the catalog - would make the argument shape depend on something the caller cannot see. + ``arguments`` is the nested form every action's ``example_request`` + shows — ``{"query": {...}, "path": {...}, "body": {...}}``. A + ``fetch_tools()`` tool takes the keys its own served schema names instead; + routing by whether an id happened to be in the catalog would make the + argument shape depend on something the caller cannot see. + + ``arguments["headers"]`` is forwarded to the action: ``*_execute_action`` serves an + open ``headers`` object, so any header name is accepted except Authorization, + x-account-id, User-Agent and x-end-user-id, which the SDK sets itself and which are + dropped. + + ``session_id`` is the value a :meth:`search` hit carries. Passing it links + this call to that search server-side. + + ``account_ids`` restricts routing to these accounts. Pass a search hit's + ``account_id`` to run the action on the account that found it. Raises: - ToolsetLoadError: If no connector matches. + ToolsetConfigError: If ``action_id``, ``arguments`` or ``session_id`` is malformed, + or the action's connector is linked on more than one account in scope. + ToolArgumentsError: If the arguments cannot be encoded as JSON. + ToolsetLoadError: If no connector matches, or an account in scope that could serve + the action failed to list. An account left out after failing whose provider in + ``GET /accounts`` is the action's connector, or whose provider no + ``GET /accounts`` has named, is listed again first, without waiting out + ``FAILED_ACCOUNT_RETRY_SECONDS``; if it still fails, the action is not run on + another account in its place. When a failed account's provider is not known, + one ``GET /accounts`` is made first to learn it; if that fails, it stays + unknown, and is neither looked up nor listed again early for + ``FAILED_ACCOUNT_RETRY_SECONDS``. + StackOneAPIError: If the action fails, including when the server rejects the + arguments. """ if not isinstance(action_id, str) or not action_id: - raise ToolsetConfigError(f"action_id must be a non-empty string, got {action_id!r}") + raise ToolsetConfigError(f"action_id must be a non-empty string, got {_json_text(action_id)}") if arguments is not None and not isinstance(arguments, dict): - raise ToolsetConfigError(f"arguments must be a JSON object, got {type(arguments).__name__}") + raise ToolsetConfigError(f"arguments must be a JSON object, got {_json_type(arguments)}") + if session_id is not None and (not isinstance(session_id, str) or not session_id): + raise ToolsetConfigError(f"session_id must be a non-empty string, got {_json_text(session_id)}") + + suffix = "_execute_action" + # At most one GET /accounts per call, to learn the providers of failed accounts: with + # explicit ids none was made, and an unknown provider could serve any action. + learnt = False + + def _learn_providers(accounts: list[str]) -> set[str]: + nonlocal learnt + if learnt: + return set() + missed = self._learn_providers(accounts) + learnt = missed is not None + return missed or set() + + def _retry_failed(cached: _Catalog) -> list[str | None]: + # An account marked failed that could serve the action is listed again now: left + # out, it would leave the action to whichever account did list, which may be + # another end user's. One whose provider cannot serve it is not, so it does not + # cost every other connector's actions its timeout. Nor is one a lookup recently + # failed to name: it would pay its timeout on every call in the window. + failed = [account for account in cached.failed_at if account is not None] + missed = self._recently_missed(failed) | _learn_providers(failed) + listed = [ + self._connector_of_name(tool_def.name, account, suffix) + for account, listings in cached.listings.items() + for tool_def, _, _ in listings + if tool_def.name.endswith(suffix) + ] + return [ + account + for account in failed + if account not in missed and self._may_serve(account, action_id, listed) + ] - meta_tools = self._meta_tools("_execute_action", account_ids) + meta_tools, failed = self._meta_tools_and_failures(suffix, account_ids, retry_failed=_retry_failed) + _learn_providers([account for account in failed if account is not None]) + connectors = [self._connector_of(tool, suffix) for tool in meta_tools] + blocking = sorted( + ( + (account, failure) + for account, failure in failed.items() + if account is not None and self._may_serve(account, action_id, connectors) + ), + key=lambda pair: pair[0], + ) + if blocking: + raise ToolsetLoadError( + f"{_json_text(action_id)} may be served by an account that failed to list (" + + "; ".join(f"{account}: {failure}" for account, failure in blocking) + + "). Pass the account id to use, such as a search hit's account_id." + ) matches = [ tool for tool in meta_tools @@ -399,33 +811,109 @@ def execute( best = max(len(self._connector_of(t, "_execute_action")) for t in matches) finalists = [t for t in matches if len(self._connector_of(t, "_execute_action")) == best] if len(finalists) > 1: - # Same provider linked twice. Picking one silently would run the action - # against an account the caller never chose. - logger.warning( - "%r matches %d connectors (%s); using %s. Pass account_ids to choose.", - action_id, - len(finalists), - ", ".join(t.name for t in finalists), - finalists[0].name, + # The same provider linked twice, which discovery makes common. Picking one + # would run the action against an account the caller never chose — another + # end user's, possibly. + ordered = sorted(finalists, key=lambda t: t.get_account_id() or "") + raise ToolsetConfigError( + f"{_json_text(action_id)} matches {len(finalists)} connectors on different accounts " + f"({', '.join(f'{t.name} on {t.get_account_id()}' for t in ordered)}). " + "Pass the account id to use, such as a search hit's account_id." ) tool = finalists[0] - # action_id LAST. Spreading arguments over it let a model-supplied - # "action_id" silently replace the action the caller pinned — the exact - # thing a host app pins it for. - result = tool.execute({**(arguments or {}), "action_id": action_id}) - # The meta tool wraps its payload as {"isError": ..., "result": ...}, but an - # isError response has already raised by this point — so the flag could - # only ever be False, and the wrapper just made this surface return a - # different shape from tool.execute() for the same action. Unwrap it. - if isinstance(result, dict) and set(result) == {"isError", "result"}: - inner = result["result"] - return inner if isinstance(inner, dict) else {"result": inner} - return result + # action_id LAST, removed first so it is last in key order too, as in Node. + # Spreading arguments over it let a model-supplied "action_id" silently replace + # the action the caller pinned — the exact thing a host app pins it for. + # session_id only when given: the served schema makes it an optional string, so + # an absent key is valid and a null is not. + call_arguments: JsonDict = dict(arguments or {}) + call_arguments.pop("action_id", None) + if session_id is not None: + call_arguments.pop("session_id", None) + call_arguments["session_id"] = session_id + call_arguments["action_id"] = action_id + # Returned as the server wrote it, the same shape tool.execute() returns. + return tool.execute(call_arguments) raise ToolsetLoadError( - f'No connector found for "{action_id}". Use search() to discover valid action ids.' + f"No connector found for {_json_text(action_id)}. Use search() to discover valid action ids." ) + def submit_feedback( + self, + rating: FeedbackRating, + tool_names: Sequence[str], + *, + feedback: str | None = None, + category: FeedbackCategory | None = None, + session_id: str | None = None, + source: FeedbackSource = "model", + account_ids: list[str] | None = None, + action_run_id: str | None = None, + ) -> JsonDict: + """Record a verdict on how well the tools served this session. + + Calls the server's ``stackone_submit_feedback`` tool once, through the account with + the lowest id among those the call is scoped to: ``account_ids``, or else the + toolset's, or else every active shared one (non-shared ones too with + ``include_non_shared=True``). Pass the ``session_id`` from a :meth:`search` + hit to attach the feedback to that session. + + Args: + rating: ``"positive"``, ``"negative"`` or ``"neutral"``. + tool_names: The tools or action ids the feedback is about. + feedback: An optional one-line reason. + category: What the feedback is about, e.g. ``"search"`` or ``"execute"``. + session_id: The session to link this feedback to. + source: Who produced the feedback. + account_ids: Accounts to choose from; only the lowest id is used. Defaults to + the toolset's own. + action_run_id: The action run the feedback is about. Sent only when given. + + Raises: + ToolsetConfigError: If ``tool_names`` is a string or ``session_id`` is empty or + not a string. + ToolsetLoadError: If feedback is not enabled for this project. + """ + if isinstance(tool_names, str): + raise ToolsetConfigError( + "tool_names must be a list of tool names, not a string. " + f"Did you mean [{_json_text(tool_names)}]?" + ) + if session_id is not None and (not isinstance(session_id, str) or not session_id): + raise ToolsetConfigError(f"session_id must be a non-empty string, got {_json_text(session_id)}") + + # Found in the served catalog, never built here: the server only serves the tool + # when the org flag and the project setting are both on, and a client-side stand-in + # would report success for feedback that went nowhere. It is global and served in + # every mode, so one account's search_execute listing — two tools per connector, + # not one per action — is enough to find it. + # The lowest id is the one a caller can predict, whatever order GET /accounts lists + # them in. + scope = self._resolve_account_ids(account_ids) + tool = self.fetch_tools(account_ids=[min(scope)], mode="search_execute").get_tool( + SUBMIT_FEEDBACK_TOOL_NAME + ) + if tool is None: + raise ToolsetLoadError( + f"The server did not serve {SUBMIT_FEEDBACK_TOOL_NAME}: feedback is not enabled " + "for this project." + ) + + # Unset optionals are omitted, never sent as null: the served schema declares them + # as optional strings, so a null is a present key of the wrong type and fails + # validation where an absent one would not. + arguments: JsonDict = {"rating": rating, "tool_names": list(tool_names)} + optional = { + "feedback": feedback, + "category": category, + "session_id": session_id, + "action_run_id": action_run_id, + "source": source, + } + arguments.update({key: value for key, value in optional.items() if value is not None}) + return tool.execute(arguments) + def openai(self, *, account_ids: list[str] | None = None) -> list[JsonDict]: """Get tools in OpenAI function calling format.""" return self.fetch_tools(account_ids=account_ids).to_openai() @@ -451,12 +939,34 @@ def _connector_of(tool: StackOneTool, suffix: str) -> str: made every action on that account unroutable — with an error blaming the caller's action id. """ - stem = tool.name[: -len(suffix)] if tool.name.endswith(suffix) else tool.name - account = tool.get_account_id() + return StackOneToolSet._connector_of_name(tool.name, tool.get_account_id(), suffix) + + @staticmethod + def _connector_of_name(name: str, account: str | None, suffix: str) -> str: + """:meth:`_connector_of` for a meta tool's served name and the account that listed it.""" + stem = name[: -len(suffix)] if name.endswith(suffix) else name if account and stem.endswith(f"_{account}"): stem = stem[: -(len(account) + 1)] return stem.lower() + def _may_serve(self, account_id: str, action_id: str, listed_connectors: list[str]) -> bool: + """Whether an account could serve an action: its provider is the action's connector, + or no GET /accounts has named its provider. + + The connector is the longest prefix of the action among the connectors listed and the + providers GET /accounts named, as execute() routes: a failed ``browser`` account cannot + serve ``browser_linkedin_search_people``. + """ + with self._cache_lock: + provider = self._providers.get(account_id) + known = [p.lower() for p in self._providers.values()] + if provider is None: + return True + lowered = action_id.lower() + matching = [c for c in [*listed_connectors, *known] if lowered.startswith(c + "_")] + connector = max(matching, key=len, default=None) + return provider.lower() == connector + def _filter_by_provider(self, tool_name: str, providers: list[str]) -> bool: """Whether a tool belongs to one of the given providers (case-insensitive). @@ -477,17 +987,60 @@ def fetch_accounts(self) -> list[JsonDict]: Each entry carries at least ``id``, ``provider`` and ``status``. Only accounts with ``status == "active"`` can serve tools. + + For each account with ``shared`` false and an ``origin_username``, that username is + recorded as the account's end-user id, and each account's ``provider`` as its + provider. The record replaces that of any call that started before this one; a call + that started earlier but finishes after this one records nothing. Every MCP request + this toolset's tools make for that account then carries it in ``x-end-user-id``, + which the API requires for a non-shared account. An account passed by id is listed + here only when the API refuses one of its requests for want of an end user: the + toolset then calls this once, and retries the request with the end user it recorded. + + Raises: + StackOneAPIError: If the API answers with an error, including a 429 that + outlasted its retries. """ + with self._cache_lock: + self._accounts_sequence += 1 + sequence = self._accounts_sequence + fetching: concurrent.futures.Future[list[JsonDict]] = concurrent.futures.Future() + self._fetching_accounts = fetching + return self._settle_accounts(fetching, sequence) + + def _settle_accounts( + self, fetching: concurrent.futures.Future[list[JsonDict]], sequence: int + ) -> list[JsonDict]: + """Make the GET /accounts started as ``sequence``, settling ``fetching`` with it.""" + try: + accounts = self._request_accounts(sequence) + except BaseException as exc: + fetching.set_exception(exc) + raise + else: + fetching.set_result(accounts) + return accounts + finally: + with self._cache_lock: + if self._fetching_accounts is fetching: + self._fetching_accounts = None + + def _request_accounts(self, sequence: int) -> list[JsonDict]: url = f"{self.base_url.rstrip('/')}/accounts" try: - response = httpx.get( - url, - headers={ - "Authorization": build_auth_header(self.api_key), - "User-Agent": USER_AGENT, - }, - timeout=self._timeout, - ) + with RateLimitRetryingClient(timeout=self._timeout, retry_within=self._timeout) as client: + response = client.get( + url, + headers=build_request_headers(api_key=self.api_key, extra_headers=self._headers), + ) + except RateLimitTimeout as exc: + # Timing out while a 429 is retried is still the rate limit, as it is over MCP. + raise StackOneAPIError( + f"Listing accounts at {url} was rate limited (429) and timed out after " + f"{_js_number(self._timeout)}s while retrying", + 429, + None, + ) from exc except httpx.HTTPError as exc: # The only public method with no error handling at all: a dead host, a bad # scheme or a timeout leaked httpx's own exception type straight out of the @@ -497,12 +1050,17 @@ def fetch_accounts(self) -> list[JsonDict]: if response.is_error: # Without this the catch-all in fetch_tools flattens it to a message and # the status is lost, so a caller cannot tell 401 from 429. - raise StackOneAPIError( + error = StackOneAPIError( f"Listing accounts at {url} failed with " f"{response.status_code} {response.reason_phrase}: {response.text.strip()}", response.status_code, response.text, ) + if response.status_code == 429: + # Chained to the HTTP 429 so is_rate_limited() reads it as the rate limit it + # is: fatal to a fan-out, as a 429 over MCP is. + raise error from httpx.HTTPStatusError("429", request=response.request, response=response) + raise error try: body = response.json() except (ValueError, UnicodeDecodeError) as exc: @@ -512,10 +1070,122 @@ def fetch_accounts(self) -> list[JsonDict]: # list(dict) yields the KEYS, so coercing here turned an unexpected wrapper # into a list of strings that blew up much later as an AttributeError. raise ToolsetLoadError( - f"Unexpected /accounts response shape: expected a list, got {type(accounts).__name__}" + f"Unexpected /accounts response shape: expected a list, got {_json_type(accounts)}" ) + end_user_ids = { + account["id"]: account["origin_username"] + for account in accounts + if isinstance(account, dict) + and isinstance(account.get("id"), str) + and account["id"] + and account.get("shared") is False + and isinstance(account.get("origin_username"), str) + and account["origin_username"] + } + providers = { + account["id"]: account["provider"] + for account in accounts + if isinstance(account, dict) + and isinstance(account.get("id"), str) + and account["id"] + and isinstance(account.get("provider"), str) + and account["provider"] + } + with self._cache_lock: + # Only if newer, by start order, than the result last written: a call that + # started first but finished last is answering a question a newer call has + # since answered better, and must not clobber it. Compared with the last + # write rather than the last start, so a newer call that fails leaves this + # one to write. + if sequence > self._end_user_ids_sequence: + self._end_user_ids = end_user_ids + self._providers = providers + self._end_user_ids_sequence = sequence return accounts + def _look_up_end_user(self, account_id: str, refusal: Exception) -> bool: + """Given the API's refusal of an account's request for want of its end user, list the + accounts and say whether that one now has an end user recorded to retry with. + + Only the API's 400 naming this account counts. A GET /accounts already in flight is + joined rather than another made. If it fails, the refusal is raised with that failure + as its cause: a key without the scope to list accounts would otherwise be told only + that the header does not match. + """ + if not is_end_user_refusal(refusal, account_id): + return False + try: + self._join_accounts_listing() + except Exception as exc: + # A rate limit is the key's, not the account's, and stays fatal to a fan-out. + if is_rate_limited(exc): + raise + raise refusal from exc + return self._end_user_id(account_id) is not None + + def _join_accounts_listing(self) -> None: + """Wait for the GET /accounts in flight, or make one: what it records is then current. + + Raises: + Exception: Whatever that GET /accounts failed with. + """ + with self._cache_lock: + fetching = self._fetching_accounts + sequence = 0 + if fetching is None: + self._accounts_sequence += 1 + sequence = self._accounts_sequence + fetching = self._fetching_accounts = concurrent.futures.Future() + if sequence: + with contextlib.suppress(Exception): + self._settle_accounts(fetching, sequence) + fetching.result() + + def _learn_providers(self, account_ids: list[str]) -> set[str] | None: + """If any of these accounts' providers is unknown, and no lookup has missed it within + ``FAILED_ACCOUNT_RETRY_SECONDS``, join or make one GET /accounts to learn it, and return + those it still could not name; ``None`` if it made no lookup. If that fails the providers + stay unknown, except that a rate limit is raised. Each one still unknown is recorded as + missed, unless clear_catalog_cache() ran meanwhile, which promises to forget misses. + """ + missed = self._recently_missed(account_ids) + with self._cache_lock: + unknown = [a for a in account_ids if a not in self._providers and a not in missed] + generation = self._cache_generation + if not unknown: + return None + try: + self._join_accounts_listing() + except Exception as exc: + if is_rate_limited(exc): + raise + missed_at = _clock() + with self._cache_lock: + still_unknown = {account for account in unknown if account not in self._providers} + if generation == self._cache_generation: + for account in still_unknown: + self._provider_misses[account] = missed_at + return still_unknown + + def _recently_missed(self, account_ids: list[str]) -> set[str]: + """Those of these accounts whose provider is unknown and a lookup failed to name + within ``FAILED_ACCOUNT_RETRY_SECONDS``. + """ + now = _clock() + with self._cache_lock: + return { + account + for account in account_ids + if account not in self._providers + and (missed_at := self._provider_misses.get(account)) is not None + and now - missed_at < FAILED_ACCOUNT_RETRY_SECONDS + } + + def _end_user_id(self, account_id: str) -> str | None: + """The end-user id GET /accounts recorded for an account, if it is not shared.""" + with self._cache_lock: + return self._end_user_ids.get(account_id) + def _discover_account_ids(self) -> list[str]: """List the linked accounts this API key can use. @@ -528,12 +1198,47 @@ def _discover_account_ids(self) -> list[str]: accounts, it is highly recommended to supply specific ``account_id`` or ``account_ids`` to avoid excessive API round trips and huge model contexts. + Non-shared accounts are skipped, with one warning naming them, unless the toolset was + built with ``include_non_shared=True``: each belongs to a single end user. + Raises: - ToolsetConfigError: If the key has no accounts, or none are usable. + ToolsetConfigError: If the key has no accounts, none are active, or every active + one is non-shared and skipped. """ if self._discovered_account_ids is not None: return self._discovered_account_ids + with self._cache_lock: + if self._discovered_account_ids is not None: + return self._discovered_account_ids + future = self._discovering + owner = future is None + if owner: + future = self._discovering = concurrent.futures.Future() + + if future is None: + raise RuntimeError("account discovery future was not initialized") + if not owner: + # A discovery already in flight: wait for it rather than issue a second + # GET /accounts. Its result, or its exception, is shared with every waiter. + return future.result() + + try: + active = self._fetch_active_account_ids() + except BaseException as exc: + with self._cache_lock: + if self._discovering is future: + self._discovering = None + future.set_exception(exc) + raise + with self._cache_lock: + if self._discovering is future: + self._discovering = None + future.set_result(active) + return active + + def _fetch_active_account_ids(self) -> list[str]: + generation = self._cache_generation accounts = self.fetch_accounts() active = [a["id"] for a in accounts if a.get("status") == "active" and a.get("id")] @@ -541,44 +1246,70 @@ def _discover_account_ids(self) -> list[str]: if not accounts: raise ToolsetConfigError( "This API key has no linked accounts. Link one in the StackOne " - "dashboard, or pass account_id explicitly." + "dashboard, or pass an account id explicitly." ) listed = ", ".join(f"{a.get('provider')} ({a.get('status')})" for a in accounts) raise ToolsetConfigError( f"None of this API key's {len(accounts)} linked accounts are active: {listed}. " - "Re-link them in the StackOne dashboard, or pass account_id explicitly." + "Re-link them in the StackOne dashboard, or pass an account id explicitly." ) - self._discovered_account_ids = active + # Each non-shared account belongs to a single end user: discovering it would put one + # end user's tools in front of whoever uses this toolset. + if not self._include_non_shared: + skipped = sorted( + {a["id"] for a in accounts if a.get("id") in active and a.get("shared") is False} + ) + if skipped and len(skipped) == len(set(active)): + raise ToolsetConfigError( + f"None of this API key's {len(skipped)} active account(s) are shared: each " + "belongs to a single end user. Pass their account ids, or opt in to non-shared " + "accounts, to use them." + ) + if skipped: + logger.warning( + "Discovery skipped %d non-shared account(s) (%s): each belongs to a single end " + "user. Pass their account ids, or opt in to non-shared accounts, to use them.", + len(skipped), + ", ".join(skipped), + ) + active = [account for account in active if account not in skipped] + + # Not kept if clear_catalog_cache() ran while the list was in flight: it may name + # accounts the clear was meant to forget. + with self._cache_lock: + if generation == self._cache_generation: + self._discovered_account_ids = active return active def _build_mcp_headers(self, account_id: str | None) -> dict[str, str]: - headers = { - "Authorization": build_auth_header(self.api_key), - "User-Agent": USER_AGENT, - } - if account_id: - headers["x-account-id"] = account_id - return headers + end_user_id = self._end_user_id(account_id) if account_id else None + return build_request_headers( + api_key=self.api_key, + account_id=account_id, + end_user_id=end_user_id, + extra_headers=self._headers, + ) def _create_tool( self, tool_def: McpToolDefinition, account_id: str | None, endpoint: str, - headers: dict[str, str], - mode: ToolMode | None = None, ) -> StackOneTool: """Build an executable tool from a served catalog entry. - In ``search_execute`` mode the served tools are MCP meta tools with no - action behind them on ``/actions/rpc``, so they are executed over - ``tools/call`` instead. + Every tool, in every mode, executes over MCP ``tools/call`` on the endpoint that + listed it, with the ``x-account-id`` of the account that listed it, and the + ``x-end-user-id`` recorded for that account when the request is made. """ schema = tool_def.input_schema or {} # Pop keys we explicitly override to avoid "multiple values for keyword argument" rest = dict(schema) - schema_type = str(rest.pop("type", "object")) + # A string type is passed through; anything else, such as ["object", "null"], is + # "object", as in Node. str() handed the model "['object', 'null']". + served_type = rest.pop("type", "object") + schema_type = served_type if isinstance(served_type, str) else "object" rest.pop("properties", None) schema_properties = self._normalize_schema_properties(schema) @@ -587,29 +1318,19 @@ def _create_tool( type=schema_type, properties=schema_properties, ) - if mode == "search_execute": - return StackOneMcpTool( - name=tool_def.name, - description=tool_def.description or "", - parameters=parameters, - api_key=self.api_key, - endpoint=endpoint, - # A copy: one headers dict is shared by every tuple in a cached listing, - # so handing it straight to the tool leaked one caller's mutation into - # every other caller's tools. - headers=dict(headers), - account_id=account_id, - timeout=self._timeout, - ) - return StackOneRpcTool( + tool = StackOneMcpTool( name=tool_def.name, description=tool_def.description or "", parameters=parameters, api_key=self.api_key, - base_url=self.base_url, + endpoint=endpoint, account_id=account_id, + headers=self._headers, timeout=self._timeout, ) + tool._end_user_id_of = self._end_user_id + tool._look_up_end_user = self._look_up_end_user + return tool def _normalize_schema_properties(self, schema: dict[str, Any]) -> dict[str, Any]: """Mirror the served schema's properties, recording requiredness per property. @@ -624,8 +1345,11 @@ def _normalize_schema_properties(self, schema: dict[str, Any]) -> dict[str, Any] raw_required = schema.get("required") # A string `required` would iterate as characters and mark every real property - # optional; a null would raise and fail the whole catalog over one bad tool. - required_fields = {str(name) for name in raw_required} if isinstance(raw_required, list) else set() + # optional; a null would raise and fail the whole catalog over one bad tool. A + # non-string entry is dropped, as to_openai_function() drops it, so the marker and + # the `required` a model is shown cannot disagree. + served_required = raw_required if isinstance(raw_required, list) else [] + required_fields = {name for name in served_required if isinstance(name, str)} normalized: dict[str, Any] = {} for name, details in properties.items(): diff --git a/stackone_ai/types.py b/stackone_ai/types.py index 0ff40c5a..1ac6b0a3 100644 --- a/stackone_ai/types.py +++ b/stackone_ai/types.py @@ -2,15 +2,18 @@ from __future__ import annotations -import ntpath -import posixpath -import re -import unicodedata -from enum import Enum +from collections.abc import Sequence from typing import Annotated, Any, Literal, TypeAlias, TypedDict -from urllib.parse import unquote -from pydantic import BaseModel, BeforeValidator, ConfigDict, Field +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PlainSerializer, + SerializerFunctionWrapHandler, + SkipValidation, + model_serializer, +) JsonDict: TypeAlias = dict[str, Any] Headers: TypeAlias = dict[str, str] @@ -29,6 +32,17 @@ how many accounts are linked, which is what keeps it inside a model's context. """ +SUBMIT_FEEDBACK_TOOL_NAME: str = "stackone_submit_feedback" +"""The one global tool the MCP endpoint serves in every mode when feedback is enabled. + +It is not a connector action, and it is global rather than account-scoped, so the toolset +keeps one copy of it however many accounts list it. +""" + +FeedbackRating = Literal["positive", "negative", "neutral"] +FeedbackSource = Literal["model", "user", "system"] +FeedbackCategory = Literal["search", "execute", "defender", "connection", "general"] + class StackOneError(Exception): """Base exception for StackOne errors""" @@ -45,6 +59,24 @@ def __init__(self, message: str, status_code: int, response_body: Any) -> None: self.response_body = response_body +class ToolArgumentsError(StackOneError, ValueError): + """Raised when a tool's arguments are unusable, before any request is made: + + - a JSON string argument that does not parse, or parses to something other than + an object + - an argument that cannot be encoded as JSON (a set, bytes, NaN/Infinity, a lone + surrogate, or a value that refers to itself, directly or through a cycle) + - ``call()`` given both positional and keyword arguments, or more than one + positional argument + + A subclass of both StackOneError, so ``except StackOneError`` catches it, and + ValueError, which is what these errors were before, so existing + ``except ValueError`` clauses still do. + """ + + pass + + class ToolsetError(StackOneError): """Base exception for toolset errors. @@ -65,9 +97,15 @@ class ToolsetConfigError(ToolsetError): class ToolsetLoadError(ToolsetError): - """Raised when there is an error loading tools""" + """Raised when there is an error loading tools. - pass + ``failures`` holds each account's error when every account failed to list tools for + differing reasons, in account order; otherwise it is empty. + """ + + def __init__(self, message: str, *, failures: Sequence[Exception] = ()) -> None: + super().__init__(message) + self.failures: list[Exception] = list(failures) class ExecuteToolsConfig(TypedDict, total=False): @@ -84,104 +122,21 @@ class ExecuteToolsConfig(TypedDict, total=False): constructor param which takes precedence.""" -class ParameterLocation(str, Enum): - """Valid locations for parameters in requests""" - - HEADER = "header" - QUERY = "query" - PATH = "path" - BODY = "body" - FILE = "file" # For file uploads - - -def validate_method(v: str) -> str: - """Validate HTTP method is uppercase and supported""" - if not isinstance(v, str): - raise ValueError(f"Unsupported HTTP method: {v}") - method = v.upper() - if method not in {"GET", "POST", "PUT", "DELETE", "PATCH"}: - raise ValueError(f"Unsupported HTTP method: {method}") - return method - - -def is_json_content_type(content_type: str) -> bool: - """Whether a response body should be parsed as JSON based on its Content-Type. - - Only genuine JSON media types are parsed (``application/json`` and structured - suffixes such as ``application/problem+json``). Anything else - including a - missing Content-Type - is treated as opaque content (a file download), so the - raw bytes are returned instead of being force-decoded as UTF-8/JSON. This mirrors - how the StackOne generated SDKs default unknown bodies to ``application/octet-stream``. - """ - media_type = content_type.split(";", 1)[0].strip().lower() - return media_type == "application/json" or media_type.endswith("+json") - - -def _safe_basename(name: str | None) -> str | None: - """Reduce a server-supplied filename to a bare, writable basename. +class ExecuteConfig(BaseModel): + """How a tool is called: its name, any extra request headers, and the timeout. - The value comes from a remote ``Content-Disposition``, which in practice is chosen - by whoever uploaded the file to the connected provider — so it is attacker-controlled. - Returned unsanitised it is an arbitrary-file-write primitive for any caller that does - the obvious thing and passes it to ``open()``: ``../../.ssh/authorized_keys`` and - ``/etc/cron.d/x`` both round-trip. The RFC 5987 branch percent-decodes, so a filter - applied before this point would be bypassed anyway; sanitise last, here, once. + Every tool executes over MCP ``tools/call``, so there is no HTTP method, URL, body + type or parameter location to configure. Extra fields are refused rather than ignored, + so code still passing them finds out here instead of assuming they took effect. """ - if name is None: - return None - # ntpath too: posixpath.basename leaves "C:evil.exe" intact, and on Windows that - # writes to drive C:'s current directory rather than the process CWD. - base = ntpath.basename(posixpath.basename(name.replace("\\", "/"))) - # Strip control characters (log and header injection) and Unicode format characters - # — U+202E renders "\u202egnp.exe" as "…exe.png", the classic extension spoof. - base = "".join(ch for ch in base if unicodedata.category(ch) not in ("Cc", "Cf")).strip() - if base in ("", ".", ".."): - return None - if len(base.encode("utf-8", "ignore")) > 255: - stem, dot, suffix = base.rpartition(".") - keep = 255 - len(dot + suffix) - base = (stem.encode("utf-8")[: max(keep, 1)].decode("utf-8", "ignore")) + dot + suffix - return base - - -def filename_from_content_disposition(value: str | None) -> str | None: - """Extract the filename from a Content-Disposition header value, if present. - - Handles both the plain ``filename="example.pdf"`` form and the RFC 5987 extended - ``filename*=UTF-8''example%20file.pdf`` form (which takes precedence when present). - The extended form is percent-decoded using its declared charset (RFC 5987 permits - both ``UTF-8`` and ``ISO-8859-1``); an unknown or empty charset falls back to UTF-8. - """ - if not value: - return None - extended = re.search(r"(?:^|;)\s*filename\*\s*=\s*([^']*)'[^']*'([^;]+)", value, re.IGNORECASE) - if extended: - charset = extended.group(1).strip() or "utf-8" - encoded = extended.group(2).strip().strip('"') - try: - return _safe_basename(unquote(encoded, encoding=charset, errors="replace")) - except LookupError: - # Unrecognised charset label - decode as UTF-8 rather than failing. - return _safe_basename(unquote(encoded, encoding="utf-8", errors="replace")) - quoted = re.search(r'(?:^|;)\s*filename\s*=\s*"([^"]*)"', value, re.IGNORECASE) - if quoted: - return _safe_basename(quoted.group(1)) - bare = re.search(r"(?:^|;)\s*filename\s*=\s*([^;]+)", value, re.IGNORECASE) - if bare: - return _safe_basename(bare.group(1).strip('"')) - return None + model_config = ConfigDict(extra="forbid") -class ExecuteConfig(BaseModel): - """Configuration for executing a tool against an API endpoint""" - - headers: Headers = Field(default_factory=dict, description="HTTP headers to include in the request") - method: Annotated[str, BeforeValidator(validate_method)] = Field(description="HTTP method to use") - url: str = Field(description="API endpoint URL") name: str = Field(description="Tool name") - body_type: str | None = Field(default=None, description="Content type for request body") - parameter_locations: dict[str, ParameterLocation] = Field( - default_factory=dict, description="Maps parameter names to their location in the request" + headers: Headers = Field( + default_factory=dict, + description="Extra HTTP headers for the MCP request. Authorization, x-account-id and " + "User-Agent are always the SDK's own and cannot be replaced here.", ) timeout: float = Field(default=60.0, description="Request timeout in seconds") @@ -190,12 +145,28 @@ class ToolParameters(BaseModel): """Schema definition for tool parameters. ``properties`` is a faithful mirror of the ``inputSchema`` the MCP server served, - with the SDK's internal ``nullable`` marker added per property. Consumers that - need the raw served schema (for example the ADK plugin) read this directly, so - nothing here may be invented or dropped. + with the SDK's internal ``nullable`` marker added per property. Every other root + keyword the server sent, ``required`` included, is kept verbatim as an extra field. + Consumers that need the raw served schema (for example the ADK plugin) read this + directly, so nothing here may be invented or dropped. """ model_config = ConfigDict(extra="allow") type: str = Field(description="JSON Schema type") properties: JsonDict = Field(description="JSON Schema properties") + # Typed for callers building a tool by hand, but neither validated nor converted: a + # served `required` is kept exactly as sent, even when malformed, and the adapters + # drop what is not a list of names. + required: Annotated[SkipValidation[list[str] | None], PlainSerializer(lambda value: value)] = Field( + default=None, description="The names of the required properties, as served" + ) + + @model_serializer(mode="wrap") + def _dump_as_served(self, handler: SerializerFunctionWrapHandler) -> JsonDict: + # Omitted when it was never given, so a schema served without `required` dumps + # without one, as it did when `required` was only an extra field. + dumped = handler(self) + if "required" not in self.model_fields_set: + dumped.pop("required", None) + return dumped diff --git a/tests/conftest.py b/tests/conftest.py index 9d1a2372..5db33b02 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,16 +6,67 @@ import socket import subprocess import time -from collections.abc import Generator +from collections.abc import Generator, Iterator +from contextlib import contextmanager from pathlib import Path +from typing import Any import pytest -# Test base URL - used instead of production URLs in all test mocks. -# Since respx intercepts at the HTTP client level before DNS resolution, -# any URL string works for matching; http://localhost avoids exposing -# real infrastructure URLs. -TEST_BASE_URL = "http://localhost" + +@pytest.fixture(autouse=True) +def _no_account_id_in_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + """A STACKONE_ACCOUNT_ID in the developer's shell would make every toolset warn.""" + monkeypatch.delenv("STACKONE_ACCOUNT_ID", raising=False) + + +_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"} + + +@pytest.fixture(autouse=True) +def _no_network(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[str]]: + """A test that reaches a real host passes or fails on that host, not on the code. + + Loopback stays open for the MCP mock server. The refusal is raised, and the test also + fails afterwards: code under test that swallows a failed request would otherwise hide it. + Yields the hosts refused so far. + """ + refused: list[str] = [] + + def _refuse_unless_loopback(address: object) -> None: + host = address[0] if isinstance(address, tuple) else address + if isinstance(host, str) and host not in _LOOPBACK_HOSTS: + refused.append(host) + raise RuntimeError( + f"Tests must not reach the network: a connection to {host!r} was refused. Stub it." + ) + + connect, connect_ex, create_connection = ( + socket.socket.connect, + socket.socket.connect_ex, + socket.create_connection, + ) + + def guarded_connect(self: socket.socket, address: Any) -> None: + _refuse_unless_loopback(address) + return connect(self, address) + + def guarded_connect_ex(self: socket.socket, address: Any) -> int: + _refuse_unless_loopback(address) + return connect_ex(self, address) + + def guarded_create_connection(address: Any, *args: Any, **kwargs: Any) -> socket.socket: + _refuse_unless_loopback(address) + return create_connection(address, *args, **kwargs) + + monkeypatch.setattr(socket.socket, "connect", guarded_connect) + monkeypatch.setattr(socket.socket, "connect_ex", guarded_connect_ex) + monkeypatch.setattr(socket, "create_connection", guarded_create_connection) + yield refused + if refused: + pytest.fail( + f"Tests must not reach the network: connections to {', '.join(refused)} were refused. Stub them." + ) def _find_free_port() -> int: @@ -38,23 +89,9 @@ def _wait_for_server(host: str, port: int, timeout: float = 10.0) -> bool: return False -@pytest.fixture(scope="session") -def mcp_mock_server() -> Generator[str, None, None]: - """ - Start the Node MCP mock server for integration tests. - - This fixture starts the Hono-based MCP mock server using tsx. - - Requires: pnpm install (provides tsx and the Hono dependencies). - - Usage: - def test_mcp_integration(mcp_mock_server): - toolset = StackOneToolSet( - api_key="test-key", - base_url=mcp_mock_server, - ) - tools = toolset.fetch_tools() - """ +@contextmanager +def _run_mcp_mock_server(extra_env: dict[str, str] | None = None) -> Iterator[str]: + """Start the Node MCP mock server, yield its base URL, and stop it afterwards.""" project_root = Path(__file__).parent.parent serve_script = project_root / "tests" / "mocks" / "serve.ts" @@ -75,6 +112,7 @@ def test_mcp_integration(mcp_mock_server): # Start the server from project root env = os.environ.copy() env["PORT"] = str(port) + env.update(extra_env or {}) process = subprocess.Popen( [str(serve_script)], @@ -107,3 +145,42 @@ def test_mcp_integration(mcp_mock_server): except subprocess.TimeoutExpired: process.kill() process.wait() + + +@pytest.fixture(scope="session") +def mcp_mock_server() -> Generator[str, None, None]: + """ + Start the Node MCP mock server for integration tests. + + This fixture starts the Hono-based MCP mock server using tsx. It serves the + global feedback tool, as a project with feedback enabled does. + + Requires: pnpm install (provides tsx and the Hono dependencies). + + Usage: + def test_mcp_integration(mcp_mock_server): + toolset = StackOneToolSet( + api_key="test-key", + base_url=mcp_mock_server, + ) + tools = toolset.fetch_tools() + """ + with _run_mcp_mock_server() as base_url: + yield base_url + + +@pytest.fixture(scope="session") +def mcp_mock_server_without_feedback() -> Generator[str, None, None]: + """The same mock, serving a project with feedback disabled: the tool is absent.""" + with _run_mcp_mock_server({"MOCK_SUBMIT_FEEDBACK": "off"}) as base_url: + yield base_url + + +@pytest.fixture(scope="session") +def mcp_mock_server_with_end_users() -> Generator[str, None, None]: + """The same mock, listing acc1 as non-shared (end user ``end-user-1``) and acc2 as shared. + + Like the real API, it refuses an MCP request for acc1 that lacks acc1's ``x-end-user-id``. + """ + with _run_mcp_mock_server({"MOCK_END_USERS": "on"}) as base_url: + yield base_url diff --git a/tests/mocks/mcp-server.ts b/tests/mocks/mcp-server.ts index b0875041..d5492602 100644 --- a/tests/mocks/mcp-server.ts +++ b/tests/mocks/mcp-server.ts @@ -2,13 +2,20 @@ * Mock MCP server for testing using Hono's app.request() method. * This creates an MCP-compatible handler that can be used with MSW * without starting a real HTTP server. + * + * It refuses what the real endpoint refuses — an unscoped request, an unknown account — so a + * client bug that sends no account, or the wrong one, fails here instead of in production. */ import type { Hono as HonoApp } from 'hono'; import { StreamableHTTPTransport } from '@hono/mcp'; -import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { + CallToolRequestSchema, + type CallToolResult, + ListToolsRequestSchema, +} from '@modelcontextprotocol/sdk/types.js'; import { Hono } from 'hono'; import { basicAuth } from 'hono/basic-auth'; -import { z } from 'zod'; export interface McpToolDefinition { name: string; @@ -18,39 +25,190 @@ export interface McpToolDefinition { properties?: Record; required?: string[]; additionalProperties?: boolean; + [keyword: string]: unknown; }; } export interface MockMcpServerOptions { - /** Tools available per account ID. Use 'default' for tools when no account header is provided. */ + /** + * Tools available per account ID. An account not listed here is refused with a 404, except a + * `ratelimit-*` id, which serves the `default` catalog after its 429s. + */ accountTools: Record; + /** + * Serve the global `stackone_submit_feedback` tool, in every tool mode, the way the real + * endpoint does when the org flag and project setting are on. Off models a project without + * it: the tool is simply absent, so the SDK has nothing to call. + */ + submitFeedback?: boolean; } +/** Mirrors the real `session_id` minted per search, so tests can assert it is carried through. */ +export const MOCK_SEARCH_SESSION_ID = 'mock-session-1'; + +/** The schema the real endpoint serves for its feedback tool, field for field. */ +const submitFeedbackTool = { + name: 'stackone_submit_feedback', + description: 'Records a structured verdict on how well the tools served this session.', + inputSchema: { + type: 'object', + properties: { + rating: { type: 'string', enum: ['positive', 'negative', 'neutral'] }, + feedback: { type: 'string' }, + tool_names: { type: 'array', items: { type: 'string' } }, + source: { type: 'string', enum: ['model', 'user', 'system'], default: 'model' }, + category: { + type: 'string', + enum: ['search', 'execute', 'defender', 'connection', 'general'], + }, + session_id: { type: 'string' }, + }, + required: ['rating', 'tool_names'], + }, +} as const satisfies McpToolDefinition; + +/** + * The two meta tools the real endpoint serves per connector under `tool-mode=search_execute`. + * Without them the SDK's search/execute path has nothing to talk to and goes untested. + */ +const metaTools = (accountId: string): McpToolDefinition[] => [ + { + name: `mock_${accountId}_search_actions`, + description: 'Search for available actions in natural language.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string' }, + top_k: { type: 'integer', minimum: 1, maximum: 50 }, + session_id: { type: 'string' }, + }, + required: ['query'], + }, + }, + { + name: `mock_${accountId}_execute_action`, + description: 'Execute an action by its action_id.', + inputSchema: { + type: 'object', + properties: { + action_id: { type: 'string' }, + path: { type: 'object' }, + query: { type: 'object' }, + body: { type: 'object' }, + headers: { type: 'object' }, + session_id: { type: 'string' }, + }, + required: ['action_id'], + }, + }, +]; + +const text = (payload: unknown, isError = false): CallToolResult => ({ + isError, + content: [{ type: 'text', text: JSON.stringify(payload) }], +}); + +/** + * A result the way UCA's `createStructuredOutput` shapes it: `{ isError, result }` as + * structuredContent, and the same JSON as a text part. + */ +const uca = (result: unknown, isError = false, extra: Record = {}): CallToolResult => { + const structured = { isError, result, ...extra }; + return { + isError, + content: [{ type: 'text', text: JSON.stringify(structured) }], + structuredContent: structured, + }; +}; + +/** The download-link shape UCA returns for a file action over tools/call (UCA #8861). */ +export const MOCK_DOWNLOAD_LINK = { + download_url: 'https://downloads.example.com/f/abc123?sig=xyz', + expires_at: '2026-01-01T00:00:00.000Z', + file: { name: 'report.pdf', content_type: 'application/pdf', content_length: 1024 }, +}; + +/** + * How a per-action tool answers, modelled on UCA's per-action handler: a success wrapper + * around the action's payload, or an isError wrapper carrying the status. + */ +const callActionTool = (name: string, args: Record): CallToolResult => { + switch (name) { + case 'bamboohr_list_employees': + return uca({ + data: [ + { id: '1', name: 'Employee 1' }, + { id: '2', name: 'Employee 2' }, + ], + }); + case 'bamboohr_get_employee': + return uca({ data: { id: args.id ?? 'test-id', name: 'Test Employee' } }); + case 'files_download_file': + return uca(MOCK_DOWNLOAD_LINK); + case 'files_download_unavailable': + return uca( + { statusCode: 501, message: 'Download links are not available for this action' }, + true, + ); + case 'files_list_defended': + return uca({ data: [] }, false, { defenderMetadata: { scanned: true, flagged: 0 } }); + case 'files_count': + return uca(3); + default: + return uca({ data: { action: name, received: args } }); + } +}; + +const callMetaTool = (name: string, args: Record): CallToolResult | undefined => { + if (name.endsWith('_search_actions')) { + return text({ + session_id: MOCK_SEARCH_SESSION_ID, + actions: [ + { + action_id: 'mock_list_items', + description: 'List items', + example_request: { query: { page_size: 25 } }, + }, + ], + }); + } + if (name.endsWith('_execute_action')) { + // An unknown action must come back as isError — a normal response with the flag set — + // the way the real endpoint reports it. + if (args.action_id !== 'mock_list_items') { + return text({ error: `Unknown action ${String(args.action_id)}` }, true); + } + return uca({ data: { nodes: [] }, echoed_query: args.query ?? null }); + } + return undefined; +}; + +const hasMethod = async (request: Request, method: string): Promise => { + try { + const payload = (await request.clone().json()) as { method?: string } | { method?: string }[]; + const messages = Array.isArray(payload) ? payload : [payload]; + return messages.some((message) => message?.method === method); + } catch { + return false; + } +}; + /** - * Creates an MSW handler for mocking MCP protocol requests. - * Uses Hono's app.request() to handle requests without starting a server. + * Creates a Hono app speaking the MCP protocol at `/mcp`. * * @example * ```ts - * import { server } from './mocks/node'; - * import { createMcpHandler, defaultMcpTools, accountMcpTools } from './mocks/mcp-server'; + * import { createMcpApp, accountMcpTools } from './mcp-server'; * - * // In your test setup - * server.use( - * createMcpHandler({ - * accountTools: { - * default: defaultMcpTools, - * 'account-1': accountMcpTools.acc1, - * }, - * }) - * ); + * const app = createMcpApp({ accountTools: { acc1: accountMcpTools.acc1 } }); * ``` */ export function createMcpApp(options: MockMcpServerOptions): HonoApp { - const { accountTools } = options; + const { accountTools, submitFeedback = false } = options; // Create a Hono app that handles MCP protocol const app = new Hono(); + const rateLimitCounts = new Map(); // Apply Basic Auth middleware with hardcoded test credentials app.use( @@ -76,95 +234,82 @@ export function createMcpApp(options: MockMcpServerOptions): HonoApp { // that does not exist is the same permissiveness that hid the missing-header bug // one line further up: any bug that sends a wrong, stale or mangled account id // would be invisible. The real API refuses. - if (!(accountId in accountTools)) { - return c.json({ statusCode: 404, message: `Unknown account ${accountId}` }, 404); - } - let tools = accountTools[accountId] ?? []; - - // The real endpoint swaps the per-action catalog for two meta tools per - // connector under this mode. Without it the SDK's search/execute path has - // nothing to talk to and goes untested. - - // Create a new MCP server instance per request - const mcp = new McpServer({ name: 'test-mcp-server', version: '1.0.0' }); - const searchExecute = c.req.query('tool-mode') === 'search_execute'; - const transport = new StreamableHTTPTransport(); - - if (searchExecute) { - mcp.registerTool( - `mock_${accountId}_search_actions`, - { - description: 'Search for available actions in natural language.', - inputSchema: { query: z.string(), top_k: z.number().optional() }, - }, - async () => ({ - content: [ - { - type: 'text' as const, - text: JSON.stringify({ - actions: [ - { - action_id: 'mock_list_items', - description: 'List items', - example_request: { query: { page_size: 25 } }, - }, - ], - }), - }, - ], - }), - ); - mcp.registerTool( - `mock_${accountId}_execute_action`, + // What the real endpoint answers for an account linked before MCP existed, verbatim. + if (accountId === 'legacy-account') { + return c.json( { - description: 'Execute an action by its action_id.', - inputSchema: { - action_id: z.string(), - path: z.record(z.string(), z.unknown()).optional(), - query: z.record(z.string(), z.unknown()).optional(), - body: z.record(z.string(), z.unknown()).optional(), - headers: z.record(z.string(), z.string()).optional(), + jsonrpc: '2.0', + error: { + code: -32600, + message: 'Legacy accounts cannot be used with MCP. Account provider: mock, version: 2', }, + id: null, }, - async ({ action_id, query }: { action_id: string; query?: Record }) => { - // An unknown action must come back as isError — a normal response with - // the flag set — the way the real endpoint reports it. - const known = action_id === 'mock_list_items'; - return { - isError: !known, - content: [ - { - type: 'text' as const, - text: JSON.stringify( - known - ? { data: { nodes: [] }, echoed_query: query ?? null } - : { error: `Unknown action ${action_id}` }, - ), - }, - ], - }; - }, + 400, ); - await mcp.connect(transport); - return transport.handleRequest(c); } - - for (const tool of tools) { - mcp.registerTool( - tool.name, - { - description: tool.description, - // MCP SDK expects Zod-like schema but accepts JSON Schema objects - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- MCP SDK type mismatch - inputSchema: tool.inputSchema as any, - }, - async ({ params }: { params: { arguments?: Record } }) => { - const args = params.arguments ?? {}; - return { content: [], structuredContent: args, _meta: undefined }; - }, - ); + // `ratelimit---[after-]` answers 429 (`Retry-After: 0`, + // or ``) to the first n requests for that exact id, or every one, and then serves the + // default catalog. `all` counts every request, `list` only tools/list and `call` only + // tools/call; the tag keeps tests' counters apart. + const rateLimit = /^ratelimit-(all|list|call)-(\d+|always)-(?:after(\d+)-)?/.exec(accountId); + if (rateLimit) { + const [, scope, times, retryAfter = '0'] = rateLimit; + const method = { list: 'tools/list', call: 'tools/call' }[scope]; + if (!method || (await hasMethod(c.req.raw, method))) { + const seen = (rateLimitCounts.get(accountId) ?? 0) + 1; + rateLimitCounts.set(accountId, seen); + if (times === 'always' || seen <= Number(times)) { + return c.json({ statusCode: 429, message: 'Too many requests' }, 429, { + 'Retry-After': retryAfter, + }); + } + } + } else if (!Object.hasOwn(accountTools, accountId)) { + return c.json({ statusCode: 404, message: `Unknown account ${accountId}` }, 404); } + const accountCatalog = (rateLimit ? accountTools.default : accountTools[accountId]) ?? []; + + // The feedback tool comes first because the real endpoint does the same: it is one + // global tool, served identically in both modes and once per account listing. Under + // search_execute the per-action catalog is swapped for two meta tools per connector. + const searchExecute = c.req.query('tool-mode') === 'search_execute'; + const listed: McpToolDefinition[] = [ + ...(submitFeedback ? [submitFeedbackTool] : []), + ...(searchExecute ? metaTools(accountId) : accountCatalog), + ]; + + // A low-level Server rather than McpServer: McpServer.registerTool expects a Zod shape + // and lists a plain JSON Schema as `properties: {}`, so no test ever saw a declared + // parameter of a per-action tool. The real endpoint serves each schema verbatim, and + // so does this. + const mcp = new Server( + { name: 'test-mcp-server', version: '1.0.0' }, + { capabilities: { tools: {} } }, + ); + mcp.setRequestHandler(ListToolsRequestSchema, () => ({ + tools: listed.map((tool) => ({ ...tool, inputSchema: structuredClone(tool.inputSchema) })), + })); + mcp.setRequestHandler(CallToolRequestSchema, (request): CallToolResult => { + const { name } = request.params; + const args = request.params.arguments ?? {}; + if (!listed.some((tool) => tool.name === name)) { + return text({ error: `Unknown tool ${name}` }, true); + } + if (name === submitFeedbackTool.name) { + return text({ + message: 'Feedback recorded', + submitted_at: new Date(0).toISOString(), + session_id: args.session_id ?? null, + }); + } + // Only search_execute serves meta tools: in individual mode a per-action tool whose + // name ends in `_execute_action` is still an action. + return (searchExecute ? callMetaTool(name, args) : undefined) ?? callActionTool(name, args); + }); + + const transport = new StreamableHTTPTransport(); await mcp.connect(transport); return transport.handleRequest(c); }); @@ -308,6 +453,47 @@ export const exampleBamboohrTools = [ }, ] as const satisfies McpToolDefinition[]; +/** File actions: a download link, a download that cannot be issued, and metadata beside a result. */ +export const fileTools = [ + { + name: 'files_download_file', + description: 'Download a file', + inputSchema: { + type: 'object', + properties: { id: { type: 'string' } }, + required: ['id'], + }, + }, + { + name: 'files_download_unavailable', + description: 'Download a file no link can be issued for', + inputSchema: { + type: 'object', + properties: { id: { type: 'string' } }, + required: ['id'], + }, + }, + { + name: 'files_list_defended', + description: 'List files, with defender metadata', + inputSchema: { type: 'object', properties: {} }, + }, + { + name: 'files_count', + description: 'Count files: a result that is not an object', + inputSchema: { type: 'object', properties: {} }, + }, +] as const satisfies McpToolDefinition[]; + +/** A per-action tool whose name ends like a meta tool's, served in individual mode. */ +export const metaLookalikeTools = [ + { + name: 'lookalike_execute_action', + description: 'An action named like a meta tool', + inputSchema: { type: 'object', properties: { action_id: { type: 'string' } } }, + }, +] as const satisfies McpToolDefinition[]; + export const mixedProviderTools = [ { name: 'hibob_list_employees', diff --git a/tests/mocks/serve.ts b/tests/mocks/serve.ts index 7fb73392..6b0bf517 100755 --- a/tests/mocks/serve.ts +++ b/tests/mocks/serve.ts @@ -1,7 +1,7 @@ #!/usr/bin/env -S pnpm exec tsx /** * Standalone HTTP server for MCP mock testing. - * Imports createMcpApp from stackone-ai-node vendor submodule. + * Serves the MCP endpoint every tool is listed from and executed on, plus /accounts. * * Usage: * ./tests/mocks/serve.ts [port] @@ -16,11 +16,45 @@ import { createMcpApp, defaultMcpTools, exampleBamboohrTools, + fileTools, + metaLookalikeTools, mixedProviderTools, } from "./mcp-server"; const port = parseInt(process.env.PORT || process.argv[2] || "8787", 10); +// On by default, as it is for a project with feedback enabled. MOCK_SUBMIT_FEEDBACK=off +// serves a project without it, so the SDK's handling of its absence can be tested. +const submitFeedback = process.env.MOCK_SUBMIT_FEEDBACK !== "off"; + +// MOCK_END_USERS=on lists acc1 as a non-shared account and acc2 as a shared one, and +// refuses an MCP request for acc1 without acc1's end-user id, as the real API does. +const endUsers = process.env.MOCK_END_USERS === "on"; +const END_USER_IDS: Record = { acc1: "end-user-1" }; + +interface RecordedRequest { + path: "/mcp"; + /** The query string the call was made with, e.g. "" or "?tool-mode=search_execute". */ + search: string; + accountId: string | null; + method: string; + name?: string; + arguments?: unknown; +} + +interface RecordedHeaders { + method: string; + accountId: string | null; + endUserId: string | null; +} + +// The SDK-set headers of every JSON-RPC message, initialize and notifications included. +const recordedHeaders: RecordedHeaders[] = []; + +// What actually reached the wire: each JSON-RPC payload exactly as the SDK serialised it, +// so a test of the SDK's wire shape sees nulls and key order as sent. +const recorded: RecordedRequest[] = []; + // Create the MCP app with all test tool configurations const mcpApp = createMcpApp({ accountTools: { @@ -30,9 +64,12 @@ const mcpApp = createMcpApp({ acc3: accountMcpTools.acc3, "test-account": accountMcpTools["test-account"], mixed: mixedProviderTools, + files: fileTools, + lookalike: metaLookalikeTools, "your-bamboohr-account-id": exampleBamboohrTools, "your-stackone-account-id": exampleBamboohrTools, }, + submitFeedback, }); // Create the main app with CORS and mount the MCP app @@ -44,101 +81,70 @@ app.use("/*", cors()); // Health check endpoint app.get("/health", (c) => c.json({ status: "ok" })); -// The SDK discovers accounts here when none is supplied. Returned as a bare list -// with an inactive entry, matching the shape and statuses the real API serves. -app.get("/accounts", (c) => - c.json([ - { id: "default", provider: "testprovider", status: "active" }, - { id: "dead", provider: "brokenprovider", status: "error" }, - ]), -); - -// Mount the MCP app (handles /mcp endpoint) -app.route("/", mcpApp); - -// RPC endpoint for tool execution -app.post("/actions/rpc", async (c) => { - const authHeader = c.req.header("Authorization"); - const accountIdHeader = c.req.header("x-account-id"); +app.get("/__requests", (c) => c.json(recorded)); +app.get("/__headers", (c) => c.json(recordedHeaders)); +app.delete("/__requests", (c) => { + recorded.length = 0; + recordedHeaders.length = 0; + return c.json({ cleared: true }); +}); - // Check for authentication - if (!authHeader || !authHeader.startsWith("Basic ")) { - return c.json( - { error: "Unauthorized", message: "Missing or invalid authorization header" }, - 401, - ); +app.use("/mcp", async (c, next) => { + const accountId = c.req.header("x-account-id") ?? c.req.query("x-account-id") ?? null; + const endUserId = c.req.header("x-end-user-id") ?? null; + if (c.req.method === "POST") { + try { + const payload = (await c.req.raw.clone().json()) as unknown; + const messages = Array.isArray(payload) ? payload : [payload]; + for (const message of messages as { method?: string; params?: Record }[]) { + if (typeof message?.method === "string") { + recordedHeaders.push({ method: message.method, accountId, endUserId }); + } + if (message?.method !== "tools/call" && message?.method !== "tools/list") continue; + recorded.push({ + path: "/mcp", + search: new URL(c.req.url).search, + accountId, + method: message.method, + name: message.params?.name as string | undefined, + arguments: message.params?.arguments, + }); + } + } catch { + // Not JSON: the MCP handler rejects it, so there is nothing to record. + } } - - // Execution is account-scoped too. This endpoint used to accept anything with a - // "Basic " prefix and no account at all, so the sibling of the bug that shipped — - // an unscoped execution request — could not be caught by any test. - if (!accountIdHeader) { + const expected = endUsers && accountId ? END_USER_IDS[accountId] : undefined; + if (expected !== undefined && endUserId !== expected) { return c.json( - { error: "Bad Request", message: "Missing x-account-id header in request" }, + { + statusCode: 400, + message: `x-end-user-id header does not match account end user id for account ${accountId}`, + }, 400, ); } + await next(); +}); - const body = (await c.req.json()) as { - action?: string; - body?: Record; - headers?: Record; - path?: Record; - query?: Record; - }; - - // Validate action is provided - if (!body.action) { - return c.json({ error: "Bad Request", message: "Action is required" }, 400); - } - - // Test action to verify x-account-id is sent as HTTP header - if (body.action === "test_account_id_header") { - return c.json({ - data: { - httpHeader: accountIdHeader, - bodyHeader: body.headers?.["x-account-id"], - }, - }); - } - - // Return mock response based on action - if (body.action === "bamboohr_get_employee") { - return c.json({ - data: { - id: body.path?.id || "test-id", - name: "Test Employee", - ...body.body, - }, - }); - } - - if (body.action === "bamboohr_list_employees") { - return c.json({ - data: [ - { id: "1", name: "Employee 1" }, - { id: "2", name: "Employee 2" }, - ], - }); - } - - if (body.action === "test_error_action") { - return c.json({ error: "Internal Server Error", message: "Test error response" }, 500); - } +// The SDK discovers accounts here when none is supplied. Returned as a bare list +// with an inactive entry, matching the shape and statuses the real API serves. +app.get("/accounts", (c) => + c.json( + endUsers + ? [ + { id: "acc1", provider: "p", status: "active", shared: false, origin_username: "end-user-1" }, + { id: "acc2", provider: "p", status: "active", shared: true, origin_username: "owner" }, + ] + : [ + { id: "default", provider: "testprovider", status: "active" }, + { id: "dead", provider: "brokenprovider", status: "error" }, + ], + ), +); - // Default response for other actions - return c.json({ - data: { - action: body.action, - received: { - body: body.body, - headers: body.headers, - path: body.path, - query: body.query, - }, - }, - }); -}); +// Mount the MCP app (handles /mcp endpoint) +app.route("/", mcpApp); console.log(`MCP Mock Server starting on port ${port}...`); @@ -148,4 +154,6 @@ console.log(`MCP Mock Server running at http://localhost:${port}`); console.log("Endpoints:"); console.log(` - GET /health - Health check`); console.log(` - ALL /mcp - MCP protocol endpoint`); -console.log(` - POST /actions/rpc - RPC execution endpoint`); +console.log(` - GET /accounts - Linked accounts`); +console.log(` - GET /__requests - tools/call requests received`); +console.log(` - GET /__headers - the headers of every JSON-RPC message received`); diff --git a/tests/test_end_user_id.py b/tests/test_end_user_id.py new file mode 100644 index 00000000..408b9df2 --- /dev/null +++ b/tests/test_end_user_id.py @@ -0,0 +1,772 @@ +"""x-end-user-id: the end user a non-shared account belongs to. + +The API refuses an MCP request for a non-shared account that does not carry the account's +end-user id. GET /accounts names it (``origin_username``), so whenever the SDK lists +accounts it records it, and sends it on every MCP request for that account. +""" + +from __future__ import annotations + +import concurrent.futures +import json +import logging +import threading +from typing import Any + +import httpx +import pytest + +from stackone_ai.tools import McpToolDefinition, StackOneMcpTool +from stackone_ai.toolset import StackOneToolSet +from stackone_ai.types import StackOneAPIError, ToolParameters, ToolsetConfigError + +UCA_REFUSAL = "x-end-user-id header does not match account end user id for account acc1" + + +def _reset(base_url: str) -> None: + httpx.delete(f"{base_url}/__requests").raise_for_status() + + +def _headers(base_url: str) -> list[dict[str, Any]]: + """Every JSON-RPC message the mock received: its method, x-account-id and x-end-user-id.""" + response = httpx.get(f"{base_url}/__headers") + response.raise_for_status() + return response.json() + + +def _end_user_ids(base_url: str, account: str) -> set[str | None]: + return {r["endUserId"] for r in _headers(base_url) if r["accountId"] == account} + + +def _methods(base_url: str, account: str) -> set[str]: + return {r["method"] for r in _headers(base_url) if r["accountId"] == account} + + +class TestOverTheWire: + """Against a mock that, like the API, refuses acc1's requests without acc1's end user.""" + + def test_a_query_scoped_request_without_the_header_is_refused(self, mcp_mock_server_with_end_users: str): + """The account id can arrive as a query parameter, as the real mounted MCP app allows.""" + base_url = mcp_mock_server_with_end_users + response = httpx.post( + f"{base_url}/mcp?x-account-id=acc1", + auth=("test-key", ""), + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + ) + assert response.status_code == 400 + assert response.json()["message"] == UCA_REFUSAL + + def test_discovery_sends_it_on_every_listing_request(self, mcp_mock_server_with_end_users: str): + base_url = mcp_mock_server_with_end_users + _reset(base_url) + tools = StackOneToolSet(api_key="test-key", base_url=base_url, include_non_shared=True).fetch_tools() + + assert {"acc1_tool_1", "acc2_tool_1"} <= {t.name for t in tools} + assert {"initialize", "tools/list"} <= _methods(base_url, "acc1") + assert _end_user_ids(base_url, "acc1") == {"end-user-1"} + # Shared: no end user, so no header. + assert _end_user_ids(base_url, "acc2") == {None} + + def test_a_listed_tool_sends_it_on_every_call_request(self, mcp_mock_server_with_end_users: str): + base_url = mcp_mock_server_with_end_users + tools = StackOneToolSet(api_key="test-key", base_url=base_url, include_non_shared=True).fetch_tools() + _reset(base_url) + + for name in ("acc1_tool_1", "acc2_tool_1"): + tool = tools.get_tool(name) + assert tool is not None + tool.execute({"fields": "id"}) + + assert {"initialize", "tools/call"} <= _methods(base_url, "acc1") + assert _end_user_ids(base_url, "acc1") == {"end-user-1"} + assert _end_user_ids(base_url, "acc2") == {None} + + def test_meta_tools_and_feedback_send_it(self, mcp_mock_server_with_end_users: str): + base_url = mcp_mock_server_with_end_users + toolset = StackOneToolSet(api_key="test-key", base_url=base_url, include_non_shared=True) + _reset(base_url) + + hits = toolset.search("list items", account_ids=None) + assert any(hit["account_id"] == "acc1" for hit in hits) + toolset.execute("mock_list_items", {}, account_ids=["acc1"]) + toolset.submit_feedback("positive", ["mock_list_items"], account_ids=["acc1"]) + + called = { + r.get("name") for r in httpx.get(f"{base_url}/__requests").json() if r["accountId"] == "acc1" + } + assert {"mock_acc1_search_actions", "mock_acc1_execute_action", "stackone_submit_feedback"} <= called + assert _end_user_ids(base_url, "acc1") == {"end-user-1"} + + def test_a_direct_fetch_accounts_records_it_for_explicit_ids(self, mcp_mock_server_with_end_users: str): + base_url = mcp_mock_server_with_end_users + toolset = StackOneToolSet(api_key="test-key", base_url=base_url, account_id="acc1") + toolset.fetch_accounts() + _reset(base_url) + + tool = toolset.fetch_tools().get_tool("acc1_tool_1") + assert tool is not None + tool.execute({}) + + assert _end_user_ids(base_url, "acc1") == {"end-user-1"} + + def test_an_explicit_id_looks_its_end_user_up_when_the_api_asks( + self, mcp_mock_server_with_end_users: str + ): + """No GET /accounts was made, so the first request goes without; the 400 prompts one.""" + base_url = mcp_mock_server_with_end_users + toolset = StackOneToolSet(api_key="test-key", base_url=base_url) + _reset(base_url) + + assert toolset.fetch_tools(account_ids=["acc2"]).get_tool("acc2_tool_1") is not None + assert _end_user_ids(base_url, "acc2") == {None} + assert toolset.fetch_tools(account_ids=["acc1"]).get_tool("acc1_tool_1") is not None + # Refused once without it, then listed with it. + assert _end_user_ids(base_url, "acc1") == {None, "end-user-1"} + assert toolset._end_user_id("acc1") == "end-user-1" + + def test_a_model_supplied_end_user_is_not_forwarded(self, mcp_mock_server_with_end_users: str): + base_url = mcp_mock_server_with_end_users + toolset = StackOneToolSet(api_key="test-key", base_url=base_url) + toolset.fetch_accounts() + _reset(base_url) + + toolset.execute( + "mock_list_items", + {"headers": {"x-custom": "kept", "X-End-User-Id": "victim"}, "headers_x-end-user-id": "victim"}, + account_ids=["acc1"], + ) + + [call] = [r for r in httpx.get(f"{base_url}/__requests").json() if r["method"] == "tools/call"] + assert call["arguments"] == {"headers": {"x-custom": "kept"}, "action_id": "mock_list_items"} + assert _end_user_ids(base_url, "acc1") == {"end-user-1"} + + +def _accounts_response(monkeypatch: pytest.MonkeyPatch, *bodies: Any) -> list[int]: + """Answer each GET /accounts with the next body: a JSON value, or an int status to fail with.""" + queue = list(bodies) + calls: list[int] = [] + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + calls.append(1) + body = queue.pop(0) + if isinstance(body, int): + return httpx.Response(body, text="nope", request=request) + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + return calls + + +def _account(account_id: str, **fields: Any) -> dict[str, Any]: + return {"id": account_id, "provider": "p", "status": "active", **fields} + + +class _Mcp: + """A fake MCP transport recording the headers of every listing and call.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.listed: list[dict[str, str]] = [] + self.called: list[dict[str, str]] = [] + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", self._list) + monkeypatch.setattr("stackone_ai.tools.call_mcp_tool", self._call) + + def _list(self, _endpoint: str, headers: dict[str, str], **_kwargs: Any) -> list[McpToolDefinition]: + self.listed.append(headers) + return [McpToolDefinition(name=f"tool_{headers['x-account-id']}", description="", input_schema={})] + + def _call(self, _endpoint: str, headers: dict[str, str], *_args: Any, **_kwargs: Any) -> dict[str, Any]: + self.called.append(headers) + return {"isError": False} + + +class TestRecording: + def test_only_a_non_shared_account_with_a_username_is_recorded(self, monkeypatch): + _accounts_response( + monkeypatch, + [ + _account("private", shared=False, origin_username="user-1"), + _account("shared", shared=True, origin_username="user-2"), + _account("unknown", origin_username="user-3"), + _account("truthy", shared="false", origin_username="user-4"), + _account("zero", shared=0, origin_username="user-5"), + _account("empty", shared=False, origin_username=""), + _account("null", shared=False, origin_username=None), + _account("number", shared=False, origin_username=7), + {"shared": False, "origin_username": "no-id"}, + "not-an-account", + ], + ) + toolset = StackOneToolSet(api_key="k") + toolset.fetch_accounts() + recorded = { + account: toolset._end_user_id(account) + for account in ("private", "shared", "unknown", "truthy", "zero", "empty", "null", "number") + } + assert recorded == { + "private": "user-1", + "shared": None, + "unknown": None, + "truthy": None, + "zero": None, + "empty": None, + "null": None, + "number": None, + } + + def test_each_successful_listing_replaces_the_record(self, monkeypatch): + _accounts_response( + monkeypatch, + {"data": [_account("a", shared=False, origin_username="user-a")]}, + [_account("b", shared=False, origin_username="user-b"), _account("a", shared=True)], + 500, + ) + toolset = StackOneToolSet(api_key="k") + toolset.fetch_accounts() + assert toolset._end_user_id("a") == "user-a" + + toolset.fetch_accounts() + assert (toolset._end_user_id("a"), toolset._end_user_id("b")) == (None, "user-b") + + # A failed listing records nothing, so it keeps the last one. + with pytest.raises(StackOneAPIError): + toolset.fetch_accounts() + assert toolset._end_user_id("b") == "user-b" + + def test_clearing_the_catalog_cache_keeps_the_record(self, monkeypatch): + _accounts_response(monkeypatch, [_account("a", shared=False, origin_username="user-a")]) + toolset = StackOneToolSet(api_key="k") + toolset.fetch_accounts() + toolset.clear_catalog_cache() + toolset.set_accounts(["a"]) + assert toolset._end_user_id("a") == "user-a" + + def test_an_older_started_call_does_not_overwrite_a_newer_ones_record(self, monkeypatch): + """fetch_accounts() racing another, e.g. via discovery: start order wins, not finish order.""" + toolset = StackOneToolSet(api_key="k") + first_started = threading.Event() + second_started = threading.Event() + release_first = threading.Event() + release_second = threading.Event() + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + if not first_started.is_set(): + first_started.set() + assert release_first.wait(timeout=5) + body = [_account("a", shared=False, origin_username="user-a")] + else: + second_started.set() + assert release_second.wait(timeout=5) + body = [_account("b", shared=False, origin_username="user-b")] + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + + failures: list[BaseException] = [] + completed: list[str] = [] + + def run(name: str) -> None: + # Thread swallows exceptions, so a failure in either call is collected and asserted. + try: + toolset.fetch_accounts() + completed.append(name) + except BaseException as exc: # noqa: BLE001 + failures.append(exc) + + first = threading.Thread(target=run, args=("first",)) + first.start() + assert first_started.wait(timeout=5) + + second = threading.Thread(target=run, args=("second",)) + second.start() + assert second_started.wait(timeout=5) + + # Started second, finishes first. + release_second.set() + second.join(timeout=5) + # Started first, finishes last: its stale response must not overwrite the + # newer-started call's record. + release_first.set() + first.join(timeout=5) + + assert not first.is_alive() and not second.is_alive() + assert failures == [] + # Both calls returned, so the stale first one really did finish last. + assert completed == ["second", "first"] + assert toolset._end_user_id("b") == "user-b" + assert toolset._end_user_id("a") is None + + +class TestHeaders: + def test_discovery_sends_it_on_listing_and_execution(self, monkeypatch): + _accounts_response( + monkeypatch, + [_account("a", shared=False, origin_username="user-a"), _account("b", shared=True)], + ) + mcp = _Mcp(monkeypatch) + tools = StackOneToolSet(api_key="k", include_non_shared=True).fetch_tools() + for tool in tools: + tool.execute({}) + + by_account = {h["x-account-id"]: h.get("x-end-user-id") for h in mcp.listed} + assert by_account == {"a": "user-a", "b": None} + assert {h["x-account-id"]: h.get("x-end-user-id") for h in mcp.called} == by_account + + def test_explicit_ids_make_no_accounts_request_and_send_none(self, monkeypatch): + calls = _accounts_response(monkeypatch) + mcp = _Mcp(monkeypatch) + StackOneToolSet(api_key="k", account_id="a").fetch_tools()[0].execute({}) + + assert calls == [] + assert "x-end-user-id" not in mcp.listed[0] + assert "x-end-user-id" not in mcp.called[0] + + def test_a_tool_built_before_the_listing_picks_it_up(self, monkeypatch): + _accounts_response( + monkeypatch, + [ + _account("a", shared=False, origin_username="user-a"), + _account("b", shared=False, origin_username="user-b"), + ], + ) + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k", account_id="a") + tool = toolset.fetch_tools()[0] + toolset.fetch_accounts() + + tool.execute({}) + # And follows the tool's account when it is rebound. + tool.set_account_id("b") + tool.execute({}) + + assert [h.get("x-end-user-id") for h in mcp.called] == ["user-a", "user-b"] + + def test_it_overrides_a_configured_header_which_is_otherwise_sent_as_given(self, monkeypatch): + _accounts_response(monkeypatch, [_account("a", shared=False, origin_username="user-a")]) + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k") + + def configured(account_id: str) -> StackOneMcpTool: + tool = StackOneMcpTool( + name="t", + description="", + parameters=ToolParameters(type="object", properties={}), + api_key="k", + endpoint="https://api.example.com/mcp", + account_id=account_id, + headers={" X-End-User-Id": "caller"}, + ) + tool._end_user_id_of = toolset._end_user_id + return tool + + with_record, without_record = configured("a"), configured("other") + without_record.execute({}) + toolset.fetch_accounts() + with_record.execute({}) + + assert mcp.called[0][" X-End-User-Id"] == "caller" + assert "x-end-user-id" not in mcp.called[0] + assert [name for name in mcp.called[1] if name.strip().lower() == "x-end-user-id"] == [ + "x-end-user-id" + ] + assert mcp.called[1]["x-end-user-id"] == "user-a" + + +class TestModelSuppliedHeader: + """A model may never pick the end user, even when the schema declares the header.""" + + @pytest.fixture + def called(self, monkeypatch) -> list[dict[str, Any]]: + seen: list[dict[str, Any]] = [] + monkeypatch.setattr( + "stackone_ai.tools.call_mcp_tool", + lambda _endpoint, headers, _name, arguments, **_kwargs: seen.append( + {"headers": headers, "arguments": arguments} + ) + or {}, + ) + return seen + + @staticmethod + def _tool(properties: dict[str, Any]) -> StackOneMcpTool: + return StackOneMcpTool( + name="t", + description="", + parameters=ToolParameters(type="object", properties=properties), + api_key="k", + endpoint="https://api.example.com/mcp", + account_id="a", + ) + + @pytest.mark.parametrize("name", ["x-end-user-id", "X-End-User-Id", " X-END-USER-ID\t"]) + @pytest.mark.parametrize( + "headers_schema", + [ + pytest.param({"type": "object"}, id="open"), + pytest.param( + {"type": "object", "properties": {"x-end-user-id": {"type": "string"}}}, id="declared" + ), + ], + ) + def test_a_nested_entry_is_dropped(self, called, caplog, name, headers_schema): + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + self._tool({"headers": headers_schema}).execute({"headers": {name: "victim", "x-other": "1"}}) + + expected = {"x-other": "1"} if headers_schema.get("properties") is None else {} + assert called[0]["arguments"] == {"headers": expected} + assert "x-end-user-id" not in called[0]["headers"] + assert f'Dropping header "{name.strip()}" from a tool call: set by the SDK' in caplog.text + + @pytest.mark.parametrize( + "key", ["headers_x-end-user-id", "headers_X-End-User-Id", "headers_ x-end-user-id"] + ) + def test_a_declared_flat_argument_is_dropped(self, called, caplog, key): + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + self._tool({key: {"type": "string"}}).execute({key: "victim", "q": 1}) + + assert called[0]["arguments"] == {"q": 1} + assert f'Dropping header argument "{key}" from a tool call: set by the SDK' in caplog.text + + +class TestRecordOrdering: + def test_an_older_call_records_when_a_newer_one_fails(self, monkeypatch): + """Ordered against the last write, not the last start: a newer call that fails must + not leave the older call's result unrecorded.""" + toolset = StackOneToolSet(api_key="k") + first_started = threading.Event() + release_first = threading.Event() + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + if not first_started.is_set(): + first_started.set() + assert release_first.wait(timeout=5) + body = [_account("a", shared=False, origin_username="user-a", provider="hris")] + return httpx.Response(200, json=body, request=request) + return httpx.Response(500, text="nope", request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + first = threading.Thread(target=toolset.fetch_accounts) + first.start() + assert first_started.wait(timeout=5) + + with pytest.raises(StackOneAPIError): + toolset.fetch_accounts() + release_first.set() + first.join(timeout=5) + + assert not first.is_alive() + assert toolset._end_user_id("a") == "user-a" + assert toolset._providers == {"a": "hris"} + + +class TestDiscoverySkipsNonSharedAccounts: + SKIPPED = ( + "Discovery skipped 2 non-shared account(s) (a, c): each belongs to a single end user. " + "Pass their account ids, or opt in to non-shared accounts, to use them." + ) + + @pytest.fixture + def accounts(self, monkeypatch: pytest.MonkeyPatch) -> list[int]: + return _accounts_response( + monkeypatch, + *[ + [ + _account("c", shared=False, origin_username="user-c"), + _account("b", shared=True), + _account("a", shared=False, origin_username="user-a"), + _account("d"), + _account("e", shared=False, status="error"), + ] + ] + * 3, + ) + + def test_by_default_with_one_warning_per_discovery(self, accounts, monkeypatch, caplog): + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k") + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + assert sorted(t.name for t in toolset.fetch_tools()) == ["tool_b", "tool_d"] + toolset.fetch_tools() + assert [r.getMessage() for r in caplog.records] == [self.SKIPPED] + assert sorted(h["x-account-id"] for h in mcp.listed) == ["b", "d"] + # Their end users are recorded all the same, for when their ids are passed. + assert (toolset._end_user_id("a"), toolset._end_user_id("c")) == ("user-a", "user-c") + + caplog.clear() + toolset.clear_catalog_cache() + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + toolset.fetch_tools() + assert [r.getMessage() for r in caplog.records] == [self.SKIPPED] + assert len(accounts) == 2 + + def test_passed_ids_are_used(self, accounts, monkeypatch): + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k") + toolset.fetch_tools() + toolset.fetch_tools(account_ids=["a"]) + assert mcp.listed[-1] == {**mcp.listed[-1], "x-account-id": "a", "x-end-user-id": "user-a"} + + def test_an_opt_in_includes_them(self, accounts, monkeypatch, caplog): + _Mcp(monkeypatch) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + tools = StackOneToolSet(api_key="k", include_non_shared=True).fetch_tools() + assert sorted(t.name for t in tools) == ["tool_a", "tool_b", "tool_c", "tool_d"] + assert caplog.records == [] + + def test_a_key_with_only_non_shared_accounts_is_refused(self, monkeypatch, caplog): + """Rather than serving nothing: 0 tools, no search hits and no feedback tool.""" + _accounts_response( + monkeypatch, + *[ + [ + _account("a", shared=False, origin_username="user-a"), + _account("b", shared=False, origin_username="user-b"), + _account("c", shared=True, status="error"), + ] + ] + * 3, + ) + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k") + message = ( + "None of this API key's 2 active account(s) are shared: each belongs to a single end " + "user. Pass their account ids, or opt in to non-shared accounts, to use them." + ) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + for call in ( + toolset.fetch_tools, + lambda: toolset.search("anything"), + lambda: toolset.submit_feedback("positive", ["x"]), + ): + with pytest.raises(ToolsetConfigError) as excinfo: + call() + assert str(excinfo.value) == message + assert mcp.listed == [] + assert caplog.records == [] + + +def _refusal(account: str) -> StackOneAPIError: + body = json.dumps({"statusCode": 400, "message": f"{UCA_REFUSAL[: -len('acc1')]}{account}"}) + return StackOneAPIError(f"MCP request failed with 400 Bad Request: {body}", 400, body) + + +class _GuardedMcp: + """A fake MCP transport that, like the API, refuses a non-shared account without its end user.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, end_users: dict[str, str]) -> None: + self.end_users = end_users + self.listed: list[dict[str, str]] = [] + self.called: list[dict[str, str]] = [] + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", self._list) + monkeypatch.setattr("stackone_ai.tools.call_mcp_tool", self._call) + + def _guard(self, headers: dict[str, str]) -> None: + account = headers["x-account-id"] + expected = self.end_users.get(account) + if expected is not None and headers.get("x-end-user-id") != expected: + raise _refusal(account) + + def _list(self, _endpoint: str, headers: dict[str, str], **_kwargs: Any) -> list[McpToolDefinition]: + self.listed.append(headers) + self._guard(headers) + return [McpToolDefinition(name=f"tool_{headers['x-account-id']}", description="", input_schema={})] + + def _call(self, _endpoint: str, headers: dict[str, str], *_args: Any, **_kwargs: Any) -> dict[str, Any]: + self.called.append(headers) + self._guard(headers) + return {"isError": False} + + +class TestAPassedIdLooksItsEndUserUp: + """An account passed by id has no end user recorded until something lists accounts; the + API's 400 asking for one prompts that listing, and one retry with what it recorded.""" + + def test_a_listing_is_retried_once_with_it(self, monkeypatch): + calls = _accounts_response(monkeypatch, [_account("a", shared=False, origin_username="user-a")]) + mcp = _GuardedMcp(monkeypatch, {"a": "user-a"}) + tools = StackOneToolSet(api_key="k", account_id="a").fetch_tools() + assert [t.name for t in tools] == ["tool_a"] + assert [h.get("x-end-user-id") for h in mcp.listed] == [None, "user-a"] + assert len(calls) == 1 + + def test_a_call_is_retried_once_with_it(self, monkeypatch): + calls = _accounts_response(monkeypatch, [_account("a", shared=False, origin_username="user-a")]) + mcp = _GuardedMcp(monkeypatch, {}) + tool = StackOneToolSet(api_key="k", account_id="a").fetch_tools()[0] + mcp.end_users["a"] = "user-a" + + tool.execute({}) + assert [h.get("x-end-user-id") for h in mcp.called] == [None, "user-a"] + assert len(calls) == 1 + + @pytest.mark.parametrize( + "accounts_response", + [ + pytest.param([_account("a", shared=True)], id="no-end-user-listed"), + pytest.param(500, id="listing-fails"), + ], + ) + def test_with_no_end_user_to_retry_with_the_400_is_raised(self, monkeypatch, accounts_response): + calls = _accounts_response(monkeypatch, accounts_response) + mcp = _GuardedMcp(monkeypatch, {"a": "user-a"}) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k", account_id="a").fetch_tools() + assert excinfo.value.status_code == 400 + assert excinfo.value.response_body == _refusal("a").response_body + assert len(mcp.listed) == 1 + assert len(calls) == 1 + + def test_a_failed_lookup_is_the_400s_cause(self, monkeypatch): + """A key without the scope to list accounts is told so, not only that the header is wrong.""" + _accounts_response(monkeypatch, 403, 403) + mcp = _GuardedMcp(monkeypatch, {}) + tool = StackOneToolSet(api_key="k", account_id="a").fetch_tools()[0] + mcp.end_users["a"] = "user-a" + with pytest.raises(StackOneAPIError) as called: + tool.execute({}) + + toolset = StackOneToolSet(api_key="k", account_id="a") + with pytest.raises(StackOneAPIError) as listed: + toolset.fetch_tools() + + for excinfo in (called, listed): + assert excinfo.value.status_code == 400 + cause = excinfo.value.__cause__ + assert isinstance(cause, StackOneAPIError) + assert cause.status_code == 403 + + def test_a_request_sent_with_a_recorded_end_user_is_not_retried(self, monkeypatch): + calls = _accounts_response(monkeypatch, [_account("a", shared=False, origin_username="stale")]) + mcp = _GuardedMcp(monkeypatch, {"a": "user-a"}) + toolset = StackOneToolSet(api_key="k", account_id="a") + toolset.fetch_accounts() + with pytest.raises(StackOneAPIError): + toolset.fetch_tools() + assert len(mcp.listed) == 1 + assert len(calls) == 1 + + @pytest.mark.parametrize( + "error", + [ + pytest.param(_refusal("a1"), id="another-account"), + pytest.param( + StackOneAPIError("400", 400, json.dumps({"message": "bad request"})), id="another-400" + ), + pytest.param(StackOneAPIError("401", 401, _refusal("a").response_body), id="not-a-400"), + ], + ) + def test_any_other_failure_is_not_retried(self, monkeypatch, error): + calls = _accounts_response(monkeypatch) + listed: list[int] = [] + + def refuse(*_args: Any, **_kwargs: Any) -> list[McpToolDefinition]: + listed.append(1) + raise error + + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", refuse) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k", account_id="a").fetch_tools() + assert excinfo.value is error + assert (calls, listed) == ([], [1]) + + def test_a_rate_limited_lookup_stays_fatal_to_a_fan_out(self, monkeypatch): + """Skipping the account would hand back a partial catalog, so the 429 is raised.""" + monkeypatch.setattr("stackone_ai.tools._sleep", lambda _delay: None) + _accounts_response(monkeypatch, 429, 429, 429, 429) + _GuardedMcp(monkeypatch, {"a": "user-a"}) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k").fetch_tools(account_ids=["a", "b"]) + assert excinfo.value.status_code == 429 + + def test_a_refusal_quoting_the_account_id_is_still_looked_up(self, monkeypatch): + calls = _accounts_response(monkeypatch, [_account("a", shared=False, origin_username="user-a")]) + + def guarded(_endpoint: str, headers: dict[str, str], **_kwargs: Any) -> list[McpToolDefinition]: + if headers.get("x-end-user-id") != "user-a": + body = json.dumps({"statusCode": 400, "message": f'{UCA_REFUSAL[: -len("acc1")]}"a"'}) + raise StackOneAPIError(f"MCP request failed with 400 Bad Request: {body}", 400, body) + return [McpToolDefinition(name="tool_a", description="", input_schema={})] + + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", guarded) + tools = StackOneToolSet(api_key="k", account_id="a").fetch_tools() + assert [tool.name for tool in tools] == ["tool_a"] + assert len(calls) == 1 + + def test_requests_refused_together_share_one_accounts_listing(self, monkeypatch): + joined = threading.Event() + + class Listing(concurrent.futures.Future): # type: ignore[type-arg] + def result(self, timeout: float | None = None) -> Any: + # The owner waits only once its own GET /accounts is done, so a wait while it + # is still held is the other account's, joining it. + joined.set() + return super().result(timeout) + + monkeypatch.setattr(concurrent.futures, "Future", Listing) + calls: list[int] = [] + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + calls.append(1) + assert joined.wait(timeout=5) + body = [ + _account("a", shared=False, origin_username="user-a"), + _account("b", shared=False, origin_username="user-b"), + ] + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + mcp = _GuardedMcp(monkeypatch, {"a": "user-a", "b": "user-b"}) + tools = StackOneToolSet(api_key="k").fetch_tools(account_ids=["a", "b"]) + assert sorted(t.name for t in tools) == ["tool_a", "tool_b"] + assert len(calls) == 1 + assert sorted((h["x-account-id"], h.get("x-end-user-id", "")) for h in mcp.listed) == [ + ("a", ""), + ("a", "user-a"), + ("b", ""), + ("b", "user-b"), + ] + + def test_a_lookup_joins_the_latest_accounts_listing_still_in_flight(self, monkeypatch): + """An older listing finishing must not stop a lookup joining a newer one in flight.""" + joined = threading.Event() + + class Listing(concurrent.futures.Future): # type: ignore[type-arg] + def result(self, timeout: float | None = None) -> Any: + joined.set() + return super().result(timeout) + + monkeypatch.setattr(concurrent.futures, "Future", Listing) + calls: list[int] = [] + started = [threading.Event(), threading.Event()] + release = [threading.Event(), threading.Event()] + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + index = len(calls) + calls.append(index) + if index < 2: + started[index].set() + assert release[index].wait(timeout=5) + body = [_account("a", shared=False, origin_username="user-a")] + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + toolset = StackOneToolSet(api_key="k") + older = threading.Thread(target=toolset.fetch_accounts) + older.start() + assert started[0].wait(timeout=5) + newer = threading.Thread(target=toolset.fetch_accounts) + newer.start() + assert started[1].wait(timeout=5) + release[0].set() + older.join(timeout=5) + + found: list[bool] = [] + lookup = threading.Thread(target=lambda: found.append(toolset._look_up_end_user("a", _refusal("a")))) + lookup.start() + assert joined.wait(timeout=5) + try: + assert len(calls) == 2 + finally: + release[1].set() + lookup.join(timeout=5) + newer.join(timeout=5) + assert found == [True] diff --git a/tests/test_feedback.py b/tests/test_feedback.py new file mode 100644 index 00000000..35073450 --- /dev/null +++ b/tests/test_feedback.py @@ -0,0 +1,374 @@ +"""The feedback tool and session_id linking, against the MCP mock server. + +Assertions about the wire read the mock's request log, which records the JSON-RPC +payload exactly as the SDK serialised it, so a key sent as null or out of order shows. +""" + +from __future__ import annotations + +import logging +from typing import Any + +import httpx +import pytest + +from stackone_ai.tools import McpToolDefinition, StackOneMcpTool +from stackone_ai.toolset import StackOneToolSet +from stackone_ai.types import SUBMIT_FEEDBACK_TOOL_NAME, ToolsetConfigError, ToolsetLoadError + +# Must match MOCK_SEARCH_SESSION_ID in tests/mocks/mcp-server.ts. +MOCK_SESSION_ID = "mock-session-1" + + +def _reset_requests(base_url: str) -> None: + httpx.delete(f"{base_url}/__requests").raise_for_status() + + +def _requests(base_url: str) -> list[dict[str, Any]]: + response = httpx.get(f"{base_url}/__requests") + response.raise_for_status() + return response.json() + + +def _tool_calls(base_url: str, name: str) -> list[dict[str, Any]]: + return [r for r in _requests(base_url) if r["path"] == "/mcp" and r.get("name") == name] + + +def _execute_calls(base_url: str) -> list[dict[str, Any]]: + return [ + r + for r in _requests(base_url) + if r["path"] == "/mcp" and str(r.get("name", "")).endswith("_execute_action") + ] + + +class TestFeedbackToolIsAnMcpTool: + """Contract 1: built as an MCP tool in every mode.""" + + @pytest.mark.parametrize("mode", [None, "individual", "search_execute"]) + def test_built_as_an_mcp_tool_in_every_mode(self, mcp_mock_server: str, mode): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, tool_mode=mode) + tool = toolset.fetch_tools(account_ids=["acc1"]).get_tool(SUBMIT_FEEDBACK_TOOL_NAME) + assert isinstance(tool, StackOneMcpTool) + + @pytest.mark.parametrize("mode", [None, "search_execute"]) + def test_calling_it_sends_one_tools_call(self, mcp_mock_server: str, mode): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, tool_mode=mode) + tool = toolset.fetch_tools(account_ids=["acc1"]).get_tool(SUBMIT_FEEDBACK_TOOL_NAME) + assert tool is not None + + _reset_requests(mcp_mock_server) + result = tool.execute({"rating": "positive", "tool_names": ["acc1_tool_1"]}) + + assert result["message"] == "Feedback recorded" + assert len(_tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME)) == 1 + + +class TestFeedbackToolIsListedOnce: + """Contract 2: one tool however many accounts list it, and no duplicate warning.""" + + @pytest.mark.parametrize("mode", [None, "search_execute"]) + def test_multi_account_listing_yields_exactly_one(self, mcp_mock_server: str, mode, caplog): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, tool_mode=mode) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + tools = toolset.fetch_tools(account_ids=["acc1", "acc2", "acc3"]) + + feedback = [t for t in tools if t.name == SUBMIT_FEEDBACK_TOOL_NAME] + assert len(feedback) == 1 + assert "more than one account" not in caplog.text + + def test_the_first_listing_is_kept(self, monkeypatch): + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda _e, _h, **_k: [ + McpToolDefinition(name=SUBMIT_FEEDBACK_TOOL_NAME, description="", input_schema={}) + ], + ) + toolset = StackOneToolSet(api_key="test-key") + tool = toolset.fetch_tools(account_ids=["acc2", "acc1"]).get_tool(SUBMIT_FEEDBACK_TOOL_NAME) + assert tool is not None + # Listings are merged in sorted account order, so "first" is stable across calls. + assert tool.get_account_id() == "acc1" + + def test_real_duplicates_still_warn(self, monkeypatch, caplog): + """Only the global tool is deduped; a genuine clash must still be reported.""" + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda _e, _h, **_k: [ + McpToolDefinition(name="linear_list_issues", description="", input_schema={}) + ], + ) + toolset = StackOneToolSet(api_key="test-key") + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + tools = toolset.fetch_tools(account_ids=["acc2", "acc1"]) + assert len(tools) == 2 + assert "more than one account (linear_list_issues)" in caplog.text + assert "The first one listed, from the lowest account id, is used" in caplog.text + + # The first listed, as Node's getTool() does. Listings merge in sorted account order. + assert [t.get_account_id() for t in tools] == ["acc1", "acc2"] + tool = tools.get_tool("linear_list_issues") + assert tool is tools[0] + + +class TestFeedbackToolAbsent: + """Contract 3: never invented client-side.""" + + @pytest.mark.parametrize("mode", [None, "search_execute"]) + def test_absent_when_the_server_does_not_serve_it(self, mcp_mock_server_without_feedback: str, mode): + base_url = mcp_mock_server_without_feedback + toolset = StackOneToolSet(api_key="test-key", base_url=base_url, tool_mode=mode) + tools = toolset.fetch_tools(account_ids=["acc1", "acc2"]) + assert len(tools) > 0 + assert tools.get_tool(SUBMIT_FEEDBACK_TOOL_NAME) is None + + def test_submit_feedback_raises_saying_not_enabled(self, mcp_mock_server_without_feedback: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server_without_feedback) + _reset_requests(mcp_mock_server_without_feedback) + + with pytest.raises(ToolsetLoadError, match="feedback is not enabled for this project"): + toolset.submit_feedback("negative", ["mock_list_items"]) + assert _tool_calls(mcp_mock_server_without_feedback, SUBMIT_FEEDBACK_TOOL_NAME) == [] + + def test_search_and_execute_are_unaffected(self, mcp_mock_server_without_feedback: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server_without_feedback) + hit = toolset.search("list items")[0] + result = toolset.execute(hit["action_id"], session_id=hit["session_id"]) + assert result["result"]["data"] == {"nodes": []} + + +class TestSearchSessionId: + """Contract 4: each hit carries the session_id of the search that produced it.""" + + def test_every_hit_carries_the_search_session_id(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + hits = toolset.search("list items") + assert hits + assert all(hit["session_id"] == MOCK_SESSION_ID for hit in hits) + + def test_each_hit_keeps_its_own_search_session_across_connectors(self, monkeypatch): + per_connector = { + "a_acc1_search_actions": {"session_id": "s-a", "actions": [{"action_id": "a_x"}]}, + "b_acc2_search_actions": {"session_id": "s-b", "actions": [{"action_id": "b_x"}]}, + } + monkeypatch.setattr(StackOneMcpTool, "execute", lambda self, _args: per_connector[self.name]) + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda _e, headers, **_k: [ + McpToolDefinition(name=name, description="", input_schema={}) + for name in per_connector + if headers["x-account-id"] in name + ], + ) + toolset = StackOneToolSet(api_key="test-key") + toolset.set_accounts(["acc1", "acc2"]) + assert {h["action_id"]: h["session_id"] for h in toolset.search("x")} == {"a_x": "s-a", "b_x": "s-b"} + + def test_the_field_is_omitted_when_the_server_returned_none(self, monkeypatch): + monkeypatch.setattr( + StackOneMcpTool, "execute", lambda self, _args: {"actions": [{"action_id": "a_x"}]} + ) + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda _e, _h, **_k: [ + McpToolDefinition(name="a_acc1_search_actions", description="", input_schema={}) + ], + ) + toolset = StackOneToolSet(api_key="test-key", account_id="acc1") + assert toolset.search("x") == [{"action_id": "a_x", "account_id": "acc1"}] + + +class TestExecuteSessionId: + """Contract 5: forwarded top-level when given, never when not, action_id still last.""" + + def test_forwarded_when_given(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + hit = toolset.search("list items")[0] + + _reset_requests(mcp_mock_server) + toolset.execute(hit["action_id"], hit["example_request"], session_id=hit["session_id"]) + + [call] = _execute_calls(mcp_mock_server) + assert call["arguments"] == { + "query": {"page_size": 25}, + "session_id": MOCK_SESSION_ID, + "action_id": "mock_list_items", + } + + def test_not_forwarded_when_not_given(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + _reset_requests(mcp_mock_server) + toolset.execute("mock_list_items") + + [call] = _execute_calls(mcp_mock_server) + assert call["arguments"] == {"action_id": "mock_list_items"} + + def test_pinning_and_header_sanitising_survive_alongside_it(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + _reset_requests(mcp_mock_server) + toolset.execute( + "mock_list_items", + { + "action_id": "mock_delete_everything", + "headers": {"x-account-id": "victim"}, + "session_id": "model-invented", + }, + session_id=MOCK_SESSION_ID, + ) + + [call] = _execute_calls(mcp_mock_server) + # Exact items, in order: the pinned keys replace the model's and move to the end. + assert list(call["arguments"].items()) == [ + ("headers", {}), + ("session_id", MOCK_SESSION_ID), + ("action_id", "mock_list_items"), + ] + + @pytest.mark.parametrize("bad", ["", 42]) + def test_rejects_a_non_string_or_empty_session_id(self, bad): + toolset = StackOneToolSet(api_key="test-key", account_id="acc1") + with pytest.raises(ToolsetConfigError, match="session_id"): + toolset.execute("mock_list_items", session_id=bad) + + +class TestSubmitFeedback: + """Contract 6: the toolset method.""" + + @pytest.mark.parametrize("mode", [None, "search_execute"]) + def test_sends_only_the_given_keys_over_tools_call(self, mcp_mock_server: str, mode): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, tool_mode=mode) + _reset_requests(mcp_mock_server) + + result = toolset.submit_feedback("negative", ["mock_list_items"], session_id=MOCK_SESSION_ID) + + [call] = _tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME) + # Exact equality: no feedback/category keys at all, rather than keys set to null. + assert call["arguments"] == { + "rating": "negative", + "tool_names": ["mock_list_items"], + "session_id": MOCK_SESSION_ID, + "source": "model", + } + assert all(value is not None for value in call["arguments"].values()) + assert result["session_id"] == MOCK_SESSION_ID + + def test_sends_every_field_in_contract_order(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + _reset_requests(mcp_mock_server) + + toolset.submit_feedback( + "positive", + ["mock_list_items"], + feedback="Found it first try", + category="search", + session_id=MOCK_SESSION_ID, + source="user", + ) + + [call] = _tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME) + assert list(call["arguments"].items()) == [ + ("rating", "positive"), + ("tool_names", ["mock_list_items"]), + ("feedback", "Found it first try"), + ("category", "search"), + ("session_id", MOCK_SESSION_ID), + ("source", "user"), + ] + + def test_session_id_is_omitted_when_not_given(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + _reset_requests(mcp_mock_server) + toolset.submit_feedback("neutral", ["mock_list_items"]) + + [call] = _tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME) + assert "session_id" not in call["arguments"] + + def test_runs_once_across_many_accounts(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + _reset_requests(mcp_mock_server) + toolset.submit_feedback("positive", ["acc1_tool_1"], account_ids=["acc1", "acc2"]) + + calls = _tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME) + assert [c["accountId"] for c in calls] == ["acc1"] + + @pytest.mark.parametrize("mode", [None, "individual"]) + def test_lists_only_the_lowest_given_account_in_search_execute_mode(self, mcp_mock_server: str, mode): + """The lowest account id, not the first given; and the small listing whatever the mode.""" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, tool_mode=mode) + _reset_requests(mcp_mock_server) + toolset.submit_feedback("positive", ["acc1_tool_1"], account_ids=["acc2", "acc1"]) + + listings = [r for r in _requests(mcp_mock_server) if r.get("method") == "tools/list"] + assert [(r["accountId"], r["search"]) for r in listings] == [("acc1", "?tool-mode=search_execute")] + calls = _tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME) + assert [c["accountId"] for c in calls] == ["acc1"] + + def test_sends_action_run_id_only_when_given(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, account_id="acc1") + _reset_requests(mcp_mock_server) + toolset.submit_feedback("positive", ["a"], session_id=MOCK_SESSION_ID, action_run_id="run-1") + toolset.submit_feedback("positive", ["a"]) + + first, second = _tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME) + assert list(first["arguments"].items()) == [ + ("rating", "positive"), + ("tool_names", ["a"]), + ("session_id", MOCK_SESSION_ID), + ("action_run_id", "run-1"), + ("source", "model"), + ] + assert "action_run_id" not in second["arguments"] + + def test_defaults_to_the_lowest_discovered_account_id(self, monkeypatch): + monkeypatch.setattr( + StackOneToolSet, + "fetch_accounts", + lambda self: [ + {"id": "zeta", "status": "active"}, + {"id": "dead", "status": "inactive"}, + {"id": "alpha", "status": "active"}, + ], + ) + listed: list[tuple[str, str]] = [] + + def fake_fetch(endpoint: str, headers: dict[str, str], **_k: object) -> list[McpToolDefinition]: + listed.append((headers["x-account-id"], endpoint)) + return [McpToolDefinition(name=SUBMIT_FEEDBACK_TOOL_NAME, description="", input_schema={})] + + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", fake_fetch) + called: list[str | None] = [] + monkeypatch.setattr( + StackOneMcpTool, "execute", lambda self, _args: called.append(self.get_account_id()) or {} + ) + + StackOneToolSet(api_key="test-key").submit_feedback("positive", ["x"]) + + assert listed == [("alpha", "https://api.stackone.com/mcp?tool-mode=search_execute")] + assert called == ["alpha"] + + def test_accepts_any_sequence_of_tool_names(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + _reset_requests(mcp_mock_server) + toolset.submit_feedback("positive", ("a", "b")) + + [call] = _tool_calls(mcp_mock_server, SUBMIT_FEEDBACK_TOOL_NAME) + assert call["arguments"]["tool_names"] == ["a", "b"] + + def test_the_feedback_types_are_exported(self): + import stackone_ai + + for name in ("FeedbackRating", "FeedbackCategory", "FeedbackSource", "StackOneMcpTool"): + assert name in stackone_ai.__all__ + assert hasattr(stackone_ai, name) + + def test_rejects_a_bare_string_of_tool_names(self): + toolset = StackOneToolSet(api_key="test-key", account_id="acc1") + with pytest.raises(ToolsetConfigError, match="Did you mean"): + toolset.submit_feedback("positive", "mock_list_items") # ty: ignore[invalid-argument-type] + + @pytest.mark.parametrize("bad", ["", 42]) + def test_rejects_a_non_string_or_empty_session_id(self, bad): + """Checked as execute() checks it, before any account is listed.""" + toolset = StackOneToolSet(api_key="test-key", account_id="acc1") + with pytest.raises(ToolsetConfigError, match="session_id"): + toolset.submit_feedback("positive", ["mock_list_items"], session_id=bad) diff --git a/tests/test_fetch_tools.py b/tests/test_fetch_tools.py index 968b797b..92886137 100644 --- a/tests/test_fetch_tools.py +++ b/tests/test_fetch_tools.py @@ -2,21 +2,47 @@ from __future__ import annotations +import asyncio +import contextlib import threading -from contextlib import asynccontextmanager +import time +from collections.abc import Iterator +from contextlib import asynccontextmanager, contextmanager +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest -from stackone_ai.tools import McpToolDefinition, fetch_mcp_tools +from stackone_ai.tools import McpToolDefinition, StackOneMcpTool, fetch_mcp_tools from stackone_ai.toolset import StackOneToolSet from stackone_ai.types import ( + SUBMIT_FEEDBACK_TOOL_NAME, StackOneAPIError, + ToolParameters, ToolsetConfigError, ToolsetError, ToolsetLoadError, ) +# Must match MOCK_DOWNLOAD_LINK in tests/mocks/mcp-server.ts. +MOCK_DOWNLOAD_LINK = { + "download_url": "https://downloads.example.com/f/abc123?sig=xyz", + "expires_at": "2026-01-01T00:00:00.000Z", + "file": {"name": "report.pdf", "content_type": "application/pdf", "content_length": 1024}, +} + + +def _reset_requests(base_url: str) -> None: + httpx.delete(f"{base_url}/__requests").raise_for_status() + + +def _tool_calls(base_url: str, name: str) -> list[dict[str, Any]]: + """The tools/call requests the mock received for ``name``, as they were on the wire.""" + response = httpx.get(f"{base_url}/__requests") + response.raise_for_status() + return [r for r in response.json() if r["method"] == "tools/call" and r["name"] == name] + class TestAccountFiltering: """Test account filtering functionality with real MCP server.""" @@ -31,7 +57,8 @@ def test_fetch_tools_without_account_filtering(self, mcp_mock_server: str): """Test fetching tools without account filtering""" toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) tools = toolset.fetch_tools() - assert len(tools) == 2 + # Plus the mock's global feedback tool, served alongside every account's catalog. + assert len(tools) == 3 tool_names = [t.name for t in tools.to_list()] assert "default_tool_1" in tool_names assert "default_tool_2" in tool_names @@ -40,7 +67,7 @@ def test_fetch_tools_with_account_ids(self, mcp_mock_server: str): """Test fetching tools with specific account IDs""" toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) tools = toolset.fetch_tools(account_ids=["acc1"]) - assert len(tools) == 2 + assert len(tools) == 3 tool_names = [t.name for t in tools.to_list()] assert "acc1_tool_1" in tool_names assert "acc1_tool_2" in tool_names @@ -50,8 +77,8 @@ def test_fetch_tools_uses_set_accounts(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) toolset.set_accounts(["acc1", "acc2"]) tools = toolset.fetch_tools() - # acc1 has 2 tools, acc2 has 2 tools, total should be 4 - assert len(tools) == 4 + # acc1 has 2 tools, acc2 has 2 tools, plus one feedback tool however many accounts list it + assert len(tools) == 5 tool_names = [t.name for t in tools.to_list()] assert "acc1_tool_1" in tool_names assert "acc1_tool_2" in tool_names @@ -63,8 +90,8 @@ def test_fetch_tools_overrides_set_accounts(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) toolset.set_accounts(["acc1", "acc2"]) tools = toolset.fetch_tools(account_ids=["acc3"]) - # Should fetch tools only for acc3 (ignoring acc1, acc2) - assert len(tools) == 1 + # Should fetch tools only for acc3 (ignoring acc1, acc2), plus the feedback tool + assert len(tools) == 2 tool_names = [t.name for t in tools.to_list()] assert "acc3_tool_1" in tool_names # Verify set_accounts state is preserved @@ -74,8 +101,8 @@ def test_fetch_tools_multiple_account_ids(self, mcp_mock_server: str): """Test fetching tools for multiple account IDs""" toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) tools = toolset.fetch_tools(account_ids=["acc1", "acc2", "acc3"]) - # acc1: 2 tools, acc2: 2 tools, acc3: 1 tool = 5 total - assert len(tools) == 5 + # acc1: 2 tools, acc2: 2 tools, acc3: 1 tool, feedback: 1 = 6 total + assert len(tools) == 6 def test_fetch_tools_preserves_account_context(self, mcp_mock_server: str): """Test that tools preserve their account context""" @@ -163,7 +190,8 @@ def test_account_id_header_is_sent(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) # When we fetch with acc1, we should get acc1's tools, proving header was sent tools = toolset.fetch_tools(account_ids=["acc1"]) - tool_names = [t.name for t in tools.to_list()] + tool_names = [t.name for t in tools.to_list() if t.name != SUBMIT_FEEDBACK_TOOL_NAME] + assert tool_names assert all("acc1" in name for name in tool_names) @@ -189,6 +217,29 @@ def test_tool_has_parameters_type(self, mcp_mock_server: str): assert tool.parameters.type == "object" +class TestRootType: + """The root type the model sees: a served string as is, anything else "object", as in Node.""" + + @pytest.mark.parametrize( + ("served", "expected"), + [ + ({"type": "object"}, "object"), + ({}, "object"), + ({"type": ["object", "null"]}, "object"), + ({"type": None}, "object"), + ({"type": 7}, "object"), + ], + ) + def test_root_type(self, monkeypatch, served: dict[str, Any], expected: str): + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda *_a, **_k: [McpToolDefinition(name="t", description="", input_schema=served)], + ) + tool = StackOneToolSet(api_key="test-key", account_id="acc1").fetch_tools()[0] + assert tool.parameters.type == expected + assert tool.to_openai_function()["function"]["parameters"]["type"] == expected + + class TestSchemaPropertyNormalization: """Test schema property normalization with monkeypatch (for precise schema control).""" @@ -271,42 +322,223 @@ def fake_fetch(_: str, headers: dict[str, str], **_kwargs: object) -> list[McpTo assert tool.parameters.properties["optional_field"].get("nullable") is True -class TestRpcToolExecution: - """Test RPC tool execution through the MCP server.""" +# Served `required` -> the `required` every adapter emits (absent means the key is omitted). +# Mirrors the Node SDK's toJsonSchema() and toolParametersFromInputSchema(). +_ABSENT = object() +_SERVED_REQUIRED_CASES = [ + pytest.param(["b", "a"], ["b", "a"], id="served-order-not-property-order"), + pytest.param(["a", "b"], ["a", "b"], id="already-in-property-order"), + pytest.param(["c", "a", "b"], ["c", "a", "b"], id="three-reversed"), + pytest.param(["ghost"], ["ghost"], id="undeclared-name-kept"), + pytest.param(["a", "a"], ["a", "a"], id="duplicates-kept"), + pytest.param(["b", 1, "a"], ["b", "a"], id="non-string-entry-dropped"), + pytest.param([], None, id="empty"), + pytest.param(_ABSENT, None, id="absent"), + pytest.param(None, None, id="null"), + pytest.param("a string", None, id="string"), + pytest.param([1, 2], None, id="only-non-strings"), +] + + +class TestServedRequiredOrder: + """`required` is the served list, verbatim and in the served order, on every adapter. + + It was rebuilt from the per-property `nullable` markers, which walked the properties + and so re-sorted it into property order: a model was shown a list the server never + sent, and the Node SDK's schema differed from this one for the same tool. + """ + + @staticmethod + def _tool(monkeypatch, served_required: object): + schema: dict[str, object] = { + "type": "object", + "properties": {"a": {"type": "string"}, "b": {"type": "string"}, "c": {"type": "string"}}, + } + if served_required is not _ABSENT: + schema["required"] = served_required + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda _e, _h, **_k: [McpToolDefinition(name="t", description="", input_schema=schema)], + ) + tool = StackOneToolSet(api_key="k", account_id="acc1").fetch_tools().get_tool("t") + assert tool is not None + return tool + + @pytest.mark.parametrize(("served", "expected"), _SERVED_REQUIRED_CASES) + def test_openai(self, monkeypatch, served, expected): + parameters = self._tool(monkeypatch, served).to_openai_function()["function"]["parameters"] + assert parameters.get("required") == expected + if expected is None: + assert "required" not in parameters + + @pytest.mark.parametrize(("served", "expected"), _SERVED_REQUIRED_CASES) + def test_langchain(self, monkeypatch, served, expected): + schema = self._tool(monkeypatch, served).to_langchain().args_schema + assert schema.get("required") == expected + if expected is None: + assert "required" not in schema + + @pytest.mark.parametrize(("served", "expected"), _SERVED_REQUIRED_CASES) + def test_pydantic_ai(self, monkeypatch, served, expected): + pytest.importorskip("pydantic_ai") + schema = self._tool(monkeypatch, served).to_pydantic_ai_tool().function_schema.json_schema + assert schema.get("required") == expected + if expected is None: + assert "required" not in schema + + def test_parameters_still_carry_the_served_list_and_markers(self, monkeypatch): + """The ADK plugin reads `tool.parameters` directly, so it must not change shape.""" + tool = self._tool(monkeypatch, ["b", "a"]) + dumped = tool.parameters.model_dump() + assert dumped["required"] == ["b", "a"] + assert {name: prop["nullable"] for name, prop in dumped["properties"].items()} == { + "a": False, + "b": False, + "c": True, + } + + def test_non_string_entry_does_not_mark_a_property_required(self, monkeypatch): + """The marker agrees with the emitted list: `1` is not the property named "1".""" + schema = {"type": "object", "properties": {"1": {"type": "string"}}, "required": [1]} + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda _e, _h, **_k: [McpToolDefinition(name="t", description="", input_schema=schema)], + ) + tool = StackOneToolSet(api_key="k", account_id="acc1").fetch_tools().get_tool("t") + assert tool is not None + assert tool.parameters.properties["1"]["nullable"] is True + + +class TestPerActionToolsExecuteOverToolsCall: + """Every per-action tool runs over MCP tools/call on the endpoint and account that listed it. + + Assertions about the wire read the mock's request log, not a handler's view of the call. + """ - def test_execute_tool_returns_response(self, mcp_mock_server: str): - """Test executing a tool via RPC returns response""" + def test_execute_returns_the_result_as_the_server_wrote_it(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) - tools = toolset.fetch_tools(account_ids=["your-bamboohr-account-id"]) - tool = tools.get_tool("bamboohr_list_employees") + tool = toolset.fetch_tools(account_ids=["your-bamboohr-account-id"]).get_tool( + "bamboohr_list_employees" + ) assert tool is not None - result = tool.execute() - assert result is not None - assert "data" in result + assert tool.execute() == { + "isError": False, + "result": {"data": [{"id": "1", "name": "Employee 1"}, {"id": "2", "name": "Employee 2"}]}, + } - def test_execute_tool_with_arguments(self, mcp_mock_server: str): - """Test executing a tool with arguments""" + def test_one_tools_call_on_the_listing_endpoint_and_no_param_style(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) - tools = toolset.fetch_tools(account_ids=["your-bamboohr-account-id"]) - tool = tools.get_tool("bamboohr_get_employee") + tool = toolset.fetch_tools(account_ids=["your-bamboohr-account-id"]).get_tool("bamboohr_get_employee") assert tool is not None + _reset_requests(mcp_mock_server) result = tool.execute({"id": "emp-123"}) - assert result is not None - assert result.get("data", {}).get("id") == "emp-123" - def test_execute_tool_sends_account_id_header(self, mcp_mock_server: str): - """Test that tool execution sends x-account-id header""" + assert result == {"isError": False, "result": {"data": {"id": "emp-123", "name": "Test Employee"}}} + [call] = _tool_calls(mcp_mock_server, "bamboohr_get_employee") + assert call["path"] == "/mcp" + assert call["search"] == "" + + @pytest.mark.parametrize( + "arguments", + [ + pytest.param({"fields": "a,b"}, id="flat"), + pytest.param({"path_id": "1", "query_limit": 5, "body_name": "x"}, id="prefix-lookalikes"), + pytest.param({"path": {"id": "1"}, "query": {"limit": 5}, "body": {"n": [1, None]}}, id="nested"), + pytest.param({"query": "not-an-object", "header_x": "y", "unicode": "é😀"}, id="odd-shapes"), + pytest.param({}, id="empty"), + ], + ) + def test_arguments_are_sent_verbatim(self, mcp_mock_server: str, arguments: dict[str, Any]): + """No envelope splitting and no prefix routing: the server maps them itself.""" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=["mixed"]).get_tool("hibob_list_employees") + assert tool is not None + + _reset_requests(mcp_mock_server) + result = tool.execute(arguments) + + [call] = _tool_calls(mcp_mock_server, "hibob_list_employees") + assert call["arguments"] == arguments + assert result == { + "isError": False, + "result": {"data": {"action": "hibob_list_employees", "received": arguments}}, + } + + def test_a_call_does_not_relist_the_catalog(self, mcp_mock_server: str): + # The mcp client's call_tool lists tools first, to validate structuredContent against + # an output schema, and every call opens a fresh session — so each call relisted the + # whole catalog. The Node SDK sends only the tools/call. + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=["acc1"]).get_tool("acc1_tool_1") + assert tool is not None + _reset_requests(mcp_mock_server) + + tool.execute({"param": "x"}) + + response = httpx.get(f"{mcp_mock_server}/__requests") + assert [r["method"] for r in response.json()] == ["tools/call"] + + def test_x_account_id_is_the_listing_accounts(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tools = toolset.fetch_tools(account_ids=["acc1", "acc2"]) + + _reset_requests(mcp_mock_server) + for name in ("acc1_tool_1", "acc2_tool_1"): + tool = tools.get_tool(name) + assert tool is not None + tool.execute({"fields": "id"}) + + assert [c["accountId"] for c in _tool_calls(mcp_mock_server, "acc1_tool_1")] == ["acc1"] + assert [c["accountId"] for c in _tool_calls(mcp_mock_server, "acc2_tool_1")] == ["acc2"] + + def test_a_download_link_is_returned_unchanged(self, mcp_mock_server: str): + """A file action over tools/call returns a link; the SDK neither follows nor reshapes it.""" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=["files"]).get_tool("files_download_file") + assert tool is not None + + assert tool.execute({"id": "f1"}) == {"isError": False, "result": MOCK_DOWNLOAD_LINK} + + def test_a_download_with_no_link_raises_with_status_501(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=["files"]).get_tool("files_download_unavailable") + assert tool is not None + + with pytest.raises(StackOneAPIError) as excinfo: + tool.execute({"id": "f1"}) + assert excinfo.value.status_code == 501 + assert excinfo.value.response_body["isError"] is True + + def test_defender_metadata_is_kept_beside_the_result(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=["files"]).get_tool("files_list_defended") + assert tool is not None + + assert tool.execute() == { + "isError": False, + "result": {"data": []}, + "defenderMetadata": {"scanned": True, "flagged": 0}, + } + + def test_a_non_object_result_is_returned_as_served(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) - tools = toolset.fetch_tools(account_ids=["test-account"]) - tool = tools.get_tool("dummy_action") + tool = toolset.fetch_tools(account_ids=["files"]).get_tool("files_count") assert tool is not None - assert tool.get_account_id() == "test-account" - # Execute and verify account context is preserved - result = tool.execute({"foo": "bar"}) - assert result is not None + assert tool.execute() == {"isError": False, "result": 3} + + def test_an_action_named_like_a_meta_tool_runs_as_an_action(self, mcp_mock_server: str): + """Outside search_execute mode, a name ending in `_execute_action` is just an action.""" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=["lookalike"]).get_tool("lookalike_execute_action") + assert tool is not None + + assert tool.execute({"action_id": "other"}) == { + "isError": False, + "result": {"data": {"action": "lookalike_execute_action", "received": {"action_id": "other"}}}, + } class TestAccountIdFallback: @@ -380,20 +612,31 @@ def test_fetch_mcp_tools_single_page(self): mock_session.__aexit__ = AsyncMock(return_value=None) # Create mock streamable client + seen_clients: list[httpx.AsyncClient] = [] + @asynccontextmanager - async def mock_streamable_client(endpoint, headers, **_kwargs: object): + async def mock_streamable_client(endpoint, *, http_client, **_kwargs: object): + seen_clients.append(http_client) yield (MagicMock(), MagicMock(), MagicMock()) # Patch at the module where imports happen with ( patch( - "mcp.client.streamable_http.streamablehttp_client", + "mcp.client.streamable_http.streamable_http_client", side_effect=mock_streamable_client, ), patch("mcp.client.session.ClientSession", return_value=mock_session), patch("mcp.types.Implementation", MagicMock()), ): - result = fetch_mcp_tools("https://api.example.com/mcp", {"Authorization": "Basic test"}) + result = fetch_mcp_tools( + "https://api.example.com/mcp", {"Authorization": "Basic test"}, timeout=7 + ) + + # The caller's headers and timeout reach the HTTP client, on every leg. + [client] = seen_clients + assert client.headers["Authorization"] == "Basic test" + assert client.timeout == httpx.Timeout(7) + assert client.follow_redirects is True assert len(result) == 1 assert result[0].name == "test_tool" @@ -430,12 +673,12 @@ def test_fetch_mcp_tools_with_pagination(self): mock_session.__aexit__ = AsyncMock(return_value=None) @asynccontextmanager - async def mock_streamable_client(endpoint, headers, **_kwargs: object): + async def mock_streamable_client(endpoint, **_kwargs: object): yield (MagicMock(), MagicMock(), MagicMock()) with ( patch( - "mcp.client.streamable_http.streamablehttp_client", + "mcp.client.streamable_http.streamable_http_client", side_effect=mock_streamable_client, ), patch("mcp.client.session.ClientSession", return_value=mock_session), @@ -647,14 +890,27 @@ def always_fails( monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", always_fails) toolset = StackOneToolSet(api_key="test-key", account_id="acc1") - with pytest.raises(ToolsetLoadError, match="No account returned tools"): - toolset.fetch_tools(account_ids=["a", "b"]) - - -class TestMcpParamStylePinning: - """The /mcp listing URL must pin param-style so the schema matches the RPC unwrap.""" - - def test_fetch_tools_pins_flat_prefixed_param_style(self, monkeypatch): + with pytest.raises(ToolsetLoadError) as excinfo: + toolset.fetch_tools(account_ids=["b", "a"]) + assert str(excinfo.value) == "Every account failed to list tools: a: boom for a; b: boom for b" + assert [str(f) for f in excinfo.value.failures] == ["boom for a", "boom for b"] + cause = excinfo.value.__cause__ + assert isinstance(cause, ExceptionGroup) + assert list(cause.exceptions) == excinfo.value.failures + + +class TestMcpEndpoint: + """No param-style pin: the endpoint is {base}/mcp, plus ?tool-mode= when a mode is set.""" + + @pytest.mark.parametrize( + ("mode", "expected"), + [ + (None, "https://api.example.com/mcp"), + ("individual", "https://api.example.com/mcp?tool-mode=individual"), + ("search_execute", "https://api.example.com/mcp?tool-mode=search_execute"), + ], + ) + def test_endpoint(self, monkeypatch, mode, expected): captured: dict[str, str] = {} def fake_fetch(endpoint: str, headers: dict[str, str], **_kwargs: object) -> list[McpToolDefinition]: @@ -663,10 +919,10 @@ def fake_fetch(endpoint: str, headers: dict[str, str], **_kwargs: object) -> lis monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", fake_fetch) - toolset = StackOneToolSet(api_key="test-key", base_url="https://api.example.com") + toolset = StackOneToolSet(api_key="test-key", base_url="https://api.example.com/", tool_mode=mode) toolset.fetch_tools(account_ids=["acc1"]) - assert captured["endpoint"] == "https://api.example.com/mcp?param-style=flat_prefixed" + assert captured["endpoint"] == expected class TestAccountDiscovery: @@ -687,6 +943,263 @@ def test_inactive_accounts_are_skipped(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) assert toolset._discover_account_ids() == ["default"] + def test_a_discovery_in_flight_during_a_clear_is_not_cached(self, monkeypatch): + """An account list fetched before clear_catalog_cache() must not outlive it.""" + toolset = StackOneToolSet(api_key="test-key") + responses = iter([["before"], ["after"]]) + + def fetch_accounts() -> list[dict[str, Any]]: + listed = next(responses) + if listed == ["before"]: + # The clear lands while this GET /accounts is still in flight. + toolset.clear_catalog_cache() + return [{"id": account, "provider": "p", "status": "active"} for account in listed] + + monkeypatch.setattr(toolset, "fetch_accounts", fetch_accounts) + + assert toolset._discover_account_ids() == ["before"] + assert toolset._discover_account_ids() == ["after"] + assert toolset._discover_account_ids() == ["after"] + + def test_a_catalog_resolved_before_a_mid_discovery_clear_is_not_cached(self, monkeypatch): + """The catalog for the pre-clear discovered account must not be cached under the new generation.""" + toolset = StackOneToolSet(api_key="test-key") + account_responses = iter([["before"], ["after"]]) + + def fetch_accounts() -> list[dict[str, Any]]: + listed = next(account_responses) + if listed == ["before"]: + # The clear lands while this GET /accounts is still in flight. + toolset.clear_catalog_cache() + return [{"id": account, "provider": "p", "status": "active"} for account in listed] + + monkeypatch.setattr(toolset, "fetch_accounts", fetch_accounts) + + calls: list[str | None] = [] + + def fake_fetch(_endpoint: str, headers: dict[str, str], **_kwargs: object) -> list[McpToolDefinition]: + calls.append(headers.get("x-account-id")) + return [] + + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", fake_fetch) + + toolset.fetch_tools() # discovers "before", races the clear + toolset.fetch_tools() # discovers "after", caches it cleanly + + calls.clear() + toolset.fetch_tools(account_ids=["before"]) + + # Must refetch: the catalog for "before" was resolved against a generation + # the clear had already moved past, so it must not have been cached. + assert calls == ["before"] + + def test_concurrent_discovery_shares_one_in_flight_request(self, monkeypatch): + """Two threads discovering accounts at once make one GET /accounts between them.""" + toolset = StackOneToolSet(api_key="test-key") + call_count = 0 + started = threading.Event() + proceed = threading.Event() + + def fetch_accounts() -> list[dict[str, Any]]: + nonlocal call_count + call_count += 1 + started.set() + assert proceed.wait(timeout=5) + return [{"id": "acc", "provider": "p", "status": "active"}] + + monkeypatch.setattr(toolset, "fetch_accounts", fetch_accounts) + + results: list[list[str]] = [] + + def worker() -> None: + results.append(toolset._discover_account_ids()) + + t1 = threading.Thread(target=worker) + t1.start() + assert started.wait(timeout=5) + + t2 = threading.Thread(target=worker) + t2.start() + # t1 cannot finish before proceed is set, so this just gives the scheduler a + # chance to get t2 into its wait on the shared future before we release t1. + time.sleep(0.05) + proceed.set() + + t1.join(timeout=5) + t2.join(timeout=5) + + assert call_count == 1 + assert results == [["acc"], ["acc"]] + + def test_concurrent_discovery_failure_reaches_both_waiters_and_is_not_cached(self, monkeypatch): + toolset = StackOneToolSet(api_key="test-key") + call_count = 0 + started = threading.Event() + proceed = threading.Event() + t2_entered = threading.Event() + + def failing_fetch_accounts() -> list[dict[str, Any]]: + nonlocal call_count + call_count += 1 + started.set() + assert proceed.wait(timeout=5) + assert t2_entered.wait(timeout=5) + raise StackOneAPIError("boom", 500, "boom") + + monkeypatch.setattr(toolset, "fetch_accounts", failing_fetch_accounts) + + errors: list[StackOneAPIError] = [] + + def worker() -> None: + try: + toolset._discover_account_ids() + except StackOneAPIError as exc: + errors.append(exc) + + def waiter() -> None: + # Signalled right before becoming a waiter, so the owner's raise is held + # back until this thread is certain to be the second call into + # _discover_account_ids(), rather than racing a fixed sleep. + t2_entered.set() + worker() + + t1 = threading.Thread(target=worker) + t1.start() + assert started.wait(timeout=5) + + t2 = threading.Thread(target=waiter) + t2.start() + proceed.set() + + t1.join(timeout=5) + t2.join(timeout=5) + + assert call_count == 1 + assert len(errors) == 2 + assert toolset._discovered_account_ids is None + + # A later call retries rather than reusing the failure. + monkeypatch.setattr( + toolset, "fetch_accounts", lambda: [{"id": "acc", "provider": "p", "status": "active"}] + ) + assert toolset._discover_account_ids() == ["acc"] + + def test_clear_during_a_shared_discovery_discards_the_result(self, monkeypatch): + """A clear_catalog_cache() while a shared discovery is in flight is not undone + by that discovery writing its result back after the clear.""" + toolset = StackOneToolSet(api_key="test-key") + call_count = 0 + started = threading.Event() + proceed = threading.Event() + + def fetch_accounts() -> list[dict[str, Any]]: + nonlocal call_count + call_count += 1 + started.set() + assert proceed.wait(timeout=5) + return [{"id": "acc", "provider": "p", "status": "active"}] + + monkeypatch.setattr(toolset, "fetch_accounts", fetch_accounts) + + result_holder: dict[str, list[str]] = {} + + def worker() -> None: + result_holder["result"] = toolset._discover_account_ids() + + t = threading.Thread(target=worker) + t.start() + assert started.wait(timeout=5) + + toolset.clear_catalog_cache() + proceed.set() + t.join(timeout=5) + + # The in-flight call still resolves for whoever was waiting on it... + assert result_holder["result"] == ["acc"] + assert call_count == 1 + # ...but its result is discarded rather than cached past the clear. + assert toolset._discovered_account_ids is None + + monkeypatch.setattr( + toolset, "fetch_accounts", lambda: [{"id": "acc2", "provider": "p", "status": "active"}] + ) + assert toolset._discover_account_ids() == ["acc2"] + + def test_a_discovery_after_a_clear_does_not_join_one_from_before_it(self, monkeypatch): + toolset = StackOneToolSet(api_key="test-key") + listed = iter(["before", "after"]) + started = threading.Event() + proceed = threading.Event() + + def fetch_accounts() -> list[dict[str, Any]]: + account = next(listed) + if account == "before": + started.set() + assert proceed.wait(timeout=5) + return [{"id": account, "provider": "p", "status": "active"}] + + monkeypatch.setattr(toolset, "fetch_accounts", fetch_accounts) + before: dict[str, list[str]] = {} + thread = threading.Thread(target=lambda: before.setdefault("ids", toolset._discover_account_ids())) + thread.start() + assert started.wait(timeout=5) + + toolset.clear_catalog_cache() + try: + assert toolset._discover_account_ids() == ["after"] + finally: + proceed.set() + thread.join(timeout=5) + assert before["ids"] == ["before"] + assert toolset._discover_account_ids() == ["after"] + + @pytest.mark.parametrize("before_fails", [False, True], ids=["before-succeeds", "before-fails"]) + def test_a_discovery_from_before_a_clear_does_not_unshare_the_one_after_it( + self, monkeypatch, before_fails + ): + """Finishing, it must leave the in-flight discovery alone unless it is its own: else a + third caller starts another GET /accounts rather than joining the one after the clear.""" + toolset = StackOneToolSet(api_key="test-key") + calls: list[str] = [] + release = {"before": threading.Event(), "after": threading.Event()} + started = {"before": threading.Event(), "after": threading.Event()} + + def fetch_accounts() -> list[dict[str, Any]]: + name = ["before", "after", "extra"][len(calls)] + calls.append(name) + if name != "extra": + started[name].set() + assert release[name].wait(timeout=5) + if name == "before" and before_fails: + raise ToolsetLoadError("boom") + return [{"id": name, "provider": "p", "status": "active"}] + + monkeypatch.setattr(toolset, "fetch_accounts", fetch_accounts) + + def discover() -> None: + with contextlib.suppress(ToolsetLoadError): + toolset._discover_account_ids() + + before = threading.Thread(target=discover) + before.start() + assert started["before"].wait(timeout=5) + toolset.clear_catalog_cache() + after = threading.Thread(target=discover) + after.start() + assert started["after"].wait(timeout=5) + + release["before"].set() + before.join(timeout=5) + third: dict[str, list[str]] = {} + joining = threading.Thread(target=lambda: third.setdefault("ids", toolset._discover_account_ids())) + joining.start() + release["after"].set() + joining.join(timeout=5) + after.join(timeout=5) + + assert calls == ["before", "after"] + assert third["ids"] == ["after"] + def test_missing_account_header_is_rejected_by_the_server(self, mcp_mock_server: str): """Guards the mock itself: if it stops enforcing this, these tests go hollow.""" import httpx @@ -716,44 +1229,49 @@ def test_http_error_carries_status_and_is_not_taskgroup_noise(self, monkeypatch) # error arrives wrapped in an ExceptionGroup. grouped = ExceptionGroup("unhandled errors in a TaskGroup", [inner]) - err = _describe_mcp_failure(grouped, "https://api.example.com/mcp") + err = _describe_mcp_failure(grouped, "https://api.example.com/mcp", 60.0) assert "412" in str(err) assert "TaskGroup" not in str(err) assert getattr(err, "status_code", None) == 412 + def test_a_real_error_response_carries_the_servers_explanation(self, mcp_mock_server: str): + """Through the real transport, whose stream is closed by the time the error surfaces. + + The test above hands in an already-read response, so it passed while every live + failure said only "400 Bad Request" and dropped why, e.g. "Legacy accounts cannot + be used with MCP". + """ + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + with pytest.raises(StackOneAPIError) as excinfo: + toolset.fetch_tools(account_ids=["legacy-account"]) + assert excinfo.value.status_code == 400 + assert "Legacy accounts cannot be used with MCP" in str(excinfo.value) + assert "Legacy accounts cannot be used with MCP" in excinfo.value.response_body + def test_non_http_error_reports_the_leaf_not_the_group(self): from stackone_ai.tools import _describe_mcp_failure grouped = ExceptionGroup("unhandled errors in a TaskGroup", [ConnectionError("no route")]) - err = _describe_mcp_failure(grouped, "https://api.example.com/mcp") + err = _describe_mcp_failure(grouped, "https://api.example.com/mcp", 60.0) assert "no route" in str(err) assert "TaskGroup" not in str(err) class TestToolModeRouting: - """search_execute tools are MCP-native: they must not be sent to /actions/rpc.""" - - def test_default_mode_builds_rpc_tools(self, monkeypatch): - from stackone_ai.tools import StackOneRpcTool - - monkeypatch.setattr( - "stackone_ai.toolset.fetch_mcp_tools", - lambda _e, _h, **_k: [McpToolDefinition(name="t", description="", input_schema={})], - ) - toolset = StackOneToolSet(api_key="k", account_id="acc1") - assert isinstance(toolset.fetch_tools().get_tool("t"), StackOneRpcTool) + """Every tool is an MCP tool, whatever the mode.""" - def test_search_execute_mode_builds_mcp_tools(self, monkeypatch): + @pytest.mark.parametrize("mode", [None, "individual", "search_execute"]) + def test_every_mode_builds_mcp_tools(self, monkeypatch, mode): from stackone_ai.tools import StackOneMcpTool monkeypatch.setattr( "stackone_ai.toolset.fetch_mcp_tools", lambda _e, _h, **_k: [McpToolDefinition(name="t", description="", input_schema={})], ) - toolset = StackOneToolSet(api_key="k", account_id="acc1", tool_mode="search_execute") + toolset = StackOneToolSet(api_key="k", account_id="acc1", tool_mode=mode) assert isinstance(toolset.fetch_tools().get_tool("t"), StackOneMcpTool) def test_search_execute_mode_is_requested_on_the_url(self, monkeypatch): @@ -799,6 +1317,174 @@ class _Result: assert parse_tool_result(_Result(), "t")["actions"][0]["action_id"] == "linear_list_comments" +def _text(text: str) -> dict[str, str]: + return {"type": "text", "text": text} + + +_IMAGE = {"type": "image", "data": "AAAA", "mimeType": "image/png"} +_AUDIO = { + "type": "audio", + "data": "AAAA", + "mimeType": "audio/wav", + "annotations": {"audience": ["user"], "priority": 0.5}, + "_meta": {"k": 1}, +} +_RESOURCE = { + "type": "resource", + "resource": {"uri": "file:///report.txt", "mimeType": "text/plain", "text": "hi"}, +} +_RESOURCE_LINK = {"type": "resource_link", "uri": "file:///report.txt", "name": "report"} + + +class TestToolResultParity: + """parse_tool_result() matches the Node SDK's parseToolResult() case for case. + + A result carrying only structuredContent came back as `{}` — a success with the + whole payload missing. Text wins when both are present, in success and in error. + """ + + @pytest.mark.parametrize( + ("result", "expected"), + [ + pytest.param({"content": [_text('{"a":1}')]}, {"a": 1}, id="text-object"), + pytest.param({"content": [_text("[1,2]")]}, {"result": [1, 2]}, id="text-array"), + pytest.param({"content": [_text("plain")]}, {"result": "plain"}, id="text-plain"), + pytest.param({"content": [_text("NaN")]}, {"result": "NaN"}, id="text-nan-is-not-json"), + pytest.param({"content": [_text('{"a":'), _text("1}")]}, {"a": 1}, id="text-parts-joined"), + pytest.param( + {"content": [], "structuredContent": {"ok": True}}, {"ok": True}, id="structured-only" + ), + pytest.param( + {"content": [_text('{"a":1}')], "structuredContent": {"b": 2}}, + {"a": 1}, + id="text-wins-over-structured", + ), + pytest.param( + {"content": [_text("")], "structuredContent": {"ok": True}}, + {"ok": True, "content_parts": [_text("")]}, + id="empty-text-is-not-text", + ), + pytest.param({"content": []}, {}, id="neither"), + pytest.param( + {"content": [_text('{"a":1}'), _IMAGE]}, + {"a": 1, "content_parts": [_IMAGE]}, + id="text-and-image", + ), + pytest.param( + {"content": [_IMAGE], "structuredContent": {"ok": True}}, + {"ok": True, "content_parts": [_IMAGE]}, + id="structured-and-image", + ), + pytest.param({"content": [_AUDIO]}, {"content_parts": [_AUDIO]}, id="audio"), + pytest.param({"content": [_RESOURCE]}, {"content_parts": [_RESOURCE]}, id="embedded-resource"), + pytest.param( + {"content": [_RESOURCE_LINK]}, {"content_parts": [_RESOURCE_LINK]}, id="resource-link" + ), + ], + ) + def test_success(self, result: dict, expected: dict): + self._assert_parses_to(result, expected) + + @staticmethod + def _assert_parses_to(result: dict, expected: dict) -> None: + from mcp.types import CallToolResult + + from stackone_ai.tools import parse_tool_result + + parsed = parse_tool_result(CallToolResult.model_validate(result), "t") + assert parsed == expected + + @pytest.mark.parametrize( + ("result", "message", "status", "body"), + [ + pytest.param( + {"content": [_text('{"error":"Lambda execution failed","status_code":502}')]}, + 'Tool "t" failed: {"error":"Lambda execution failed","status_code":502}', + 502, + {"error": "Lambda execution failed", "status_code": 502}, + id="text", + ), + pytest.param( + {"content": [], "structuredContent": {"error": "boom", "status_code": 503}}, + 'Tool "t" failed: {"error":"boom","status_code":503}', + 503, + {"error": "boom", "status_code": 503}, + id="structured-only", + ), + pytest.param( + {"content": [_text('{"statusCode":409}')], "structuredContent": {"status_code": 500}}, + 'Tool "t" failed: {"statusCode":409}', + 409, + {"statusCode": 409}, + id="text-wins-over-structured", + ), + pytest.param({"content": []}, 'Tool "t" failed: {}', 0, {}, id="neither"), + pytest.param( + {"content": [_text("oops")]}, 'Tool "t" failed: oops', 0, {"result": "oops"}, id="plain-text" + ), + pytest.param( + {"content": [_IMAGE], "structuredContent": {"status_code": 404}}, + 'Tool "t" failed: {"status_code":404}', + 404, + {"status_code": 404}, + id="structured-and-image", + ), + ], + ) + def test_is_error(self, result: dict, message: str, status: int, body: dict): + from mcp.types import CallToolResult + + from stackone_ai.tools import parse_tool_result + + with pytest.raises(StackOneAPIError) as excinfo: + parse_tool_result(CallToolResult.model_validate({**result, "isError": True}), "t") + assert str(excinfo.value) == message + assert excinfo.value.status_code == status + assert excinfo.value.response_body == body + + @pytest.mark.parametrize( + "payload", + [ + pytest.param({"isError": False, "result": {"data": {"id": "1"}}}, id="wrapper"), + pytest.param( + { + "isError": False, + "result": {"data": []}, + "defenderMetadata": {"flagged": 0}, + "policyMetadata": {"policy": "p1"}, + }, + id="wrapper-with-metadata", + ), + pytest.param({"isError": False, "result": [1, 2]}, id="wrapper-with-a-list"), + pytest.param( + {"session_id": "s1", "actions": [{"action_id": "a", "similarity_score": 0.9}]}, + id="search-result", + ), + pytest.param({"message": "Feedback recorded", "session_id": None}, id="feedback-receipt"), + ], + ) + def test_a_result_is_returned_as_the_server_wrote_it(self, payload: dict): + """UCA's { isError: false, result, ...metadata } wrapper included, as Node returns it.""" + import json + + self._assert_parses_to({"content": [_text(json.dumps(payload))]}, payload) + self._assert_parses_to({"content": [], "structuredContent": payload}, payload) + self._assert_parses_to( + {"content": [_text(json.dumps(payload))], "structuredContent": payload}, payload + ) + + def test_uca_wrapper_over_the_wire(self, mcp_mock_server: str): + """The mock answers a per-action tools/call as UCA does: wrapper as structuredContent and text.""" + from stackone_ai.tools import build_auth_header, call_mcp_tool + + headers = {"Authorization": build_auth_header("test-key"), "x-account-id": "test-account"} + result = call_mcp_tool(f"{mcp_mock_server}/mcp", headers, "dummy_action", {"foo": "bar"}) + assert result == { + "isError": False, + "result": {"data": {"action": "dummy_action", "received": {"foo": "bar"}}}, + } + + class TestSearchAndExecuteApi: """The recommended surface: search() then execute(), no account id needed.""" @@ -810,7 +1496,7 @@ def test_search_returns_actions(self, mcp_mock_server: str): def test_execute_runs_a_searched_action(self, mcp_mock_server: str): toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) action_id = toolset.search("list items")[0]["action_id"] - assert toolset.execute(action_id)["data"] == {"nodes": []} + assert toolset.execute(action_id)["result"]["data"] == {"nodes": []} def test_execute_passes_the_nested_envelope_through(self, mcp_mock_server: str): """execute() takes the envelope an action's example_request shows, verbatim. @@ -823,7 +1509,31 @@ def test_execute_passes_the_nested_envelope_through(self, mcp_mock_server: str): assert action["example_request"] == {"query": {"page_size": 25}} result = toolset.execute(action["action_id"], action["example_request"]) - assert result["echoed_query"] == {"page_size": 25} + assert result["result"]["echoed_query"] == {"page_size": 25} + + def test_execute_forwards_host_headers_but_cannot_switch_tenant(self, mcp_mock_server: str): + """The served `headers` object is open, so a host header reaches the action — but + neither form of header argument can carry the SDK's own headers onto the wire.""" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + _reset_requests(mcp_mock_server) + + toolset.execute( + "mock_list_items", + { + "query": {"page_size": 25}, + "headers": {"x-custom": "kept", "x-account-id": "victim", "Authorization": "Basic stolen"}, + "headers_x-account-id": "victim", + }, + account_ids=["default"], + ) + + [call] = _tool_calls(mcp_mock_server, "mock_default_execute_action") + assert call["accountId"] == "default" + assert call["arguments"] == { + "query": {"page_size": 25}, + "headers": {"x-custom": "kept"}, + "action_id": "mock_list_items", + } def test_unknown_action_raises_rather_than_returning_an_error_body(self, mcp_mock_server: str): from stackone_ai.types import StackOneAPIError @@ -849,7 +1559,7 @@ def test_unknown_account_is_rejected_rather_than_served_a_default_catalog(self, toolset.fetch_tools(account_ids=["no-such-account"]) def test_execution_without_an_account_is_rejected_by_the_server(self, mcp_mock_server: str): - """/actions/rpc is account-scoped too — the sibling of the bug that shipped.""" + """tools/call is account-scoped too — the sibling of the bug that shipped.""" toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) tool = toolset.fetch_tools(account_ids=["test-account"]).get_tool("dummy_action") assert tool is not None @@ -860,6 +1570,27 @@ def test_execution_without_an_account_is_rejected_by_the_server(self, mcp_mock_s assert excinfo.value.status_code == 400 +class TestMockServesSchemasVerbatim: + """The mock once listed every per-action JSON Schema as `properties: {}`. + + MCP's high-level McpServer expects a Zod shape, and given a plain JSON Schema it + served an empty object, so no schema test against the mock ever saw a declared + parameter. A catalog with every field missing still looked like a working one. + """ + + def test_declared_properties_arrive(self, mcp_mock_server: str): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=["test-account"]).get_tool("dummy_action") + assert tool is not None + + assert tool.to_openai_function()["function"]["parameters"] == { + "type": "object", + "properties": {"foo": {"type": "string", "description": "A string parameter"}}, + "required": ["foo"], + "additionalProperties": False, + } + + class TestRecentlyFixedBehaviour: """Pins for fixes that could otherwise be reverted with the suite still green.""" @@ -905,10 +1636,10 @@ def test_fetch_accounts_rejects_a_non_list_body(self, monkeypatch): """list(dict) yields the keys, which blew up much later as an AttributeError.""" import httpx - def fake_get(*_args, **_kwargs): + def fake_send(*_args, **_kwargs): return httpx.Response(200, json={"results": [{"id": "a"}]}) - monkeypatch.setattr("stackone_ai.toolset.httpx.get", fake_get) + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", fake_send) toolset = StackOneToolSet(api_key="test-key") with pytest.raises(ToolsetLoadError, match="Unexpected /accounts response shape"): toolset.fetch_accounts() @@ -917,10 +1648,10 @@ def test_fetch_accounts_handles_invalid_or_malformed_encoding(self, monkeypatch) """Non-JSON or invalid UTF-8 (e.g. b'[\xff]') must raise ToolsetLoadError, not escape.""" import httpx - def fake_get(*_args, **_kwargs): + def fake_send(*_args, **_kwargs): return httpx.Response(200, content=b"[\xff]") - monkeypatch.setattr("stackone_ai.toolset.httpx.get", fake_get) + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", fake_send) toolset = StackOneToolSet(api_key="test-key") with pytest.raises(ToolsetLoadError, match="Invalid JSON returned by"): toolset.fetch_accounts() @@ -948,6 +1679,39 @@ def fake_fetch( toolset.fetch_tools() assert toolset._catalog_cache == {} + def test_a_fetch_after_a_clear_does_not_adopt_a_listing_from_before_it(self, monkeypatch): + toolset = StackOneToolSet(api_key="test-key", account_id="acc1") + served = iter(["stale_tool", "fresh_tool"]) + started = threading.Event() + proceed = threading.Event() + + def fake_fetch( + _endpoint: str, _headers: dict[str, str], **_kwargs: object + ) -> list[McpToolDefinition]: + name = next(served) + if name == "stale_tool": + started.set() + assert proceed.wait(timeout=5) + return [McpToolDefinition(name=name, description="", input_schema={})] + + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", fake_fetch) + before: dict[str, list[str]] = {} + thread = threading.Thread( + target=lambda: before.setdefault("names", [t.name for t in toolset.fetch_tools()]) + ) + thread.start() + assert started.wait(timeout=5) + + toolset.clear_catalog_cache() + try: + assert [t.name for t in toolset.fetch_tools()] == ["fresh_tool"] + finally: + proceed.set() + thread.join(timeout=5) + assert before["names"] == ["stale_tool"] + # The listing from before the clear finished last, and did not replace the fresh one. + assert [t.name for t in toolset.fetch_tools()] == ["fresh_tool"] + def test_nested_schema_is_not_shared_between_callers(self, monkeypatch): """Rebuilding tools per call copied the tool but not the schema graph under it.""" @@ -973,22 +1737,62 @@ def fake_fetch( class TestExecuteReturnShape: - def test_execute_unwraps_the_meta_tool_envelope(self, monkeypatch): - """Both surfaces return the payload itself, not a wrapper whose flag is always False.""" - from stackone_ai.tools import StackOneMcpTool + def test_execute_returns_the_same_shape_as_the_tool(self, mcp_mock_server: str): + """Both return the result as the server wrote it, UCA's wrapper included.""" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + assert toolset.execute("mock_list_items") == { + "isError": False, + "result": {"data": {"nodes": []}, "echoed_query": None}, + } - def fake_execute(self, arguments): - return {"isError": False, "result": {"data": {"nodes": [1, 2]}}} - monkeypatch.setattr(StackOneMcpTool, "execute", fake_execute) - monkeypatch.setattr( - "stackone_ai.toolset.fetch_mcp_tools", - lambda _e, _h, **_k: [ - McpToolDefinition(name="linear_acc1_execute_action", description="", input_schema={}) - ], - ) - toolset = StackOneToolSet(api_key="test-key", account_id="acc1") - assert toolset.execute("linear_list_issues") == {"data": {"nodes": [1, 2]}} +@contextmanager +def _silent_host() -> Iterator[str]: + """The base URL of a host that accepts connections and never answers.""" + import socket + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(8) + port = listener.getsockname()[1] + accepted: list[socket.socket] = [] + + def accept_and_stay_silent() -> None: + try: + while True: + connection, _ = listener.accept() + accepted.append(connection) + except OSError: + return + + threading.Thread(target=accept_and_stay_silent, daemon=True).start() + try: + yield f"http://127.0.0.1:{port}" + finally: + listener.close() + for connection in accepted: + connection.close() + + +def _within(seconds: float, fn: Any) -> BaseException | None: + """Run ``fn`` on a thread with a join deadline, and return what it raised. + + If a timeout regresses, the test must FAIL, not hang the whole suite. + """ + outcome: list[BaseException | None] = [] + + def attempt() -> None: + try: + fn() + outcome.append(None) + except BaseException as exc: + outcome.append(exc) + + worker = threading.Thread(target=attempt, daemon=True) + worker.start() + worker.join(timeout=seconds) + assert not worker.is_alive(), "the call ignored its timeout and is still hanging" + return outcome[0] class TestTimeoutIsHonoured: @@ -996,50 +1800,95 @@ def test_mcp_listing_respects_timeout_against_a_host_that_never_answers(self): """The MCP client's own defaults are a 300s SSE read, and timeout= was never passed through — so timeout=2 against a silent host hung for five minutes. """ - import socket import time - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(8) - port = listener.getsockname()[1] - accepted: list[socket.socket] = [] + with _silent_host() as base_url: + toolset = StackOneToolSet(api_key="k", account_id="a", base_url=base_url, timeout=1) + started = time.monotonic() + assert isinstance(_within(15, toolset.fetch_tools), ToolsetError) + assert time.monotonic() - started < 10 - def accept_and_stay_silent() -> None: - try: - while True: - connection, _ = listener.accept() - accepted.append(connection) - except OSError: - return + @pytest.mark.parametrize("in_a_running_loop", [False, True]) + def test_a_timeout_says_it_timed_out(self, in_a_running_loop: bool): + """Not "RuntimeError: no running event loop", or "WouldBlock:" inside a loop. - threading.Thread(target=accept_and_stay_silent, daemon=True).start() - try: - toolset = StackOneToolSet( - api_key="k", account_id="a", base_url=f"http://127.0.0.1:{port}", timeout=1 - ) - # Run it on a thread with a join deadline: if the timeout regresses, this - # must FAIL, not hang the whole suite for five minutes the way the bug did. - outcome: list[BaseException | None] = [] + The failure walk followed __context__ to run_async's own probe for a loop, and + reported that as the cause of every timeout. + """ + with _silent_host() as base_url: + toolset = StackOneToolSet(api_key="k", account_id="a", base_url=base_url, timeout=0.5) - def attempt() -> None: - try: + def fetch() -> None: + if not in_a_running_loop: toolset.fetch_tools() - outcome.append(None) - except BaseException as exc: - outcome.append(exc) + return - started = time.monotonic() - worker = threading.Thread(target=attempt, daemon=True) - worker.start() - worker.join(timeout=15) - assert not worker.is_alive(), "fetch_tools ignored timeout=1 and is still hanging" - assert isinstance(outcome[0], ToolsetError) - assert time.monotonic() - started < 10 - finally: - listener.close() - for connection in accepted: - connection.close() + async def inside_a_loop() -> None: + toolset.fetch_tools() + + asyncio.run(inside_a_loop()) + + error = _within(15, fetch) + + assert isinstance(error, ToolsetLoadError) + assert str(error) == f"MCP request to {base_url}/mcp timed out after 0.5s" + + def test_a_tool_call_timeout_says_it_timed_out(self): + with _silent_host() as base_url: + tool = StackOneMcpTool( + name="t", + description="", + parameters=ToolParameters(type="object", properties={}), + api_key="k", + endpoint=f"{base_url}/mcp", + account_id="a", + timeout=0.5, + ) + error = _within(15, tool.execute) + + assert isinstance(error, ToolsetLoadError) + assert str(error) == f"MCP request to {base_url}/mcp timed out after 0.5s" + + +class TestInterruptsPropagate: + """Ctrl-C and SystemExit stop the caller; they are not reported as a failed MCP request. + + Caught as BaseException they came out as a ToolsetLoadError, which LangChain's + handle_tool_error or a LangGraph ToolNode hands the model as a tool result. + """ + + @staticmethod + def _interrupted(monkeypatch, interrupt: BaseException) -> None: + def run_async(awaitable: Any) -> Any: + awaitable.close() + raise interrupt + + monkeypatch.setattr("stackone_ai.tools.run_async", run_async) + + @pytest.mark.parametrize("interrupt", [KeyboardInterrupt, SystemExit]) + def test_listing(self, monkeypatch, interrupt: type[BaseException]): + self._interrupted(monkeypatch, interrupt()) + with pytest.raises(interrupt): + fetch_mcp_tools("https://api.example.com/mcp", {}) + + @pytest.mark.parametrize("interrupt", [KeyboardInterrupt, SystemExit]) + def test_tool_call(self, monkeypatch, interrupt: type[BaseException]): + self._interrupted(monkeypatch, interrupt()) + tool = StackOneMcpTool( + name="t", + description="", + parameters=ToolParameters(type="object", properties={}), + api_key="k", + endpoint="https://api.example.com/mcp", + account_id="a", + ) + with pytest.raises(interrupt): + tool.execute({}) + + def test_an_ordinary_failure_is_still_described(self, monkeypatch): + self._interrupted(monkeypatch, RuntimeError("boom")) + with pytest.raises(ToolsetLoadError, match="failed: RuntimeError: boom"): + fetch_mcp_tools("https://api.example.com/mcp", {}) def test_every_sdk_error_is_a_stackone_error(): @@ -1048,9 +1897,9 @@ def test_every_sdk_error_is_a_stackone_error(): The toolset errors used to be unrelated siblings, so the obvious catch-all missed the two errors a user is most likely to hit first. """ - from stackone_ai.types import StackOneError + from stackone_ai.types import StackOneError, ToolArgumentsError - for error in (StackOneAPIError, ToolsetError, ToolsetConfigError, ToolsetLoadError): + for error in (StackOneAPIError, ToolArgumentsError, ToolsetError, ToolsetConfigError, ToolsetLoadError): assert issubclass(error, StackOneError), error with pytest.raises(StackOneError): @@ -1141,15 +1990,15 @@ class TestAdapterErrorsReachTheModel: @staticmethod def _failing_tool(): - from stackone_ai.tools import StackOneRpcTool + from stackone_ai.tools import StackOneMcpTool from stackone_ai.types import ToolParameters - tool = StackOneRpcTool( + tool = StackOneMcpTool( name="linear_get_issue", description="", parameters=ToolParameters(type="object", properties={"path_id": {"type": "string"}}), api_key="k", - base_url="https://api.example.com", + endpoint="https://api.example.com/mcp", account_id="acc1", ) @@ -1161,21 +2010,21 @@ def reject(_self, _arguments=None): def test_langchain_raises_tool_exception_carrying_the_response_body(self, monkeypatch): from langchain_core.tools import ToolException - from stackone_ai.tools import StackOneRpcTool + from stackone_ai.tools import StackOneMcpTool tool, reject = self._failing_tool() - monkeypatch.setattr(StackOneRpcTool, "execute", reject) + monkeypatch.setattr(StackOneMcpTool, "execute", reject) with pytest.raises(ToolException, match="path.id is missing") as excinfo: tool.to_langchain()._run(path_id="x") assert excinfo.value.status_code == 400 def test_langchain_does_not_forward_unsupplied_optionals_as_null(self, monkeypatch): """The API reads an explicit null as "required field missing", so this 400'd every call.""" - from stackone_ai.tools import StackOneRpcTool + from stackone_ai.tools import StackOneMcpTool seen: dict[str, object] = {} tool, _ = self._failing_tool() - monkeypatch.setattr(StackOneRpcTool, "execute", lambda _self, args=None: seen.update(args=args) or {}) + monkeypatch.setattr(StackOneMcpTool, "execute", lambda _self, args=None: seen.update(args=args) or {}) tool.to_langchain()._run(path_id="x", body_after=None) assert seen["args"] == {"path_id": "x"} @@ -1183,9 +2032,9 @@ def test_pydantic_ai_raises_model_retry(self, monkeypatch): pytest.importorskip("pydantic_ai") from pydantic_ai.exceptions import ModelRetry - from stackone_ai.tools import StackOneRpcTool + from stackone_ai.tools import StackOneMcpTool tool, reject = self._failing_tool() - monkeypatch.setattr(StackOneRpcTool, "execute", reject) + monkeypatch.setattr(StackOneMcpTool, "execute", reject) with pytest.raises(ModelRetry, match="path.id is missing"): tool.to_pydantic_ai_tool().function(path_id="x") diff --git a/tests/test_integrations_pydantic_ai.py b/tests/test_integrations_pydantic_ai.py index 9c98239f..cab20a31 100644 --- a/tests/test_integrations_pydantic_ai.py +++ b/tests/test_integrations_pydantic_ai.py @@ -27,8 +27,6 @@ def sample_tool() -> StackOneTool: ), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com/employees", name="bamboohr_list_employees", ), _api_key="test_key", @@ -42,8 +40,6 @@ def second_tool() -> StackOneTool: parameters=ToolParameters(type="object", properties={}), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com/employees/{id}", name="bamboohr_get_employee", ), _api_key="test_key", diff --git a/tests/test_models.py b/tests/test_models.py index 90e37c64..db281345 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,44 +1,18 @@ from collections.abc import Sequence -from unittest.mock import MagicMock, patch +from typing import Any -import httpx import pytest from hypothesis import given, settings from hypothesis import strategies as st from langchain_core.tools import BaseTool as LangChainBaseTool from pydantic import ValidationError -from stackone_ai.tools import StackOneTool, Tools +from stackone_ai.tools import StackOneMcpTool, StackOneTool, Tools from stackone_ai.types import ( ExecuteConfig, - ParameterLocation, StackOneAPIError, StackOneError, ToolParameters, - validate_method, -) - -# Hypothesis strategies for PBT -VALID_HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "PATCH"] - -# Strategy for case variations of valid HTTP methods -valid_method_case_variants = st.sampled_from(VALID_HTTP_METHODS).flatmap( - lambda method: st.sampled_from( - [ - method.lower(), - method.upper(), - method.capitalize(), - method.lower().capitalize(), - ] - ) -) - -# Strategy for invalid HTTP methods -invalid_method_strategy = st.one_of( - st.sampled_from(["OPTIONS", "HEAD", "TRACE", "CONNECT", "COPY", "MOVE", "INVALID", "FOO"]), - st.text(min_size=1, max_size=10, alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ").filter( - lambda m: m.upper() not in VALID_HTTP_METHODS - ), ) # Strategy for invalid JSON strings (must not be parseable as valid JSON at all) @@ -78,6 +52,32 @@ ) +def _mcp_tool(name: str = "test_tool", account_id: str | None = None) -> StackOneMcpTool: + return StackOneMcpTool( + name=name, + description="Test tool", + parameters=ToolParameters(type="object", properties={"id": {"type": "string"}}), + api_key="test_key", + endpoint="https://api.example.com/mcp", + account_id=account_id, + ) + + +@pytest.fixture +def mcp_calls(monkeypatch) -> list[dict[str, Any]]: + """Replace the MCP transport; each call is recorded and answered with a fixed payload.""" + calls: list[dict[str, Any]] = [] + + def fake_call(endpoint, headers, name, arguments, **kwargs): + calls.append( + {"endpoint": endpoint, "headers": headers, "name": name, "arguments": arguments, **kwargs} + ) + return {"id": arguments.get("id"), "name": "Test User"} + + monkeypatch.setattr("stackone_ai.tools.call_mcp_tool", fake_call) + return calls + + @pytest.fixture def mock_tool() -> StackOneTool: """Create a mock tool for testing""" @@ -89,42 +89,34 @@ def mock_tool() -> StackOneTool: ), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com/test/{id}", name="test_tool", ), _api_key="test_key", ) -def test_tool_execution(mock_tool): - """Test tool execution with parameters""" - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = {"id": "123", "name": "Test User"} - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response +def test_base_tool_has_no_executor(mock_tool): + """A hand-built StackOneTool has nothing to call; it must say so, not send anything.""" + with pytest.raises(StackOneError, match=r'Tool "test_tool" has no executor\. Override execute\(\)'): + mock_tool.execute({"id": "123"}) - result = mock_tool.execute({"id": "123"}) - assert result == {"id": "123", "name": "Test User"} - mock_request.assert_called_once() +def test_tool_execution(mcp_calls): + """Test tool execution with parameters""" + result = _mcp_tool().execute({"id": "123"}) + assert result == {"id": "123", "name": "Test User"} + assert len(mcp_calls) == 1 + assert mcp_calls[0]["name"] == "test_tool" + assert mcp_calls[0]["arguments"] == {"id": "123"} -def test_tool_execution_with_string_args(mock_tool): - """Test tool execution with string arguments""" - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = {"id": "123", "name": "Test User"} - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response - result = mock_tool.execute('{"id": "123"}') +def test_tool_execution_with_string_args(mcp_calls): + """Test tool execution with string arguments""" + result = _mcp_tool().execute('{"id": "123"}') - assert result == {"id": "123", "name": "Test User"} - mock_request.assert_called_once() + assert result == {"id": "123", "name": "Test User"} + assert mcp_calls[0]["arguments"] == {"id": "123"} def test_tool_openai_function_conversion(mock_tool): @@ -176,26 +168,14 @@ def test_to_langchain_conversion(mock_tool): @pytest.mark.asyncio -async def test_langchain_tool_execution(mock_tool): +async def test_langchain_tool_execution(mcp_calls): """Test execution of converted LangChain tools""" - tools = Tools(tools=[mock_tool]) - langchain_tools = tools.to_langchain() - langchain_tool = langchain_tools[0] + langchain_tool = Tools(tools=[_mcp_tool()]).to_langchain()[0] - # Mock the HTTP request - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = {"id": "test_value", "name": "Test User"} - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response + result = langchain_tool._run(id="test_value") - # Test sync execution with correct parameter name - test_args = {"id": "test_value"} - result = langchain_tool._run(**test_args) - - assert result == {"id": "test_value", "name": "Test User"} - mock_request.assert_called_once() + assert result == {"id": "test_value", "name": "Test User"} + assert len(mcp_calls) == 1 def test_to_langchain_empty_tools(): @@ -213,9 +193,7 @@ def test_to_langchain_multiple_tools(mock_tool): second_tool = mock_tool.__class__( description="Second test tool", parameters=ToolParameters(type="object", properties={"other_param": "string"}), - _execute_config=ExecuteConfig( - headers={}, method="GET", url="https://test.com/api/v2", name="second_test_tool" - ), + _execute_config=ExecuteConfig(headers={}, name="second_test_tool"), _api_key="test_key", ) @@ -231,276 +209,149 @@ def test_to_langchain_multiple_tools(mock_tool): assert set(langchain_tools[1].args_schema["properties"]) == set(second_tool.parameters.properties.keys()) -class TestValidateMethod: - """Test validate_method function""" +class TestToolParametersRequired: + """`required` is a declared field, so a hand-built tool's `required=[...]` type-checks.""" - def test_valid_methods(self): - """Test valid HTTP methods""" - assert validate_method("get") == "GET" - assert validate_method("POST") == "POST" - assert validate_method("put") == "PUT" - assert validate_method("DELETE") == "DELETE" - assert validate_method("patch") == "PATCH" + def test_it_defaults_to_none_and_is_left_out_of_a_dump(self): + parameters = ToolParameters(type="object", properties={}) + assert parameters.required is None + assert "required" not in parameters.model_dump() - def test_unsupported_method(self): - """Test unsupported HTTP method raises ValueError""" - with pytest.raises(ValueError, match="Unsupported HTTP method"): - validate_method("OPTIONS") + @pytest.mark.parametrize("served", [["b", "a"], None, "id", [1]]) + def test_a_served_value_is_kept_verbatim_even_when_malformed(self, served): + parameters = ToolParameters(type="object", properties={}, required=served) + assert parameters.required == served + assert parameters.model_dump()["required"] == served - @given(method=valid_method_case_variants) - @settings(max_examples=50) - def test_valid_methods_case_variations_pbt(self, method: str): - """PBT: Test valid HTTP methods with various case combinations.""" - result = validate_method(method) - assert result in VALID_HTTP_METHODS - assert result == method.upper() + def test_the_migration_recipe_runs(self): + class GetEmployee(StackOneTool): + def __init__(self) -> None: + super().__init__( + description="Get an employee", + parameters=ToolParameters( + type="object", properties={"id": {"type": "string"}}, required=["id"] + ), + _execute_config=ExecuteConfig(name="get_employee"), + ) - @given(method=invalid_method_strategy) - @settings(max_examples=50) - def test_invalid_methods_pbt(self, method: str): - """PBT: Test that invalid HTTP methods raise ValueError.""" - with pytest.raises(ValueError, match="Unsupported HTTP method"): - validate_method(method) + def execute(self, arguments=None): + return {"id": dict(arguments or {})["id"]} + + tool = GetEmployee() + assert tool.name == "get_employee" + assert tool.execute({"id": "e1"}) == {"id": "e1"} + assert tool.to_openai_function()["function"]["parameters"]["required"] == ["id"] class TestExecuteConfig: - """Test ExecuteConfig validation""" + """ExecuteConfig carries only what an MCP call uses.""" - def test_invalid_method_in_config(self): - """Test that invalid method in ExecuteConfig raises ValidationError""" + def test_defaults(self): + config = ExecuteConfig(name="test") + assert config.headers == {} + assert config.timeout == 60.0 + + @pytest.mark.parametrize("field", ["method", "url", "body_type", "parameter_locations"]) + def test_http_fields_are_refused_not_ignored(self, field): + """Silently ignoring them would let a caller believe a URL or method took effect.""" with pytest.raises(ValidationError): - ExecuteConfig( - method="INVALID", - url="https://api.example.com", - name="test", - ) + ExecuteConfig(name="test", **{field: "x"}) class TestStackOneToolExecution: """Test StackOneTool execution edge cases""" - @pytest.fixture - def tool_with_locations(self) -> StackOneTool: - """Create a tool with explicit parameter locations""" - return StackOneTool( - description="Test tool with param locations", - parameters=ToolParameters( - type="object", - properties={ - "path_param": {"type": "string"}, - "query_param": {"type": "string"}, - "body_param": {"type": "string"}, - }, - ), - _execute_config=ExecuteConfig( - headers={}, - method="POST", - url="https://api.example.com/resource/{path_param}", - name="test_tool", - parameter_locations={ - "path_param": ParameterLocation.PATH, - "query_param": ParameterLocation.QUERY, - "body_param": ParameterLocation.BODY, - }, - ), - _api_key="test_key", - ) - - def test_parameter_location_path(self, tool_with_locations): - """Test PATH parameter location handling""" - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = {"success": True} - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response - - tool_with_locations.execute( - { - "path_param": "test_id", - "query_param": "filter", - "body_param": "data", - } - ) - - call_kwargs = mock_request.call_args[1] - assert "resource/test_id" in call_kwargs["url"] - assert call_kwargs["params"] == {"query_param": "filter"} - assert call_kwargs["json"] == {"body_param": "data"} - - def test_account_id_in_headers(self): + def test_account_id_in_headers(self, mcp_calls): """Test account ID is added to headers""" - tool = StackOneTool( - description="Test", + _mcp_tool(account_id="acc123").execute({}) + assert mcp_calls[0]["headers"]["x-account-id"] == "acc123" + + def test_no_account_id_sends_no_account_header(self, mcp_calls): + _mcp_tool().execute({}) + assert "x-account-id" not in mcp_calls[0]["headers"] + + def test_set_account_id_retargets_the_call(self, mcp_calls): + tool = _mcp_tool(account_id="acc1") + tool.set_account_id("acc2") + tool.execute({}) + assert mcp_calls[0]["headers"]["x-account-id"] == "acc2" + + def test_sdk_headers_are_set_after_caller_headers(self, mcp_calls): + """Configured headers cannot replace the credential or retarget the account.""" + tool = StackOneMcpTool( + name="t", + description="", parameters=ToolParameters(type="object", properties={}), - _execute_config=ExecuteConfig( - headers={}, - method="GET", - url="https://api.example.com/test", - name="test", - ), - _api_key="test_key", - _account_id="acc123", + api_key="test_key", + endpoint="https://api.example.com/mcp", + account_id="acc1", + headers={ + "authorization": "Bearer stolen", + " X-Account-Id ": "victim", + "USER-AGENT": "spoof", + "X-Trace": "abc", + }, ) + tool.execute({}) + headers = mcp_calls[0]["headers"] + assert headers["X-Trace"] == "abc" + assert headers["Authorization"].startswith("Basic ") + assert headers["x-account-id"] == "acc1" + assert headers["User-Agent"].startswith("stackone-ai-python/") + assert {name.strip().lower() for name in headers} == { + "x-trace", + "authorization", + "x-account-id", + "user-agent", + } + assert list(headers)[-3:] == ["User-Agent", "Authorization", "x-account-id"] - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = {} - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response - - tool.execute({}) - - call_kwargs = mock_request.call_args[1] - assert call_kwargs["headers"]["x-account-id"] == "acc123" + def test_the_timeout_reaches_the_call(self, mcp_calls): + tool = StackOneMcpTool( + name="t", + description="", + parameters=ToolParameters(type="object", properties={}), + api_key="k", + endpoint="https://api.example.com/mcp", + account_id=None, + timeout=7.5, + ) + tool.execute({}) + assert mcp_calls[0]["timeout"] == 7.5 - def test_invalid_json_arguments(self, mock_tool): + def test_invalid_json_arguments(self): """Test invalid JSON string raises ValueError""" with pytest.raises(ValueError, match="Invalid JSON"): - mock_tool.execute("not valid json") + _mcp_tool().execute("not valid json") - def test_non_dict_arguments(self, mock_tool): + def test_non_dict_arguments(self): """Test non-dict JSON raises ValueError""" - with pytest.raises(ValueError, match="Tool arguments must be a JSON object"): - mock_tool.execute("[1, 2, 3]") + with pytest.raises(ValueError, match=r'^Tool arguments for "[^"]+" must be a JSON object$'): + _mcp_tool().execute("[1, 2, 3]") @given(invalid_json=invalid_json_strategy) @settings(max_examples=50) def test_invalid_json_arguments_pbt(self, invalid_json: str): """PBT: Test various invalid JSON strings raise ValueError.""" - # Create tool inside the test to avoid fixture issues with Hypothesis - tool = StackOneTool( - description="Test tool", - parameters=ToolParameters( - type="object", - properties={"id": {"type": "string"}}, - ), - _execute_config=ExecuteConfig( - headers={}, - method="GET", - url="https://api.example.com/test/{id}", - name="test_tool", - ), - _api_key="test_key", - ) with pytest.raises(ValueError, match="Invalid JSON"): - tool.execute(invalid_json) + _mcp_tool().execute(invalid_json) @given(non_dict_json=non_dict_json_strategy) @settings(max_examples=50) def test_non_dict_arguments_pbt(self, non_dict_json: str): """PBT: Test non-dict JSON values raise ValueError.""" - # Create tool inside the test to avoid fixture issues with Hypothesis - tool = StackOneTool( - description="Test tool", - parameters=ToolParameters( - type="object", - properties={"id": {"type": "string"}}, - ), - _execute_config=ExecuteConfig( - headers={}, - method="GET", - url="https://api.example.com/test/{id}", - name="test_tool", - ), - _api_key="test_key", - ) - with pytest.raises(ValueError, match="Tool arguments must be a JSON object"): - tool.execute(non_dict_json) + with pytest.raises(ValueError, match=r'^Tool arguments for "[^"]+" must be a JSON object$'): + _mcp_tool().execute(non_dict_json) - def test_form_body_type(self): - """Test form body type handling""" - tool = StackOneTool( - description="Test", - parameters=ToolParameters( - type="object", - properties={"field": {"type": "string"}}, - ), - _execute_config=ExecuteConfig( - headers={}, - method="POST", - url="https://api.example.com/test", - name="test", - body_type="form", - ), - _api_key="test_key", - ) + def test_api_error_propagates(self, monkeypatch): + def reject(*_args, **_kwargs): + raise StackOneAPIError("Tool failed", 400, {"error": "Bad request"}) - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = {} - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response - - tool.execute({"field": "value"}) - - call_kwargs = mock_request.call_args[1] - assert call_kwargs["data"] == {"field": "value"} - assert "json" not in call_kwargs - - def test_http_status_error_with_json_body(self, mock_tool): - """Test HTTP error with JSON response body""" - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.status_code = 400 - mock_response.text = '{"error": "Bad request"}' - mock_response.json.return_value = {"error": "Bad request"} - mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( - "Bad Request", - request=MagicMock(), - response=mock_response, - ) - mock_request.return_value = mock_response - - with pytest.raises(StackOneAPIError) as exc_info: - mock_tool.execute({"id": "123"}) - - assert exc_info.value.status_code == 400 - assert exc_info.value.response_body == {"error": "Bad request"} - - def test_http_status_error_with_text_body(self, mock_tool): - """Test HTTP error with plain text response body""" - import json as json_module - - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.status_code = 500 - mock_response.text = "Internal Server Error" - mock_response.json.side_effect = json_module.JSONDecodeError("No JSON", "", 0) - mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( - "Server Error", - request=MagicMock(), - response=mock_response, - ) - mock_request.return_value = mock_response - - with pytest.raises(StackOneAPIError) as exc_info: - mock_tool.execute({"id": "123"}) - - assert exc_info.value.status_code == 500 - assert exc_info.value.response_body == "Internal Server Error" - - def test_request_error(self, mock_tool): - """Test network/request error handling""" - with patch("httpx.request") as mock_request: - mock_request.side_effect = httpx.RequestError("Connection failed") - - with pytest.raises(StackOneError, match="Request failed"): - mock_tool.execute({"id": "123"}) - - def test_non_dict_response(self, mock_tool): - """Test non-dict JSON response is wrapped""" - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = ["item1", "item2"] - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response - - result = mock_tool.execute({"id": "123"}) - assert result == {"result": ["item1", "item2"]} + monkeypatch.setattr("stackone_ai.tools.call_mcp_tool", reject) + with pytest.raises(StackOneAPIError) as exc_info: + _mcp_tool().execute({"id": "123"}) + assert exc_info.value.status_code == 400 + assert exc_info.value.response_body == {"error": "Bad request"} class TestStackOneToolOpenAIConversion: @@ -522,8 +373,6 @@ def test_enum_property(self): ), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com", name="test", ), _api_key="test_key", @@ -548,8 +397,6 @@ def test_array_type_property(self): ), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com", name="test", ), _api_key="test_key", @@ -578,8 +425,6 @@ def test_object_type_property(self): ), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com", name="test", ), _api_key="test_key", @@ -602,8 +447,6 @@ def test_non_dict_property(self): ), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com", name="test", ), _api_key="test_key", @@ -627,9 +470,7 @@ def _tool(properties: dict) -> StackOneTool: return StackOneTool( description="Test", parameters=ToolParameters(type="object", properties=properties), - _execute_config=ExecuteConfig( - headers={}, method="GET", url="https://api.example.com", name="test" - ), + _execute_config=ExecuteConfig(headers={}, name="test"), _api_key="test_key", ) @@ -670,34 +511,12 @@ def test_internal_nullable_marker_is_not_exposed(self): assert "nullable" not in tool.to_langchain().args_schema["properties"]["a"] @pytest.mark.asyncio - async def test_arun_method(self): + async def test_arun_method(self, mcp_calls): """Test async _arun method""" - tool = StackOneTool( - description="Test", - parameters=ToolParameters( - type="object", - properties={"id": {"type": "string"}}, - ), - _execute_config=ExecuteConfig( - headers={}, - method="GET", - url="https://api.example.com", - name="test", - ), - _api_key="test_key", - ) + lc_tool = _mcp_tool().to_langchain() - lc_tool = tool.to_langchain() - - with patch("httpx.request") as mock_request: - mock_response = MagicMock() - mock_response.json.return_value = {"result": "async_test"} - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_request.return_value = mock_response - - result = await lc_tool._arun(id="123") - assert result == {"result": "async_test"} + result = await lc_tool._arun(id="123") + assert result == {"id": "123", "name": "Test User"} class TestStackOneToolAccountId: @@ -710,8 +529,6 @@ def test_set_and_get_account_id(self): parameters=ToolParameters(type="object", properties={}), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com", name="test", ), _api_key="test_key", @@ -734,8 +551,6 @@ def test_account_id_round_trip_pbt(self, account_id: str | None): parameters=ToolParameters(type="object", properties={}), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com", name="test", ), _api_key="test_key", @@ -756,8 +571,6 @@ def sample_tools(self) -> list[StackOneTool]: parameters=ToolParameters(type="object", properties={}), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com/1", name="tool_1", ), _api_key="key", @@ -768,8 +581,6 @@ def sample_tools(self) -> list[StackOneTool]: parameters=ToolParameters(type="object", properties={}), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com/2", name="tool_2", ), _api_key="key", @@ -804,8 +615,6 @@ def test_get_account_id_returns_none_when_all_none(self): parameters=ToolParameters(type="object", properties={}), _execute_config=ExecuteConfig( headers={}, - method="GET", - url="https://api.example.com", name="test", ), _api_key="key", @@ -828,9 +637,7 @@ def _tool(properties: dict) -> StackOneTool: return StackOneTool( description="Test tool", parameters=ToolParameters(type="object", properties=properties), - _execute_config=ExecuteConfig( - headers={}, method="POST", url="https://api.example.com/x", name="schema_tool" - ), + _execute_config=ExecuteConfig(headers={}, name="schema_tool"), _api_key="key", ) @@ -882,8 +689,8 @@ def test_preserves_unknown_keywords(self): assert props["x"]["x-vendor-hint"] == "something" - def test_strips_internal_nullable_marker_and_derives_required(self): - """`nullable` is an SDK-internal marker; it becomes JSON Schema `required`.""" + def test_strips_internal_nullable_marker_without_deriving_required(self): + """`nullable` is an SDK-internal marker; `required` comes only from the root.""" tool = self._tool( { "needed": {"type": "string", "nullable": False}, @@ -895,7 +702,7 @@ def test_strips_internal_nullable_marker_and_derives_required(self): assert "nullable" not in params["properties"]["needed"] assert "nullable" not in params["properties"]["optional"] - assert params["required"] == ["needed"] + assert "required" not in params def test_strips_nested_internal_marker(self): tool = self._tool( @@ -934,17 +741,15 @@ class TestExecuteOpenAIToolCalls: @staticmethod def _tools(monkeypatch, behaviour): - from stackone_ai.tools import StackOneRpcTool - - tool = StackOneRpcTool( + tool = StackOneMcpTool( name="linear_list_issues", description="", parameters=ToolParameters(type="object", properties={"body_variables": {"type": "object"}}), api_key="k", - base_url="https://api.example.com", + endpoint="https://api.example.com/mcp", account_id="acc1", ) - monkeypatch.setattr(StackOneRpcTool, "execute", lambda _self, arguments=None: behaviour(arguments)) + monkeypatch.setattr(StackOneMcpTool, "execute", lambda _self, arguments=None: behaviour(arguments)) return Tools([tool]) def test_runs_each_call_and_pairs_the_result_with_its_id(self, monkeypatch): @@ -989,16 +794,35 @@ def test_an_unknown_tool_is_reported_not_raised(self, monkeypatch): ) assert "Unknown tool" in message["content"] - def test_binary_results_serialise_instead_of_crashing(self, monkeypatch): - """A download returns raw bytes, which json.dumps cannot encode.""" - import base64 + def test_non_text_content_parts_reach_the_model_as_json(self, monkeypatch): + """An image part went to the model as its repr ("type='image' data=..."), not JSON.""" import json - tools = self._tools(monkeypatch, lambda _args: {"content": b"%PDF-1.4"}) + from mcp.types import CallToolResult + + from stackone_ai.tools import parse_tool_result + + image = {"type": "image", "data": "AAAA", "mimeType": "image/png"} + served = CallToolResult.model_validate({"content": [{"type": "text", "text": '{"a":1}'}, image]}) + tools = self._tools(monkeypatch, lambda _args: parse_tool_result(served, "linear_list_issues")) + [message] = tools.execute_openai_tool_calls( + [{"id": "c", "function": {"name": "linear_list_issues", "arguments": "{}"}}] + ) + assert json.loads(message["content"]) == {"a": 1, "content_parts": [image]} + + def test_a_download_link_is_passed_through(self, monkeypatch): + import json + + link = { + "download_url": "https://downloads.example.com/f/1", + "expires_at": "2026-01-01T00:00:00.000Z", + "file": {"name": "a.pdf", "content_type": "application/pdf", "content_length": 3}, + } + tools = self._tools(monkeypatch, lambda _args: link) [message] = tools.execute_openai_tool_calls( [{"id": "c", "function": {"name": "linear_list_issues", "arguments": "{}"}}] ) - assert json.loads(message["content"])["content"] == base64.b64encode(b"%PDF-1.4").decode() + assert json.loads(message["content"]) == link def test_no_tool_calls_is_no_messages(self, monkeypatch): tools = self._tools(monkeypatch, lambda _args: {}) diff --git a/tests/test_multi_account.py b/tests/test_multi_account.py new file mode 100644 index 00000000..c4cd01c2 --- /dev/null +++ b/tests/test_multi_account.py @@ -0,0 +1,675 @@ +"""More than one account: the catalog cache, search hits, routing and adapters. + +With discovery as the default, the same provider linked on two accounts is common, so +every surface has to say which account it means. +""" + +from __future__ import annotations + +import logging +import threading +from typing import Any + +import httpx +import pytest + +from stackone_ai import tools as tools_module +from stackone_ai import toolset as toolset_module +from stackone_ai.tools import McpToolDefinition, StackOneMcpTool, Tools +from stackone_ai.toolset import FAILED_ACCOUNT_RETRY_SECONDS, StackOneToolSet +from stackone_ai.types import ( + StackOneAPIError, + ToolParameters, + ToolsetConfigError, + ToolsetLoadError, +) + + +def _tool(name: str) -> McpToolDefinition: + return McpToolDefinition(name=name, description="", input_schema={}) + + +class _Accounts: + """A fake MCP listing per account: a list of tool names, or an exception to raise.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, **accounts: list[str] | Exception) -> None: + self.accounts = accounts + self.listed: list[str] = [] + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", self._fetch) + + def _fetch(self, _endpoint: str, headers: dict[str, str], **_kwargs: object) -> list[McpToolDefinition]: + account = headers["x-account-id"] + self.listed.append(account) + served = self.accounts[account] + if isinstance(served, Exception): + raise served + return [_tool(name) for name in served] + + +@pytest.fixture +def clock(monkeypatch: pytest.MonkeyPatch) -> list[float]: + """The failed-account clock, as a one-element list a test moves on by hand.""" + now = [1000.0] + monkeypatch.setattr(toolset_module, "_clock", lambda: now[0]) + return now + + +class TestPartialFailureCaching: + """The healthy accounts are cached; a failed one is left out until it is due a retry. + + Not caching at all made every call re-list every account, and wait out the whole + timeout on one that hangs. + """ + + def test_the_healthy_accounts_are_served_from_the_cache(self, monkeypatch, clock, caplog): + accounts = _Accounts(monkeypatch, a=["tool_a"], b=RuntimeError("boom"), c=["tool_c"]) + toolset = StackOneToolSet(api_key="k") + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + first = toolset.fetch_tools(account_ids=["a", "b", "c"]) + clock[0] += FAILED_ACCOUNT_RETRY_SECONDS - 0.001 + second = toolset.fetch_tools(account_ids=["a", "b", "c"]) + + assert [t.name for t in first] == [t.name for t in second] == ["tool_a", "tool_c"] + assert sorted(accounts.listed) == ["a", "b", "c"] + # Warned once, when it failed; not again while it is left out. + assert [r.getMessage() for r in caplog.records] == [ + "Skipping account that failed to list tools — b: boom" + ] + + def test_only_the_failed_account_is_retried_once_it_is_due(self, monkeypatch, clock, caplog): + accounts = _Accounts(monkeypatch, a=["tool_a"], b=RuntimeError("boom")) + toolset = StackOneToolSet(api_key="k") + toolset.fetch_tools(account_ids=["a", "b"]) + accounts.listed.clear() + + clock[0] += FAILED_ACCOUNT_RETRY_SECONDS + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + toolset.fetch_tools(account_ids=["a", "b"]) + assert accounts.listed == ["b"] + assert "b: boom" in caplog.text + + # Failed again: left out for another full window from now. + accounts.listed.clear() + clock[0] += FAILED_ACCOUNT_RETRY_SECONDS - 0.001 + toolset.fetch_tools(account_ids=["a", "b"]) + assert accounts.listed == [] + + def test_a_recovered_account_joins_the_cached_catalog_in_account_order(self, monkeypatch, clock): + accounts = _Accounts(monkeypatch, a=RuntimeError("boom"), b=["tool_b"]) + toolset = StackOneToolSet(api_key="k") + assert [t.name for t in toolset.fetch_tools(account_ids=["a", "b"])] == ["tool_b"] + + accounts.accounts["a"] = ["tool_a"] + clock[0] += FAILED_ACCOUNT_RETRY_SECONDS + assert [t.name for t in toolset.fetch_tools(account_ids=["a", "b"])] == ["tool_a", "tool_b"] + + accounts.listed.clear() + clock[0] += 3600 + assert [t.name for t in toolset.fetch_tools(account_ids=["a", "b"])] == ["tool_a", "tool_b"] + assert accounts.listed == [] + + def test_a_clear_forgets_the_failure_too(self, monkeypatch, clock): + accounts = _Accounts(monkeypatch, a=["tool_a"], b=RuntimeError("boom")) + toolset = StackOneToolSet(api_key="k") + toolset.fetch_tools(account_ids=["a", "b"]) + accounts.listed.clear() + toolset.clear_catalog_cache() + toolset.fetch_tools(account_ids=["a", "b"]) + assert sorted(accounts.listed) == ["a", "b"] + + +class TestConcurrentCallsDoNotClobberEachOthersPartialCatalog: + """Two concurrent fetch_tools() calls for the same scope must not overwrite each + other's cache entry: a transient failure in one must not hide a success the other + just cached for FAILED_ACCOUNT_RETRY_SECONDS. + """ + + def test_a_slower_failure_does_not_clobber_a_faster_success(self, monkeypatch, clock): + b_started = threading.Event() + second_call_done = threading.Event() + b_calls: list[int] = [] + + def fetch(_endpoint: str, headers: dict[str, str], **_kwargs: object) -> list[McpToolDefinition]: + account = headers["x-account-id"] + if account == "a": + return [_tool("tool_a")] + b_calls.append(1) + if len(b_calls) == 1: + # The first caller to reach "b" blocks until the second caller's whole + # fetch_tools() call has finished and cached both accounts, then fails — + # so its own store is the last one to run. + b_started.set() + assert second_call_done.wait(timeout=5) + raise RuntimeError("boom") + return [_tool("tool_b")] + + monkeypatch.setattr(toolset_module, "fetch_mcp_tools", fetch) + toolset = StackOneToolSet(api_key="k") + results: dict[str, list[str]] = {} + + def call_first() -> None: + results["first"] = [t.name for t in toolset.fetch_tools(account_ids=["a", "b"])] + + def call_second() -> None: + assert b_started.wait(timeout=5) + results["second"] = [t.name for t in toolset.fetch_tools(account_ids=["a", "b"])] + second_call_done.set() + + first = threading.Thread(target=call_first) + second = threading.Thread(target=call_second) + first.start() + second.start() + first.join(timeout=5) + second.join(timeout=5) + + assert results["second"] == ["tool_a", "tool_b"] + # The slower, failing call must not have evicted "b" from the cache behind it. + assert [t.name for t in toolset.fetch_tools(account_ids=["a", "b"])] == ["tool_a", "tool_b"] + + +class TestEveryAccountFails: + def test_the_same_status_everywhere_is_raised_as_itself(self, monkeypatch): + """A revoked key is a 401 with two accounts, as it is with one.""" + first, second = StackOneAPIError("a: 401", 401, None), StackOneAPIError("b: 401", 401, None) + _Accounts(monkeypatch, b=second, a=first) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k").fetch_tools(account_ids=["b", "a"]) + assert excinfo.value is first + + def test_differing_failures_are_kept(self, monkeypatch): + unauthorised, broken = StackOneAPIError("401", 401, None), StackOneAPIError("412", 412, None) + _Accounts(monkeypatch, a=unauthorised, b=broken) + with pytest.raises(ToolsetLoadError) as excinfo: + StackOneToolSet(api_key="k").fetch_tools(account_ids=["a", "b"]) + assert str(excinfo.value) == "Every account failed to list tools: a: 401; b: 412" + assert excinfo.value.failures == [unauthorised, broken] + cause = excinfo.value.__cause__ + assert isinstance(cause, ExceptionGroup) + assert list(cause.exceptions) == [unauthorised, broken] + + def test_a_401_and_a_403_are_kept_rather_than_either_raised(self, monkeypatch): + """Differing statuses are not one cause: raising the first would hide the other.""" + unauthorised, forbidden = StackOneAPIError("401", 401, None), StackOneAPIError("403", 403, None) + _Accounts(monkeypatch, a=unauthorised, b=forbidden) + with pytest.raises(ToolsetLoadError) as excinfo: + StackOneToolSet(api_key="k").fetch_tools(account_ids=["a", "b"]) + assert str(excinfo.value) == "Every account failed to list tools: a: 401; b: 403" + assert excinfo.value.failures == [unauthorised, forbidden] + + def test_an_account_that_lists_nothing_has_not_failed(self, monkeypatch): + _Accounts(monkeypatch, a=[], b=RuntimeError("boom")) + assert len(StackOneToolSet(api_key="k").fetch_tools(account_ids=["a", "b"])) == 0 + + def test_nothing_is_cached(self, monkeypatch, clock): + accounts = _Accounts(monkeypatch, a=RuntimeError("x"), b=RuntimeError("y")) + toolset = StackOneToolSet(api_key="k") + for _ in range(2): + with pytest.raises(ToolsetLoadError): + toolset.fetch_tools(account_ids=["a", "b"]) + assert len(accounts.listed) == 4 + + +def _meta_tools(monkeypatch: pytest.MonkeyPatch, per_account: dict[str, list[str]]) -> dict[str, Any]: + """Serve ``per_account``'s meta tools, and record what each one is called with.""" + seen: dict[str, Any] = {} + + def fake_execute(self: StackOneMcpTool, arguments: Any) -> dict[str, Any]: + seen["tool"], seen["account"], seen["arguments"] = self.name, self.get_account_id(), arguments + if self.name.endswith("_search_actions"): + return {"session_id": f"s-{self.get_account_id()}", "actions": seen["hits"][self.name]} + return {"isError": False, "result": {}} + + monkeypatch.setattr(StackOneMcpTool, "execute", fake_execute) + monkeypatch.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + lambda _e, headers, **_k: [_tool(name) for name in per_account[headers["x-account-id"]]], + ) + return seen + + +class TestSearchHits: + def test_each_hit_names_its_account_and_top_k_cuts_the_merged_ranking(self, monkeypatch): + seen = _meta_tools( + monkeypatch, + { + "acc1": ["hris_acc1_search_actions"], + "acc2": ["hris_acc2_search_actions", "crm_acc2_search_actions"], + }, + ) + seen["hits"] = { + "hris_acc1_search_actions": [ + {"action_id": "hris_list", "similarity_score": 0.9}, + {"action_id": "hris_get", "similarity_score": 0.4}, + ], + "hris_acc2_search_actions": [ + {"action_id": "hris_list", "similarity_score": 0.8}, + {"action_id": "hris_get", "similarity_score": 0.3}, + ], + "crm_acc2_search_actions": [{"action_id": "crm_list", "similarity_score": 0.7}], + } + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + + hits = toolset.search("list", top_k=3) + + # top_k is the total, not per connector: 3 connectors asked for 3 each served 5. + assert seen["arguments"]["top_k"] == 3 + assert [(h["action_id"], h["account_id"], h["session_id"]) for h in hits] == [ + ("hris_list", "acc1", "s-acc1"), + ("hris_list", "acc2", "s-acc2"), + ("crm_list", "acc2", "s-acc2"), + ] + + +class TestExecuteRouting: + PER_ACCOUNT = { + "acc1": ["hris_acc1_execute_action"], + "acc2": ["hris_acc2_execute_action"], + } + + def test_a_connector_on_two_accounts_is_refused(self, monkeypatch): + seen = _meta_tools(monkeypatch, self.PER_ACCOUNT) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc2", "acc1"]}) + with pytest.raises(ToolsetConfigError) as excinfo: + toolset.execute("hris_list_employees") + assert str(excinfo.value) == ( + '"hris_list_employees" matches 2 connectors on different accounts ' + "(hris_acc1_execute_action on acc1, hris_acc2_execute_action on acc2). " + "Pass the account id to use, such as a search hit's account_id." + ) + assert "tool" not in seen + + @pytest.mark.parametrize("account", ["acc1", "acc2"]) + def test_a_hits_account_id_routes_to_it(self, monkeypatch, account): + seen = _meta_tools(monkeypatch, self.PER_ACCOUNT) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + toolset.execute("hris_list_employees", account_ids=[account]) + assert (seen["tool"], seen["account"]) == (f"hris_{account}_execute_action", account) + + +def _providers(monkeypatch: pytest.MonkeyPatch, status: int = 200, **providers: str) -> list[int]: + """Answer GET /accounts with each account on its provider, or fail with ``status``.""" + body = [ + {"id": account, "provider": provider, "status": "active"} for account, provider in providers.items() + ] + calls: list[int] = [] + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + calls.append(1) + if status != 200: + return httpx.Response(status, text="nope", request=request) + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + return calls + + +class TestExecuteWithAFailedAccount: + """An account that failed to list could have served the action: running it on another + account instead would run it on an account the caller never chose. + """ + + def _execute(self, monkeypatch: pytest.MonkeyPatch, **accounts: list[str] | Exception) -> Any: + listing = _Accounts(monkeypatch, **accounts) + seen: dict[str, Any] = {} + + def fake_execute(tool: StackOneMcpTool, arguments: Any) -> dict[str, Any]: + seen["account"] = tool.get_account_id() + return {"isError": False} + + monkeypatch.setattr(StackOneMcpTool, "execute", fake_execute) + return listing, seen + + def test_an_account_on_the_actions_connector_refuses(self, monkeypatch): + _, seen = self._execute(monkeypatch, acc1=RuntimeError("boom"), acc2=["hris_acc2_execute_action"]) + _providers(monkeypatch, acc1="HRIS", acc2="hris") + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc2", "acc1"]}) + toolset.fetch_accounts() + with pytest.raises(ToolsetLoadError) as excinfo: + toolset.execute("hris_list_employees") + assert str(excinfo.value) == ( + '"hris_list_employees" may be served by an account that failed to list (acc1: boom). ' + "Pass the account id to use, such as a search hit's account_id." + ) + assert seen == {} + + def test_accounts_with_no_known_provider_refuse_in_account_order(self, monkeypatch): + """GET /accounts failed, so either could be on the action's connector.""" + _, seen = self._execute( + monkeypatch, + acc3=RuntimeError("down"), + acc1=RuntimeError("boom"), + acc2=["hris_acc2_execute_action"], + ) + calls = _providers(monkeypatch, status=403) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc3", "acc2", "acc1"]}) + with pytest.raises(ToolsetLoadError, match=r"failed to list \(acc1: boom; acc3: down\)\."): + toolset.execute("hris_list_employees") + assert seen == {} + assert len(calls) == 1 + + def test_explicit_ids_learn_their_providers_with_one_accounts_listing(self, monkeypatch, clock): + """With explicit ids no GET /accounts was made, so execute() makes one to learn the + failed account's provider before deciding.""" + listing, seen = self._execute( + monkeypatch, acc1=RuntimeError("dead"), acc2=["hris_acc2_execute_action"] + ) + calls = _providers(monkeypatch, acc1="crm", acc2="hris") + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + toolset.execute("hris_list_employees") + assert seen == {"account": "acc2"} + assert len(calls) == 1 + + # Known now: the next call neither lists accounts nor re-lists the dead one. + listing.listed.clear() + toolset.execute("hris_list_employees") + assert (len(calls), listing.listed) == (1, []) + + def test_a_rate_limited_lookup_is_raised(self, monkeypatch): + """The 429 is the key's: swallowed, it would read as a provider no lookup has named.""" + monkeypatch.setattr(tools_module, "_sleep", lambda _delay: None) + self._execute(monkeypatch, acc1=RuntimeError("boom"), acc2=["hris_acc2_execute_action"]) + _providers(monkeypatch, status=429) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + with pytest.raises(StackOneAPIError) as excinfo: + toolset.execute("hris_list_employees") + assert excinfo.value.status_code == 429 + + @pytest.mark.parametrize( + "accounts", + [{"status": 403}, {"acc1": "hris"}], + ids=["the lookup fails", "the lookup does not list it"], + ) + def test_a_lookup_that_misses_waits_out_the_failure_window(self, monkeypatch, clock, accounts): + """Looking up and listing it again on every call would pay a GET /accounts each + time, and the listing's timeout if it hangs.""" + listing, seen = self._execute( + monkeypatch, acc1=["hris_acc1_execute_action"], acc2=RuntimeError("dead") + ) + calls = _providers(monkeypatch, **accounts) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + for _ in range(3): + with pytest.raises(ToolsetLoadError, match=r"failed to list \(acc2: dead\)"): + toolset.execute("hris_list_employees") + assert seen == {} + assert (len(calls), listing.listed.count("acc2")) == (1, 1) + + clock[0] += FAILED_ACCOUNT_RETRY_SECONDS + with pytest.raises(ToolsetLoadError, match=r"failed to list \(acc2: dead\)"): + toolset.execute("hris_list_employees") + assert (len(calls), listing.listed.count("acc2")) == (2, 2) + + def test_clearing_the_cache_forgets_a_lookups_miss(self, monkeypatch, clock): + self._execute(monkeypatch, acc1=["hris_acc1_execute_action"], acc2=RuntimeError("dead")) + calls = _providers(monkeypatch, status=403) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + with pytest.raises(ToolsetLoadError): + toolset.execute("hris_list_employees") + toolset.clear_catalog_cache() + with pytest.raises(ToolsetLoadError): + toolset.execute("hris_list_employees") + assert len(calls) == 2 + + def test_the_call_whose_lookup_misses_does_not_list_it_again(self, monkeypatch, clock): + """Its own miss holds it out at once, so concurrent calls sharing the lookup don't + each pay its timeout.""" + listing, _ = self._execute(monkeypatch, acc1=["hris_acc1_execute_action"], acc2=RuntimeError("dead")) + _providers(monkeypatch, status=403) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + toolset.fetch_tools(mode="search_execute") + clock[0] += 1 + with pytest.raises(ToolsetLoadError, match=r"failed to list \(acc2: dead\)"): + toolset.execute("hris_list_employees") + assert listing.listed.count("acc2") == 1 + + def test_a_miss_is_not_recorded_across_a_clear_during_its_lookup(self, monkeypatch, clock): + self._execute(monkeypatch, acc1=["hris_acc1_execute_action"], acc2=RuntimeError("dead")) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + calls: list[int] = [] + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + calls.append(1) + if len(calls) == 1: + toolset.clear_catalog_cache() + return httpx.Response(403, text="nope", request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + for _ in range(2): + with pytest.raises(ToolsetLoadError): + toolset.execute("hris_list_employees") + assert len(calls) == 2 + + def test_a_miss_is_forgotten_once_get_accounts_names_the_account(self, monkeypatch, clock): + listing, _ = self._execute(monkeypatch, acc1=["hris_acc1_execute_action"], acc2=RuntimeError("dead")) + _providers(monkeypatch, status=403) + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + with pytest.raises(ToolsetLoadError): + toolset.execute("hris_list_employees") + _providers(monkeypatch, acc1="hris", acc2="hris") + toolset.fetch_accounts() + + clock[0] += 1 + with pytest.raises(ToolsetLoadError): + toolset.execute("hris_list_employees") + # acc2 is hris's, so it is listed again at once rather than after the window. + assert listing.listed.count("acc2") == 2 + + def test_an_account_on_another_provider_does_not(self, monkeypatch): + _, seen = self._execute(monkeypatch, acc1=RuntimeError("boom"), acc2=["hris_acc2_execute_action"]) + _providers(monkeypatch, acc1="crm", acc2="hris") + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + toolset.fetch_accounts() + toolset.execute("hris_list_employees") + assert seen == {"account": "acc2"} + + def test_a_failed_account_on_a_shorter_provider_prefix_does_not(self, monkeypatch): + """acc1's provider ``hris`` prefixes the action, but its connector is ``hris_beta``.""" + _, seen = self._execute( + monkeypatch, acc1=RuntimeError("boom"), acc2=["hris_beta_acc2_execute_action"] + ) + _providers(monkeypatch, acc1="hris", acc2="hris_beta") + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + toolset.fetch_accounts() + toolset.execute("hris_beta_list_employees") + assert seen == {"account": "acc2"} + + def test_a_failed_account_is_listed_again_at_once_and_then_routes_as_usual(self, monkeypatch, clock): + listing, _ = self._execute(monkeypatch, acc1=RuntimeError("boom"), acc2=["hris_acc2_execute_action"]) + _providers(monkeypatch, acc1="hris", acc2="hris") + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + # Left out of the search_execute catalog, which execute() lists in, for 30s from now. + assert [t.name for t in toolset.fetch_tools(mode="search_execute")] == ["hris_acc2_execute_action"] + + listing.accounts["acc1"] = ["hris_acc1_execute_action"] + listing.listed.clear() + with pytest.raises(ToolsetConfigError, match="matches 2 connectors on different accounts"): + toolset.execute("hris_list_employees") + # Only the failed account, and without waiting for it to be due. + assert listing.listed == ["acc1"] + + # Its listing is cached for the next call, like any other. + listing.listed.clear() + toolset.fetch_tools(mode="search_execute") + assert listing.listed == [] + + def test_an_account_that_fails_during_the_call_is_not_listed_again_in_it(self, monkeypatch, clock): + """Listing it again at once, as Node once did, would only double the wait.""" + listing, _ = self._execute(monkeypatch, acc1=RuntimeError("hangs"), acc2=["hris_acc2_execute_action"]) + _providers(monkeypatch, acc1="hris", acc2="hris") + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + with pytest.raises(ToolsetLoadError): + toolset.execute("hris_list_employees") + assert sorted(listing.listed) == ["acc1", "acc2"] + + def test_a_failed_account_on_another_provider_is_not_listed_again(self, monkeypatch, clock): + """A hanging crm account must not cost every hris action its timeout.""" + listing, seen = self._execute( + monkeypatch, acc1=RuntimeError("hangs"), acc2=["hris_acc2_execute_action"] + ) + _providers(monkeypatch, acc1="crm", acc2="hris") + toolset = StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", "acc2"]}) + toolset.fetch_accounts() + toolset.execute("hris_list_employees") + + listing.listed.clear() + toolset.execute("hris_list_employees") + assert listing.listed == [] + assert seen == {"account": "acc2"} + + # One that could serve the action is still listed again at once. + with pytest.raises(ToolsetLoadError, match=r"failed to list \(acc1: hangs\)"): + toolset.execute("crm_list_contacts") + assert listing.listed == ["acc1"] + + +class TestPinnedArgumentsGoLast: + """A pinned session_id and action_id are moved to the end, as in Node.""" + + def test_a_supplied_session_id_is_moved_before_action_id(self, monkeypatch): + seen = _meta_tools(monkeypatch, {"acc1": ["hris_acc1_execute_action"]}) + toolset = StackOneToolSet(api_key="k", account_id="acc1") + toolset.execute( + "hris_list_employees", + {"session_id": "model-chosen", "action_id": "x", "query": {"a": 1}}, + session_id="s-1", + ) + assert list(seen["arguments"].items()) == [ + ("query", {"a": 1}), + ("session_id", "s-1"), + ("action_id", "hris_list_employees"), + ] + + def test_an_unpinned_session_id_stays_where_it_was(self, monkeypatch): + seen = _meta_tools(monkeypatch, {"acc1": ["hris_acc1_execute_action"]}) + toolset = StackOneToolSet(api_key="k", account_id="acc1") + toolset.execute("hris_list_employees", {"session_id": "s-0", "query": {}}) + assert list(seen["arguments"]) == ["session_id", "query", "action_id"] + + +class TestExtraHeadersCannotReplaceTheSdksOwn: + def test_a_case_or_space_variant_of_an_sdk_owned_header_is_dropped(self): + tool = StackOneMcpTool( + name="t", + description="", + parameters=ToolParameters(type="object", properties={}), + api_key="k", + endpoint="https://api.example.com/mcp", + account_id="acc1", + headers={ + "authorization": "Bearer stolen", + " X-Account-Id ": "someone-else", + "USER-AGENT": "spoofed", + "x-custom": "kept", + }, + ) + headers = tool._prepare_headers() + assert headers == { + "x-custom": "kept", + "User-Agent": headers["User-Agent"], + "Authorization": "Basic azo=", + "x-account-id": "acc1", + } + assert headers["User-Agent"].startswith("stackone-ai-python/") + + +def _duplicated() -> Tools: + """hris_list on acc1 and acc2. Constructing it warns, so a test clears caplog after.""" + + def tool(name: str, account: str) -> StackOneMcpTool: + return StackOneMcpTool( + name=name, + description=f"on {account}", + parameters=ToolParameters(type="object", properties={}), + api_key="k", + endpoint="https://api.example.com/mcp", + account_id=account, + ) + + return Tools([tool("hris_list", "acc1"), tool("other", "acc1"), tool("hris_list", "acc2")]) + + +class TestAdaptersKeepTheFirstOfEachName: + DUPLICATE_WARNING = ( + "1 tool name(s) are served by more than one account (hris_list). The first one listed, " + "from the lowest account id, is used — pass account ids to choose." + ) + + def test_openai(self, caplog): + tools = _duplicated() + caplog.clear() + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + functions = tools.to_openai() + assert [(f["function"]["name"], f["function"]["description"]) for f in functions] == [ + ("hris_list", "on acc1"), + ("other", "on acc1"), + ] + assert [r.getMessage() for r in caplog.records] == [self.DUPLICATE_WARNING] + + def test_langchain(self, caplog): + tools = _duplicated() + caplog.clear() + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + converted = tools.to_langchain() + assert [(t.name, t.description) for t in converted] == [ + ("hris_list", "on acc1"), + ("other", "on acc1"), + ] + assert [r.getMessage() for r in caplog.records] == [self.DUPLICATE_WARNING] + + def test_pydantic_ai(self, caplog): + tools = _duplicated() + caplog.clear() + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + converted = tools.to_pydantic_ai() + assert [(t.name, t.description) for t in converted] == [ + ("hris_list", "on acc1"), + ("other", "on acc1"), + ] + assert [r.getMessage() for r in caplog.records] == [self.DUPLICATE_WARNING] + + def test_no_warning_without_a_clash(self, caplog): + unique = _duplicated().tools[:2] + caplog.clear() + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + Tools(unique).to_openai() + assert caplog.records == [] + + +class TestAccountIdEnvironmentVariable: + """3.x never reads STACKONE_ACCOUNT_ID; a 2.x setup relying on it is warned, once.""" + + WARNING = ( + "STACKONE_ACCOUNT_ID is set, but the SDK does not read it: with no account id passed, every " + "active shared account on this API key is used. Pass an account id to scope the toolset." + ) + + @pytest.mark.parametrize( + ("value", "kwargs", "warned"), + [ + pytest.param("acc1", {}, True, id="set-and-unscoped"), + pytest.param("", {}, False, id="empty"), + pytest.param(None, {}, False, id="unset"), + pytest.param("acc1", {"account_id": "acc2"}, False, id="account-id"), + pytest.param("acc1", {"execute": {"account_ids": ["acc2"]}}, False, id="account-ids"), + pytest.param("acc1", {"execute": {"timeout": 5}}, True, id="execute-without-account-ids"), + # An empty account_ids list is unset too, the same as _resolve_account_ids() + # treats it: it must not swallow the warning. + pytest.param("acc1", {"execute": {"account_ids": []}}, True, id="empty-account-ids"), + ], + ) + def test_warning(self, monkeypatch, caplog, value, kwargs, warned): + if value is None: + monkeypatch.delenv("STACKONE_ACCOUNT_ID", raising=False) + else: + monkeypatch.setenv("STACKONE_ACCOUNT_ID", value) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + StackOneToolSet(api_key="k", **kwargs) + assert [r.getMessage() for r in caplog.records] == ([self.WARNING] if warned else []) + + def test_the_variable_is_still_never_read(self, monkeypatch): + monkeypatch.setenv("STACKONE_ACCOUNT_ID", "from-env") + accounts = _Accounts(monkeypatch, discovered=["t"]) + monkeypatch.setattr( + StackOneToolSet, "fetch_accounts", lambda _self: [{"id": "discovered", "status": "active"}] + ) + StackOneToolSet(api_key="k").fetch_tools() + assert accounts.listed == ["discovered"] diff --git a/tests/test_no_network.py b/tests/test_no_network.py new file mode 100644 index 00000000..10b4be59 --- /dev/null +++ b/tests/test_no_network.py @@ -0,0 +1,23 @@ +"""The suite cannot reach the network: a test that did would pass or fail on a real host.""" + +from __future__ import annotations + +import httpx +import pytest + + +def test_a_request_to_a_real_host_is_refused(_no_network: list[str]) -> None: + # Which name the guard sees depends on the connect path (a resolved IP, say), so only the + # refusal is pinned. No proxy, so the attempt is at the host itself. + with ( + httpx.Client(trust_env=False) as client, + pytest.raises(RuntimeError, match="must not reach the network"), + ): + client.get("https://api.stackone.com/accounts") + assert _no_network + # Cleared, so the fixture does not also fail this test for the attempt it expected. + _no_network.clear() + + +def test_loopback_is_still_reachable(mcp_mock_server: str) -> None: + assert httpx.get(mcp_mock_server).status_code < 500 diff --git a/tests/test_rate_limit.py b/tests/test_rate_limit.py new file mode 100644 index 00000000..6302d5e5 --- /dev/null +++ b/tests/test_rate_limit.py @@ -0,0 +1,595 @@ +"""429 handling: every request the SDK makes retries a rate limit, and one that outlasts +its retries ends the whole call rather than being skipped like a per-account failure.""" + +from __future__ import annotations + +import asyncio +import logging +from datetime import UTC, datetime, timedelta +from email.utils import format_datetime +from typing import Any + +import httpx +import pytest + +from stackone_ai import tools as tools_module +from stackone_ai.tools import ( + RATE_LIMIT_BACKOFF_SECONDS, + RATE_LIMIT_MAX_DELAY_SECONDS, + RateLimitRetryingAsyncClient, + RateLimitRetryingClient, + _buffer_error_body, + _rate_limit_delay, + _retry_after_seconds, + _Throttle, + fetch_mcp_tools, + is_rate_limited, +) +from stackone_ai.toolset import StackOneToolSet +from stackone_ai.types import StackOneAPIError, ToolsetLoadError + +URL = "https://api.example.com/mcp" + + +@pytest.fixture +def sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]: + """Record every retry delay, sync or async, instead of waiting it out. + + Each recorded delay still moves the retry clocks on, so a deadline sees the time a + real wait would have taken. + """ + recorded: list[float] = [] + real_clock, real_async_clock = tools_module._clock, tools_module._async_clock + + def fake_sleep(delay: float) -> None: + recorded.append(delay) + + async def fake_async_sleep(delay: float) -> None: + recorded.append(delay) + + monkeypatch.setattr(tools_module, "_sleep", fake_sleep) + monkeypatch.setattr(tools_module, "_async_sleep", fake_async_sleep) + monkeypatch.setattr(tools_module, "_clock", lambda: real_clock() + sum(recorded)) + monkeypatch.setattr(tools_module, "_async_clock", lambda: real_async_clock() + sum(recorded)) + return recorded + + +def _responder(*responses: httpx.Response) -> tuple[list[httpx.Request], Any]: + """A handler that answers with ``responses`` in turn, repeating the last, and the requests it saw.""" + seen: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return responses[min(len(seen), len(responses)) - 1] + + return seen, handler + + +def _429(retry_after: str | None = None) -> httpx.Response: + headers = {"Retry-After": retry_after} if retry_after is not None else {} + return httpx.Response(429, headers=headers, json={"message": "Too many requests"}) + + +def _recorded(base_url: str, account_id: str, method: str) -> list[dict[str, Any]]: + response = httpx.get(f"{base_url}/__requests") + response.raise_for_status() + return [r for r in response.json() if r["accountId"] == account_id and r["method"] == method] + + +class TestRetryDelay: + def test_retry_after_as_an_http_date(self): + when = datetime.now(UTC) + timedelta(seconds=10) + assert 8.0 <= (_retry_after_seconds(format_datetime(when, usegmt=True)) or 0) <= 10.0 + + def test_retry_after_with_an_ordinary_leap_second(self): + now = datetime(2015, 6, 30, 23, 59, 0, tzinfo=UTC) + assert _retry_after_seconds("Tue, 30 Jun 2015 23:59:60 GMT", now) == 60.0 + + def test_retry_after_with_a_leap_second_at_the_maximum_year_does_not_overflow(self): + now = datetime(9999, 12, 31, 23, 59, 0, tzinfo=UTC) + assert _retry_after_seconds("Fri, 31 Dec 9999 23:59:60 GMT", now) == 60.0 + + @pytest.mark.parametrize( + "retry_after", [format_datetime(datetime.now(UTC) + timedelta(hours=1), usegmt=True)] + ) + def test_an_http_date_retry_after_is_capped(self, retry_after: str): + assert _rate_limit_delay(_429(retry_after), 1) == RATE_LIMIT_MAX_DELAY_SECONDS == 30.0 + + @pytest.mark.parametrize("retry", [1, 2, 3]) + def test_backoff_without_a_header_is_jittered_within_bounds(self, retry: int): + base = RATE_LIMIT_BACKOFF_SECONDS[retry - 1] + assert base == 2.0 ** (retry - 1) + delays = [_rate_limit_delay(_429(), retry) for _ in range(200)] + assert all(base * 0.5 <= delay <= base for delay in delays) + assert len(set(delays)) > 1 + + +class TestSyncClient: + def test_a_429_then_success(self, sleeps: list[float]): + seen, handler = _responder(_429("2"), httpx.Response(200, json={"ok": True})) + with RateLimitRetryingClient(transport=httpx.MockTransport(handler)) as client: + response = client.get(URL) + assert response.json() == {"ok": True} + assert len(seen) == 2 + assert sleeps == [2.0] + + def test_a_persistent_429_is_returned_after_four_attempts(self, sleeps: list[float]): + seen, handler = _responder(_429()) + with RateLimitRetryingClient(transport=httpx.MockTransport(handler)) as client: + response = client.get(URL) + assert response.status_code == 429 + assert response.json() == {"message": "Too many requests"} + assert len(seen) == 4 + assert len(sleeps) == 3 + + @pytest.mark.parametrize("status", [400, 412, 500, 503]) + def test_no_retry_on_any_other_status(self, sleeps: list[float], status: int): + seen, handler = _responder(httpx.Response(status, headers={"Retry-After": "1"})) + with RateLimitRetryingClient(transport=httpx.MockTransport(handler)) as client: + assert client.get(URL).status_code == status + assert len(seen) == 1 + assert sleeps == [] + + def test_each_retry_logs_one_warning(self, sleeps: list[float], caplog: pytest.LogCaptureFixture): + _, handler = _responder(_429("3")) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + with RateLimitRetryingClient(transport=httpx.MockTransport(handler)) as client: + client.get(URL) + messages = [r.getMessage() for r in caplog.records] + assert messages == [ + f"GET {URL} was rate limited (429) on attempt {n} of 4; retrying in 3s" for n in (1, 2, 3) + ] + + +class TestAsyncClient: + def _run(self, handler: Any, *, stream: bool) -> tuple[int, bytes]: + async def go() -> tuple[int, bytes]: + async with RateLimitRetryingAsyncClient( + transport=httpx.MockTransport(handler), + event_hooks={"response": [_buffer_error_body]}, + ) as client: + if not stream: + response = await client.post(URL, json={"jsonrpc": "2.0"}) + return response.status_code, response.content + async with client.stream("POST", URL, json={"jsonrpc": "2.0"}) as response: + return response.status_code, await response.aread() + + return asyncio.run(go()) + + @pytest.mark.parametrize("stream", [False, True]) + def test_a_429_then_success_resends_the_same_body(self, sleeps: list[float], stream: bool): + seen, handler = _responder(_429("1"), _429(), httpx.Response(200, json={"ok": True})) + assert self._run(handler, stream=stream) == (200, b'{"ok":true}') + assert [r.content for r in seen] == [b'{"jsonrpc":"2.0"}'] * 3 + assert sleeps[0] == 1.0 + assert 1.0 <= sleeps[1] <= 2.0 + + @pytest.mark.parametrize("stream", [False, True]) + def test_the_last_429_keeps_its_body(self, sleeps: list[float], stream: bool): + seen, handler = _responder(_429("0")) + assert self._run(handler, stream=stream) == (429, b'{"message":"Too many requests"}') + assert len(seen) == 4 + assert sleeps == [0.0, 0.0, 0.0] + + @pytest.mark.parametrize("status", [400, 500]) + def test_no_retry_on_any_other_status(self, sleeps: list[float], status: int): + seen, handler = _responder(httpx.Response(status)) + assert self._run(handler, stream=True)[0] == status + assert len(seen) == 1 + assert sleeps == [] + + +class TestFetchAccounts: + @pytest.fixture + def accounts_api(self, monkeypatch: pytest.MonkeyPatch) -> Any: + def install(*responses: httpx.Response) -> list[httpx.Request]: + seen, handler = _responder(*responses) + monkeypatch.setattr( + httpx.HTTPTransport, "handle_request", lambda _self, request: handler(request) + ) + return seen + + return install + + def test_a_429_then_success(self, sleeps: list[float], accounts_api: Any): + accounts = [{"id": "a", "provider": "p", "status": "active"}] + seen = accounts_api(_429("4"), httpx.Response(200, json=accounts)) + assert StackOneToolSet(api_key="k").fetch_accounts() == accounts + assert len(seen) == 2 + assert sleeps == [4.0] + + def test_a_persistent_429_raises_after_four_attempts(self, sleeps: list[float], accounts_api: Any): + seen = accounts_api(_429()) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k").fetch_accounts() + assert excinfo.value.status_code == 429 + assert "Too many requests" in excinfo.value.response_body + assert len(seen) == 4 + assert len(sleeps) == 3 + + def test_no_retry_on_500(self, sleeps: list[float], accounts_api: Any): + seen = accounts_api(httpx.Response(500, text="boom")) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k").fetch_accounts() + assert excinfo.value.status_code == 500 + assert len(seen) == 1 + assert sleeps == [] + + def test_a_wait_longer_than_the_timeout_is_not_started(self, sleeps: list[float], accounts_api: Any): + """Retry-After 3 against timeout=1 raises the 429 at once, rather than after 9s.""" + seen = accounts_api(_429("3")) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k", timeout=1).fetch_accounts() + assert excinfo.value.status_code == 429 + assert len(seen) == 1 + assert sleeps == [] + + def test_the_timeout_counts_from_the_first_attempt(self, sleeps: list[float], accounts_api: Any): + # 2s fits in 5s; a second 2s would end at 4s and fits; a third would end at 6s. + seen = accounts_api(_429("2")) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k", timeout=5).fetch_accounts() + assert excinfo.value.status_code == 429 + assert len(seen) == 3 + assert sleeps == [2.0, 2.0] + + +class TestATimeoutAfterARetried429IsThe429: + """The retry's wait ends before the deadline, but the retried request can still run past it. + + As a timeout, a multi-account call skipped the account like any other failure and + returned a partial catalog that looked complete. + """ + + def test_fetch_accounts(self, sleeps: list[float], monkeypatch: pytest.MonkeyPatch): + seen: list[httpx.Request] = [] + + def handler(_self: Any, request: httpx.Request) -> httpx.Response: + seen.append(request) + if len(seen) == 1: + return _429("0") + raise httpx.ReadTimeout("timed out", request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler) + with pytest.raises(StackOneAPIError) as excinfo: + StackOneToolSet(api_key="k").fetch_accounts() + assert excinfo.value.status_code == 429 + assert str(excinfo.value) == ( + "Listing accounts at https://api.stackone.com/accounts was rate limited (429) and " + "timed out after 60s while retrying" + ) + assert len(seen) == 2 + + @staticmethod + def _429_then_hang(monkeypatch: pytest.MonkeyPatch, account: str) -> None: + """The MCP host answers ``account`` 429 once, then never answers its retry.""" + real = httpx.AsyncHTTPTransport.handle_async_request + attempts: dict[str, int] = {} + + async def handler(self: Any, request: httpx.Request) -> httpx.Response: + if request.headers.get("x-account-id") != account: + return await real(self, request) + attempts[account] = attempts.get(account, 0) + 1 + if attempts[account] == 1: + return _429("0") + await asyncio.sleep(60) + raise AssertionError("the exchange's deadline should have cancelled this") + + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", handler) + + def test_listing(self, sleeps: list[float], monkeypatch: pytest.MonkeyPatch): + self._429_then_hang(monkeypatch, "throttled") + with pytest.raises(StackOneAPIError) as excinfo: + fetch_mcp_tools(URL, {"x-account-id": "throttled"}, timeout=0.5) + assert excinfo.value.status_code == 429 + assert str(excinfo.value) == ( + f"MCP request to {URL} was rate limited (429) and timed out after 0.5s while retrying" + ) + assert is_rate_limited(excinfo.value) + + def test_a_timeout_with_no_429_is_still_a_timeout(self, monkeypatch: pytest.MonkeyPatch): + async def hang(_self: Any, _request: httpx.Request) -> httpx.Response: + await asyncio.sleep(60) + raise AssertionError("unreachable") + + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", hang) + with pytest.raises(ToolsetLoadError, match="timed out after 0.5s"): + fetch_mcp_tools(URL, {"x-account-id": "a"}, timeout=0.5) + + def test_a_multi_account_listing_is_aborted( + self, mcp_mock_server: str, sleeps: list[float], monkeypatch: pytest.MonkeyPatch + ): + self._429_then_hang(monkeypatch, "throttled") + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, timeout=1) + with pytest.raises(StackOneAPIError) as excinfo: + toolset.fetch_tools(account_ids=["acc1", "throttled"]) + assert excinfo.value.status_code == 429 + + def test_a_tool_call(self, mcp_mock_server: str, sleeps: list[float], monkeypatch: pytest.MonkeyPatch): + from stackone_ai.tools import StackOneMcpTool + from stackone_ai.types import ToolParameters + + self._429_then_hang(monkeypatch, "throttled") + tool = StackOneMcpTool( + name="default_tool_1", + description="", + parameters=ToolParameters(type="object", properties={}), + api_key="test-key", + endpoint=f"{mcp_mock_server}/mcp", + account_id="throttled", + timeout=0.5, + ) + with pytest.raises(StackOneAPIError) as excinfo: + tool.execute({}) + assert excinfo.value.status_code == 429 + assert is_rate_limited(excinfo.value) + + +class TestAnOverlappingRequestDoesNotClearTheThrottle: + """Another request answered while a 429's retry hangs leaves that 429 on record. + + The MCP client's requests overlap, its event-stream GET with a ``tools/list`` say: if any + answer cleared the marker, the retry's timeout would be reported as an ordinary timeout. + """ + + def test_the_throttle_survives_another_requests_answer(self, sleeps: list[float]): + throttle = _Throttle() + list_attempts = 0 + retry_started = asyncio.Event() + + async def handler(request: httpx.Request) -> httpx.Response: + nonlocal list_attempts + if request.method == "GET": + await retry_started.wait() + return httpx.Response(405) + list_attempts += 1 + if list_attempts == 1: + return _429("0") + retry_started.set() + await asyncio.sleep(60) + raise AssertionError("cancelled before this") + + async def go() -> httpx.Response | None: + async with RateLimitRetryingAsyncClient( + transport=httpx.MockTransport(handler), throttle=throttle + ) as client: + listing = asyncio.create_task(client.post(URL, json={"method": "tools/list"})) + # Bounded, so a change that never retries fails here rather than hanging the run. + await asyncio.wait_for(client.get(URL), timeout=5) + listing.cancel() + return throttle.response + + assert asyncio.run(go()) is not None + + +class TestAStaleThrottleMarkerDoesNotTaintALaterTimeout: + """A retried 429 that then succeeds must not mark a later, unrelated timeout as a 429. + + ``initialize`` and ``tools/list`` share one ``_Throttle`` for the whole MCP exchange. + Without clearing the marker after the retry succeeds, a plain timeout on ``tools/list`` + was reported as the ``initialize`` 429 instead, and a multi-account call aborted the + whole listing rather than skipping the one account. + """ + + @staticmethod + def _429_once_on_initialize_then_hang_on_list(monkeypatch: pytest.MonkeyPatch, account: str) -> None: + import json as _json + + real = httpx.AsyncHTTPTransport.handle_async_request + attempts: dict[str, int] = {} + + async def handler(self: Any, request: httpx.Request) -> httpx.Response: + if request.headers.get("x-account-id") != account: + return await real(self, request) + method = _json.loads(request.content or b"{}").get("method") + if method == "initialize": + attempts["initialize"] = attempts.get("initialize", 0) + 1 + if attempts["initialize"] == 1: + return _429("0") + return await real(self, request) + if method == "tools/list": + await asyncio.sleep(60) + raise AssertionError("the exchange's deadline should have cancelled this") + return await real(self, request) + + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", handler) + + def test_fetch_mcp_tools_times_out_rather_than_reporting_a_stale_429( + self, mcp_mock_server: str, sleeps: list[float], monkeypatch: pytest.MonkeyPatch + ): + self._429_once_on_initialize_then_hang_on_list(monkeypatch, "acc1") + with pytest.raises(ToolsetLoadError, match="timed out after 0.5s") as excinfo: + fetch_mcp_tools( + f"{mcp_mock_server}/mcp", + {"x-account-id": "acc1", "Authorization": "Basic dGVzdC1rZXk6"}, + timeout=0.5, + ) + assert not is_rate_limited(excinfo.value) + + def test_a_two_account_listing_skips_only_the_timed_out_account( + self, mcp_mock_server: str, sleeps: list[float], monkeypatch: pytest.MonkeyPatch + ): + self._429_once_on_initialize_then_hang_on_list(monkeypatch, "acc2") + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, timeout=0.5) + tools = toolset.fetch_tools(account_ids=["acc1", "acc2"]) + assert tools.get_tool("acc1_tool_1") is not None + assert tools.get_tool("acc2_tool_1") is None + + +class TestA429ThenAStalledStreamIsATimeout: + """The retry of a 429 is answered 200, and then that response's stream stalls: the 429 + cleared, so the deadline that ends it is an ordinary timeout, as in Node.""" + + @staticmethod + def _429_then_stall_on_list(monkeypatch: pytest.MonkeyPatch, account: str) -> None: + import json as _json + + real = httpx.AsyncHTTPTransport.handle_async_request + attempts: list[int] = [] + + class Stalled(httpx.AsyncByteStream): + async def __aiter__(self) -> Any: + yield b"event: message\n" + await asyncio.sleep(60) + raise AssertionError("the exchange's deadline should have cancelled this") + + async def handler(self: Any, request: httpx.Request) -> httpx.Response: + if request.headers.get("x-account-id") != account: + return await real(self, request) + if _json.loads(request.content or b"{}").get("method") != "tools/list": + return await real(self, request) + attempts.append(1) + if len(attempts) == 1: + return _429("0") + return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=Stalled()) + + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", handler) + + def test_listing(self, mcp_mock_server: str, sleeps: list[float], monkeypatch: pytest.MonkeyPatch): + self._429_then_stall_on_list(monkeypatch, "acc1") + with pytest.raises(ToolsetLoadError, match="timed out after 0.5s") as excinfo: + fetch_mcp_tools( + f"{mcp_mock_server}/mcp", + {"x-account-id": "acc1", "Authorization": "Basic dGVzdC1rZXk6"}, + timeout=0.5, + ) + assert not is_rate_limited(excinfo.value) + assert len(sleeps) == 1 + + def test_a_two_account_listing_skips_only_that_account( + self, mcp_mock_server: str, sleeps: list[float], monkeypatch: pytest.MonkeyPatch + ): + self._429_then_stall_on_list(monkeypatch, "acc2") + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, timeout=0.5) + tools = toolset.fetch_tools(account_ids=["acc1", "acc2"]) + assert tools.get_tool("acc1_tool_1") is not None + assert tools.get_tool("acc2_tool_1") is None + + +class TestMcpAgainstMockServer: + """The mock answers 429 for `ratelimit---` account ids.""" + + @pytest.mark.parametrize("account", ["ratelimit-list-2-listing", "ratelimit-all-2-initialize"]) + def test_listing_recovers_from_429s(self, mcp_mock_server: str, sleeps: list[float], account: str): + """On tools/list itself, and on the initialize that precedes it.""" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tools = toolset.fetch_tools(account_ids=[account]) + assert tools.get_tool("default_tool_1") is not None + assert sleeps == [0.0, 0.0] + + def test_tools_call_recovers_from_a_429(self, mcp_mock_server: str, sleeps: list[float]): + account = "ratelimit-call-1-call" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=[account]).get_tool("default_tool_1") + assert tool is not None + + result = tool.execute({"fields": "id"}) + + assert result["isError"] is False + assert len(_recorded(mcp_mock_server, account, "tools/call")) == 2 + assert sleeps == [0.0] + + def test_a_persistent_429_on_tools_call_raises_after_four_attempts( + self, mcp_mock_server: str, sleeps: list[float] + ): + account = "ratelimit-call-always-call" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + tool = toolset.fetch_tools(account_ids=[account]).get_tool("default_tool_1") + assert tool is not None + + with pytest.raises(StackOneAPIError) as excinfo: + tool.execute({"fields": "id"}) + + assert excinfo.value.status_code == 429 + assert "Too many requests" in excinfo.value.response_body + assert len(_recorded(mcp_mock_server, account, "tools/call")) == 4 + assert sleeps == [0.0, 0.0, 0.0] + + def test_a_persistent_429_on_listing_raises(self, mcp_mock_server: str, sleeps: list[float]): + headers = {"Authorization": tools_module.build_auth_header("test-key")} + with pytest.raises(StackOneAPIError) as excinfo: + fetch_mcp_tools( + f"{mcp_mock_server}/mcp", {**headers, "x-account-id": "ratelimit-all-always-listing"} + ) + assert excinfo.value.status_code == 429 + assert "Too many requests" in excinfo.value.response_body + assert sleeps == [0.0, 0.0, 0.0] + + def test_a_rate_limited_account_aborts_a_multi_account_listing( + self, mcp_mock_server: str, sleeps: list[float], caplog: pytest.LogCaptureFixture + ): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + with pytest.raises(StackOneAPIError) as excinfo: + toolset.fetch_tools(account_ids=["acc1", "legacy-account", "ratelimit-all-always-multi"]) + assert excinfo.value.status_code == 429 + assert not any("Skipping account" in r.getMessage() for r in caplog.records) + # Nothing was cached: the next call lists again rather than serving a partial catalog. + assert toolset._catalog_cache == {} + + def test_other_failures_are_still_skipped(self, mcp_mock_server: str, caplog: pytest.LogCaptureFixture): + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + tools = toolset.fetch_tools(account_ids=["acc1", "legacy-account"]) + assert tools.get_tool("acc1_tool_1") is not None + assert any("Skipping account" in r.getMessage() for r in caplog.records) + + def test_a_rate_limited_account_aborts_discovery( + self, mcp_mock_server: str, sleeps: list[float], monkeypatch: pytest.MonkeyPatch + ): + discovered = [ + {"id": "acc1", "provider": "p", "status": "active"}, + {"id": "ratelimit-all-always-discovery", "provider": "p", "status": "active"}, + ] + monkeypatch.setattr(StackOneToolSet, "fetch_accounts", lambda _self: discovered) + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + with pytest.raises(StackOneAPIError) as excinfo: + toolset.fetch_tools() + assert excinfo.value.status_code == 429 + + def test_a_wait_past_the_deadline_raises_the_429(self, mcp_mock_server: str, sleeps: list[float]): + """Retry-After 2 against timeout=3: one wait fits, a second would not. + + Sleeping into the deadline turned the 429 into a timeout, which was skipped like + any per-account failure, and the call returned acc1's tools alone. + """ + account = "ratelimit-all-always-after2-deadline" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, timeout=3) + with pytest.raises(StackOneAPIError) as excinfo: + toolset.fetch_tools(account_ids=["acc1", account]) + assert excinfo.value.status_code == 429 + assert sleeps == [2.0] + + def test_a_wait_longer_than_the_timeout_is_not_started(self, mcp_mock_server: str, sleeps: list[float]): + account = "ratelimit-call-always-after5-deadline" + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server, timeout=3) + tool = toolset.fetch_tools(account_ids=[account]).get_tool("default_tool_1") + assert tool is not None + + with pytest.raises(StackOneAPIError) as excinfo: + tool.execute({"fields": "id"}) + + assert excinfo.value.status_code == 429 + assert len(_recorded(mcp_mock_server, account, "tools/call")) == 1 + assert sleeps == [] + + def test_a_rate_limited_connector_aborts_search(self, mcp_mock_server: str, sleeps: list[float]): + # Listing succeeds for both; only the rate-limited account's search_actions call 429s. + toolset = StackOneToolSet(api_key="test-key", base_url=mcp_mock_server) + with pytest.raises(StackOneAPIError) as excinfo: + toolset.search("anything", account_ids=["acc1", "ratelimit-call-always-search"]) + assert excinfo.value.status_code == 429 + + +class TestOnlyAnHttp429IsFatal: + """A 429 in a tool result's payload was never retried, so it does not end the call.""" + + def test_an_http_429_inside_a_task_group_is_fatal(self): + request = httpx.Request("POST", "https://api.example.com/mcp") + http = httpx.HTTPStatusError("429", request=request, response=httpx.Response(429, request=request)) + error = StackOneAPIError("rate limited", 429, None) + error.__cause__ = ExceptionGroup("unhandled errors in a TaskGroup", [http]) + assert is_rate_limited(error) + + def test_a_payload_429_is_not(self): + assert not is_rate_limited(StackOneAPIError("Tool failed", 429, {"isError": True})) diff --git a/tests/test_shared_vectors.py b/tests/test_shared_vectors.py new file mode 100644 index 00000000..a1556da7 --- /dev/null +++ b/tests/test_shared_vectors.py @@ -0,0 +1,844 @@ +"""sdk-conformance's shared unit-test vectors, run against this SDK's own code. + +``tests/vectors/`` is a byte-identical copy of sdk-conformance's ``vectors/`` at the commit +CI pins; ``scripts/sync_vectors.sh`` refreshes it and CI fails if it drifts. The Node SDK +loads the same files, so both are checked against the same inputs and expected outputs. +Every case runs through the code path the SDK uses for real: a header case through +``StackOneMcpTool.execute``, an account-id case through the toolset, a message through +whatever raises or logs it. ``unresolved`` and ``known_differences`` are not graded. +""" + +from __future__ import annotations + +import json +import logging +import math +import re +from collections.abc import Callable, Iterator +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import httpx +import pytest + +from stackone_ai import tools as tools_module +from stackone_ai.tools import ( + McpToolDefinition, + RateLimitRetryingClient, + StackOneMcpTool, + StackOneTool, + Tools, + _backoff_delay, + _describe_mcp_failure, + _outlasts_deadline, + _raise_mcp_failure, + _retry_after_seconds, + _Throttle, + parse_tool_result, +) +from stackone_ai.toolset import StackOneToolSet +from stackone_ai.types import ( + ExecuteConfig, + StackOneAPIError, + StackOneError, + ToolArgumentsError, + ToolParameters, + ToolsetConfigError, + ToolsetLoadError, +) + +VECTORS = Path(__file__).parent / "vectors" + +# Bumped by sdk-conformance when an existing case changes meaning. A bump fails here until +# someone has read what changed and updated the SDK to match. +VERSIONS = { + "account-ids.json": 1, + "argument-encoding.json": 1, + "backoff.json": 1, + "header-arguments.json": 1, + "header-names.json": 1, + "header-values.json": 1, + "messages.json": 1, + "retry-after.json": 1, +} + +ENDPOINT = "https://api.example.com/mcp" +TOOL_NAME = "linear_acct_execute_action" + + +def _decode(value: Any) -> Any: + """Replace every ``{"$number": "NaN" | "Infinity" | "-Infinity"}`` with its float.""" + if isinstance(value, dict): + if value.keys() == {"$number"}: + return float(value["$number"].replace("Infinity", "inf")) + return {key: _decode(item) for key, item in value.items()} + if isinstance(value, list): + return [_decode(item) for item in value] + return value + + +def _load(name: str) -> dict[str, Any]: + return json.loads((VECTORS / name).read_text(encoding="utf-8")) + + +def _cases(name: str, key: str = "cases") -> list[Any]: + return [pytest.param(case, id=case["id"]) for case in _load(name)[key]] + + +def _seconds_close(actual: float | None, expected: float | None) -> bool: + """Equal within the README's relative tolerance of 1e-9; ``None`` only equals ``None``.""" + if actual is None or expected is None: + return actual is expected + if math.isinf(expected): + return actual == expected + return math.isclose(actual, expected, rel_tol=1e-9, abs_tol=0.0) + + +# --- messages.json: rendering a template ------------------------------------------------ + +MESSAGES = _load("messages.json") +TEMPLATES = {message["id"]: message for message in MESSAGES["messages"]} + + +def _js_seconds(value: float) -> str: + """A number as JavaScript's String(n) writes it, for the ordinary values the tests use.""" + return str(int(value)) if float(value).is_integer() else repr(float(value)) + + +def _json_type(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "boolean" + if isinstance(value, int | float): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, list): + return "array" + return "object" + + +FORMATS: dict[str, Callable[[Any], str]] = { + "text": str, + "json": lambda v: json.dumps(v, ensure_ascii=False, separators=(",", ":")), + "integer": lambda v: str(int(v)), + "seconds": _js_seconds, + "seconds-2dp": lambda v: _js_seconds(math.floor(v * 100 + 0.5) / 100), + "json-type": _json_type, + "reason": lambda v: MESSAGES["reasons"][v], +} + + +def _render(message_id: str, **values: Any) -> str: + """The canonical text of a message, its placeholders written in their formats.""" + message = TEMPLATES[message_id] + placeholders = message["placeholders"] + assert set(values) == set(placeholders), f"{message_id} takes {sorted(placeholders)}" + return re.sub( + r"\{(\w+)\}", + lambda m: FORMATS[placeholders[m[1]]["format"]](values[m[1]]), + message["template"], + ) + + +def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + r.getMessage() + for r in caplog.records + if r.name.startswith("stackone") and r.levelno == logging.WARNING + ] + + +# --- shared fixtures --------------------------------------------------------------------- + + +@pytest.fixture +def sent(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """Replace the MCP transport and record the arguments a tools/call would have sent.""" + captured: dict[str, Any] = {} + + def fake_call(endpoint, headers, name, arguments, **_kwargs): + captured["arguments"] = arguments + return {"isError": False, "result": {}} + + monkeypatch.setattr(tools_module, "call_mcp_tool", fake_call) + return captured + + +@pytest.fixture +def warnings_log(caplog: pytest.LogCaptureFixture) -> Iterator[pytest.LogCaptureFixture]: + with caplog.at_level(logging.WARNING, logger="stackone"): + yield caplog + + +def _served_tool(input_schema: dict[str, Any], name: str = TOOL_NAME) -> StackOneTool: + """A tool built the way fetch_tools() builds one from a served inputSchema.""" + toolset = StackOneToolSet(api_key="test-key", account_id="acc-1") + return toolset._create_tool(McpToolDefinition(name, "", input_schema), "acc-1", ENDPOINT) + + +OPEN_HEADERS = {"type": "object", "properties": {"headers": {"type": "object"}}} + + +def test_every_vector_file_is_graded(): + """A new vector file fails here until a test loads it.""" + assert sorted(p.name for p in VECTORS.glob("*.json")) == sorted(VERSIONS) + + +@pytest.mark.parametrize(("name", "version"), sorted(VERSIONS.items())) +def test_vector_file_versions(name: str, version: int): + assert _load(name)["version"] == version, f"{name} changed meaning: read what changed, then bump" + + +# --- retry-after.json -------------------------------------------------------------------- + + +@pytest.mark.parametrize("case", _cases("retry-after.json")) +def test_retry_after(case: dict[str, Any]): + now = datetime.fromisoformat(case["now"].replace("Z", "+00:00")) + expected = _decode(case["expected_seconds"]) + actual = _retry_after_seconds(case["header"], now) + assert _seconds_close(actual, expected), f"{case['header']!r} read as {actual}, expected {expected}" + + +# --- backoff.json ------------------------------------------------------------------------ + +BACKOFF = _load("backoff.json") + + +def test_backoff_constants(): + constants = BACKOFF["constants"] + assert tools_module.RATE_LIMIT_MAX_RETRIES == constants["max_retries"] + assert list(tools_module.RATE_LIMIT_BACKOFF_SECONDS) == constants["backoff_seconds"] + assert tools_module.RATE_LIMIT_JITTER == (constants["jitter"]["min"], constants["jitter"]["max"]) + assert tools_module.RATE_LIMIT_MAX_DELAY_SECONDS == constants["max_delay_seconds"] + + +@pytest.mark.parametrize("case", _cases("backoff.json")) +def test_backoff(case: dict[str, Any], monkeypatch: pytest.MonkeyPatch): + # Through _rate_limit_delay, as the client calls it: the 429's Retry-After header, the + # jitter drawn from the patched random source. + monkeypatch.setattr(tools_module.random, "random", lambda: case["random"]) + retry_after = case["retry_after_seconds"] + headers = {} if retry_after is None else {"Retry-After": str(retry_after)} + actual = tools_module._rate_limit_delay(httpx.Response(429, headers=headers), case["retry"]) + assert _seconds_close(actual, case["expected_delay_seconds"]) + assert _seconds_close( + _backoff_delay(case["retry"], retry_after, case["random"]), case["expected_delay_seconds"] + ) + + +@pytest.mark.parametrize("case", _cases("backoff.json", "deadline_cases")) +def test_backoff_deadline(case: dict[str, Any], warnings_log: pytest.LogCaptureFixture): + request = httpx.Request("GET", ENDPOINT) + gives_up = _outlasts_deadline(request, 1, case["delay_seconds"], case["remaining_seconds"]) + assert ("return-429" if gives_up else "wait") == case["expected"] + expected_warnings = ( + [ + _render( + "rate-limit-deadline", + method="GET", + url=ENDPOINT, + attempt=1, + max_attempts=BACKOFF["constants"]["max_attempts"], + delay=case["delay_seconds"], + ) + ] + if gives_up + else [] + ) + assert _warnings(warnings_log) == expected_warnings + + +# --- header-values.json ------------------------------------------------------------------ + + +@pytest.mark.parametrize("case", _cases("header-values.json")) +def test_header_value(case: dict[str, Any], sent: dict[str, Any]): + _served_tool(OPEN_HEADERS).execute({"headers": {"x-probe": _decode(case["value"])}}) + expected = case["expected"] + forwarded = {} if expected == {"dropped": True} else {"x-probe": expected} + assert sent["arguments"] == {"headers": forwarded} + + +# --- header-names.json ------------------------------------------------------------------- + + +@pytest.mark.parametrize("case", _cases("header-names.json")) +def test_header_name(case: dict[str, Any], sent: dict[str, Any], warnings_log: pytest.LogCaptureFixture): + _served_tool(OPEN_HEADERS).execute({"headers": {case["name"]: "v"}}) + expected = case["expected"] + if "forwarded" in expected: + assert sent["arguments"] == {"headers": {expected["forwarded"]: "v"}} + assert _warnings(warnings_log) == [] + else: + assert sent["arguments"] == {"headers": {}} + header = tools_module._trim_header_name(case["name"]) + assert _warnings(warnings_log) == [ + _render("header-dropped", header=header, reason=expected["dropped"]) + ] + + +# --- header-arguments.json --------------------------------------------------------------- + + +def _render_header_warning(warning: dict[str, str]) -> str: + if "header" in warning: + return _render("header-dropped", header=warning["header"], reason=warning["reason"]) + return _render("header-argument-dropped", argument=warning["argument"], reason=warning["reason"]) + + +@pytest.mark.parametrize("case", _cases("header-arguments.json")) +def test_header_arguments(case: dict[str, Any], sent: dict[str, Any], warnings_log: pytest.LogCaptureFixture): + _served_tool(case["schema"]).execute(_decode(case["arguments"])) + assert sent["arguments"] == _decode(case["expected_arguments"]) + assert _warnings(warnings_log) == [_render_header_warning(w) for w in case["expected_warnings"]] + + +# --- argument-encoding.json -------------------------------------------------------------- + + +def _first_non_finite(value: Any) -> float | None: + if isinstance(value, float) and not math.isfinite(value): + return value + children = value.values() if isinstance(value, dict) else value if isinstance(value, list) else [] + return next((found for child in children if (found := _first_non_finite(child)) is not None), None) + + +@pytest.mark.parametrize("case", _cases("argument-encoding.json")) +def test_argument_encoding(case: dict[str, Any], sent: dict[str, Any]): + tool = _served_tool({"type": "object", "properties": {}}) + arguments = case["arguments_json"] if "arguments_json" in case else _decode(case["arguments"]) + expected = case["expected"] + if expected == "ok": + tool.execute(arguments) + assert sent["arguments"] == (json.loads(arguments) if isinstance(arguments, str) else arguments) + return + + assert expected["error"] == "ToolArgumentsError" + with pytest.raises(ToolArgumentsError) as raised: + tool.execute(arguments) + message = str(raised.value) + reason = expected["reason"] + if reason == "not-finite": + parsed = json.loads(arguments) if isinstance(arguments, str) else arguments + value = {"nan": "NaN", "inf": "Infinity", "-inf": "-Infinity"}[repr(_first_non_finite(parsed))] + assert message == _render("arguments-not-finite", tool=TOOL_NAME, value=value) + elif reason == "not-an-object": + assert message == _render("arguments-not-an-object", tool=TOOL_NAME) + else: + message_id = {"invalid-json": "arguments-invalid-json", "unencodable": "arguments-not-encodable"}[ + reason + ] + detail = str(raised.value.__cause__) + assert message == _render(message_id, tool=TOOL_NAME, detail=detail) + assert not message.endswith("is not a JSON number") + + +# --- account-ids.json -------------------------------------------------------------------- + +ACCOUNT_IDS = _load("account-ids.json") + + +def _account_id_messages(parameter: str, value: Any) -> set[str]: + """Every canonical refusal of an account-id argument.""" + if parameter == "account_id": + return {_render("empty-account-id", parameter=parameter)} + rendered = { + _render("account-ids-not-strings", parameter=parameter), + _render("account-ids-empty-id", parameter=parameter), + } + if isinstance(value, str): + rendered.add(_render("account-ids-not-a-list", parameter=parameter, value=value)) + return rendered + + +def _account_id_paths(parameter: str, value: Any) -> list[Callable[[], object]]: + """Every way a caller hands the toolset this argument.""" + if parameter == "account_id": + return [lambda: StackOneToolSet(api_key="test-key", account_id=value)] + return [ + lambda: StackOneToolSet(api_key="test-key", execute={"account_ids": value}), + lambda: StackOneToolSet(api_key="test-key").set_accounts(value), + lambda: StackOneToolSet(api_key="test-key", account_id="acc-0")._resolve_account_ids(value), + ] + + +@pytest.mark.parametrize("case", _cases("account-ids.json")) +def test_account_ids(case: dict[str, Any]): + parameter, value, expected = case["parameter"], case["value"], case["expected"] + for path in _account_id_paths(parameter, value): + if expected == "ok": + path() + continue + error_class = expected["error"]["python"] + assert error_class == ToolsetConfigError.__name__ == ACCOUNT_IDS["error_classes"]["python"] + with pytest.raises(ToolsetConfigError) as raised: + path() + assert str(raised.value) in _account_id_messages(parameter, value) + + +# --- messages.json: every message through the code that emits it ------------------------ + + +def _listing(*names: str) -> Callable[..., list[McpToolDefinition]]: + def fake_fetch(_endpoint: str, headers: dict[str, str], **_kwargs: object) -> list[McpToolDefinition]: + account = headers.get("x-account-id", "") + return [ + McpToolDefinition(name.format(account=account), "", {"type": "object", "properties": {}}) + for name in names + ] + + return fake_fetch + + +def _failing_for(failing: set[str], error: Exception, *names: str) -> Callable[..., list[McpToolDefinition]]: + listed = _listing(*names) + + def fake_fetch(endpoint: str, headers: dict[str, str], **kwargs: object) -> list[McpToolDefinition]: + if headers.get("x-account-id") in failing: + raise error + return listed(endpoint, headers, **kwargs) + + return fake_fetch + + +def _accounts_answer( + monkeypatch: pytest.MonkeyPatch, answer: Callable[[httpx.Request], httpx.Response] +) -> str: + """Answer GET /accounts with ``answer``; the URL it is requested at.""" + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", lambda _self, request: answer(request)) + return "https://api.stackone.com/accounts" + + +def _raised(error_class: type[BaseException], call: Callable[[], object]) -> BaseException: + with pytest.raises(error_class) as raised: + call() + return raised.value + + +# The warning or tool-result text, or the exception raised; and the placeholder values. +Emitted = tuple[str | BaseException, dict[str, Any]] + + +def _header_dropped(mp, log) -> Emitted: + mp.setattr(tools_module, "call_mcp_tool", lambda *_args, **_kwargs: {}) + _served_tool(OPEN_HEADERS).execute({"headers": {"Authorization": "Bearer x"}}) + return _warnings(log)[0], {"header": "Authorization", "reason": "set-by-sdk"} + + +def _header_argument_dropped(mp, log) -> Emitted: + mp.setattr(tools_module, "call_mcp_tool", lambda *_args, **_kwargs: {}) + _served_tool({"type": "object", "properties": {}}).execute({"headers_x-foo": "1"}) + return _warnings(log)[0], {"argument": "headers_x-foo", "reason": "not-declared"} + + +def _rate_limit_retry(mp, log) -> Emitted: + mp.setattr(tools_module.random, "random", lambda: 0.5) + mp.setattr(tools_module, "_sleep", lambda _delay: None) + answers = iter([httpx.Response(429), httpx.Response(200)]) + with RateLimitRetryingClient(transport=httpx.MockTransport(lambda _r: next(answers))) as client: + client.get(ENDPOINT) + values = {"method": "GET", "url": ENDPOINT, "attempt": 1, "max_attempts": 4, "delay": 0.75} + return _warnings(log)[0], values + + +def _rate_limit_deadline(mp, log) -> Emitted: + answer = httpx.Response(429, headers={"Retry-After": "5"}) + with RateLimitRetryingClient(transport=httpx.MockTransport(lambda _r: answer), retry_within=1) as client: + client.get(ENDPOINT) + values = {"method": "GET", "url": ENDPOINT, "attempt": 1, "max_attempts": 4, "delay": 5} + return _warnings(log)[0], values + + +def _skip_account(mp, log) -> Emitted: + mp.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + _failing_for({"acc-2"}, ToolsetLoadError("boom"), "t_{account}"), + ) + StackOneToolSet(api_key="k").fetch_tools(account_ids=["acc-1", "acc-2"]) + return _warnings(log)[0], {"account_id": "acc-2", "error": "boom"} + + +def _skip_connector(mp, log) -> Emitted: + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", _listing("c_{account}_search_actions")) + + def search(self: StackOneTool, _arguments: Any = None) -> dict[str, Any]: + if self.name == "c_acc-2_search_actions": + raise StackOneAPIError("boom", 500, None) + return {"actions": [{"action_id": "c_a"}]} + + mp.setattr(StackOneMcpTool, "execute", search) + StackOneToolSet(api_key="k").search("q", account_ids=["acc-1", "acc-2"]) + return _warnings(log)[0], {"tool": "c_acc-2_search_actions", "error": "boom"} + + +def _duplicate_tool_names(mp, log) -> Emitted: + tools = Tools([_served_tool({"type": "object", "properties": {}}, "t") for _ in range(2)]) + # Once by the collection, and again by each adapter that keeps the first of each name. + tools.to_openai() + collection, adapter = _warnings(log) + assert collection == adapter + return adapter, {"count": 1, "names": "t"} + + +def _ambiguous_connector(mp, log) -> Emitted: + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", _listing("linear_{account}_execute_action")) + mp.setattr(StackOneMcpTool, "execute", lambda _self, _arguments=None: {}) + error = _raised( + ToolsetConfigError, + lambda: StackOneToolSet(api_key="k").execute("linear_list_issues", account_ids=["acc-2", "acc-1"]), + ) + tools = "linear_acc-1_execute_action on acc-1, linear_acc-2_execute_action on acc-2" + return error, {"action_id": "linear_list_issues", "count": 2, "tools": tools} + + +def _account_id_env_ignored(mp, log) -> Emitted: + mp.setenv("STACKONE_ACCOUNT_ID", "acc-1") + StackOneToolSet(api_key="k") + [warning] = _warnings(log) + return warning, {} + + +def _connector_account_unavailable(mp, log) -> Emitted: + mp.setattr( + "stackone_ai.toolset.fetch_mcp_tools", + _failing_for({"acc-2"}, ToolsetLoadError("boom"), "linear_{account}_execute_action"), + ) + # Explicit ids, so execute() makes one GET /accounts to learn acc-2's provider. It does + # not list acc-2, so its provider stays unknown and it blocks. + _accounts_answer(mp, lambda _r: httpx.Response(200, json=[{"id": "acc-1", "provider": "linear"}])) + error = _raised( + ToolsetLoadError, + lambda: StackOneToolSet(api_key="k").execute("linear_list_issues", account_ids=["acc-1", "acc-2"]), + ) + return error, {"action_id": "linear_list_issues", "failures": "acc-2: boom"} + + +def _non_shared_accounts_skipped(mp, log) -> Emitted: + accounts = [ + {"id": "acc-1", "provider": "linear", "status": "active", "shared": True}, + {"id": "acc-3", "provider": "linear", "status": "active", "shared": False, "origin_username": "u3"}, + {"id": "acc-2", "provider": "linear", "status": "active", "shared": False, "origin_username": "u2"}, + ] + mp.setattr(StackOneToolSet, "fetch_accounts", lambda _self: accounts) + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", _listing("linear_{account}_execute_action")) + StackOneToolSet(api_key="k").fetch_tools() + [warning] = _warnings(log) + return warning, {"count": 2, "accounts": "acc-2, acc-3"} + + +def _mcp_timeout(mp, log) -> Emitted: + return _describe_mcp_failure(TimeoutError(), ENDPOINT, 0.5), {"endpoint": ENDPOINT, "timeout": 0.5} + + +def _mcp_http_failure(mp, log) -> Emitted: + response = httpx.Response(503, text=" down ", request=httpx.Request("POST", ENDPOINT)) + failure = httpx.HTTPStatusError("503", request=response.request, response=response) + values = {"endpoint": ENDPOINT, "status": 503, "reason_phrase": "Service Unavailable", "body": "down"} + return _describe_mcp_failure(failure, ENDPOINT, 60), values + + +def _mcp_failure(mp, log) -> Emitted: + failure = ConnectionRefusedError("refused") + values = {"endpoint": ENDPOINT, "error_type": "ConnectionRefusedError", "error_message": "refused"} + return _describe_mcp_failure(failure, ENDPOINT, 60), values + + +def _mcp_rate_limit_timeout(mp, log) -> Emitted: + throttled = httpx.Response(429, request=httpx.Request("POST", ENDPOINT)) + throttle = _Throttle(response=throttled) + error = _raised(StackOneAPIError, lambda: _raise_mcp_failure(TimeoutError(), ENDPOINT, 0.5, throttle)) + assert error.status_code == 429 + return error, {"endpoint": ENDPOINT, "timeout": 0.5} + + +def _accounts_http_failure(mp, log) -> Emitted: + url = _accounts_answer(mp, lambda _r: httpx.Response(401, text="bad key\n")) + error = _raised(StackOneAPIError, StackOneToolSet(api_key="k").fetch_accounts) + return error, {"url": url, "status": 401, "reason_phrase": "Unauthorized", "body": "bad key"} + + +def _accounts_rate_limit_timeout(mp, log) -> Emitted: + seen: list[httpx.Request] = [] + + def answer(request: httpx.Request) -> httpx.Response: + seen.append(request) + if len(seen) == 1: + return httpx.Response(429, headers={"Retry-After": "0"}) + raise httpx.ReadTimeout("timed out", request=request) + + url = _accounts_answer(mp, answer) + error = _raised(StackOneAPIError, StackOneToolSet(api_key="k").fetch_accounts) + assert error.status_code == 429 + return error, {"url": url, "timeout": 60.0} + + +def _accounts_unreachable(mp, log) -> Emitted: + def refuse(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("refused", request=request) + + url = _accounts_answer(mp, refuse) + return _raised(ToolsetLoadError, StackOneToolSet(api_key="k").fetch_accounts), { + "url": url, + "detail": "refused", + } + + +def _accounts_invalid_json(mp, log) -> Emitted: + url = _accounts_answer(mp, lambda _r: httpx.Response(200, content=b"{")) + error = _raised(ToolsetLoadError, StackOneToolSet(api_key="k").fetch_accounts) + return error, {"url": url, "detail": str(error.__cause__)} + + +def _accounts_unexpected_shape(mp, log) -> Emitted: + _accounts_answer(mp, lambda _r: httpx.Response(200, json="accounts")) + return _raised(ToolsetLoadError, StackOneToolSet(api_key="k").fetch_accounts), {"json_type": "accounts"} + + +def _no_linked_accounts(mp, log) -> Emitted: + _accounts_answer(mp, lambda _r: httpx.Response(200, json=[])) + return _raised(ToolsetConfigError, StackOneToolSet(api_key="k").fetch_tools), {} + + +def _no_active_accounts(mp, log) -> Emitted: + accounts = [ + {"id": "a", "provider": "linear", "status": "inactive"}, + {"id": "b", "provider": "jira", "status": "error"}, + ] + _accounts_answer(mp, lambda _r: httpx.Response(200, json=accounts)) + error = _raised(ToolsetConfigError, StackOneToolSet(api_key="k").fetch_tools) + return error, {"count": 2, "accounts": "linear (inactive), jira (error)"} + + +def _no_shared_accounts(mp, log) -> Emitted: + accounts = [ + {"id": "a", "provider": "linear", "status": "active", "shared": False, "origin_username": "u1"}, + {"id": "b", "provider": "jira", "status": "active", "shared": False, "origin_username": "u2"}, + {"id": "c", "provider": "jira", "status": "error", "shared": True}, + ] + _accounts_answer(mp, lambda _r: httpx.Response(200, json=accounts)) + error = _raised(ToolsetConfigError, StackOneToolSet(api_key="k").fetch_tools) + return error, {"count": 2} + + +def _all_accounts_failed(mp, log) -> Emitted: + failures = {"acc-1": StackOneAPIError("gone", 412, None), "acc-2": ToolsetLoadError("boom")} + + def fail(_endpoint: str, headers: dict[str, str], **_kwargs: object) -> list[McpToolDefinition]: + raise failures[headers["x-account-id"]] + + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", fail) + error = _raised( + ToolsetLoadError, lambda: StackOneToolSet(api_key="k").fetch_tools(account_ids=["acc-2", "acc-1"]) + ) + return error, {"failures": "acc-1: gone; acc-2: boom"} + + +def _no_connector_returned_results(mp, log) -> Emitted: + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", _listing("c_{account}_search_actions")) + + def fail(self: StackOneTool, _arguments: Any = None) -> dict[str, Any]: + raise StackOneAPIError("boom", 500, None) + + mp.setattr(StackOneMcpTool, "execute", fail) + error = _raised( + ToolsetLoadError, lambda: StackOneToolSet(api_key="k").search("q", account_ids=["acc-1", "acc-2"]) + ) + return error, {"failures": "c_acc-1_search_actions: boom | c_acc-2_search_actions: boom"} + + +def _fetch_tools_failed(mp, log) -> Emitted: + def explode(*_args: object, **_kwargs: object) -> list[McpToolDefinition]: + raise RuntimeError("kaput") + + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", explode) + error = _raised(ToolsetLoadError, StackOneToolSet(api_key="k", account_id="acc-1").fetch_tools) + return error, {"detail": "kaput"} + + +def _missing_api_key(mp, log) -> Emitted: + mp.delenv("STACKONE_API_KEY", raising=False) + return _raised(ToolsetConfigError, StackOneToolSet), {} + + +def _empty_account_id(mp, log) -> Emitted: + error = _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k", account_id="")) + return error, {"parameter": "account_id"} + + +def _account_ids_not_a_list(mp, log) -> Emitted: + error = _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k").set_accounts("acc-1")) + return error, {"parameter": "account_ids", "value": "acc-1"} + + +def _account_ids_not_strings(mp, log) -> Emitted: + error = _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k").set_accounts([1])) + return error, {"parameter": "account_ids"} + + +def _account_ids_empty_id(mp, log) -> Emitted: + error = _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k").set_accounts(["acc-1", ""])) + return error, {"parameter": "account_ids"} + + +def _invalid_session_id(mp, log) -> Emitted: + error = _raised( + ToolsetConfigError, lambda: StackOneToolSet(api_key="k").execute("linear_x", session_id="") + ) + return error, {"parameter": "session_id", "value": ""} + + +def _invalid_action_id(mp, log) -> Emitted: + return _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k").execute("")), { + "parameter": "action_id", + "value": "", + } + + +def _execute_arguments_not_an_object(mp, log) -> Emitted: + error = _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k").execute("linear_x", [1])) + return error, {"json_type": [1]} + + +def _top_k_out_of_range(mp, log) -> Emitted: + error = _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k").search("q", top_k=0)) + return error, {"parameter": "top_k", "max": 50, "value": 0} + + +def _tool_names_not_a_list(mp, log) -> Emitted: + error = _raised(ToolsetConfigError, lambda: StackOneToolSet(api_key="k").submit_feedback("positive", "t")) + return error, {"parameter": "tool_names", "value": "t"} + + +def _feedback_not_enabled(mp, log) -> Emitted: + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", _listing("c_{account}_search_actions")) + toolset = StackOneToolSet(api_key="k", account_id="acc-1") + return _raised(ToolsetLoadError, lambda: toolset.submit_feedback("positive", ["t"])), {} + + +def _no_connector_for_action(mp, log) -> Emitted: + mp.setattr("stackone_ai.toolset.fetch_mcp_tools", _listing("jira_{account}_execute_action")) + toolset = StackOneToolSet(api_key="k", account_id="acc-1") + return _raised(ToolsetLoadError, lambda: toolset.execute("linear_list_issues")), { + "action_id": "linear_list_issues" + } + + +def _arguments_not_finite(mp, log) -> Emitted: + tool = _served_tool({"type": "object", "properties": {}}) + error = _raised(ToolArgumentsError, lambda: tool.execute({"a": [1, float("-inf")]})) + return error, {"tool": TOOL_NAME, "value": "-Infinity"} + + +def _arguments_not_encodable(mp, log) -> Emitted: + tool = _served_tool({"type": "object", "properties": {}}) + error = _raised(ToolArgumentsError, lambda: tool.execute({"a": {1, 2}})) + return error, {"tool": TOOL_NAME, "detail": str(error.__cause__)} + + +def _arguments_invalid_json(mp, log) -> Emitted: + tool = _served_tool({"type": "object", "properties": {}}) + error = _raised(ToolArgumentsError, lambda: tool.execute("{")) + return error, {"tool": TOOL_NAME, "detail": str(error.__cause__)} + + +def _arguments_not_an_object(mp, log) -> Emitted: + tool = _served_tool({"type": "object", "properties": {}}) + return _raised(ToolArgumentsError, lambda: tool.execute("[1]")), {"tool": TOOL_NAME} + + +def _tool_call_failed(mp, log) -> Emitted: + result = SimpleNamespace(content=[SimpleNamespace(text='{"error":"bad"}')], isError=True) + error = _raised(StackOneAPIError, lambda: parse_tool_result(result, "t")) + return error, {"tool": "t", "detail": '{"error":"bad"}'} + + +def _no_executor(mp, log) -> Emitted: + tool = StackOneTool( + description="", + parameters=ToolParameters(type="object", properties={}), + _execute_config=ExecuteConfig(headers={}, name="t"), + ) + return _raised(StackOneError, lambda: tool.execute({})), {"tool": "t"} + + +def _toolset_headers_ignored(mp, log) -> Emitted: + StackOneToolSet(api_key="k", headers={"Authorization": "x", " X-Account-Id ": "y", "x-trace": "z"}) + [warning] = _warnings(log) + return warning, {"names": '"Authorization", " X-Account-Id "'} + + +def _unknown_tool(mp, log) -> Emitted: + [message] = Tools([]).execute_openai_tool_calls( + [{"id": "1", "function": {"name": "nope", "arguments": "{}"}}] + ) + return json.loads(message["content"])["error"], {"name": "nope"} + + +EMITTERS: dict[str, Callable[[pytest.MonkeyPatch, pytest.LogCaptureFixture], Emitted]] = { + "header-dropped": _header_dropped, + "header-argument-dropped": _header_argument_dropped, + "rate-limit-retry": _rate_limit_retry, + "rate-limit-deadline": _rate_limit_deadline, + "skip-account": _skip_account, + "skip-connector": _skip_connector, + "duplicate-tool-names": _duplicate_tool_names, + "ambiguous-connector": _ambiguous_connector, + "account-id-env-ignored": _account_id_env_ignored, + "connector-account-unavailable": _connector_account_unavailable, + "non-shared-accounts-skipped": _non_shared_accounts_skipped, + "mcp-timeout": _mcp_timeout, + "mcp-http-failure": _mcp_http_failure, + "mcp-failure": _mcp_failure, + "mcp-rate-limit-timeout": _mcp_rate_limit_timeout, + "accounts-http-failure": _accounts_http_failure, + "accounts-rate-limit-timeout": _accounts_rate_limit_timeout, + "accounts-unreachable": _accounts_unreachable, + "accounts-invalid-json": _accounts_invalid_json, + "accounts-unexpected-shape": _accounts_unexpected_shape, + "no-linked-accounts": _no_linked_accounts, + "no-active-accounts": _no_active_accounts, + "no-shared-accounts": _no_shared_accounts, + "all-accounts-failed": _all_accounts_failed, + "no-connector-returned-results": _no_connector_returned_results, + "fetch-tools-failed": _fetch_tools_failed, + "missing-api-key": _missing_api_key, + "empty-account-id": _empty_account_id, + "account-ids-not-a-list": _account_ids_not_a_list, + "account-ids-not-strings": _account_ids_not_strings, + "account-ids-empty-id": _account_ids_empty_id, + "invalid-session-id": _invalid_session_id, + "invalid-action-id": _invalid_action_id, + "execute-arguments-not-an-object": _execute_arguments_not_an_object, + "top-k-out-of-range": _top_k_out_of_range, + "tool-names-not-a-list": _tool_names_not_a_list, + "feedback-not-enabled": _feedback_not_enabled, + "no-connector-for-action": _no_connector_for_action, + "arguments-not-finite": _arguments_not_finite, + "arguments-not-encodable": _arguments_not_encodable, + "arguments-invalid-json": _arguments_invalid_json, + "arguments-not-an-object": _arguments_not_an_object, + "tool-call-failed": _tool_call_failed, + "no-executor": _no_executor, + "unknown-tool": _unknown_tool, + "toolset-headers-ignored": _toolset_headers_ignored, +} + + +def test_every_message_has_an_emitter(): + assert sorted(EMITTERS) == sorted(TEMPLATES) + + +@pytest.mark.parametrize("message", [pytest.param(m, id=m["id"]) for m in MESSAGES["messages"]]) +def test_message( + message: dict[str, Any], monkeypatch: pytest.MonkeyPatch, warnings_log: pytest.LogCaptureFixture +): + emitted, values = EMITTERS[message["id"]](monkeypatch, warnings_log) + if message["kind"] == "error": + assert isinstance(emitted, BaseException) + assert message["error"]["python"] in {cls.__name__ for cls in type(emitted).__mro__} + assert str(emitted) == _render(message["id"], **values) diff --git a/tests/test_tool_calling.py b/tests/test_tool_calling.py index 975f1368..f25396df 100644 --- a/tests/test_tool_calling.py +++ b/tests/test_tool_calling.py @@ -1,112 +1,83 @@ """Tests for tool calling functionality""" -import json +import logging +from typing import Any -import httpx import pytest -import respx - -from stackone_ai import StackOneTool -from stackone_ai.tools import StackOneRpcTool -from stackone_ai.types import ( - ExecuteConfig, - ToolParameters, - filename_from_content_disposition, - is_json_content_type, -) -from tests.conftest import TEST_BASE_URL +from stackone_ai import StackOneError, StackOneTool, ToolArgumentsError +from stackone_ai.tools import StackOneMcpTool, _header_text +from stackone_ai.types import ExecuteConfig, ToolParameters -@pytest.fixture -def mock_tool(): - """Create a mock tool for testing""" - execute_config = ExecuteConfig( - name="test_tool", - method="POST", - url="https://api.example.com/test", - headers={"Content-Type": "application/json"}, - ) - parameters = ToolParameters( - type="object", - properties={ - "name": {"type": "string", "description": "Name parameter"}, - "value": {"type": "number", "description": "Value parameter"}, - }, +def _mcp_tool(properties: dict[str, Any] | None = None, account_id: str | None = "real-account"): + return StackOneMcpTool( + name="linear_acct_execute_action", + description="Execute", + parameters=ToolParameters(type="object", properties=properties or {}), + api_key="test_key", + endpoint="https://api.example.com/mcp", + account_id=account_id, ) - tool = StackOneTool( + +@pytest.fixture +def seen(monkeypatch) -> dict[str, Any]: + """Replace the MCP transport and record what each call would have sent.""" + captured: dict[str, Any] = {} + + def fake_call(endpoint, headers, name, arguments, **_kwargs): + captured.update(endpoint=endpoint, headers=headers, name=name, arguments=arguments) + return {"success": True, "result": "test_result"} + + monkeypatch.setattr("stackone_ai.tools.call_mcp_tool", fake_call) + return captured + + +@pytest.fixture +def mock_tool(): + """An MCP tool with two declared parameters""" + return StackOneMcpTool( + name="test_tool", description="Test tool", - parameters=parameters, - _execute_config=execute_config, - _api_key="test_api_key", + parameters=ToolParameters( + type="object", + properties={ + "name": {"type": "string", "description": "Name parameter"}, + "value": {"type": "number", "description": "Value parameter"}, + }, + ), + api_key="test_api_key", + endpoint="https://api.example.com/mcp", + account_id="acc1", ) - return tool - class TestToolCalling: """Test tool calling functionality""" - @respx.mock - def test_call_with_kwargs(self, mock_tool): + def test_call_with_kwargs(self, mock_tool, seen): """Test calling a tool with keyword arguments""" - # Mock the API response - route = respx.post("https://api.example.com/test").mock( - return_value=httpx.Response(200, json={"success": True, "result": "test_result"}) - ) - - # Call the tool with kwargs result = mock_tool.call(name="test", value=42) - # Verify the result assert result == {"success": True, "result": "test_result"} + assert seen["name"] == "test_tool" + assert seen["endpoint"] == "https://api.example.com/mcp" + assert seen["arguments"] == {"name": "test", "value": 42} - # Verify the request was made correctly - assert route.called - assert route.call_count == 1 - request = route.calls[0].request - assert json.loads(request.content) == {"name": "test", "value": 42} - - @respx.mock - def test_call_with_dict_arg(self, mock_tool): + def test_call_with_dict_arg(self, mock_tool, seen): """Test calling a tool with a dictionary argument""" - # Mock the API response - route = respx.post("https://api.example.com/test").mock( - return_value=httpx.Response(200, json={"success": True, "result": "test_result"}) - ) - - # Call the tool with a dict result = mock_tool.call({"name": "test", "value": 42}) - # Verify the result assert result == {"success": True, "result": "test_result"} + assert seen["arguments"] == {"name": "test", "value": 42} - # Verify the request - assert route.called - assert route.call_count == 1 - request = route.calls[0].request - assert json.loads(request.content) == {"name": "test", "value": 42} - - @respx.mock - def test_call_with_json_string(self, mock_tool): + def test_call_with_json_string(self, mock_tool, seen): """Test calling a tool with a JSON string argument""" - # Mock the API response - route = respx.post("https://api.example.com/test").mock( - return_value=httpx.Response(200, json={"success": True, "result": "test_result"}) - ) - - # Call the tool with a JSON string result = mock_tool.call('{"name": "test", "value": 42}') - # Verify the result assert result == {"success": True, "result": "test_result"} - - # Verify the request - assert route.called - assert route.call_count == 1 - request = route.calls[0].request - assert json.loads(request.content) == {"name": "test", "value": 42} + assert seen["arguments"] == {"name": "test", "value": 42} def test_call_with_both_args_and_kwargs_raises_error(self, mock_tool): """Test that providing both args and kwargs raises an error""" @@ -118,578 +89,90 @@ def test_call_with_multiple_args_raises_error(self, mock_tool): with pytest.raises(ValueError, match="Only one positional argument is allowed"): mock_tool.call({"name": "test"}, {"value": 42}) - @respx.mock - def test_call_without_arguments(self, mock_tool): - """Test calling a tool without any arguments""" - # Mock the API response - route = respx.post("https://api.example.com/test").mock( - return_value=httpx.Response(200, json={"success": True, "result": "no_args"}) - ) - - # Call the tool without arguments - result = mock_tool.call() - - # Verify the result - assert result == {"success": True, "result": "no_args"} - - # Verify the request body is empty or contains empty JSON - assert route.called - assert route.call_count == 1 - request = route.calls[0].request - # Handle case where body might be None for empty POST - if request.content: - assert json.loads(request.content) == {} - else: - assert request.content == b"" - - -class TestStackOneRpcTool: - """Test StackOneRpcTool functionality""" - - @pytest.fixture - def rpc_tool(self): - """Create a mock RPC tool for testing""" - parameters = ToolParameters( - type="object", - properties={ - "employee_id": {"type": "string", "description": "Employee ID"}, - }, - ) - return StackOneRpcTool( - name="hibob_get_employee", - description="Get employee details", - parameters=parameters, - api_key="test_api_key", - base_url=TEST_BASE_URL, - account_id="test_account", - ) - - @respx.mock - def test_execute_basic(self, rpc_tool): - """Test basic RPC tool execution""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"data": {"id": "123", "name": "John"}}) - ) - - result = rpc_tool.execute({"employee_id": "123"}) - - assert result == {"data": {"id": "123", "name": "John"}} - assert route.called - request = route.calls[0].request - body = json.loads(request.content) - assert body["action"] == "hibob_get_employee" - assert body["body"]["employee_id"] == "123" - assert body["headers"]["x-account-id"] == "test_account" - - @respx.mock - def test_execute_with_json_string(self, rpc_tool): - """Test RPC tool execution with JSON string arguments""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute('{"employee_id": "456"}') - - assert result == {"success": True} - assert route.called - body = json.loads(route.calls[0].request.content) - assert body["body"]["employee_id"] == "456" + def test_call_without_arguments(self, mock_tool, seen): + """Test calling a tool without any arguments sends an empty object""" + mock_tool.call() + assert seen["arguments"] == {} - @respx.mock - def test_execute_with_body_payload(self, rpc_tool): - """Test RPC tool execution with nested body payload""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute({"body": {"name": "Jane", "email": "jane@example.com"}}) - - assert result == {"success": True} - body = json.loads(route.calls[0].request.content) - assert body["body"]["name"] == "Jane" - assert body["body"]["email"] == "jane@example.com" - - @respx.mock - def test_execute_with_path_payload(self, rpc_tool): - """Test RPC tool execution with path parameters""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute({"path": {"id": "emp123"}}) - - assert result == {"success": True} - body = json.loads(route.calls[0].request.content) - assert body["path"] == {"id": "emp123"} - - @respx.mock - def test_execute_with_query_payload(self, rpc_tool): - """Test RPC tool execution with query parameters""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute({"query": {"limit": "10", "offset": "0"}}) - - assert result == {"success": True} - body = json.loads(route.calls[0].request.content) - assert body["query"] == {"limit": "10", "offset": "0"} - - @respx.mock - def test_execute_with_headers_payload(self, rpc_tool): - """Test RPC tool execution with custom headers""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute({"headers": {"X-Custom-Header": "custom_value"}}) - - assert result == {"success": True} - body = json.loads(route.calls[0].request.content) - assert "X-Custom-Header" not in body["headers"] # undeclared by the served schema - assert body["headers"]["x-account-id"] == "test_account" - - @respx.mock - def test_execute_headers_strips_authorization(self, rpc_tool): - """Test that Authorization header is stripped from action headers""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute({"headers": {"Authorization": "Bearer token", "X-Other": "value"}}) - - assert result == {"success": True} - body = json.loads(route.calls[0].request.content) - assert "Authorization" not in body["headers"] - assert "X-Other" not in body["headers"] # undeclared by the served schema - - @respx.mock - @pytest.mark.parametrize( - "header_name", - ["authorization", "AUTHORIZATION", "AuThOrIzAtIon"], - ) - def test_execute_headers_strips_authorization_any_case(self, rpc_tool, header_name): - """Reserved headers are stripped case-insensitively. - - HTTP header names are case-insensitive, and tool arguments are model-controlled, - so a case variant must not smuggle a credential into the RPC envelope. - """ - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) + def test_execute_with_none_arguments(self, mock_tool, seen): + mock_tool.execute(None) + assert seen["arguments"] == {} - rpc_tool.execute({"headers": {header_name: "Bearer attacker-token"}}) - - body = json.loads(route.calls[0].request.content) - assert all(key.lower() != "authorization" for key in body["headers"]) - - @respx.mock - def test_execute_headers_cannot_override_account_id(self, rpc_tool): - """A tool call must not be able to retarget another account. - - x-account-id scopes the request to a tenant; letting model-supplied headers - override it would allow lateral movement across every account the key reaches. - """ - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - rpc_tool.execute({"headers": {"x-account-id": "victim_account"}}) - - body = json.loads(route.calls[0].request.content) - assert body["headers"]["x-account-id"] == "test_account" - - @respx.mock - def test_execute_headers_skips_none_values(self, rpc_tool): - """Test that None header values are skipped""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute({"headers": {"X-Present": "value", "X-Absent": None}}) - - assert result == {"success": True} - body = json.loads(route.calls[0].request.content) - assert "X-Present" not in body["headers"] # undeclared by the served schema - assert "X-Absent" not in body["headers"] - - @respx.mock - def test_execute_without_account_id_sends_no_account_anywhere(self): - """A tool built with no account scopes nothing — and the server refuses it. - - This used to assert only that the envelope omitted x-account-id, which is the - same shape as the assertion that pinned the bug that shipped: a green test - recording what the client happened to send. The point worth pinning is that - an unscoped request is not a usable request, so the HTTP header is checked - too — that is the one the API actually reads. - """ - parameters = ToolParameters( - type="object", - properties={}, - ) - tool = StackOneRpcTool( - name="test_tool", - description="Test", - parameters=parameters, - api_key="test_key", - base_url=TEST_BASE_URL, - account_id=None, - ) - - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = tool.execute({}) - - assert result == {"success": True} - request = route.calls[0].request - body = json.loads(request.content) - assert "x-account-id" not in body["headers"] - # The header the API reads. The mock server 400s when it is absent, which is - # what makes tests/test_fetch_tools.py::TestRpcToolExecution meaningful. - assert "x-account-id" not in request.headers - - @respx.mock - def test_execute_with_none_arguments(self, rpc_tool): - """Test RPC tool execution with None arguments""" - route = respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response(200, json={"success": True}) - ) - - result = rpc_tool.execute(None) - - assert result == {"success": True} - body = json.loads(route.calls[0].request.content) - assert body["action"] == "hibob_get_employee" - assert body["body"] == {} - - def test_parse_arguments_invalid_json(self, rpc_tool): + def test_parse_arguments_invalid_json(self, mock_tool): """Test that invalid JSON raises ValueError""" - with pytest.raises(ValueError): - rpc_tool._parse_arguments("not valid json") + with pytest.raises(ValueError, match="Invalid JSON"): + mock_tool._parse_arguments("not valid json") - def test_parse_arguments_non_dict(self, rpc_tool): + def test_parse_arguments_non_dict(self, mock_tool): """Test that non-dict JSON raises ValueError""" - with pytest.raises(ValueError, match="Tool arguments must be a JSON object"): - rpc_tool._parse_arguments("[1, 2, 3]") - - def test_split_envelope_params_routes_flat_prefixed_keys(self, rpc_tool): - """flat_prefixed keys are bucketed into the RPC envelope by their location prefix""" - actual = rpc_tool._split_envelope_params( - { - "path_id": "123", - "query_limit": 10, - "headers_x-custom": "value", - "body_name": "test", - } - ) - assert actual["path"] == {"id": "123"} - assert actual["query"] == {"limit": 10} - assert actual["headers"] == {"x-custom": "value"} - assert actual["body"] == {"name": "test"} - - def test_split_envelope_params_buckets_nested_and_unprefixed_keys(self, rpc_tool): - """Bare nested envelopes are accepted and unprefixed keys fall through to the body""" - actual = rpc_tool._split_envelope_params( - { - "body": {"nested": "value"}, - "path": {"id": "1"}, - "extra": "x", - } - ) - assert actual["path"] == {"id": "1"} - assert actual["query"] == {} - assert actual["body"] == {"nested": "value", "extra": "x"} - - -class TestBinaryDownloadResponse: - """File-download actions return raw bytes + metadata instead of failing on JSON parsing. - - The StackOne API serves file downloads as raw binary with the file's own MIME type - (e.g. application/pdf) and a Content-Disposition header - never JSON. The returned - shape mirrors the StackOne generated SDKs' download response (content + content_type + - status_code + headers), with content as raw bytes (the Python analog of the Java - client's byte[] body / the TypeScript client's response stream). - """ - - @respx.mock - def test_binary_response_returns_content_dict(self, mock_tool): - """A non-JSON (binary) body is returned as bytes + metadata, not JSON-parsed.""" - # Leading bytes of a real PDF; the 0xc4 byte is invalid UTF-8 and is exactly - # what makes the unconditional response.json() raise UnicodeDecodeError. - pdf_bytes = b"%PDF-1.4\n%\xc4\xe5\xf2\xe5\xeb\xa7\xf3\xa0\xd0\xc4\xc6\n1 0 obj\n" - respx.post("https://api.example.com/test").mock( - return_value=httpx.Response( - 200, - headers={ - "content-type": "application/pdf", - "content-disposition": 'attachment; filename="download.pdf"', - }, - content=pdf_bytes, - ) - ) - - result = mock_tool.execute({"name": "report", "value": 1}) - - assert result["content"] == pdf_bytes - assert result["content_type"] == "application/pdf" - assert result["status_code"] == 200 - assert result["file_name"] == "download.pdf" - assert result["headers"]["content-type"] == "application/pdf" - - @respx.mock - def test_rpc_download_action_returns_content_dict(self): - """The RPC download path (e.g. googledrive_unified_download_file) returns bytes. - - Reproduces the reported failure: a download action invoked through /actions/rpc - previously raised UnicodeDecodeError because the binary body was JSON-parsed. - """ - parameters = ToolParameters( - type="object", - properties={"id": {"type": "string", "description": "File ID"}}, - ) - tool = StackOneRpcTool( - name="googledrive_unified_download_file", - description="Download a file", - parameters=parameters, - api_key="test_api_key", - base_url=TEST_BASE_URL, - account_id="test_account", - ) - - rtf_bytes = b"{\\rtf1\\ansi\\ansicpg1252\\\xc4\xe5 hello}" - respx.post(f"{TEST_BASE_URL}/actions/rpc").mock( - return_value=httpx.Response( - 200, - headers={ - "content-type": "application/rtf", - "content-disposition": 'attachment; filename="download.rtf"', - }, - content=rtf_bytes, - ) - ) - - result = tool.execute({"path": {"id": "file-123"}}) - - assert result["content"] == rtf_bytes - assert result["content_type"] == "application/rtf" - assert result["file_name"] == "download.rtf" - - @respx.mock - def test_octet_stream_without_filename(self, mock_tool): - """A binary body with no Content-Disposition still returns content with file_name=None.""" - blob = b"\x00\x01\x02\xc4\xff\xfe" - respx.post("https://api.example.com/test").mock( - return_value=httpx.Response( - 200, - headers={"content-type": "application/octet-stream"}, - content=blob, - ) - ) - - result = mock_tool.execute({}) - - assert result["content"] == blob - assert result["content_type"] == "application/octet-stream" - assert result["file_name"] is None - - @respx.mock - def test_json_response_still_parsed(self, mock_tool): - """Regression guard: JSON responses are unchanged - parsed to a dict, not wrapped.""" - respx.post("https://api.example.com/test").mock( - return_value=httpx.Response(200, json={"id": "123", "ok": True}) - ) - - result = mock_tool.execute({"name": "x", "value": 1}) - - assert result == {"id": "123", "ok": True} - assert "content" not in result - - @respx.mock - def test_json_with_charset_param_still_parsed(self, mock_tool): - """A JSON Content-Type with parameters (charset) is still parsed as JSON.""" - respx.post("https://api.example.com/test").mock( - return_value=httpx.Response( - 200, - headers={"content-type": "application/json; charset=utf-8"}, - content=b'{"ok": true}', - ) - ) - - result = mock_tool.execute({}) - - assert result == {"ok": True} - - @respx.mock - def test_missing_content_type_returns_bytes(self, mock_tool): - """A body with no Content-Type is treated as opaque content (bytes), not JSON. - - Pins the deliberate contract: the SDK trusts Content-Type to decide JSON vs - file, so an absent Content-Type is returned as raw bytes rather than risking - a UTF-8/JSON decode of binary. (StackOne always labels JSON as application/json.) - """ - blob = b"\xff\xd8\xff\xe0\x00\x10JFIF" # JPEG magic bytes, no content-type - respx.post("https://api.example.com/test").mock(return_value=httpx.Response(200, content=blob)) - - result = mock_tool.execute({}) - - assert result["content"] == blob - assert result["content_type"] == "application/octet-stream" - assert result["file_name"] is None - - -class TestResponseHelpers: - """Unit tests for the Content-Type and Content-Disposition helpers.""" - - @pytest.mark.parametrize( - ("content_type", "expected"), - [ - ("application/json", True), - ("application/json; charset=utf-8", True), - ("APPLICATION/JSON", True), - ("application/problem+json", True), - ("application/vnd.api+json", True), - ("", False), - ("application/pdf", False), - ("application/octet-stream", False), - ("text/plain", False), - ("text/json-but-not-really", False), - ], - ) - def test_is_json_content_type(self, content_type, expected): - assert is_json_content_type(content_type) is expected - - @pytest.mark.parametrize( - ("header", "expected"), - [ - ('attachment; filename="download.pdf"', "download.pdf"), - ("attachment; filename=download.pdf", "download.pdf"), - ('inline; filename="my report.docx"', "my report.docx"), - # RFC 5987 extended form is percent-decoded and takes precedence. - ("attachment; filename=\"fallback.txt\"; filename*=UTF-8''na%C3%AFve.txt", "naïve.txt"), - # Non-UTF-8 charset is honoured: 0xA3 is "£" in ISO-8859-1, not UTF-8. - ("attachment; filename*=ISO-8859-1'en'%A3%20rates.txt", "£ rates.txt"), - # Unknown charset label falls back to UTF-8 instead of raising. - ("attachment; filename*=bogus-charset''%C2%A3.txt", "£.txt"), - # Non-conformant quoted extended value: surrounding quotes are stripped. - ("attachment; filename*=\"UTF-8''na%C3%AFve.txt\"", "naïve.txt"), - ("attachment", None), - (None, None), - ("", None), - ], - ) - def test_filename_from_content_disposition(self, header, expected): - assert filename_from_content_disposition(header) == expected - - -class TestEnvelopeSplitIsSchemaAware: - """The prefix pattern alone cannot tell a path param from a body field that - happens to start with "path_". The served schema settles it.""" + with pytest.raises(ValueError, match=r'^Tool arguments for "[^"]+" must be a JSON object$'): + mock_tool._parse_arguments("[1, 2, 3]") + + def test_arguments_are_not_split_or_renamed(self, seen): + """No envelope splitting, no prefix routing: the server maps arguments itself.""" + tool = _mcp_tool({"path_id": {"type": "string"}, "path_to_file": {"type": "string"}}) + arguments = { + "path_id": "1", + "path_to_file": "/tmp/x", + "query": {"limit": 5}, + "body_foo": 1, + "foo": 2, + "body": {"foo": 9}, + } + tool.execute(arguments) + assert seen["arguments"] == arguments + + def test_the_callers_arguments_are_not_mutated(self, seen): + tool = _mcp_tool({"headers": {"type": "object", "properties": {}}}) + arguments = {"headers": {"Authorization": "Bearer stolen"}} + tool.execute(arguments) + assert arguments == {"headers": {"Authorization": "Bearer stolen"}} + assert seen["arguments"] == {"headers": {}} + + +class TestBaseToolHasNoExecutor: + """The base class describes a tool; only a subclass can run one (mirrors Node's BaseTool).""" @pytest.fixture - def rpc_tool(self): - return StackOneRpcTool( - name="test_action", - description="Test", + def base_tool(self): + """No API key: the base class cannot execute, so it has nothing to authenticate.""" + return StackOneTool( + description="Hand-built", parameters=ToolParameters(type="object", properties={}), - api_key="test_api_key", - base_url=TEST_BASE_URL, - account_id="test-account", - ) - - def test_a_bare_schema_keeps_prefix_lookalikes_in_the_body(self, rpc_tool): - """No declared key is location-prefixed, so `path_to_file` is a real field name. - - Splitting it would send the server a path component it has no use for and drop - the argument the model supplied — silently. - """ - actual = rpc_tool._split_envelope_params({"path_to_file": "/tmp/x"}, {"path_to_file", "name"}) - assert actual["body"] == {"path_to_file": "/tmp/x"} - assert actual["path"] == {} - - def test_a_prefixed_schema_splits_every_match(self, rpc_tool): - """Under flat_prefixed every parameter is prefixed, so a match really is located.""" - actual = rpc_tool._split_envelope_params( - {"path_id": "1", "query_offset": 10}, {"path_id", "query_limit"} + _execute_config=ExecuteConfig(name="hand_built"), ) - assert actual["path"] == {"id": "1"} - # Undeclared but prefixed: the model may be working from a newer schema than the - # cached listing. Routing it to the body would silently drop the argument. - assert actual["query"] == {"offset": 10} - - def test_a_declared_reserved_word_is_a_field_not_a_container(self, rpc_tool): - """A served property literally named `query` must not be rejected as malformed.""" - actual = rpc_tool._split_envelope_params({"query": "sales"}, {"query", "id"}) - assert actual["body"] == {"query": "sales"} - def test_no_schema_trusts_every_match(self, rpc_tool): - """Direct callers without a schema keep the old, purely pattern-based behaviour.""" - actual = rpc_tool._split_envelope_params({"path_to_file": "/tmp/x"}) - assert actual["path"] == {"to_file": "/tmp/x"} + def test_execute_raises_a_stackone_error_naming_the_fix(self, base_tool): + with pytest.raises(StackOneError, match=r'Tool "hand_built" has no executor\. Override execute\(\)'): + base_tool.execute({}) - def test_scalar_under_a_reserved_key_is_rejected_not_dropped(self, rpc_tool): - with pytest.raises(ValueError, match="envelope container"): - rpc_tool._split_envelope_params({"query": "not-an-object"}) + def test_call_raises_too(self, base_tool): + with pytest.raises(StackOneError, match="has no executor"): + base_tool.call() - def test_precedence_does_not_depend_on_caller_key_order(self, rpc_tool): - """flat_prefixed beats nested beats bare, whatever order the dict is built in.""" - forwards = rpc_tool._split_envelope_params({"body_foo": 1, "foo": 2}) - backwards = rpc_tool._split_envelope_params({"foo": 2, "body_foo": 1}) - assert forwards["body"] == backwards["body"] == {"foo": 1} + def test_an_override_runs(self): + class EchoTool(StackOneTool): + def execute(self, arguments=None): + return {"echo": self._parse_arguments(arguments)} - nested_first = rpc_tool._split_envelope_params({"body": {"foo": 9}, "foo": 2}) - bare_first = rpc_tool._split_envelope_params({"foo": 2, "body": {"foo": 9}}) - assert nested_first["body"] == bare_first["body"] == {"foo": 9} - - def test_empty_schema_falls_back_to_trusting_prefixes(self, rpc_tool): - """An empty declared set means "no schema", not "nothing is declared". - - Treating it as an allowlist would route every path_* key into the body and - silently drop every path parameter. - """ - assert rpc_tool._split_envelope_params({"path_id": "1"}, set())["path"] == {"id": "1"} - - -class TestMcpToolHeaderGuard: - """The header guard on the MCP path — the one search()/execute() actually use. - - This had no coverage at all: the whole `_sanitise_headers` call could be deleted - from StackOneMcpTool.execute and every test still passed. Every existing header - test drives the RPC tool only. - """ - - @pytest.fixture - def mcp_tool(self): - from stackone_ai.tools import StackOneMcpTool - - return StackOneMcpTool( - name="linear_acct_execute_action", - description="Execute", + tool = EchoTool( + description="", parameters=ToolParameters(type="object", properties={}), - api_key="test_key", - endpoint="https://api.example.com/mcp", - headers={"Authorization": "Basic real", "x-account-id": "real-account"}, - account_id="real-account", + _execute_config=ExecuteConfig(name="echo"), + _api_key="k", ) + assert tool.call(a=1) == {"echo": {"a": 1}} + # Still accepted and kept, for overrides that authenticate with it. + assert tool._api_key == "k" - @staticmethod - def _capture(monkeypatch): - seen: dict[str, object] = {} - - def fake_call(endpoint, headers, name, arguments, **_kwargs): - seen["arguments"] = arguments - return {"ok": True} - monkeypatch.setattr("stackone_ai.tools.call_mcp_tool", fake_call) - return seen +class TestMcpToolHeaderGuard: + """The guard on a nested ``headers`` argument, which the model controls.""" - def test_undeclared_headers_are_all_dropped(self, mcp_tool, monkeypatch): + def test_undeclared_headers_are_all_dropped(self, seen): """An allowlist, not a denylist: a two-name denylist let Proxy-Authorization, - x-stackone-account-id, Cookie and X-Api-Key through. No served action declares - a headers_* property, so nothing model-supplied belongs here.""" - seen = self._capture(monkeypatch) - mcp_tool.execute( + x-stackone-account-id, Cookie and X-Api-Key through.""" + _mcp_tool().execute( { "action_id": "linear_list_issues", "headers": { @@ -705,39 +188,123 @@ def test_undeclared_headers_are_all_dropped(self, mcp_tool, monkeypatch): ) assert seen["arguments"]["headers"] == {} + def test_a_nested_declared_header_survives(self, seen): + """The allowlist is the schema itself, so a future action needing a header + works with no SDK release.""" + tool = _mcp_tool({"headers": {"type": "object", "properties": {"x-trace": {"type": "string"}}}}) + tool.execute({"headers": {"X-Trace": "abc", "X-Other": "no"}}) + assert seen["arguments"]["headers"] == {"X-Trace": "abc"} + + def test_a_flat_declaration_does_not_declare_a_nested_header(self, seen): + """Each form is declared on its own: `headers_x-trace` allows only the flat argument.""" + tool = _mcp_tool({"headers_x-trace": {"type": "string"}}) + tool.execute({"action_id": "a", "headers": {"X-Trace": "abc"}}) + assert seen["arguments"] == {"action_id": "a", "headers": {}} + @pytest.mark.parametrize( - "name", - [" authorization", "AUTHORIZATION\t", "X-Account-Id", " x-account-id "], + "schema", + [ + pytest.param({"type": "object", "additionalProperties": False}, id="closed-object"), + pytest.param({}, id="no-type"), + pytest.param({"additionalProperties": True}, id="no-type-additional-allowed"), + ], ) - def test_whitespace_and_case_variants_do_not_slip_past(self, mcp_tool, monkeypatch, name): - seen = self._capture(monkeypatch) - mcp_tool.execute({"action_id": "a", "headers": {name: "stolen"}}) + def test_any_other_schema_without_properties_declares_no_header(self, seen, schema): + """Only `type: "object"` not closed by `additionalProperties: false` is open, as in Node.""" + _mcp_tool({"headers": schema}).execute({"headers": {"x-custom": "dropped"}}) assert seen["arguments"]["headers"] == {} - @pytest.mark.parametrize("value", ["a\r\nEvil: 1", "trailing\n", "bad\rvalue"]) - def test_crlf_injection_is_rejected(self, mcp_tool, monkeypatch, value): - """`$` also matches before a trailing newline, so this needs fullmatch.""" - seen = self._capture(monkeypatch) - mcp_tool.execute({"action_id": "a", "headers": {"X-Probe": value}}) + def test_a_non_object_declared_type_is_an_ordinary_field(self, seen): + """`type: "string"` makes `headers` an ordinary field, forwarded unchanged whatever + the value actually is, not filtered as a header container.""" + _mcp_tool({"headers": {"type": "string"}}).execute({"headers": {"x-custom": "a"}}) + assert seen["arguments"]["headers"] == {"x-custom": "a"} + + @pytest.mark.parametrize("name", ["Authorization", "x-account-id", "User-Agent"]) + def test_sdk_owned_headers_are_refused_even_when_declared(self, seen, name): + tool = _mcp_tool({"headers": {"type": "object", "properties": {name.lower(): {"type": "string"}}}}) + tool.execute({"headers": {name: "stolen"}}) assert seen["arguments"]["headers"] == {} - def test_a_header_the_served_schema_declares_survives(self, monkeypatch): - """The allowlist is the schema itself, so a future action needing a header - works with no SDK release.""" - from stackone_ai.tools import StackOneMcpTool - - tool = StackOneMcpTool( - name="linear_acct_execute_action", - description="Execute", - parameters=ToolParameters(type="object", properties={"headers_x-trace": {"type": "string"}}), - api_key="test_key", - endpoint="https://api.example.com/mcp", - headers={}, - account_id="real-account", - ) - seen = self._capture(monkeypatch) - tool.execute({"action_id": "a", "headers": {"X-Trace": "abc", "X-Other": "no"}}) - assert seen["arguments"]["headers"] == {"X-Trace": "abc"} + def test_a_value_json_cannot_hold_is_an_argument_error(self, seen): + with pytest.raises(ValueError, match="could not be encoded"): + _mcp_tool({"headers": {"type": "object"}}).execute({"headers": {"x-set": [{"a"}]}}) + assert seen == {} + + def test_a_self_referential_header_value_is_an_argument_error(self, seen): + """A cycle would otherwise recurse forever in _header_text, raising RecursionError.""" + cyclic: dict[str, Any] = {} + cyclic["self"] = cyclic + with pytest.raises(ToolArgumentsError, match="could not be encoded"): + _mcp_tool({"headers": {"type": "object"}}).execute({"headers": {"x-cyclic": cyclic}}) + assert seen == {} + + def test_a_list_headers_argument_is_dropped(self, seen, caplog): + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + _mcp_tool().execute({"headers": ["a"], "q": 1}) + assert seen["arguments"] == {"q": 1} + assert 'Dropping header argument "headers" from a tool call: not an object' in caplog.text + + def test_the_request_is_scoped_to_the_tools_account(self, seen): + tool = _mcp_tool({"headers": {"type": "object"}, "headers_x-account-id": {"type": "string"}}) + tool.execute({"headers": {"x-account-id": "victim"}, "headers_x-account-id": "victim"}) + assert seen["headers"]["x-account-id"] == "real-account" + assert seen["arguments"] == {"headers": {}} + + def test_no_account_sends_no_account_header(self, seen): + """A tool built with no account scopes nothing; the server refuses such a call.""" + _mcp_tool(account_id=None).execute({}) + assert "x-account-id" not in seen["headers"] + + +@pytest.mark.parametrize( + ("value", "text"), + [ + ("as is", "as is"), + (False, "false"), + (7, "7"), + (-0.0, "0"), + (0.1, "0.1"), + (1e-7, "1e-7"), + (1.5e-7, "1.5e-7"), + (0.000001, "0.000001"), + (1e20, "100000000000000000000"), + (1e21, "1e+21"), + (1.5e300, "1.5e+300"), + (float("nan"), "NaN"), + (float("-inf"), "-Infinity"), + ({"k": [1.0, 2.5e-7, None, "é\n"]}, '{"k":[1,2.5e-7,null,"é\\n"]}'), + ({"k": float("nan")}, '{"k":null}'), + ], +) +def test_header_text_matches_node(value, text): + """Expected values are what Node's headerText returns for the same input.""" + assert _header_text(value) == text + + +class TestFlatHeaderArguments: + """A top-level `headers_` is a header argument too, and gets the same guard.""" + + @pytest.mark.parametrize("name", ["headers_x-account-id", "headers_Authorization", "headers_ user-agent"]) + def test_sdk_owned_headers_are_refused_even_when_declared(self, seen, name, caplog): + tool = _mcp_tool({name: {"type": "string"}}) + with caplog.at_level(logging.WARNING, logger="stackone.tools"): + tool.execute({name: "stolen", "q": 1}) + assert seen["arguments"] == {"q": 1} + assert "from a tool call: set by the SDK" in caplog.text + + def test_non_header_arguments_are_untouched(self, seen): + """Only `headers` and `headers_` are header arguments; bare lookalikes are not.""" + arguments = { + "x-account-id": "not-a-header", + "authorization": "not-a-header", + "header_foo": 1, + "Headers_foo": 2, + "path": {"headers_x": "y"}, + "query": {"limit": 5}, + } + _mcp_tool().execute(arguments) + assert seen["arguments"] == arguments class TestDeclaredHeaderValuesAreStillValidated: @@ -746,16 +313,8 @@ class TestDeclaredHeaderValuesAreStillValidated: @pytest.fixture def tool(self): - from stackone_ai.tools import StackOneMcpTool - - return StackOneMcpTool( - name="linear_acct_execute_action", - description="Execute", - parameters=ToolParameters(type="object", properties={"headers_x-trace": {"type": "string"}}), - api_key="k", - endpoint="https://api.example.com/mcp", - headers={}, - account_id="acct", + return _mcp_tool( + {"headers": {"type": "object", "properties": {"x-trace": {"type": "string"}}}}, account_id="acct" ) @pytest.mark.parametrize("value", ["trailing\n", "a\r\nInjected: 1", "bad\rvalue"]) @@ -767,41 +326,34 @@ def test_a_clean_declared_header_survives(self, tool): assert tool._sanitise_headers({"X-Trace": "abc-123"}) == {"X-Trace": "abc-123"} -class TestDownloadFilenamesAreSafe: - """The filename comes from a Content-Disposition an attacker can choose.""" +@pytest.mark.parametrize( + "value", ["half an emoji \ud83d", {"a", "set"}, b"bytes", float("nan"), float("inf"), float("-inf")] +) +def test_unencodable_arguments_raise_value_error(value, seen): + """These would otherwise fail inside the MCP client and surface as a transport error. - @pytest.mark.parametrize( - ("header", "expected"), - [ - ('attachment; filename="../../.ssh/authorized_keys"', "authorized_keys"), - ("attachment; filename*=UTF-8''%2e%2e%2f%2e%2e%2fetc%2fcron.d%2fx", "x"), - ('attachment; filename="/etc/passwd"', "passwd"), - ('attachment; filename="C:evil.exe"', "evil.exe"), - ('attachment; filename="..\\\\..\\\\windows\\\\x.dll"', "x.dll"), - ('attachment; filename="‮gnp.exe"', "gnp.exe"), - ('attachment; filename=".."', None), - ('attachment; notfilename="decoy.txt"', None), - ('attachment; filename="report.pdf"', "report.pdf"), - ], - ) - def test_traversal_and_spoofing_are_neutralised(self, header, expected): - assert filename_from_content_disposition(header) == expected - - def test_overlong_names_are_capped_keeping_the_extension(self): - name = filename_from_content_disposition(f'attachment; filename="{"a" * 400}.pdf"') - assert name is not None and name.endswith(".pdf") and len(name.encode()) <= 255 - - -@pytest.mark.parametrize("value", ["half an emoji \ud83d", {"a", "set"}, b"bytes"]) -def test_unencodable_arguments_raise_value_error(value): - """These escaped as a bare UnicodeEncodeError/TypeError from inside httpx.""" - tool = StackOneRpcTool( - name="linear_x", - description="", - parameters=ToolParameters(type="object", properties={"body_q": {"type": "string"}}), - api_key="k", - base_url="https://api.example.invalid", - account_id="a", - ) + NaN and Infinity are not JSON: the MCP client would send them as null. + """ with pytest.raises(ValueError, match="could not be encoded"): - tool.execute({"body_q": value}) + _mcp_tool().execute({"q": value}) + assert seen == {} + + +@pytest.mark.parametrize( + "call", + [ + pytest.param(lambda tool: tool.execute({"q": float("nan")}), id="unencodable"), + pytest.param(lambda tool: tool.execute({"headers": {"x": {"a", "set"}}}), id="unencodable-header"), + pytest.param(lambda tool: tool.execute("not json"), id="invalid-json"), + pytest.param(lambda tool: tool.execute("[1, 2]"), id="not-an-object"), + pytest.param(lambda tool: tool.call({"q": 1}, q=2), id="args-and-kwargs"), + pytest.param(lambda tool: tool.call({}, {}), id="two-positional"), + ], +) +def test_an_argument_error_is_a_stackone_error_and_a_value_error(call, seen): + """`except StackOneError` catches everything the SDK raises; `except ValueError` still works.""" + with pytest.raises(ToolArgumentsError) as excinfo: + call(_mcp_tool({"headers": {"type": "object"}})) + assert isinstance(excinfo.value, StackOneError) + assert isinstance(excinfo.value, ValueError) + assert seen == {} diff --git a/tests/test_toolset.py b/tests/test_toolset.py index ca7f082c..7ad3c79b 100644 --- a/tests/test_toolset.py +++ b/tests/test_toolset.py @@ -199,6 +199,37 @@ def test_init_with_custom_base_url(self): toolset = StackOneToolSet(api_key="test_key", base_url="https://custom.api.com") assert toolset.base_url == "https://custom.api.com" + def test_base_url_argument_takes_precedence_over_env(self): + """The base_url argument wins over STACKONE_BASE_URL.""" + with patch.dict(os.environ, {"STACKONE_BASE_URL": "https://env.api.com"}): + toolset = StackOneToolSet(api_key="test_key", base_url="https://arg.api.com") + assert toolset.base_url == "https://arg.api.com" + + def test_base_url_falls_back_to_env_var(self): + """STACKONE_BASE_URL is used when no base_url argument is given.""" + with patch.dict(os.environ, {"STACKONE_BASE_URL": "https://env.api.com"}): + toolset = StackOneToolSet(api_key="test_key") + assert toolset.base_url == "https://env.api.com" + + def test_base_url_defaults_when_no_argument_or_env(self): + """Falls back to the default when neither is given.""" + with patch.dict(os.environ, {}, clear=True): + os.environ.pop("STACKONE_BASE_URL", None) + toolset = StackOneToolSet(api_key="test_key") + assert toolset.base_url == DEFAULT_BASE_URL + + def test_base_url_empty_env_var_treated_as_unset(self): + """An empty STACKONE_BASE_URL falls back to the default, not a literal empty host.""" + with patch.dict(os.environ, {"STACKONE_BASE_URL": ""}): + toolset = StackOneToolSet(api_key="test_key") + assert toolset.base_url == DEFAULT_BASE_URL + + def test_base_url_empty_argument_falls_back_to_env(self): + """An empty base_url argument is treated as unset, so the env var is used.""" + with patch.dict(os.environ, {"STACKONE_BASE_URL": "https://env.api.com"}): + toolset = StackOneToolSet(api_key="test_key", base_url="") + assert toolset.base_url == "https://env.api.com" + class TestStackOneToolSetNormalizeSchemaProperties: """Test _normalize_schema_properties method.""" @@ -288,6 +319,62 @@ def test_set_accounts(): assert toolset._account_ids == ["acc1", "acc2"] +@pytest.mark.parametrize( + "use", + [ + lambda: StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", ""]}), + lambda: StackOneToolSet(api_key="k").set_accounts([""]), + lambda: StackOneToolSet(api_key="k").fetch_tools(account_ids=["acc1", ""]), + lambda: StackOneToolSet(api_key="k").submit_feedback("positive", ["t"], account_ids=[""]), + ], + ids=["constructor", "set_accounts", "fetch_tools", "submit_feedback"], +) +def test_an_empty_account_id_is_rejected(use): + """An empty id would be sent with no x-account-id; it fails before any request, + so no request error can stand in for it.""" + with pytest.raises(ToolsetConfigError, match="empty account id"): + use() + + +@pytest.mark.parametrize("bad", [None, 123, ["acc1"]]) +@pytest.mark.parametrize( + "use", + [ + lambda bad: StackOneToolSet(api_key="k", execute={"account_ids": ["acc1", bad]}), + lambda bad: StackOneToolSet(api_key="k").set_accounts([bad]), + lambda bad: StackOneToolSet(api_key="k").fetch_tools(account_ids=["acc1", bad]), + lambda bad: StackOneToolSet(api_key="k").submit_feedback("positive", ["t"], account_ids=[bad]), + ], + ids=["constructor", "set_accounts", "fetch_tools", "submit_feedback"], +) +def test_a_non_string_account_id_is_rejected(use, bad): + """None was accepted and sent no x-account-id at all, as in Node it is refused.""" + with pytest.raises(ToolsetConfigError, match="account_ids must be a list of account id strings"): + use(bad) + + +@pytest.mark.parametrize( + "use", + [ + lambda: StackOneToolSet(api_key="k", execute={"account_ids": ("acc1", "acc2")}), + lambda: StackOneToolSet(api_key="k").set_accounts(("acc1", "acc2")), + lambda: StackOneToolSet(api_key="k").fetch_tools(account_ids=("acc1", "acc2")), + ], + ids=["constructor", "set_accounts", "fetch_tools"], +) +def test_a_tuple_of_account_ids_is_rejected(use): + """Node rejects anything that isn't an Array, so a tuple is refused too, not silently + accepted as a list-like.""" + with pytest.raises(ToolsetConfigError, match="account_ids must be a list of account id strings"): + use() + + +def test_an_empty_constructor_account_id_is_rejected(): + """Treated as unset, it would silently widen every call to all active accounts.""" + with pytest.raises(ToolsetConfigError, match="account_id must not be an empty string"): + StackOneToolSet(api_key="k", account_id="") + + def test_filter_by_provider(): """Test provider filtering""" toolset = StackOneToolSet(api_key="test_key") diff --git a/tests/test_toolset_headers.py b/tests/test_toolset_headers.py new file mode 100644 index 00000000..73897dd8 --- /dev/null +++ b/tests/test_toolset_headers.py @@ -0,0 +1,146 @@ +"""The toolset-level ``headers`` option: extra HTTP headers sent with every request.""" + +from __future__ import annotations + +import logging +from typing import Any + +import httpx +import pytest + +from stackone_ai.tools import McpToolDefinition +from stackone_ai.toolset import StackOneToolSet + + +def _accounts_response(monkeypatch: pytest.MonkeyPatch, body: Any = None) -> list[httpx.Request]: + """Answer GET /accounts with ``body`` (``[]`` if omitted); every request it received.""" + seen: list[httpx.Request] = [] + + def handle(_self: Any, request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json=body if body is not None else [], request=request) + + monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handle) + return seen + + +class _Mcp: + """A fake MCP transport recording the headers of every listing and call.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.listed: list[dict[str, str]] = [] + self.called: list[dict[str, str]] = [] + monkeypatch.setattr("stackone_ai.toolset.fetch_mcp_tools", self._list) + monkeypatch.setattr("stackone_ai.tools.call_mcp_tool", self._call) + + def _list(self, _endpoint: str, headers: dict[str, str], **_kwargs: Any) -> list[McpToolDefinition]: + self.listed.append(headers) + return [McpToolDefinition(name=f"tool_{headers['x-account-id']}", description="", input_schema={})] + + def _call(self, _endpoint: str, headers: dict[str, str], *_args: Any, **_kwargs: Any) -> dict[str, Any]: + self.called.append(headers) + return {"isError": False} + + +def test_warns_once_about_sdk_owned_names_and_ignores_them(caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.WARNING, logger="stackone"): + toolset = StackOneToolSet( + api_key="k", headers={"Authorization": "Bearer x", "X-Account-Id": "spoofed", "X-Trace": "t"} + ) + + [message] = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert message == ( + 'Ignoring headers "Authorization", "X-Account-Id": the SDK sets them itself. ' + "Pass the API key and account ids through their own options instead." + ) + assert toolset._headers == { + "Authorization": "Bearer x", + "X-Account-Id": "spoofed", + "X-Trace": "t", + } + + +def test_does_not_warn_about_x_end_user_id_or_unrelated_names(caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.WARNING, logger="stackone"): + StackOneToolSet(api_key="k", headers={"x-end-user-id": "dave", "X-Trace": "t"}) + + assert [r for r in caplog.records if r.levelno == logging.WARNING] == [] + + +def test_reaches_get_accounts(monkeypatch: pytest.MonkeyPatch): + seen = _accounts_response(monkeypatch) + StackOneToolSet(api_key="k", headers={"X-Trace": "t"}).fetch_accounts() + + assert seen[0].headers["x-trace"] == "t" + + +def test_sdk_owned_names_are_dropped_beneath_the_sdks_own_on_get_accounts(monkeypatch: pytest.MonkeyPatch): + seen = _accounts_response(monkeypatch) + StackOneToolSet(api_key="k", headers={"Authorization": "Bearer spoofed"}).fetch_accounts() + + assert seen[0].headers["authorization"] != "Bearer spoofed" + + +def test_reaches_every_mcp_listing_and_call_request(monkeypatch: pytest.MonkeyPatch): + seen = _accounts_response(monkeypatch) + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k", headers={"X-Trace": "t"}) + + tools = toolset.fetch_tools(account_ids=["acc1", "acc2"]) + for tool in tools: + tool.execute({}) + + # Explicit ids: no account discovery, so there is nothing for the header to reach there. + assert seen == [] + assert all(h.get("X-Trace") == "t" for h in mcp.listed) + assert all(h.get("X-Trace") == "t" for h in mcp.called) + + +def test_sdk_owned_names_are_dropped_beneath_the_sdks_own_on_mcp_requests(monkeypatch: pytest.MonkeyPatch): + _accounts_response(monkeypatch) + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k", headers={"x-account-id": "spoofed", "User-Agent": "spoofed"}) + + tool = toolset.fetch_tools(account_ids=["acc1"])[0] + tool.execute({}) + + assert mcp.listed[0]["x-account-id"] == "acc1" + assert mcp.called[0]["x-account-id"] == "acc1" + assert "spoofed" not in mcp.listed[0]["User-Agent"] + assert "spoofed" not in mcp.called[0]["User-Agent"] + + +def test_x_end_user_id_is_passed_through_as_given(monkeypatch: pytest.MonkeyPatch): + _accounts_response(monkeypatch) + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k", account_id="acc1", headers={"x-end-user-id": "caller"}) + + tool = toolset.fetch_tools()[0] + tool.execute({}) + + assert mcp.listed[0]["x-end-user-id"] == "caller" + assert mcp.called[0]["x-end-user-id"] == "caller" + + +def test_a_recorded_end_user_id_overrides_the_configured_one(monkeypatch: pytest.MonkeyPatch): + _accounts_response( + monkeypatch, + [ + { + "id": "acc1", + "provider": "p", + "status": "active", + "shared": False, + "origin_username": "recorded-user", + } + ], + ) + mcp = _Mcp(monkeypatch) + toolset = StackOneToolSet(api_key="k", account_id="acc1", headers={"x-end-user-id": "caller"}) + toolset.fetch_accounts() + + tool = toolset.fetch_tools()[0] + tool.execute({}) + + assert mcp.listed[0]["x-end-user-id"] == "recorded-user" + assert mcp.called[0]["x-end-user-id"] == "recorded-user" diff --git a/tests/vectors/README.md b/tests/vectors/README.md new file mode 100644 index 00000000..24e3fad7 --- /dev/null +++ b/tests/vectors/README.md @@ -0,0 +1,144 @@ +# Shared test vectors + +Unit-test data both SDKs load, so the Python and Node unit tests check the same inputs against +the same expected outputs. The harness in this repo grades an SDK end to end, over the wire; +these grade the pure functions underneath: reading a `Retry-After`, the backoff, header +filtering, argument encoding and account-id validation. They also fix the canonical wording of +every message the two SDKs share. + +Every expected value was taken from both SDKs' behaviour at the time it was written, and only +where they agreed. Where they disagreed, the contract has since settled which is right, and the +case is in that file's `cases` array with the agreed value — a `description` on the case says what +changes. A case that is settled but still not identical across SDKs for a reason outside the rule +(for example, a JSON parser has already rounded a value before the rule ever sees it) is in a +`known_differences` array instead, not graded. A file keeps an `unresolved` array for a case the +SDKs disagree on today and the contract does not yet settle; none is open right now. + +| File | Input → expected | +|---|---| +| `retry-after.json` | a `Retry-After` value and `now` → seconds before the cap, or `null` when absent or unreadable | +| `backoff.json` | retry number, injected random value, `Retry-After` → the delay; and a delay against the time left → `wait` or `return-429` | +| `header-values.json` | a nested header value → the string forwarded, or `{"dropped": true}` | +| `header-names.json` | a nested header name → `{"forwarded": ""}` or `{"dropped": ""}` | +| `header-arguments.json` | a served `inputSchema` and the model's arguments → the arguments sent, and the warnings as `{argument, header?, reason}` | +| `argument-encoding.json` | a tool's arguments → `ok`, or `{"error": "ToolArgumentsError", "reason"}` | +| `account-ids.json` | an account-id argument → `ok`, or each language's config error class | +| `messages.json` | the canonical template of every shared warning and error, with each SDK's current text beside it | + +`schema/` holds a JSON Schema (draft 2020-12) for each file. Each file names its schema in +`$schema`. + +## File shape + +```jsonc +{ + "$schema": "./schema/.schema.json", + "description": "what the vectors are, and the context each case runs in", + "version": 1, + // any constants the cases share: max_delay_seconds, constants, reasons, error_classes + "cases": [{ "id": "…", "description": "…", /* inputs */ "expected…": … }], + // present only while a case is open: + "unresolved": [{ "id": "…", "description": "…", /* inputs */ "python": …, "node": … }], + // present only where a settled case still differs for a reason outside the rule: + "known_differences": [{ "id": "…", "description": "…", /* inputs */ "python": …, "node": … }] +} +``` + +- **`id`** is unique within a file and does not change: a test names its cases by it. +- **`version`** is bumped when an existing case's meaning or expected value changes. Adding a case + does not bump it. Versions count from the vectors' first release: every change made before it + stays at version 1. +- **`unresolved`** cases are not graded. Each one says what the two SDKs do differently. Once the + contract settles it, the case moves to `cases` with the agreed value, and the SDK that changes + fixes its code in the same release. A file only has this array while it has an open case. +- **`known_differences`** cases are not graded either. Each one is a settled exception, recorded + only in `known_differences` and not also in `cases`: the contract has chosen the behaviour, but + the two SDKs still produce different values for a reason outside the rule being tested, so + there is nothing to converge on. A file only has this array while it has such a case. + +## Reading the values + +- **Numbers JSON cannot hold.** `{"$number": "NaN"}`, `{"$number": "Infinity"}` and + `{"$number": "-Infinity"}` stand for the non-finite floats. Decode them before use: walk the + value, and replace every object whose only key is `$number` with the float it names. +- **Seconds** are compared with a relative tolerance of 1e-9. The Node SDK works in milliseconds, + and converting back can leave a last-bit difference: 99999999999999999999 s round-trips as + 99999999999999980000. +- **`now` and `random` are injected.** `retry-after.json` gives each case its `now` in RFC 3339. + `backoff.json` gives the uniform value in [0, 1) that the jitter is built from: Node's + `Math.random()`, and in Python the value `random.uniform(0.5, 1.0)` scales (`0.5 + 0.5 * r`). An + SDK whose clock or random source is not yet injectable has to make it injectable to load these. +- **Warnings** in `header-arguments.json` are listed in argument order, which both SDKs follow. A + warning's `reason` is a code. `messages.json` turns a code into text (`reasons`) and fills it + into a template, so the wording is checked once there and not in every case. +- **Error classes** are given by name. `account-ids.json` gives both, because the spelling differs: + `ToolsetConfigError` in Python, `ToolSetConfigError` in Node. + +## `messages.json` + +Each message has an `id`, a `template` with named placeholders (`{endpoint}`, `{attempt}`, …), a +`format` for each placeholder, and the text each SDK emits today (`python`, `node`). `matches` +says whether that text is already the canonical one, and `notes` say where the text depends on +more than the template. Render +a template by replacing each placeholder with its value written in the placeholder's format: + +| Format | Written as | +|---|---| +| `text` | the value as is | +| `json` | compact JSON: `JSON.stringify(v)` / `json.dumps(v, ensure_ascii=False, separators=(",", ":"))`, so a string comes out in double quotes | +| `integer` | decimal digits | +| `seconds` | a number of seconds, as JavaScript's `String(n)` writes it: `1`, `0.5`, never `1.0` | +| `seconds-2dp` | rounded half up to two decimals (`Math.floor(x * 100 + 0.5) / 100`), then as `seconds` | +| `json-type` | the value's JSON type: `object`, `array`, `string`, `number`, `boolean`, `null` | +| `reason` | the text `reasons` gives for the code | + +A prefix that a language adds stays outside the template: Node's `[@stackone/ai] ` on every +`console.warn`, and Python's logger name. The harness checks the same templates end to end: a +scenario's `expect.warnings` and `expect.error.message` must render one, or the run refuses to +start. + +`unshared` lists messages only one SDK emits. They are recorded so they are not forgotten. They +have no canonical form yet. + +## How an SDK consumes them + +An SDK vendors a copy of this directory. Its unit tests load the vendored copy and never fetch +these files over the network. The vendored copy is pinned to a commit of this repo: the one the +SDK's CI already uses to check out the conformance suite. + +1. Copy `vectors/` into the SDK, for example to `tests/conformance_vectors/` (Python) or + `src/__fixtures__/conformance-vectors/` (Node), from the commit you pin. +2. Add a test per file that iterates `cases` and checks each `id`. Skip `unresolved` and + `known_differences`. Check `version` too, so that a bump fails the test until someone has read + what changed. +3. In CI, check out this repo at the pinned commit, as the conformance job already does, and fail + unless the vendored copy is **byte-identical** to it: + + ```yaml + - name: Vendored vectors match the pinned sdk-conformance + run: diff -r sdk/tests/conformance_vectors sdk-conformance/vectors + ``` + + An edit to the copy fails CI, and so does a pin that moved without a re-copy. Neither SDK can + drift from the data the other one tests against. + +To update, bump the pin, re-copy, and fix whatever the new vectors fail. + +```python +# Python (pytest) +VECTORS = Path(__file__).parent / "conformance_vectors" +retry_after = json.loads((VECTORS / "retry-after.json").read_text()) + +@pytest.mark.parametrize("case", retry_after["cases"], ids=lambda case: case["id"]) +def test_retry_after(case): + ... +``` + +```typescript +// Node (vitest) +import retryAfter from './__fixtures__/conformance-vectors/retry-after.json'; + +describe.each(retryAfter.cases)('Retry-After $id', (testCase) => { + it('reads as the agreed seconds', () => { /* … */ }); +}); +``` diff --git a/tests/vectors/account-ids.json b/tests/vectors/account-ids.json new file mode 100644 index 00000000..37959056 --- /dev/null +++ b/tests/vectors/account-ids.json @@ -0,0 +1,198 @@ +{ + "$schema": "./schema/account-ids.schema.json", + "description": "Which account ids a toolset accepts. `parameter: \"account_ids\"` is the per-call list (fetch_tools(account_ids=…) / fetchTools({accountIds})), checked by the same validator as set_accounts/setAccounts and the constructor's execute.account_ids; `parameter: \"account_id\"` is the constructor's single id. A refusal raises the language's own config error class, before any request.", + "version": 1, + "error_classes": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "cases": [ + { + "id": "list", + "description": "a list of ids", + "parameter": "account_ids", + "value": ["acc-1", "acc-2"], + "expected": "ok" + }, + { + "id": "duplicates", + "description": "duplicates are allowed (the scope is deduplicated)", + "parameter": "account_ids", + "value": ["acc-1", "acc-1"], + "expected": "ok" + }, + { + "id": "whitespace-id", + "description": "a whitespace id is not empty, so it is accepted", + "parameter": "account_ids", + "value": [" "], + "expected": "ok" + }, + { + "id": "empty-list", + "description": "an empty list means not given: the toolset's own accounts, then discovery", + "parameter": "account_ids", + "value": [], + "expected": "ok" + }, + { + "id": "empty-string-id", + "description": "an empty id would be sent with no x-account-id", + "parameter": "account_ids", + "value": [""], + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "empty-among-valid", + "description": "one empty id among valid ones", + "parameter": "account_ids", + "value": ["acc-1", ""], + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "bare-string", + "description": "a bare string is refused, not iterated as characters", + "parameter": "account_ids", + "value": "acc-1", + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "bare-empty-string", + "description": "a bare empty string is refused as a string", + "parameter": "account_ids", + "value": "", + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "number-id", + "description": "a non-string id", + "parameter": "account_ids", + "value": [1], + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "null-id", + "description": "a null id", + "parameter": "account_ids", + "value": [null], + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "object-id", + "description": "an object id", + "parameter": "account_ids", + "value": [ + { + "id": "acc-1" + } + ], + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "nested-list", + "description": "a list id", + "parameter": "account_ids", + "value": [["acc-1"]], + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "single", + "description": "a single account id", + "parameter": "account_id", + "value": "acc-1", + "expected": "ok" + }, + { + "id": "single-empty", + "description": "an empty single account id is usually an unset variable: refused rather than widened to every account", + "parameter": "account_id", + "value": "", + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "single-whitespace", + "description": "a whitespace single account id is not empty", + "parameter": "account_id", + "value": " ", + "expected": "ok" + }, + { + "id": "null", + "description": "account_ids=None / accountIds: null means not given, the same as omitting it: the toolset's own accounts, then discovery. In both SDKs.", + "parameter": "account_ids", + "value": null, + "expected": "ok" + }, + { + "id": "not-a-list-number", + "description": "a number is refused in both: it is not a list of account id strings (account-ids-not-strings).", + "parameter": "account_ids", + "value": 5, + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + }, + { + "id": "not-a-list-object", + "description": "an object is refused in both: it is not a list of account id strings (account-ids-not-strings), not iterated for its keys.", + "parameter": "account_ids", + "value": { + "id": "acc-1" + }, + "expected": { + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + } + } + } + ] +} diff --git a/tests/vectors/argument-encoding.json b/tests/vectors/argument-encoding.json new file mode 100644 index 00000000..53f23892 --- /dev/null +++ b/tests/vectors/argument-encoding.json @@ -0,0 +1,249 @@ +{ + "$schema": "./schema/argument-encoding.schema.json", + "description": "Whether a tool's arguments can be sent: `ok`, or a ToolArgumentsError raised before any request with the given reason. `arguments` is the value handed to the tool's execute; `arguments_json` is JSON text handed instead. `{\"$number\": \"NaN\"}`, \"Infinity\" and \"-Infinity\" stand for the non-finite numbers JSON cannot hold. The tool's schema declares no header arguments, so header filtering plays no part.", + "version": 1, + "reasons": ["not-finite", "invalid-json", "not-an-object", "unencodable"], + "cases": [ + { + "id": "empty-object", + "description": "no arguments", + "arguments": {}, + "expected": "ok" + }, + { + "id": "nested-empty", + "description": "empty containers", + "arguments": { + "a": {}, + "b": [] + }, + "expected": "ok" + }, + { + "id": "scalars", + "description": "every JSON scalar", + "arguments": { + "s": "x", + "n": 1, + "f": 1.5, + "t": true, + "z": null + }, + "expected": "ok" + }, + { + "id": "deep-nesting", + "description": "deep nesting", + "arguments": { + "a": { + "b": { + "c": { + "d": { + "e": [ + [ + [ + { + "f": 1 + } + ] + ] + ] + } + } + } + } + }, + "expected": "ok" + }, + { + "id": "proto-key-nested", + "description": "a __proto__ key one level down is an ordinary key", + "arguments": { + "body": { + "__proto__": { + "polluted": true + } + } + }, + "expected": "ok" + }, + { + "id": "proto-key-top", + "description": "a top-level __proto__ key is an ordinary key to the SDK (the server's MCP stack drops it, see invariant 3)", + "arguments": { + "__proto__": { + "polluted": true + } + }, + "expected": "ok" + }, + { + "id": "constructor-key", + "description": "builtin-named keys are ordinary keys", + "arguments": { + "constructor": "x", + "toString": 1, + "valueOf": {} + }, + "expected": "ok" + }, + { + "id": "nan-top", + "description": "NaN is not JSON", + "arguments": { + "a": { + "$number": "NaN" + } + }, + "expected": { + "error": "ToolArgumentsError", + "reason": "not-finite" + } + }, + { + "id": "infinity-nested", + "description": "Infinity, however deep, is not JSON", + "arguments": { + "body": { + "limit": { + "$number": "Infinity" + } + } + }, + "expected": { + "error": "ToolArgumentsError", + "reason": "not-finite" + } + }, + { + "id": "neg-infinity-in-array", + "description": "-Infinity inside an array is not JSON", + "arguments": { + "a": [ + 1, + { + "$number": "-Infinity" + } + ] + }, + "expected": { + "error": "ToolArgumentsError", + "reason": "not-finite" + } + }, + { + "id": "big-int", + "description": "an integer above 2^53 is still encodable", + "arguments": { + "a": 12345678901234567890 + }, + "expected": "ok" + }, + { + "id": "non-ascii", + "description": "non-ASCII and astral characters are encodable", + "arguments": { + "a": "é€🙂" + }, + "expected": "ok" + }, + { + "id": "json-text-object", + "description": "arguments given as JSON text", + "arguments_json": "{\"a\":1}", + "expected": "ok" + }, + { + "id": "json-text-invalid", + "description": "JSON text that does not parse", + "arguments_json": "{", + "expected": { + "error": "ToolArgumentsError", + "reason": "invalid-json" + } + }, + { + "id": "json-text-array", + "description": "JSON text that is an array", + "arguments_json": "[1]", + "expected": { + "error": "ToolArgumentsError", + "reason": "not-an-object" + } + }, + { + "id": "json-text-string", + "description": "JSON text that is a string", + "arguments_json": "\"x\"", + "expected": { + "error": "ToolArgumentsError", + "reason": "not-an-object" + } + }, + { + "id": "json-text-overflow", + "description": "1e400 parses as Infinity in both languages, which is not JSON", + "arguments_json": "{\"a\": 1e400}", + "expected": { + "error": "ToolArgumentsError", + "reason": "not-finite" + } + }, + { + "id": "json-text-empty", + "description": "empty JSON text does not parse", + "arguments_json": "", + "expected": { + "error": "ToolArgumentsError", + "reason": "invalid-json" + } + }, + { + "id": "not-object-array", + "description": "an array is not an arguments object", + "arguments": [1, 2], + "expected": { + "error": "ToolArgumentsError", + "reason": "not-an-object" + } + }, + { + "id": "not-object-number", + "description": "a number is not an arguments object", + "arguments": 5, + "expected": { + "error": "ToolArgumentsError", + "reason": "not-an-object" + } + }, + { + "id": "lone-surrogate", + "description": "a lone UTF-16 surrogate (what a model emits when a token boundary splits an emoji), anywhere in the arguments, cannot be encoded: it is refused in both SDKs rather than sent as an escaped \\ud800.", + "arguments": { + "a": "\ud800" + }, + "expected": { + "error": "ToolArgumentsError", + "reason": "unencodable" + } + }, + { + "id": "json-text-nan-literal", + "description": "JSON text containing a bare NaN is invalid JSON in both SDKs, even though a lenient parser could read it as a number.", + "arguments_json": "{\"a\": NaN}", + "expected": { + "error": "ToolArgumentsError", + "reason": "invalid-json" + } + }, + { + "id": "json-text-bare-nan", + "description": "the JSON text NaN alone is invalid JSON in both SDKs.", + "arguments_json": "NaN", + "expected": { + "error": "ToolArgumentsError", + "reason": "invalid-json" + } + } + ] +} diff --git a/tests/vectors/backoff.json b/tests/vectors/backoff.json new file mode 100644 index 00000000..bbb33387 --- /dev/null +++ b/tests/vectors/backoff.json @@ -0,0 +1,241 @@ +{ + "$schema": "./schema/backoff.schema.json", + "description": "The delay before a retry of a 429, and whether the deadline allows it. Before retry `retry` (1-based), the delay is `retry_after_seconds` when the 429 carried a readable Retry-After, else `backoff_seconds[retry - 1] * (jitter.min + (jitter.max - jitter.min) * random)`; either way capped at `max_delay_seconds`. `random` is the injected uniform value in [0, 1) — Node's Math.random(), and in Python the value random.uniform(0.5, 1.0) is built from. Deadline cases take the delay after the cap.", + "version": 1, + "constants": { + "max_retries": 3, + "max_attempts": 4, + "backoff_seconds": [1, 2, 4], + "jitter": { + "min": 0.5, + "max": 1, + "max_exclusive": true + }, + "max_delay_seconds": 30 + }, + "cases": [ + { + "id": "backoff-retry-1-random-0", + "description": "no Retry-After: backoff 1s x jitter 0.5", + "retry": 1, + "random": 0, + "retry_after_seconds": null, + "expected_delay_seconds": 0.5 + }, + { + "id": "backoff-retry-1-random-0.25", + "description": "no Retry-After: backoff 1s x jitter 0.625", + "retry": 1, + "random": 0.25, + "retry_after_seconds": null, + "expected_delay_seconds": 0.625 + }, + { + "id": "backoff-retry-1-random-0.5", + "description": "no Retry-After: backoff 1s x jitter 0.75", + "retry": 1, + "random": 0.5, + "retry_after_seconds": null, + "expected_delay_seconds": 0.75 + }, + { + "id": "backoff-retry-1-random-0.9375", + "description": "no Retry-After: backoff 1s x jitter 0.96875", + "retry": 1, + "random": 0.9375, + "retry_after_seconds": null, + "expected_delay_seconds": 0.96875 + }, + { + "id": "backoff-retry-2-random-0", + "description": "no Retry-After: backoff 2s x jitter 0.5", + "retry": 2, + "random": 0, + "retry_after_seconds": null, + "expected_delay_seconds": 1 + }, + { + "id": "backoff-retry-2-random-0.25", + "description": "no Retry-After: backoff 2s x jitter 0.625", + "retry": 2, + "random": 0.25, + "retry_after_seconds": null, + "expected_delay_seconds": 1.25 + }, + { + "id": "backoff-retry-2-random-0.5", + "description": "no Retry-After: backoff 2s x jitter 0.75", + "retry": 2, + "random": 0.5, + "retry_after_seconds": null, + "expected_delay_seconds": 1.5 + }, + { + "id": "backoff-retry-2-random-0.9375", + "description": "no Retry-After: backoff 2s x jitter 0.96875", + "retry": 2, + "random": 0.9375, + "retry_after_seconds": null, + "expected_delay_seconds": 1.9375 + }, + { + "id": "backoff-retry-3-random-0", + "description": "no Retry-After: backoff 4s x jitter 0.5", + "retry": 3, + "random": 0, + "retry_after_seconds": null, + "expected_delay_seconds": 2 + }, + { + "id": "backoff-retry-3-random-0.25", + "description": "no Retry-After: backoff 4s x jitter 0.625", + "retry": 3, + "random": 0.25, + "retry_after_seconds": null, + "expected_delay_seconds": 2.5 + }, + { + "id": "backoff-retry-3-random-0.5", + "description": "no Retry-After: backoff 4s x jitter 0.75", + "retry": 3, + "random": 0.5, + "retry_after_seconds": null, + "expected_delay_seconds": 3 + }, + { + "id": "backoff-retry-3-random-0.9375", + "description": "no Retry-After: backoff 4s x jitter 0.96875", + "retry": 3, + "random": 0.9375, + "retry_after_seconds": null, + "expected_delay_seconds": 3.875 + }, + { + "id": "retry-after-0", + "description": "Retry-After 0s is honoured as given", + "retry": 1, + "random": 0.5, + "retry_after_seconds": 0, + "expected_delay_seconds": 0 + }, + { + "id": "retry-after-5", + "description": "Retry-After 5s is honoured as given", + "retry": 1, + "random": 0.5, + "retry_after_seconds": 5, + "expected_delay_seconds": 5 + }, + { + "id": "retry-after-30", + "description": "Retry-After 30s is honoured as given", + "retry": 1, + "random": 0.5, + "retry_after_seconds": 30, + "expected_delay_seconds": 30 + }, + { + "id": "retry-after-31", + "description": "Retry-After 31s is capped at 30s", + "retry": 1, + "random": 0.5, + "retry_after_seconds": 31, + "expected_delay_seconds": 30 + }, + { + "id": "retry-after-3600", + "description": "Retry-After 3600s is capped at 30s", + "retry": 1, + "random": 0.5, + "retry_after_seconds": 3600, + "expected_delay_seconds": 30 + }, + { + "id": "retry-after-on-retry-3", + "description": "Retry-After 7s is honoured as given", + "retry": 3, + "random": 0, + "retry_after_seconds": 7, + "expected_delay_seconds": 7 + } + ], + "deadline_cases": [ + { + "id": "delay-5-remaining-10", + "description": "a 5s wait with 10s left: the wait is less than the time left", + "delay_seconds": 5, + "remaining_seconds": 10, + "expected": "wait" + }, + { + "id": "delay-10-remaining-10", + "description": "a 10s wait with 10s left: the wait is equal to the time left — the boundary: an equal wait does not finish before the deadline", + "delay_seconds": 10, + "remaining_seconds": 10, + "expected": "return-429" + }, + { + "id": "delay-10-remaining-5", + "description": "a 10s wait with 5s left: the wait is more than the time left", + "delay_seconds": 10, + "remaining_seconds": 5, + "expected": "return-429" + }, + { + "id": "delay-0-remaining-0", + "description": "a 0s wait with 0s left: the wait is equal to the time left — the boundary: an equal wait does not finish before the deadline", + "delay_seconds": 0, + "remaining_seconds": 0, + "expected": "return-429" + }, + { + "id": "delay-0-remaining-1", + "description": "a 0s wait with 1s left: the wait is less than the time left", + "delay_seconds": 0, + "remaining_seconds": 1, + "expected": "wait" + }, + { + "id": "delay-1-remaining-1", + "description": "a 1s wait with 1s left: the wait is equal to the time left — the boundary: an equal wait does not finish before the deadline", + "delay_seconds": 1, + "remaining_seconds": 1, + "expected": "return-429" + }, + { + "id": "delay-2-remaining-1", + "description": "a 2s wait with 1s left: the wait is more than the time left", + "delay_seconds": 2, + "remaining_seconds": 1, + "expected": "return-429" + }, + { + "id": "delay-10-remaining-2", + "description": "a 10s wait with 2s left: the wait is more than the time left", + "delay_seconds": 10, + "remaining_seconds": 2, + "expected": "return-429" + }, + { + "id": "delay-0-remaining--1", + "description": "a 0s wait with -1s left: the wait is more than the time left", + "delay_seconds": 0, + "remaining_seconds": -1, + "expected": "return-429" + }, + { + "id": "delay-30-remaining-30.5", + "description": "a 30s wait with 30.5s left: the wait is less than the time left", + "delay_seconds": 30, + "remaining_seconds": 30.5, + "expected": "wait" + }, + { + "id": "delay-29-remaining-29.001", + "description": "a 29s wait with 29.001s left: the wait is less than the time left", + "delay_seconds": 29, + "remaining_seconds": 29.001, + "expected": "wait" + } + ] +} diff --git a/tests/vectors/header-arguments.json b/tests/vectors/header-arguments.json new file mode 100644 index 00000000..bdd2d202 --- /dev/null +++ b/tests/vectors/header-arguments.json @@ -0,0 +1,1484 @@ +{ + "$schema": "./schema/header-arguments.schema.json", + "description": "Invariant 4 as data: a tool's served inputSchema, the model's arguments, the arguments that reach tools/call, and the warnings the SDK emits, in argument order. A warning is {argument, header?, reason}: `argument` is the top-level key (`headers` or `headers_`); `header` is the trimmed nested name, for an entry of a `headers` object. `reason` is one of `reasons`; messages.json renders it. `x-end-user-id` is SDK-owned like authorization, x-account-id and user-agent, and dropped unconditionally — whether or not the SDK has an end user id recorded for the call's account (SDK-UCA-CONTRACT.md §9): these cases carry no account context, since the owned-name check runs the same regardless.", + "version": 1, + "reasons": ["set-by-sdk", "not-declared", "malformed", "not-an-object", "not-a-scalar"], + "cases": [ + { + "id": "open-forwards-every-name", + "description": "an open headers object (type object, no properties, additionalProperties not false) declares every name", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object" + } + } + }, + "arguments": { + "headers": { + "x-custom": "a", + "X-Other": "b" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a", + "X-Other": "b" + } + }, + "expected_warnings": [] + }, + { + "id": "open-drops-sdk-owned", + "description": "an open object still never forwards an SDK-owned name", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object" + } + } + }, + "arguments": { + "headers": { + "Authorization": "Bearer x", + "x-account-id": "victim", + "User-Agent": "evil/0", + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "Authorization", + "reason": "set-by-sdk" + }, + { + "argument": "headers", + "header": "x-account-id", + "reason": "set-by-sdk" + }, + { + "argument": "headers", + "header": "User-Agent", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "open-additional-properties-true", + "description": "additionalProperties: true is open", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "additionalProperties": true + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [] + }, + { + "id": "open-additional-properties-schema", + "description": "additionalProperties as a schema is open", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [] + }, + { + "id": "closed-declares-nothing", + "description": "a closed object (additionalProperties: false, no properties) declares no names; headers is sent empty", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "additionalProperties": false + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": {} + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-custom", + "reason": "not-declared" + } + ] + }, + { + "id": "closed-with-properties", + "description": "a closed object with properties declares exactly those", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "additionalProperties": false, + "properties": { + "x-custom": { + "type": "string" + } + } + } + } + }, + "arguments": { + "headers": { + "x-custom": "a", + "x-other": "b" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-other", + "reason": "not-declared" + } + ] + }, + { + "id": "declared-nested", + "description": "declared names are forwarded; an undeclared one is not; a declared SDK-owned one is not; a non-header argument is untouched", + "schema": { + "type": "object", + "properties": { + "path": { + "type": "object" + }, + "headers": { + "type": "object", + "properties": { + "x-custom": { + "type": "string" + }, + "Authorization": { + "type": "string" + } + } + } + } + }, + "arguments": { + "path": { + "id": "emp_1" + }, + "headers": { + "x-custom": "a", + "x-other": "b", + "authorization": "Bearer x" + } + }, + "expected_arguments": { + "path": { + "id": "emp_1" + }, + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-other", + "reason": "not-declared" + }, + { + "argument": "headers", + "header": "authorization", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "declared-nested-case-insensitive", + "description": "nested names are declared case-insensitively, and forwarded as supplied", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "properties": { + "X-Custom": { + "type": "string" + } + } + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [] + }, + { + "id": "declared-nested-trimmed", + "description": "a nested name is trimmed before it is looked up and forwarded", + "schema": { + "type": "object", + "properties": { + "path": { + "type": "object" + }, + "headers": { + "type": "object", + "properties": { + "x-custom": { + "type": "string" + }, + "Authorization": { + "type": "string" + } + } + } + } + }, + "arguments": { + "headers": { + " x-custom\t": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [] + }, + { + "id": "declared-nested-null-value", + "description": "a null nested value is dropped silently", + "schema": { + "type": "object", + "properties": { + "path": { + "type": "object" + }, + "headers": { + "type": "object", + "properties": { + "x-custom": { + "type": "string" + }, + "Authorization": { + "type": "string" + } + } + } + } + }, + "arguments": { + "headers": { + "x-custom": null + } + }, + "expected_arguments": { + "headers": {} + }, + "expected_warnings": [] + }, + { + "id": "declared-nested-scalar-value", + "description": "a nested scalar is forwarded as its text (see header-values.json)", + "schema": { + "type": "object", + "properties": { + "path": { + "type": "object" + }, + "headers": { + "type": "object", + "properties": { + "x-custom": { + "type": "string" + }, + "Authorization": { + "type": "string" + } + } + } + } + }, + "arguments": { + "headers": { + "x-custom": 1 + } + }, + "expected_arguments": { + "headers": { + "x-custom": "1" + } + }, + "expected_warnings": [] + }, + { + "id": "nested-empty-object", + "description": "an empty headers object is forwarded empty", + "schema": { + "type": "object", + "properties": { + "path": { + "type": "object" + }, + "headers": { + "type": "object", + "properties": { + "x-custom": { + "type": "string" + }, + "Authorization": { + "type": "string" + } + } + } + } + }, + "arguments": { + "headers": {} + }, + "expected_arguments": { + "headers": {} + }, + "expected_warnings": [] + }, + { + "id": "properties-not-an-object", + "description": "a properties keyword that is not an object declares nothing", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "properties": [] + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": {} + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-custom", + "reason": "not-declared" + } + ] + }, + { + "id": "untyped-declares-nothing", + "description": "an untyped headers schema with no properties declares nothing", + "schema": { + "type": "object", + "properties": { + "headers": {} + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": {} + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-custom", + "reason": "not-declared" + } + ] + }, + { + "id": "untyped-with-properties", + "description": "an untyped headers schema with properties declares those", + "schema": { + "type": "object", + "properties": { + "headers": { + "properties": { + "x-custom": {} + } + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [] + }, + { + "id": "untyped-non-object-dropped", + "description": "an untyped headers schema is not an ordinary field: a non-object value is dropped", + "schema": { + "type": "object", + "properties": { + "headers": {} + } + }, + "arguments": { + "headers": "abc" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "object-or-null-with-properties", + "description": "[\"object\",\"null\"] with properties declares those", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": ["object", "null"], + "properties": { + "x-custom": {} + } + } + } + }, + "arguments": { + "headers": { + "x-custom": "a", + "x-other": "b" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-other", + "reason": "not-declared" + } + ] + }, + { + "id": "object-or-null-non-object-dropped", + "description": "[\"object\",\"null\"] is still a header container: a non-object value is dropped", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": ["object", "null"] + } + } + }, + "arguments": { + "headers": "abc" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "string-typed-is-ordinary", + "description": "a headers property typed string is an ordinary field: its value is forwarded unchanged", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "string" + } + } + }, + "arguments": { + "headers": "abc" + }, + "expected_arguments": { + "headers": "abc" + }, + "expected_warnings": [] + }, + { + "id": "string-typed-array-value-is-ordinary", + "description": "an ordinary field's value is forwarded whatever its type", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "string" + } + } + }, + "arguments": { + "headers": ["a"] + }, + "expected_arguments": { + "headers": ["a"] + }, + "expected_warnings": [] + }, + { + "id": "string-or-null-is-ordinary", + "description": "[\"string\",\"null\"] is an ordinary field", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": ["string", "null"] + } + } + }, + "arguments": { + "headers": "abc" + }, + "expected_arguments": { + "headers": "abc" + }, + "expected_warnings": [] + }, + { + "id": "array-typed-is-ordinary", + "description": "a headers property typed array is an ordinary field", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "array" + } + } + }, + "arguments": { + "headers": [["x-account-id", "B"]] + }, + "expected_arguments": { + "headers": [["x-account-id", "B"]] + }, + "expected_warnings": [] + }, + { + "id": "string-typed-object-value-is-ordinary", + "description": "a headers property typed \"string\" is an ordinary field whatever the value: an object is forwarded unchanged, not filtered", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "string" + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [] + }, + { + "id": "no-headers-schema-object", + "description": "with no headers property served, a nested headers object declares nothing", + "schema": { + "type": "object", + "properties": { + "path_id": {} + } + }, + "arguments": { + "path_id": "e", + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "path_id": "e", + "headers": {} + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-custom", + "reason": "not-declared" + } + ] + }, + { + "id": "no-headers-schema-non-object", + "description": "with no headers property served, a non-object headers value is dropped", + "schema": { + "type": "object", + "properties": { + "path_id": {} + } + }, + "arguments": { + "path_id": "e", + "headers": "abc" + }, + "expected_arguments": { + "path_id": "e" + }, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "non-object-string", + "description": "a non-object headers argument is dropped entirely, not sent as {}", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object" + } + } + }, + "arguments": { + "headers": "x-account-id: B" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "non-object-array", + "description": "an array of pairs is not an object: dropped", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object" + } + } + }, + "arguments": { + "headers": [["x-account-id", "B"]] + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "non-object-number", + "description": "a number is not an object: dropped", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object" + } + } + }, + "arguments": { + "headers": 5 + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "non-object-boolean", + "description": "a boolean is not an object: dropped", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object" + } + } + }, + "arguments": { + "headers": true + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "non-object-null", + "description": "null is not an object: dropped, with a warning", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object" + } + } + }, + "arguments": { + "headers": null + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers", + "reason": "not-an-object" + } + ] + }, + { + "id": "flat-exact-match", + "description": "a flat headers_ is declared only by the identical top-level property, and forwarded as given", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "path_id": "e", + "headers_x-custom": "abc" + }, + "expected_arguments": { + "path_id": "e", + "headers_x-custom": "abc" + }, + "expected_warnings": [] + }, + { + "id": "flat-case-mismatch", + "description": "flat names are matched exactly, not case-insensitively", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_X-Custom": "abc" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_X-Custom", + "reason": "not-declared" + } + ] + }, + { + "id": "flat-undeclared", + "description": "an undeclared flat header is dropped", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-other": "abc" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_x-other", + "reason": "not-declared" + } + ] + }, + { + "id": "flat-number-as-given", + "description": "a flat number is forwarded as a number, not stringified", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": 7 + }, + "expected_arguments": { + "headers_x-custom": 7 + }, + "expected_warnings": [] + }, + { + "id": "flat-float-as-given", + "description": "a flat float is forwarded as given", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": 1.5 + }, + "expected_arguments": { + "headers_x-custom": 1.5 + }, + "expected_warnings": [] + }, + { + "id": "flat-boolean-as-given", + "description": "a flat boolean is forwarded as given", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": true + }, + "expected_arguments": { + "headers_x-custom": true + }, + "expected_warnings": [] + }, + { + "id": "flat-empty-string", + "description": "an empty flat string is a valid value", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": "" + }, + "expected_arguments": { + "headers_x-custom": "" + }, + "expected_warnings": [] + }, + { + "id": "flat-array-dropped", + "description": "a flat array is dropped, CRLF and all", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": ["a\r\nx-account-id: B"] + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_x-custom", + "reason": "not-a-scalar" + } + ] + }, + { + "id": "flat-object-dropped", + "description": "a flat object is dropped", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": { + "k": 1 + } + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_x-custom", + "reason": "not-a-scalar" + } + ] + }, + { + "id": "flat-array-undeclared-dropped", + "description": "the value check comes first: an undeclared flat array is reported as not-a-scalar", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-other": ["a"] + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_x-other", + "reason": "not-a-scalar" + } + ] + }, + { + "id": "flat-null-dropped-silently", + "description": "a flat null is dropped silently, declared or not", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": null, + "headers_x-other": null + }, + "expected_arguments": {}, + "expected_warnings": [] + }, + { + "id": "flat-owned-declared", + "description": "a declared flat SDK-owned header is never forwarded", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_authorization": "Bearer x" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_authorization", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "flat-owned-undeclared", + "description": "an owned name is reported as set-by-sdk before declaration is checked, whatever its case or surrounding space", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_X-Account-Id": "victim", + "headers_ user-agent": "evil/0" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_X-Account-Id", + "reason": "set-by-sdk" + }, + { + "argument": "headers_ user-agent", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "flat-malformed-value", + "description": "a declared flat header with CRLF in its value is malformed", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "headers_x-custom": "a\r\nb" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_x-custom", + "reason": "malformed" + } + ] + }, + { + "id": "flat-malformed-name", + "description": "a declared flat header whose name is not a token is malformed", + "schema": { + "type": "object", + "properties": { + "headers_x custom": { + "type": "string" + } + } + }, + "arguments": { + "headers_x custom": "a" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_x custom", + "reason": "malformed" + } + ] + }, + { + "id": "flat-empty-name", + "description": "a declared headers_ with an empty name is malformed", + "schema": { + "type": "object", + "properties": { + "headers_": { + "type": "string" + } + } + }, + "arguments": { + "headers_": "a" + }, + "expected_arguments": {}, + "expected_warnings": [ + { + "argument": "headers_", + "reason": "malformed" + } + ] + }, + { + "id": "non-header-arguments-untouched", + "description": "only `headers` and the `headers_` prefix are header arguments: Headers, header_x, headers-x and a headers key inside body are not", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-custom": { + "type": "string" + }, + "headers_authorization": { + "type": "string" + } + } + }, + "arguments": { + "path_id": "e", + "body": { + "headers": { + "authorization": "x" + } + }, + "Headers": { + "authorization": "x" + }, + "header_x": "a", + "headers-x": "b", + "x_headers_y": 1 + }, + "expected_arguments": { + "path_id": "e", + "body": { + "headers": { + "authorization": "x" + } + }, + "Headers": { + "authorization": "x" + }, + "header_x": "a", + "headers-x": "b", + "x_headers_y": 1 + }, + "expected_warnings": [] + }, + { + "id": "nested-end-user-id-is-owned", + "description": "a declared nested x-end-user-id is never forwarded, whatever the schema declares, since it is SDK-owned", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "properties": { + "x-custom": { + "type": "string" + }, + "x-end-user-id": { + "type": "string" + } + } + } + } + }, + "arguments": { + "headers": { + "x-custom": "a", + "x-end-user-id": "evil-user" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [ + { + "argument": "headers", + "header": "x-end-user-id", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "flat-end-user-id-is-owned", + "description": "a declared flat headers_x-end-user-id is never forwarded, whatever its case or surrounding space", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_x-end-user-id": { + "type": "string" + } + } + }, + "arguments": { + "path_id": "e", + "headers_x-end-user-id": "evil-user" + }, + "expected_arguments": { + "path_id": "e" + }, + "expected_warnings": [ + { + "argument": "headers_x-end-user-id", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "flat-end-user-id-is-owned-uppercase", + "description": "a declared flat headers_X-End-User-Id is dropped the same as the lowercase form: the owned-name check is case-insensitive", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_X-End-User-Id": { + "type": "string" + } + } + }, + "arguments": { + "path_id": "e", + "headers_X-End-User-Id": "evil-user" + }, + "expected_arguments": { + "path_id": "e" + }, + "expected_warnings": [ + { + "argument": "headers_X-End-User-Id", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "flat-end-user-id-is-owned-padded", + "description": "a declared flat headers_ x-end-user-id is dropped the same as the untrimmed form: the owned-name check trims surrounding space", + "schema": { + "type": "object", + "properties": { + "path_id": { + "type": "string" + }, + "headers_ x-end-user-id": { + "type": "string" + } + } + }, + "arguments": { + "path_id": "e", + "headers_ x-end-user-id": "evil-user" + }, + "expected_arguments": { + "path_id": "e" + }, + "expected_warnings": [ + { + "argument": "headers_ x-end-user-id", + "reason": "set-by-sdk" + } + ] + }, + { + "id": "flat-and-nested-together", + "description": "nested and flat header arguments are filtered independently", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "properties": { + "x-custom": {} + } + }, + "headers_x-custom": {} + } + }, + "arguments": { + "headers": { + "x-custom": "a" + }, + "headers_x-custom": "b", + "headers_x-other": "c" + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + }, + "headers_x-custom": "b" + }, + "expected_warnings": [ + { + "argument": "headers_x-other", + "reason": "not-declared" + } + ] + }, + { + "id": "object-or-null-open", + "description": "headers typed [\"object\",\"null\"] with no properties (and additionalProperties not false) is OPEN, the same as `type: \"object\"`: a type list containing \"object\" declares every name, so x-custom is forwarded.", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": ["object", "null"] + } + } + }, + "arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_arguments": { + "headers": { + "x-custom": "a" + } + }, + "expected_warnings": [] + }, + { + "id": "empty-type-list", + "description": "headers typed [] (an empty list) is an ORDINARY field: the list does not contain \"object\", so headers is not a header container and the string is forwarded unchanged, with no header filtering.", + "schema": { + "type": "object", + "properties": { + "headers": { + "type": [] + } + } + }, + "arguments": { + "headers": "abc" + }, + "expected_arguments": { + "headers": "abc" + }, + "expected_warnings": [] + } + ] +} diff --git a/tests/vectors/header-names.json b/tests/vectors/header-names.json new file mode 100644 index 00000000..b60af92c --- /dev/null +++ b/tests/vectors/header-names.json @@ -0,0 +1,407 @@ +{ + "$schema": "./schema/header-names.schema.json", + "description": "How a nested header name — an entry of a top-level `headers` object argument whose served schema is open (`{\"type\": \"object\"}`), with the value \"v\" — is treated. The name is trimmed of what JavaScript's String.prototype.trim() strips, then refused if it is SDK-owned (authorization, x-account-id, user-agent, x-end-user-id, case-insensitively — x-end-user-id unconditionally, whether or not the SDK has an end user id recorded for the call's account), then if it is not an RFC 9110 token. A forwarded name is the trimmed one.", + "version": 1, + "cases": [ + { + "id": "plain", + "description": "a plain token", + "name": "x-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "mixed-case", + "description": "case is kept as supplied", + "name": "X-Custom", + "expected": { + "forwarded": "X-Custom" + } + }, + { + "id": "token-chars", + "description": "every RFC 9110 tchar", + "name": "x_c.u!s#t$o%m&'*+^`|~", + "expected": { + "forwarded": "x_c.u!s#t$o%m&'*+^`|~" + } + }, + { + "id": "leading-space", + "description": "leading space is trimmed", + "name": " x-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "trailing-space", + "description": "trailing space is trimmed", + "name": "x-custom ", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "tab", + "description": "tabs are trimmed", + "name": "\tx-custom\t", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "newline-ends", + "description": "CR and LF at the ends are trimmed", + "name": "\nx-custom\r\n", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "vt-ff", + "description": "vertical tab and form feed are trimmed", + "name": "\u000bx-custom\f", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "nbsp", + "description": "U+00A0 NO-BREAK SPACE is JavaScript whitespace: trimmed", + "name": "\u00a0x-custom\u00a0", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "bom", + "description": "U+FEFF BYTE ORDER MARK is JavaScript whitespace: trimmed", + "name": "\ufeffx-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "ogham", + "description": "U+1680 OGHAM SPACE MARK is trimmed", + "name": "\u1680x-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "en-quad", + "description": "U+2000 EN QUAD is trimmed", + "name": "\u2000x-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "hair-space", + "description": "U+200A HAIR SPACE is trimmed", + "name": "x-custom\u200a", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "line-separator", + "description": "U+2028 LINE SEPARATOR is a JavaScript line terminator: trimmed", + "name": "\u2028x-custom\u2029", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "narrow-nbsp", + "description": "U+202F NARROW NO-BREAK SPACE is trimmed", + "name": "\u202fx-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "math-space", + "description": "U+205F MEDIUM MATHEMATICAL SPACE is trimmed", + "name": "\u205fx-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "ideographic-space", + "description": "U+3000 IDEOGRAPHIC SPACE is trimmed", + "name": "\u3000x-custom", + "expected": { + "forwarded": "x-custom" + } + }, + { + "id": "fs", + "description": "U+001C is not JavaScript whitespace (Python's str.strip() would strip it): kept, so malformed", + "name": "\u001cx-custom", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "gs", + "description": "U+001D is not trimmed: malformed", + "name": "x-custom\u001d", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "rs", + "description": "U+001E is not trimmed: malformed", + "name": "\u001ex-custom", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "us", + "description": "U+001F is not trimmed: malformed", + "name": "x-custom\u001f", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "nel", + "description": "U+0085 NEXT LINE is not JavaScript whitespace (str.strip() would strip it): malformed", + "name": "\u0085x-custom", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "zwsp", + "description": "U+200B ZERO WIDTH SPACE is not whitespace: malformed", + "name": "\u200bx-custom", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "non-ascii", + "description": "a non-ASCII letter is not a tchar", + "name": "x-cüstom", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "non-ascii-cjk", + "description": "a CJK character is not a tchar", + "name": "x-頭", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "inner-space", + "description": "a space inside the name is not trimmed: malformed", + "name": "x custom", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "colon", + "description": "a colon is not a tchar", + "name": "x-custom:", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "empty", + "description": "an empty name is malformed", + "name": "", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "whitespace-only", + "description": "whitespace only trims to empty: malformed", + "name": " ", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "owned-authorization", + "description": "SDK-owned", + "name": "authorization", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-upper", + "description": "SDK-owned, compared case-insensitively", + "name": "AUTHORIZATION", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-mixed", + "description": "SDK-owned, as usually written", + "name": "Authorization", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-account", + "description": "SDK-owned tenant selector", + "name": "X-Account-Id", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-end-user", + "description": "SDK-owned end-user tenant selector, dropped whether or not the SDK has an end user id recorded for the account", + "name": "X-End-User-Id", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-ua", + "description": "SDK-owned client identity", + "name": "User-Agent", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-padded", + "description": "SDK-owned once trimmed", + "name": " authorization ", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-tab", + "description": "SDK-owned once trimmed", + "name": "\tx-account-id", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-nbsp", + "description": "SDK-owned once U+00A0 is trimmed", + "name": "\u00a0User-Agent", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-end-user-padded", + "description": "SDK-owned once trimmed", + "name": " x-end-user-id\t", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-bom", + "description": "SDK-owned once the BOM is trimmed", + "name": "\ufeffauthorization", + "expected": { + "dropped": "set-by-sdk" + } + }, + { + "id": "owned-us-suffix", + "description": "U+001F is not trimmed, so this is not the owned name: it is malformed", + "name": "authorization\u001f", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "owned-nel", + "description": "U+0085 is not trimmed, so this is not the owned name: it is malformed", + "name": "\u0085x-account-id", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "owned-dotted-i", + "description": "U+0130 lower-cases to i + U+0307, not i: not owned, and malformed", + "name": "AUTHORİZATION", + "expected": { + "dropped": "malformed" + } + }, + { + "id": "not-owned-similar", + "description": "a name that merely starts with an owned one is not owned", + "name": "x-account-id-2", + "expected": { + "forwarded": "x-account-id-2" + } + }, + { + "id": "not-owned-end-user-similar", + "description": "a name that merely starts with x-end-user-id is not owned", + "name": "x-end-user-id-2", + "expected": { + "forwarded": "x-end-user-id-2" + } + }, + { + "id": "proxy-authorization", + "description": "only the four names are owned; others are declared by the open schema", + "name": "Proxy-Authorization", + "expected": { + "forwarded": "Proxy-Authorization" + } + }, + { + "id": "proto-name", + "description": "__proto__ is an ordinary header name", + "name": "__proto__", + "expected": { + "forwarded": "__proto__" + } + }, + { + "id": "constructor-name", + "description": "constructor is an ordinary header name", + "name": "constructor", + "expected": { + "forwarded": "constructor" + } + }, + { + "id": "upper-case", + "description": "upper case kept as supplied", + "name": "X-CUSTOM", + "expected": { + "forwarded": "X-CUSTOM" + } + }, + { + "id": "owned-long-s", + "description": "U+017F LATIN SMALL LETTER LONG S. The SDK-owned check lowercases ASCII only, so uſer-agent is never folded to user-agent: it is not owned, and ſ is not a tchar, so it is malformed.", + "name": "uſer-agent", + "expected": { + "dropped": "malformed" + } + } + ] +} diff --git a/tests/vectors/header-values.json b/tests/vectors/header-values.json new file mode 100644 index 00000000..afd55c2a --- /dev/null +++ b/tests/vectors/header-values.json @@ -0,0 +1,307 @@ +{ + "$schema": "./schema/header-values.schema.json", + "description": "How a nested header value — an entry of a top-level `headers` object argument whose served schema is open (`{\"type\": \"object\"}`), under the declared name `x-probe` — reaches the wire: the string forwarded, or dropped. Values are JSON; `{\"$number\": \"NaN\"}`, `\"Infinity\"` and `\"-Infinity\"` stand for the non-finite numbers JSON cannot hold.", + "version": 1, + "cases": [ + { + "id": "string", + "description": "a string is forwarded as is", + "value": "abc", + "expected": "abc" + }, + { + "id": "empty-string", + "description": "an empty string is a valid header value", + "value": "", + "expected": "" + }, + { + "id": "string-with-space", + "description": "a space is allowed", + "value": "a b", + "expected": "a b" + }, + { + "id": "string-with-tab", + "description": "a tab is allowed", + "value": "a\tb", + "expected": "a\tb" + }, + { + "id": "latin1", + "description": "obs-text (U+0080-U+00FF) is allowed", + "value": "café", + "expected": "café" + }, + { + "id": "non-latin1", + "description": "a character above U+00FF is malformed", + "value": "€", + "expected": { + "dropped": true + } + }, + { + "id": "emoji", + "description": "an astral character is malformed", + "value": "🙂", + "expected": { + "dropped": true + } + }, + { + "id": "crlf", + "description": "CRLF would smuggle a second header line: malformed", + "value": "a\r\nx-account-id: B", + "expected": { + "dropped": true + } + }, + { + "id": "lf-trailing", + "description": "a trailing LF is malformed (a `$` anchor must not let it through)", + "value": "abc\n", + "expected": { + "dropped": true + } + }, + { + "id": "cr", + "description": "a bare CR is malformed", + "value": "a\rb", + "expected": { + "dropped": true + } + }, + { + "id": "nul", + "description": "NUL is malformed", + "value": "a\u0000b", + "expected": { + "dropped": true + } + }, + { + "id": "del", + "description": "DEL (0x7f) is malformed", + "value": "a\u007fb", + "expected": { + "dropped": true + } + }, + { + "id": "true", + "description": "a boolean as its literal text", + "value": true, + "expected": "true" + }, + { + "id": "false", + "description": "a boolean as its literal text", + "value": false, + "expected": "false" + }, + { + "id": "zero", + "description": "zero", + "value": 0, + "expected": "0" + }, + { + "id": "int", + "description": "an integer", + "value": 7, + "expected": "7" + }, + { + "id": "negative-int", + "description": "a negative integer", + "value": -3, + "expected": "-3" + }, + { + "id": "float-integral", + "description": "an integral float is written as JavaScript writes it: 1, not 1.0", + "value": 1.0, + "expected": "1" + }, + { + "id": "float", + "description": "a float", + "value": 1.5, + "expected": "1.5" + }, + { + "id": "negative-float", + "description": "a negative float", + "value": -0.5, + "expected": "-0.5" + }, + { + "id": "negative-zero", + "description": "negative zero is written 0", + "value": -0.0, + "expected": "0" + }, + { + "id": "exp-small", + "description": "below 1e-6, exponent notation, written as JavaScript does", + "value": 1e-7, + "expected": "1e-7" + }, + { + "id": "exp-boundary-small", + "description": "1e-6 is still written in full", + "value": 1e-6, + "expected": "0.000001" + }, + { + "id": "exp-large", + "description": "from 1e21, exponent notation with an explicit +", + "value": 1e21, + "expected": "1e+21" + }, + { + "id": "exp-boundary-large", + "description": "1e20 is still written in full", + "value": 1e20, + "expected": "100000000000000000000" + }, + { + "id": "exp-large-fraction", + "description": "a large exponent with a fraction", + "value": 1.5e300, + "expected": "1.5e+300" + }, + { + "id": "float-repr", + "description": "the shortest representation that round-trips", + "value": 0.30000000000000004, + "expected": "0.30000000000000004" + }, + { + "id": "safe-int", + "description": "the largest integer both languages hold exactly", + "value": 9007199254740991, + "expected": "9007199254740991" + }, + { + "id": "nan", + "description": "NaN as its text (a JSON argument cannot hold it; see argument-encoding.json for what else happens)", + "value": { + "$number": "NaN" + }, + "expected": "NaN" + }, + { + "id": "infinity", + "description": "Infinity as its text", + "value": { + "$number": "Infinity" + }, + "expected": "Infinity" + }, + { + "id": "neg-infinity", + "description": "-Infinity as its text", + "value": { + "$number": "-Infinity" + }, + "expected": "-Infinity" + }, + { + "id": "object", + "description": "an object as compact JSON", + "value": { + "k": 1 + }, + "expected": "{\"k\":1}" + }, + { + "id": "object-nested", + "description": "nested values as compact JSON, numbers as JavaScript writes them", + "value": { + "a": { + "b": [1, 2, true, null] + } + }, + "expected": "{\"a\":{\"b\":[1,2,true,null]}}" + }, + { + "id": "object-float", + "description": "floats inside JSON as JavaScript writes them", + "value": { + "x": 1, + "y": 1e21 + }, + "expected": "{\"x\":1,\"y\":1e+21}" + }, + { + "id": "object-non-ascii", + "description": "non-ASCII stays unescaped in the JSON text, and is valid obs-text", + "value": { + "k": "é" + }, + "expected": "{\"k\":\"é\"}" + }, + { + "id": "object-euro", + "description": "non-ASCII above U+00FF stays unescaped, so the JSON text is malformed as a header", + "value": { + "k": "€" + }, + "expected": { + "dropped": true + } + }, + { + "id": "object-escapes", + "description": "JSON escapes for quote, backslash and newline keep the text on one line", + "value": { + "k": "a\"b\\c\nd" + }, + "expected": "{\"k\":\"a\\\"b\\\\c\\nd\"}" + }, + { + "id": "object-empty", + "description": "an empty object", + "value": {}, + "expected": "{}" + }, + { + "id": "array", + "description": "an array as compact JSON", + "value": ["a", "b"], + "expected": "[\"a\",\"b\"]" + }, + { + "id": "array-empty", + "description": "an empty array", + "value": [], + "expected": "[]" + }, + { + "id": "array-mixed", + "description": "mixed array values as compact JSON", + "value": [true, 1.5, null, "x"], + "expected": "[true,1.5,null,\"x\"]" + }, + { + "id": "null", + "description": "null is dropped silently, with no warning", + "value": null, + "expected": { + "dropped": true + } + } + ], + "known_differences": [ + { + "id": "big-int", + "description": "an integer above 2^53. Not graded: by the time this value reaches header filtering, the SDK's own JSON parser has already rounded it, so what is forwarded is an artifact of each language's JSON number type, not a choice the header-value rule makes. Python's JSON parser keeps every digit; Node's JSON numbers are doubles, so JSON.parse already rounds it before this code ever sees it.", + "value": 12345678901234567890, + "python": "12345678901234567890", + "node": "12345678901234567000" + } + ] +} diff --git a/tests/vectors/messages.json b/tests/vectors/messages.json new file mode 100644 index 00000000..868235b1 --- /dev/null +++ b/tests/vectors/messages.json @@ -0,0 +1,1172 @@ +{ + "$schema": "./schema/messages.schema.json", + "description": "The canonical wording of every user-facing warning and error message both SDKs emit. `template` is the canonical text; `python` and `node` are each SDK's text today, written with the same placeholders, so an SDK can see what changes. A prefix the language adds — Node's `[@stackone/ai] `, Python's logger name — is outside the template. An error's class is in `error`. `matches` says whether an SDK's text is already the canonical one; `notes` say where the text depends on more than the template.", + "version": 1, + "prefixes": { + "node": "Every warning goes through console.warn, prefixed `[@stackone/ai] ` — outside the template.", + "python": "Every warning is a WARNING record on a `stackone.*` logger; its getMessage() is the template, rendered." + }, + "formats": { + "text": "Inserted as is.", + "json": "Compact JSON text: JSON.stringify(value), json.dumps(value, ensure_ascii=False, separators=(\",\", \":\")). A string comes out in double quotes.", + "integer": "Decimal digits.", + "seconds": "A number of seconds as JavaScript's String(number) writes it: 1, 0.5, 1.25 — never 1.0, never padded, never exponent form for values in [1e-6, 1e21).", + "seconds-2dp": "The seconds rounded half up to two decimals, Math.floor(x * 100 + 0.5) / 100, then written as `seconds`: 0, 0.75, 1.5.", + "json-type": "The JSON type of the value: object, array, string, number, boolean or null.", + "reason": "The text for a header reason code, from `reasons`." + }, + "reasons": { + "set-by-sdk": "set by the SDK", + "not-declared": "not declared by the schema", + "malformed": "malformed", + "not-an-object": "not an object", + "not-a-scalar": "not a string, number or boolean" + }, + "messages": [ + { + "id": "header-dropped", + "kind": "warning", + "template": "Dropping header {header} from a tool call: {reason}", + "placeholders": { + "header": { + "format": "json", + "description": "the nested header name, trimmed" + }, + "reason": { + "format": "reason", + "description": "set-by-sdk, not-declared or malformed" + } + }, + "python": "Dropping header {header} from a tool call: {reason}", + "node": "Dropping header {header} from a tool call: {reason}", + "matches": { + "python": true, + "node": true + }, + "notes": "The name is JSON text, so a quote, a backslash or a control character in it is escaped." + }, + { + "id": "header-argument-dropped", + "kind": "warning", + "template": "Dropping header argument {argument} from a tool call: {reason}", + "placeholders": { + "argument": { + "format": "json", + "description": "the top-level argument: headers, or headers_ as supplied" + }, + "reason": { + "format": "reason", + "description": "any reason code" + } + }, + "python": "Dropping header argument {argument} from a tool call: {reason}", + "node": "Dropping header argument {argument} from a tool call: {reason}", + "matches": { + "python": true, + "node": true + }, + "notes": "The argument is JSON text, as the name in header-dropped is." + }, + { + "id": "rate-limit-retry", + "kind": "warning", + "template": "{method} {url} was rate limited (429) on attempt {attempt} of {max_attempts}; retrying in {delay}s", + "placeholders": { + "method": { + "format": "text", + "description": "the HTTP method" + }, + "url": { + "format": "text", + "description": "the request URL" + }, + "attempt": { + "format": "integer", + "description": "the attempt that was answered 429, 1-based" + }, + "max_attempts": { + "format": "integer", + "description": "4" + }, + "delay": { + "format": "seconds-2dp", + "description": "the wait before the next attempt" + } + }, + "python": "{method} {url} was rate limited (429) on attempt {attempt} of {max_attempts}; retrying in {delay}s", + "node": "{method} {url} was rate limited (429) on attempt {attempt} of {max_attempts}; retrying in {delay}s", + "matches": { + "python": true, + "node": true + }, + "notes": "Names the attempt that was answered 429, not the one about to be made." + }, + { + "id": "rate-limit-deadline", + "kind": "warning", + "template": "{method} {url} was rate limited (429) on attempt {attempt} of {max_attempts}; not retrying, because waiting {delay}s would pass the deadline", + "placeholders": { + "method": { + "format": "text", + "description": "the HTTP method" + }, + "url": { + "format": "text", + "description": "the request URL" + }, + "attempt": { + "format": "integer", + "description": "the attempt that was answered 429, 1-based" + }, + "max_attempts": { + "format": "integer", + "description": "4" + }, + "delay": { + "format": "seconds-2dp", + "description": "the wait before the next attempt that would pass the deadline" + } + }, + "python": "{method} {url} was rate limited (429) on attempt {attempt} of {max_attempts}; not retrying, because waiting {delay}s would pass the deadline", + "node": "{method} {url} was rate limited (429) on attempt {attempt} of {max_attempts}; not retrying, because waiting {delay}s would pass the deadline", + "matches": { + "python": true, + "node": true + }, + "notes": "When the next wait would not finish before the call's deadline, the SDK ends the call with the 429 it already has instead of sleeping, and warns this in place of rate-limit-retry." + }, + { + "id": "account-id-env-ignored", + "kind": "warning", + "template": "STACKONE_ACCOUNT_ID is set, but the SDK does not read it: with no account id passed, every active shared account on this API key is used. Pass an account id to scope the toolset.", + "placeholders": {}, + "python": "STACKONE_ACCOUNT_ID is set, but the SDK does not read it: with no account id passed, every active shared account on this API key is used. Pass an account id to scope the toolset.", + "node": "STACKONE_ACCOUNT_ID is set, but the SDK does not read it: with no account id passed, every active shared account on this API key is used. Pass an account id to scope the toolset.", + "matches": { + "python": true, + "node": true + }, + "notes": "Emitted once, when the toolset is constructed, if the STACKONE_ACCOUNT_ID environment variable is set and non-empty and neither an account id nor account ids were passed." + }, + { + "id": "skip-account", + "kind": "warning", + "template": "Skipping account that failed to list tools — {account_id}: {error}", + "placeholders": { + "account_id": { + "format": "text", + "description": "the account's id" + }, + "error": { + "format": "text", + "description": "the failure's own message" + } + }, + "python": "Skipping account that failed to list tools — {account_id}: {error}", + "node": "Skipping account that failed to list tools — {account_id}: {error}", + "matches": { + "python": true, + "node": true + }, + "notes": "The error part is whichever message the failure carries, so the whole text is the same in both SDKs only where that message is." + }, + { + "id": "skip-connector", + "kind": "warning", + "template": "Skipping connector that failed to search — {tool}: {error}", + "placeholders": { + "tool": { + "format": "text", + "description": "the *_search_actions tool's name" + }, + "error": { + "format": "text", + "description": "the failure's own message" + } + }, + "python": "Skipping connector that failed to search — {tool}: {error}", + "node": "Skipping connector that failed to search — {tool}: {error}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "duplicate-tool-names", + "kind": "warning", + "template": "{count} tool name(s) are served by more than one account ({names}). The first one listed, from the lowest account id, is used — pass account ids to choose.", + "placeholders": { + "count": { + "format": "integer", + "description": "how many names clash" + }, + "names": { + "format": "text", + "description": "up to five of them, sorted, joined by \", \"" + } + }, + "python": "{count} tool name(s) are served by more than one account ({names}). The first one listed, from the lowest account id, is used — pass account ids to choose.", + "node": "{count} tool name(s) are served by more than one account ({names}). The first one listed, from the lowest account id, is used — pass account ids to choose.", + "matches": { + "python": true, + "node": true + }, + "notes": "Language-neutral wording: neither SDK's method or parameter names. Emitted once when a fetched collection holds a clashing name, and again by every adapter that keeps the first tool per name and drops the rest: OpenAI, LangChain, Pydantic AI, AI SDK and Claude Agent SDK. Listing order is ascending account id, so the first is the lowest account id's." + }, + { + "id": "ambiguous-connector", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{action_id} matches {count} connectors on different accounts ({tools}). Pass the account id to use, such as a search hit's account_id.", + "placeholders": { + "action_id": { + "format": "json", + "description": "the action id" + }, + "count": { + "format": "integer", + "description": "how many connectors match: the longest connector prefix, linked on more than one account" + }, + "tools": { + "format": "text", + "description": "each matching *_execute_action tool as \"{tool} on {account_id}\", in ascending account-id order, joined by \", \"" + } + }, + "python": "{action_id} matches {count} connectors on different accounts ({tools}). Pass the account id to use, such as a search hit's account_id.", + "node": "{action_id} matches {count} connectors on different accounts ({tools}). Pass the account id to use, such as a search hit's account_id.", + "matches": { + "python": true, + "node": true + }, + "notes": "Raised by execute() when the action's connector is linked on more than one account in scope — no account id passed, or several; nothing is called. Was a warning that ran the first match." + }, + { + "id": "mcp-timeout", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "MCP request to {endpoint} timed out after {timeout}s", + "placeholders": { + "endpoint": { + "format": "text", + "description": "the MCP endpoint URL, with its ?tool-mode query when it has one" + }, + "timeout": { + "format": "seconds", + "description": "the toolset's timeout" + } + }, + "python": "MCP request to {endpoint} timed out after {timeout}s", + "node": "MCP request to {endpoint} timed out after {timeout}s", + "matches": { + "python": true, + "node": true + }, + "notes": "Seconds, written as a number: 1s, 0.5s, 60s." + }, + { + "id": "mcp-rate-limit-timeout", + "kind": "error", + "error": { + "python": "StackOneAPIError", + "node": "StackOneAPIError" + }, + "template": "MCP request to {endpoint} was rate limited (429) and timed out after {timeout}s while retrying", + "placeholders": { + "endpoint": { + "format": "text", + "description": "the MCP endpoint URL, with its ?tool-mode query when it has one" + }, + "timeout": { + "format": "seconds", + "description": "the toolset's timeout" + } + }, + "python": "MCP request to {endpoint} was rate limited (429) and timed out after {timeout}s while retrying", + "node": "MCP request to {endpoint} was rate limited (429) and timed out after {timeout}s while retrying", + "matches": { + "python": true, + "node": true + }, + "notes": "Raised in place of mcp-timeout when the request that timed out was retrying a 429. Its status is 429, so a multi-account call fails with it rather than skipping the account (invariant 14)." + }, + { + "id": "accounts-rate-limit-timeout", + "kind": "error", + "error": { + "python": "StackOneAPIError", + "node": "StackOneAPIError" + }, + "template": "Listing accounts at {url} was rate limited (429) and timed out after {timeout}s while retrying", + "placeholders": { + "url": { + "format": "text", + "description": "the /accounts URL" + }, + "timeout": { + "format": "seconds", + "description": "the toolset's timeout" + } + }, + "python": "Listing accounts at {url} was rate limited (429) and timed out after {timeout}s while retrying", + "node": "Listing accounts at {url} was rate limited (429) and timed out after {timeout}s while retrying", + "matches": { + "python": true, + "node": true + }, + "notes": "GET /accounts' counterpart of mcp-rate-limit-timeout; status 429." + }, + { + "id": "mcp-http-failure", + "kind": "error", + "error": { + "python": "StackOneAPIError", + "node": "StackOneAPIError" + }, + "template": "MCP request to {endpoint} failed with {status} {reason_phrase}: {body}", + "placeholders": { + "endpoint": { + "format": "text", + "description": "the MCP endpoint URL, with its ?tool-mode query when it has one" + }, + "status": { + "format": "integer", + "description": "the HTTP status" + }, + "reason_phrase": { + "format": "text", + "description": "the status's reason phrase" + }, + "body": { + "format": "text", + "description": "the response body, trimmed; with an empty body the \": {body}\" suffix is omitted" + } + }, + "python": "MCP request to {endpoint} failed with {status} {reason_phrase}: {body}", + "node": "MCP request to {endpoint} failed with {status} {reason_phrase}: {body}", + "matches": { + "python": true, + "node": true + }, + "notes": "For a status with no reason phrase, the space before it is left out too (`failed with 599: …`), as is the `: {body}` suffix for an empty body." + }, + { + "id": "mcp-failure", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "MCP request to {endpoint} failed: {error_type}: {error_message}", + "placeholders": { + "endpoint": { + "format": "text", + "description": "the MCP endpoint URL, with its ?tool-mode query when it has one" + }, + "error_type": { + "format": "text", + "description": "the innermost cause's class name, which is the language's own" + }, + "error_message": { + "format": "text", + "description": "its message" + } + }, + "python": "MCP request to {endpoint} failed: {error_type}: {error_message}", + "node": "MCP request to {endpoint} failed: {error_type}: {error_message}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "accounts-http-failure", + "kind": "error", + "error": { + "python": "StackOneAPIError", + "node": "StackOneAPIError" + }, + "template": "Listing accounts at {url} failed with {status} {reason_phrase}: {body}", + "placeholders": { + "url": { + "format": "text", + "description": "the /accounts URL" + }, + "status": { + "format": "integer", + "description": "the HTTP status" + }, + "reason_phrase": { + "format": "text", + "description": "the reason phrase" + }, + "body": { + "format": "text", + "description": "the response body, trimmed" + } + }, + "python": "Listing accounts at {url} failed with {status} {reason_phrase}: {body}", + "node": "Listing accounts at {url} failed with {status} {reason_phrase}: {body}", + "matches": { + "python": true, + "node": true + }, + "notes": "For a status with no reason phrase, Node leaves out the space before the colon and Python keeps it; with an empty body, both keep the trailing `: `. Neither is canonical yet." + }, + { + "id": "accounts-unreachable", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "Could not reach {url}: {detail}", + "placeholders": { + "url": { + "format": "text", + "description": "the /accounts URL" + }, + "detail": { + "format": "text", + "description": "the transport's own message" + } + }, + "python": "Could not reach {url}: {detail}", + "node": "Could not reach {url}: {detail}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "accounts-invalid-json", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "Invalid JSON returned by {url}: {detail}", + "placeholders": { + "url": { + "format": "text", + "description": "the /accounts URL" + }, + "detail": { + "format": "text", + "description": "the parser's own message" + } + }, + "python": "Invalid JSON returned by {url}: {detail}", + "node": "Invalid JSON returned by {url}: {detail}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "accounts-unexpected-shape", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "Unexpected /accounts response shape: expected a list, got {json_type}", + "placeholders": { + "json_type": { + "format": "json-type", + "description": "object, string, number, boolean or null" + } + }, + "python": "Unexpected /accounts response shape: expected a list, got {json_type}", + "node": "Unexpected /accounts response shape: expected a list, got {json_type}", + "matches": { + "python": true, + "node": true + }, + "notes": "Names the JSON type, not either language's own type name." + }, + { + "id": "no-linked-accounts", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "This API key has no linked accounts. Link one in the StackOne dashboard, or pass an account id explicitly.", + "placeholders": {}, + "python": "This API key has no linked accounts. Link one in the StackOne dashboard, or pass an account id explicitly.", + "node": "This API key has no linked accounts. Link one in the StackOne dashboard, or pass an account id explicitly.", + "matches": { + "python": true, + "node": true + }, + "notes": "Language-neutral wording: neither SDK's parameter name." + }, + { + "id": "no-active-accounts", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "None of this API key's {count} linked accounts are active: {accounts}. Re-link them in the StackOne dashboard, or pass an account id explicitly.", + "placeholders": { + "count": { + "format": "integer", + "description": "how many accounts are linked" + }, + "accounts": { + "format": "text", + "description": "each as \"{provider} ({status})\", joined by \", \"" + } + }, + "python": "None of this API key's {count} linked accounts are active: {accounts}. Re-link them in the StackOne dashboard, or pass an account id explicitly.", + "node": "None of this API key's {count} linked accounts are active: {accounts}. Re-link them in the StackOne dashboard, or pass an account id explicitly.", + "matches": { + "python": true, + "node": true + }, + "notes": "A missing provider or status is written None by Python and undefined by Node; neither is canonical yet." + }, + { + "id": "all-accounts-failed", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "Every account failed to list tools: {failures}", + "placeholders": { + "failures": { + "format": "text", + "description": "each failed account as \"{account_id}: {error}\", where {error} is the failure's own message (as in skip-account), in ascending account-id order, joined by \"; \" — e.g. \"acc-1: ; acc-2: \"" + } + }, + "python": "Every account failed to list tools: {failures}", + "node": "Every account failed to list tools: {failures}", + "matches": { + "python": true, + "node": true + }, + "notes": "Only when every account in scope failed to list, and only when the failures do not all share one HTTP status: if they do, the SDK rethrows the StackOneAPIError itself (e.g. two accounts both answering 401 raise StackOneAPIError 401). The failures are the error's cause: Python raises it `from ExceptionGroup(...)` of them, Node passes `{ cause: new AggregateError(...) }`. An account that listed zero tools has not failed. Both SDKs join the failures with \"; \", and also raise when some accounts failed and the rest listed zero tools. Replaces no-account-returned-tools." + }, + { + "id": "no-connector-returned-results", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "No connector returned results. {failures}", + "placeholders": { + "failures": { + "format": "text", + "description": "each as \"{tool}: {error}\", joined by \" | \"" + } + }, + "python": "No connector returned results. {failures}", + "node": "No connector returned results. {failures}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "fetch-tools-failed", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "Error fetching tools: {detail}", + "placeholders": { + "detail": { + "format": "text", + "description": "the unexpected failure's message" + } + }, + "python": "Error fetching tools: {detail}", + "node": "Error fetching tools: {detail}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "missing-api-key", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "An API key must be provided, either to the toolset or in the STACKONE_API_KEY environment variable", + "placeholders": {}, + "python": "An API key must be provided, either to the toolset or in the STACKONE_API_KEY environment variable", + "node": "An API key must be provided, either to the toolset or in the STACKONE_API_KEY environment variable", + "matches": { + "python": true, + "node": true + }, + "notes": "Language-neutral wording: neither SDK's parameter name." + }, + { + "id": "empty-account-id", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must not be an empty string", + "placeholders": { + "parameter": { + "format": "text", + "description": "account_id / accountId" + } + }, + "python": "{parameter} must not be an empty string", + "node": "{parameter} must not be an empty string", + "matches": { + "python": true, + "node": true + }, + "notes": "Each SDK names the parameter in its own spelling." + }, + { + "id": "account-ids-not-a-list", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must be a list of account ids, not a string. Did you mean [{value}]?", + "placeholders": { + "parameter": { + "format": "text", + "description": "the parameter's name in the SDK's own spelling: account_ids / accountIds" + }, + "value": { + "format": "json", + "description": "the string given" + } + }, + "python": "{parameter} must be a list of account ids, not a string. Did you mean [{value}]?", + "node": "{parameter} must be a list of account ids, not a string. Did you mean [{value}]?", + "matches": { + "python": true, + "node": true + }, + "notes": "\"list\", not \"array\": the language-neutral word. The suggestion is JSON, so it reads the same in both." + }, + { + "id": "account-ids-not-strings", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must be a list of account id strings", + "placeholders": { + "parameter": { + "format": "text", + "description": "the parameter's name in the SDK's own spelling: account_ids / accountIds" + } + }, + "python": "{parameter} must be a list of account id strings", + "node": "{parameter} must be a list of account id strings", + "matches": { + "python": true, + "node": true + }, + "notes": "The list given is not echoed: not every value it could hold is JSON." + }, + { + "id": "account-ids-empty-id", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must not contain an empty account id", + "placeholders": { + "parameter": { + "format": "text", + "description": "the parameter's name in the SDK's own spelling: account_ids / accountIds" + } + }, + "python": "{parameter} must not contain an empty account id", + "node": "{parameter} must not contain an empty account id", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "invalid-session-id", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must be a non-empty string, got {value}", + "placeholders": { + "parameter": { + "format": "text", + "description": "session_id / sessionId" + }, + "value": { + "format": "json", + "description": "the value given" + } + }, + "python": "{parameter} must be a non-empty string, got {value}", + "node": "{parameter} must be a non-empty string, got {value}", + "matches": { + "python": true, + "node": true + }, + "notes": "null (None) means the session id was not given, and does not raise this." + }, + { + "id": "invalid-action-id", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must be a non-empty string, got {value}", + "placeholders": { + "parameter": { + "format": "text", + "description": "action_id / actionId" + }, + "value": { + "format": "json", + "description": "the value given" + } + }, + "python": "{parameter} must be a non-empty string, got {value}", + "node": "{parameter} must be a non-empty string, got {value}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "execute-arguments-not-an-object", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "arguments must be a JSON object, got {json_type}", + "placeholders": { + "json_type": { + "format": "json-type", + "description": "array, string, number, boolean or null" + } + }, + "python": "arguments must be a JSON object, got {json_type}", + "node": "arguments must be a JSON object, got {json_type}", + "matches": { + "python": true, + "node": true + }, + "notes": "Names the JSON type; no object reaches this message." + }, + { + "id": "top-k-out-of-range", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must be an integer between 1 and {max}, got {value}", + "placeholders": { + "parameter": { + "format": "text", + "description": "top_k / topK" + }, + "max": { + "format": "integer", + "description": "50" + }, + "value": { + "format": "json", + "description": "the value given" + } + }, + "python": "{parameter} must be an integer between 1 and {max}, got {value}", + "node": "{parameter} must be an integer between 1 and {max}, got {value}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "tool-names-not-a-list", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "{parameter} must be a list of tool names, not a string. Did you mean [{value}]?", + "placeholders": { + "parameter": { + "format": "text", + "description": "tool_names / toolNames" + }, + "value": { + "format": "json", + "description": "the string given" + } + }, + "python": "{parameter} must be a list of tool names, not a string. Did you mean [{value}]?", + "node": "{parameter} must be a list of tool names, not a string. Did you mean [{value}]?", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "feedback-not-enabled", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "The server did not serve stackone_submit_feedback: feedback is not enabled for this project.", + "placeholders": {}, + "python": "The server did not serve stackone_submit_feedback: feedback is not enabled for this project.", + "node": "The server did not serve stackone_submit_feedback: feedback is not enabled for this project.", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "no-connector-for-action", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "No connector found for {action_id}. Use search() to discover valid action ids.", + "placeholders": { + "action_id": { + "format": "json", + "description": "the action id" + } + }, + "python": "No connector found for {action_id}. Use search() to discover valid action ids.", + "node": "No connector found for {action_id}. Use search() to discover valid action ids.", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "arguments-not-finite", + "kind": "error", + "error": { + "python": "ToolArgumentsError", + "node": "ToolArgumentsError" + }, + "template": "Arguments for {tool} could not be encoded as JSON: {value} is not a JSON number", + "placeholders": { + "tool": { + "format": "json", + "description": "the tool's name" + }, + "value": { + "format": "text", + "description": "NaN, Infinity or -Infinity" + } + }, + "python": "Arguments for {tool} could not be encoded as JSON: {value} is not a JSON number", + "node": "Arguments for {tool} could not be encoded as JSON: {value} is not a JSON number", + "matches": { + "python": true, + "node": true + }, + "notes": "The value is in JavaScript's spelling, and the message leaves out where in the arguments it is, which Python's encoder does not report." + }, + { + "id": "arguments-not-encodable", + "kind": "error", + "error": { + "python": "ToolArgumentsError", + "node": "ToolArgumentsError" + }, + "template": "Arguments for {tool} could not be encoded as JSON: {detail}", + "placeholders": { + "tool": { + "format": "json", + "description": "the tool's name" + }, + "detail": { + "format": "text", + "description": "the SDK's own explanation: a circular reference, a non-JSON type, …" + } + }, + "python": "Arguments for {tool} could not be encoded as JSON: {detail}", + "node": "Arguments for {tool} could not be encoded as JSON: {detail}", + "matches": { + "python": true, + "node": true + }, + "notes": "For every reason but a non-finite number, which has its own message above." + }, + { + "id": "arguments-invalid-json", + "kind": "error", + "error": { + "python": "ToolArgumentsError", + "node": "ToolArgumentsError" + }, + "template": "Invalid JSON in arguments for {tool}: {detail}", + "placeholders": { + "tool": { + "format": "json", + "description": "the tool's name" + }, + "detail": { + "format": "text", + "description": "the JSON parser's own message" + } + }, + "python": "Invalid JSON in arguments for {tool}: {detail}", + "node": "Invalid JSON in arguments for {tool}: {detail}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "arguments-not-an-object", + "kind": "error", + "error": { + "python": "ToolArgumentsError", + "node": "ToolArgumentsError" + }, + "template": "Tool arguments for {tool} must be a JSON object", + "placeholders": { + "tool": { + "format": "json", + "description": "the tool's name" + } + }, + "python": "Tool arguments for {tool} must be a JSON object", + "node": "Tool arguments for {tool} must be a JSON object", + "matches": { + "python": true, + "node": true + }, + "notes": "For every value that is not an object, including one that is not a string either." + }, + { + "id": "tool-call-failed", + "kind": "error", + "error": { + "python": "StackOneAPIError", + "node": "StackOneAPIError" + }, + "template": "Tool {tool} failed: {detail}", + "placeholders": { + "tool": { + "format": "json", + "description": "the tool's name" + }, + "detail": { + "format": "text", + "description": "the result's text, or its structured content as compact JSON" + } + }, + "python": "Tool {tool} failed: {detail}", + "node": "Tool {tool} failed: {detail}", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "no-executor", + "kind": "error", + "error": { + "python": "StackOneError", + "node": "StackOneError" + }, + "template": "Tool {tool} has no executor. Override execute() to run a hand-built tool.", + "placeholders": { + "tool": { + "format": "json", + "description": "the tool's name" + } + }, + "python": "Tool {tool} has no executor. Override execute() to run a hand-built tool.", + "node": "Tool {tool} has no executor. Override execute() to run a hand-built tool.", + "matches": { + "python": true, + "node": true + } + }, + { + "id": "unknown-tool", + "kind": "tool-result", + "template": "Unknown tool {name}", + "placeholders": { + "name": { + "format": "json", + "description": "the tool call's function name" + } + }, + "python": "Unknown tool {name}", + "node": "Unknown tool {name}", + "matches": { + "python": true, + "node": true + }, + "notes": "Not raised: the `error` of the tool message execute_openai_tool_calls / executeOpenAIToolCalls returns for a call to a tool not in the collection." + }, + { + "id": "toolset-headers-ignored", + "kind": "warning", + "template": "Ignoring headers {names}: the SDK sets them itself. Pass the API key and account ids through their own options instead.", + "placeholders": { + "names": { + "format": "text", + "description": "each ignored name as given, in double quotes, joined with \", \", in the order supplied" + } + }, + "python": "Ignoring headers {names}: the SDK sets them itself. Pass the API key and account ids through their own options instead.", + "node": "Ignoring headers {names}: the SDK sets them itself. Pass the API key and account ids through their own options instead.", + "matches": { + "python": true, + "node": true + }, + "notes": "Emitted once at construction when the toolset's headers option names Authorization, x-account-id or User-Agent (any case, surrounding whitespace ignored). x-end-user-id is allowed in this option and is not warned about. The option's other headers go on every request, GET /accounts included, beneath the SDK's own." + }, + { + "id": "connector-account-unavailable", + "kind": "error", + "error": { + "python": "ToolsetLoadError", + "node": "ToolSetLoadError" + }, + "template": "{action_id} may be served by an account that failed to list ({failures}). Pass the account id to use, such as a search hit's account_id.", + "placeholders": { + "action_id": { + "format": "json", + "description": "the action id execute() was called with, JSON-quoted as in ambiguous-connector" + }, + "failures": { + "format": "text", + "description": "each failed account as \"{account_id}: {error}\", where {error} is the failure's own message (as in skip-account), in ascending account-id order, joined by \"; \" — same format as all-accounts-failed" + } + }, + "python": "{action_id} may be served by an account that failed to list ({failures}). Pass the account id to use, such as a search hit's account_id.", + "node": "{action_id} may be served by an account that failed to list ({failures}). Pass the account id to use, such as a search hit's account_id.", + "matches": { + "python": true, + "node": true + }, + "notes": "Raised by execute() when an account in scope is marked failed and, after an immediate re-list, still fails, and that account's provider (from GET /accounts) matches the action's connector or is unknown (no GET /accounts record). A failed account on a different known provider does not block routing to the remaining account." + }, + { + "id": "non-shared-accounts-skipped", + "kind": "warning", + "template": "Discovery skipped {count} non-shared account(s) ({accounts}): each belongs to a single end user. Pass their account ids, or opt in to non-shared accounts, to use them.", + "placeholders": { + "count": { + "format": "integer", + "description": "how many non-shared accounts were skipped" + }, + "accounts": { + "format": "text", + "description": "the skipped accounts' ids, ascending, joined by \", \"" + } + }, + "python": "Discovery skipped {count} non-shared account(s) ({accounts}): each belongs to a single end user. Pass their account ids, or opt in to non-shared accounts, to use them.", + "node": "Discovery skipped {count} non-shared account(s) ({accounts}): each belongs to a single end user. Pass their account ids, or opt in to non-shared accounts, to use them.", + "matches": { + "python": true, + "node": true + }, + "notes": "Emitted once per discovery (fetchTools / search) that skips any account with shared === false from GET /accounts, unless the toolset opts in: Python include_non_shared=True, Node includeNonShared: true. End-user ids are still recorded from every GET /accounts regardless." + }, + { + "id": "no-shared-accounts", + "kind": "error", + "error": { + "python": "ToolsetConfigError", + "node": "ToolSetConfigError" + }, + "template": "None of this API key's {count} active account(s) are shared: each belongs to a single end user. Pass their account ids, or opt in to non-shared accounts, to use them.", + "placeholders": { + "count": { + "format": "integer", + "description": "how many active accounts this API key has" + } + }, + "python": "None of this API key's {count} active account(s) are shared: each belongs to a single end user. Pass their account ids, or opt in to non-shared accounts, to use them.", + "node": "None of this API key's {count} active account(s) are shared: each belongs to a single end user. Pass their account ids, or opt in to non-shared accounts, to use them.", + "matches": { + "python": true, + "node": true + }, + "notes": "Raised by discovery (fetchTools / search) instead of non-shared-accounts-skipped when every active account on this API key is non-shared and the toolset has not opted in. With no shared account to serve a catalog, discovery fails outright rather than warning and returning an empty one." + } + ], + "unshared": [ + { + "sdk": "python", + "kind": "error", + "text": "Cannot provide both positional and keyword arguments", + "notes": "call() only; Node has no call()." + }, + { + "sdk": "python", + "kind": "error", + "text": "Only one positional argument is allowed" + }, + { + "sdk": "python", + "kind": "error", + "text": "Tool \"{tool}\" has no API key to authenticate with." + }, + { + "sdk": "node", + "kind": "error", + "text": "Cannot provide both accountId and accountIds. Use accountId for a single account or accountIds for multiple accounts." + }, + { + "sdk": "node", + "kind": "error", + "text": "query must be a string, got {typeof}" + }, + { + "sdk": "node", + "kind": "error", + "text": "toolNames must be an array of tool names" + }, + { + "sdk": "node", + "kind": "error", + "text": "Could not read the response from {url}: {detail}" + }, + { + "sdk": "node", + "kind": "tool-result", + "text": "Unsupported tool call type \"{type}\"" + }, + { + "sdk": "node", + "kind": "error", + "text": "Tool {name} is not a StackOne tool" + } + ] +} diff --git a/tests/vectors/retry-after.json b/tests/vectors/retry-after.json new file mode 100644 index 00000000..3f3069c9 --- /dev/null +++ b/tests/vectors/retry-after.json @@ -0,0 +1,339 @@ +{ + "$schema": "./schema/retry-after.schema.json", + "description": "How a Retry-After header value reads as seconds from `now`: delta-seconds or an HTTP-date. `expected_seconds` is the RAW value, before the cap: null when the header is absent or unreadable (the caller falls back to its own backoff), 0 for a date in the past. An HTTP-date's second may be 60, a leap second, which reads as the first second of the next minute. The cap (`max_delay_seconds`) is applied afterwards by the caller, and graded in backoff.json.", + "version": 1, + "max_delay_seconds": 30, + "cases": [ + { + "id": "delta", + "description": "delta-seconds", + "header": "5", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 5 + }, + { + "id": "zero", + "description": "zero delta-seconds: retry at once", + "header": "0", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 0 + }, + { + "id": "leading-zeros", + "description": "delta-seconds with leading zeros", + "header": "005", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 5 + }, + { + "id": "large", + "description": "20 digits: read as a number, not refused; the cap is applied separately, so this is the raw value, compared within the seconds tolerance", + "header": "99999999999999999999", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 99999999999999999999 + }, + { + "id": "huge", + "description": "309 digits overflow to infinity in both languages; the cap then makes it 30s", + "header": "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": { + "$number": "Infinity" + } + }, + { + "id": "cap-boundary", + "description": "above the cap: recorded raw, the cap is applied separately", + "header": "31", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 31 + }, + { + "id": "imf", + "description": "IMF-fixdate (RFC 9110's preferred form), 30s ahead", + "header": "Thu, 01 Oct 2026 12:00:30 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 30 + }, + { + "id": "rfc850", + "description": "RFC 850 date, 60s ahead", + "header": "Thursday, 01-Oct-26 12:01:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 60 + }, + { + "id": "imf-past", + "description": "IMF-fixdate in the past: retry now", + "header": "Sun, 06 Nov 1994 08:49:37 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 0 + }, + { + "id": "imf-now", + "description": "IMF-fixdate equal to now: retry now", + "header": "Thu, 01 Oct 2026 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 0 + }, + { + "id": "rfc850-past", + "description": "RFC 850 date in the past (two-digit year 94 is 1994)", + "header": "Sunday, 06-Nov-94 08:49:37 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 0 + }, + { + "id": "asctime-past", + "description": "asctime date in the past; in the past in every time zone, so it is 0 wherever it is read", + "header": "Sun Nov 6 08:49:37 1994", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 0 + }, + { + "id": "weekday-mismatch", + "description": "the weekday does not match the date; it is ignored", + "header": "Mon, 01 Oct 2026 12:00:10 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 10 + }, + { + "id": "lower-weekday", + "description": "a lower-case weekday is not an HTTP-date", + "header": "thu, 01 Oct 2026 12:00:30 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "two-digit-year-imf", + "description": "an IMF-fixdate requires a four-digit year, so a two-digit one is unreadable and falls back to the backoff", + "header": "Thu, 01 Oct 26 12:00:30 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "rfc850-year-49", + "description": "RFC 850 two-digit year 49 is read as 2049 by both", + "header": "Friday, 01-Oct-49 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 725846400 + }, + { + "id": "rfc850-year-70", + "description": "RFC 850 two-digit year 70 is 2070: 44 years ahead is within RFC 9110 §5.6.7's 50-year window", + "header": "Wednesday, 01-Oct-70 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 1388534400 + }, + { + "id": "offset-zone", + "description": "an IMF-fixdate is always GMT; a numeric offset is unreadable and falls back to the backoff", + "header": "Thu, 01 Oct 2026 13:00:30 +0100", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "day-32", + "description": "day 32 is not a date", + "header": "Thu, 32 Oct 2026 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "hour-25", + "description": "hour 25 is not a time", + "header": "Thu, 01 Oct 2026 25:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "year-overflow", + "description": "a year too large to represent (20 digits) is unreadable, not an error", + "header": "Thu, 01 Oct 99999999999999999999 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "fraction", + "description": "not delta-seconds: a fraction", + "header": "1.5", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "negative", + "description": "not delta-seconds: a sign", + "header": "-1", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "plus", + "description": "not delta-seconds: a plus sign", + "header": "+5", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "exponent", + "description": "not delta-seconds: exponent notation", + "header": "1.5e3", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "superscript", + "description": "not delta-seconds: a Unicode digit that is not ASCII", + "header": "²", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "arabic-digit", + "description": "not delta-seconds: an Arabic-Indic digit", + "header": "٣", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "iso-date", + "description": "an ISO 8601 date is not an HTTP-date", + "header": "2026-10-01", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "prose-date", + "description": "a date that does not start with a weekday is not an HTTP-date, even if a date parser would read it", + "header": "March 1, 2027", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "bad-weekday", + "description": "an HTTP-date shape whose weekday is not a weekday", + "header": "X, 21 Oct 2026 07:28:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "unit", + "description": "delta-seconds with a unit", + "header": "5 seconds", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "empty", + "description": "an empty value", + "header": "", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "spaces", + "description": "whitespace only", + "header": " ", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "padded", + "description": "surrounding spaces are trimmed", + "header": " 5 ", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 5 + }, + { + "id": "tab", + "description": "surrounding tabs are trimmed", + "header": "\t5\t", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 5 + }, + { + "id": "absent", + "description": "no Retry-After header at all", + "header": null, + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "asctime", + "description": "asctime date 45s ahead, read as UTC: RFC 9110 says an HTTP-date is always GMT, so a zone-less or asctime date is never read as local time.", + "header": "Thu Oct 1 12:00:45 2026", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 45 + }, + { + "id": "rfc850-year-60", + "description": "RFC 850 two-digit year 60 resolves to 2060 (RFC 9110 §5.6.7): 34 years ahead of `now`, not more than 50, so the current century is kept.", + "header": "Friday, 01-Oct-60 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 1073001600 + }, + { + "id": "rfc850-year-76", + "description": "RFC 850 two-digit year 76 resolves to 2076: exactly 50 years ahead of `now`, the boundary — not *more than* 50, so it stays in the future.", + "header": "Thursday, 01-Oct-76 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 1577923200 + }, + { + "id": "rfc850-year-77", + "description": "RFC 850 two-digit year 77 would resolve to 2077, 51 years ahead — more than 50 — so RFC 9110 §5.6.7 moves it back a century, to 1977, which is in the past.", + "header": "Saturday, 01-Oct-77 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": 0 + }, + { + "id": "no-zone", + "description": "an IMF-fixdate without its GMT suffix is unreadable and falls back to the backoff (only the asctime form has no zone, and is read as UTC)", + "header": "Thu, 01 Oct 2026 12:00:30", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "trailing-garbage", + "description": "an IMF-fixdate followed by extra text is unreadable: trailing text after a valid date is refused, not ignored.", + "header": "Thu, 01 Oct 2026 12:00:30 GMT garbage", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "feb-30", + "description": "30 February is not a calendar date, and is refused rather than rolled over to March.", + "header": "Mon, 30 Feb 2026 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "year-10000", + "description": "a five-digit year is above 9999, and is refused.", + "header": "Fri, 01 Oct 10000 12:00:00 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + }, + { + "id": "leap-second", + "description": "second 60 is a leap second: readable, and the first second of the next minute, here of the next year", + "header": "Thu, 31 Dec 2026 23:59:60 GMT", + "now": "2026-12-31T23:59:00Z", + "expected_seconds": 60 + }, + { + "id": "leap-second-year-9999", + "description": "a leap second rolls over even past 31 December 9999: the year limit applies to the year as written, so this is 1 January 10000, not unreadable", + "header": "Fri, 31 Dec 9999 23:59:60 GMT", + "now": "9999-12-31T23:59:00Z", + "expected_seconds": 60 + }, + { + "id": "second-61", + "description": "second 61 is not a time, leap second or not", + "header": "Thu, 01 Oct 2026 12:00:61 GMT", + "now": "2026-10-01T12:00:00Z", + "expected_seconds": null + } + ] +} diff --git a/tests/vectors/schema/account-ids.schema.json b/tests/vectors/schema/account-ids.schema.json new file mode 100644 index 00000000..ec112bc2 --- /dev/null +++ b/tests/vectors/schema/account-ids.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/account-ids.schema.json", + "title": "Account id vectors", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bumped whenever a case's meaning or expected value changes; adding a case does not bump it." + }, + "error_classes": { + "$ref": "#/$defs/classes" + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/case" + } + }, + "unresolved": { + "type": "array", + "items": { + "$ref": "#/$defs/unresolved" + } + } + }, + "required": ["description", "version", "cases", "error_classes"], + "additionalProperties": false, + "$defs": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "description": "Stable and unique within the file: an SDK test names its case by it." + }, + "description": { + "type": "string", + "minLength": 1 + }, + "classes": { + "type": "object", + "properties": { + "python": { + "const": "ToolsetConfigError" + }, + "node": { + "const": "ToolSetConfigError" + } + }, + "required": ["python", "node"], + "additionalProperties": false + }, + "case": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "parameter": { + "enum": ["account_ids", "account_id"] + }, + "value": { + "description": "The value passed, as JSON." + }, + "expected": { + "oneOf": [ + { + "const": "ok" + }, + { + "type": "object", + "properties": { + "error": { + "$ref": "#/$defs/classes" + } + }, + "required": ["error"], + "additionalProperties": false + } + ] + } + }, + "required": ["id", "description", "parameter", "value", "expected"], + "additionalProperties": false + }, + "unresolved": { + "description": "A case the two SDKs disagree on today. Not graded: each SDK's observed value is recorded so the disagreement can be settled, and the case moved to `cases`.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "parameter": { + "enum": ["account_ids", "account_id"] + }, + "value": { + "description": "The value passed, as JSON." + }, + "python": { + "description": "What the Python SDK does today." + }, + "node": { + "description": "What the Node SDK does today." + } + }, + "required": ["id", "description", "parameter", "value", "python", "node"], + "additionalProperties": false + } + } +} diff --git a/tests/vectors/schema/argument-encoding.schema.json b/tests/vectors/schema/argument-encoding.schema.json new file mode 100644 index 00000000..3de1a34e --- /dev/null +++ b/tests/vectors/schema/argument-encoding.schema.json @@ -0,0 +1,182 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/argument-encoding.schema.json", + "title": "Argument encoding vectors", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bumped whenever a case's meaning or expected value changes; adding a case does not bump it." + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/case" + } + }, + "unresolved": { + "type": "array", + "items": { + "$ref": "#/$defs/unresolved" + } + } + }, + "required": ["description", "version", "cases", "reasons"], + "additionalProperties": false, + "$defs": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "description": "Stable and unique within the file: an SDK test names its case by it." + }, + "description": { + "type": "string", + "minLength": 1 + }, + "value": { + "description": "Any JSON value, in which an object with a `$number` key is a marker for a number JSON cannot hold, and nothing else.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/value" + } + }, + { + "type": "object", + "properties": { + "$number": { + "enum": ["NaN", "Infinity", "-Infinity"] + } + }, + "required": ["$number"], + "additionalProperties": false + }, + { + "type": "object", + "not": { + "required": ["$number"] + }, + "additionalProperties": { + "$ref": "#/$defs/value" + } + } + ] + }, + "case": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "arguments": { + "$ref": "#/$defs/value" + }, + "arguments_json": { + "type": "string", + "description": "JSON text handed to execute instead of a value." + }, + "expected": { + "oneOf": [ + { + "const": "ok" + }, + { + "type": "object", + "properties": { + "error": { + "const": "ToolArgumentsError" + }, + "reason": { + "enum": [ + "not-finite", + "invalid-json", + "not-an-object", + "unencodable" + ] + } + }, + "required": ["error", "reason"], + "additionalProperties": false + } + ] + } + }, + "required": ["id", "description", "expected"], + "additionalProperties": false, + "oneOf": [ + { + "required": ["arguments"] + }, + { + "required": ["arguments_json"] + } + ] + }, + "unresolved": { + "description": "A case the two SDKs disagree on today. Not graded: each SDK's observed value is recorded so the disagreement can be settled, and the case moved to `cases`.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "python": { + "description": "What the Python SDK does today." + }, + "node": { + "description": "What the Node SDK does today." + }, + "arguments": { + "$ref": "#/$defs/value" + }, + "arguments_json": { + "type": "string", + "description": "JSON text handed to execute instead of a value." + } + }, + "required": ["id", "description", "python", "node"], + "additionalProperties": false, + "oneOf": [ + { + "required": ["arguments"] + }, + { + "required": ["arguments_json"] + } + ] + } + } +} diff --git a/tests/vectors/schema/backoff.schema.json b/tests/vectors/schema/backoff.schema.json new file mode 100644 index 00000000..f58501db --- /dev/null +++ b/tests/vectors/schema/backoff.schema.json @@ -0,0 +1,204 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/backoff.schema.json", + "title": "429 backoff and deadline vectors", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bumped whenever a case's meaning or expected value changes; adding a case does not bump it." + }, + "constants": { + "type": "object", + "properties": { + "max_retries": { + "type": "integer" + }, + "max_attempts": { + "type": "integer" + }, + "backoff_seconds": { + "type": "array", + "minItems": 3, + "items": { + "type": "number", + "minimum": 0 + }, + "description": "The base delay before each retry: retry n uses entry n - 1." + }, + "jitter": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "max_exclusive": { + "type": "boolean" + } + }, + "required": ["min", "max", "max_exclusive"], + "additionalProperties": false + }, + "max_delay_seconds": { + "type": "number" + } + }, + "required": [ + "max_retries", + "max_attempts", + "backoff_seconds", + "jitter", + "max_delay_seconds" + ], + "additionalProperties": false + }, + "deadline_cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/deadlineCase" + } + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/case" + } + }, + "unresolved": { + "type": "array", + "items": { + "$ref": "#/$defs/unresolved" + } + } + }, + "required": ["description", "version", "cases", "constants", "deadline_cases"], + "additionalProperties": false, + "$defs": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "description": "Stable and unique within the file: an SDK test names its case by it." + }, + "description": { + "type": "string", + "minLength": 1 + }, + "infinity": { + "description": "Positive infinity, which JSON cannot hold, as a marker an SDK decodes before use.", + "type": "object", + "properties": { + "$number": { + "const": "Infinity" + } + }, + "required": ["$number"], + "additionalProperties": false + }, + "deadlineCase": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "delay_seconds": { + "type": "number", + "minimum": 0, + "description": "The wait before the retry, after the cap." + }, + "remaining_seconds": { + "type": "number", + "description": "Time left before the call's deadline; may be negative." + }, + "expected": { + "enum": ["wait", "return-429"] + } + }, + "required": ["id", "description", "delay_seconds", "remaining_seconds", "expected"], + "additionalProperties": false + }, + "case": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "retry": { + "type": "integer", + "minimum": 1, + "maximum": 3, + "description": "Which retry the delay precedes, 1-based." + }, + "random": { + "type": "number", + "minimum": 0, + "exclusiveMaximum": 1, + "description": "The injected uniform value in [0, 1)." + }, + "retry_after_seconds": { + "oneOf": [ + { + "type": ["number", "null"], + "minimum": 0 + }, + { + "$ref": "#/$defs/infinity" + } + ], + "description": "The 429's Retry-After as read (see retry-after.json); null when absent or unreadable. A readable value too large for a float is `{\"$number\": \"Infinity\"}`, as there." + }, + "expected_delay_seconds": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "id", + "description", + "retry", + "random", + "retry_after_seconds", + "expected_delay_seconds" + ], + "additionalProperties": false + }, + "unresolved": { + "description": "A case the two SDKs disagree on today. Not graded: each SDK's observed value is recorded so the disagreement can be settled, and the case moved to `cases`.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "python": { + "description": "What the Python SDK does today." + }, + "node": { + "description": "What the Node SDK does today." + } + }, + "required": ["id", "description", "python", "node"], + "additionalProperties": false + } + } +} diff --git a/tests/vectors/schema/header-arguments.schema.json b/tests/vectors/schema/header-arguments.schema.json new file mode 100644 index 00000000..14bc5b71 --- /dev/null +++ b/tests/vectors/schema/header-arguments.schema.json @@ -0,0 +1,162 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/header-arguments.schema.json", + "title": "Header argument vectors (invariant 4)", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bumped whenever a case's meaning or expected value changes; adding a case does not bump it." + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/case" + } + }, + "unresolved": { + "type": "array", + "items": { + "$ref": "#/$defs/unresolved" + } + } + }, + "required": ["description", "version", "cases", "reasons"], + "additionalProperties": false, + "$defs": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "description": "Stable and unique within the file: an SDK test names its case by it." + }, + "description": { + "type": "string", + "minLength": 1 + }, + "warning": { + "type": "object", + "properties": { + "argument": { + "type": "string", + "description": "The top-level argument: `headers`, or `headers_` as supplied." + }, + "header": { + "type": "string", + "description": "For an entry of a `headers` object: its name, trimmed. Present exactly when `argument` is `headers` and the entry, not the whole argument, was dropped." + }, + "reason": { + "enum": [ + "set-by-sdk", + "not-declared", + "malformed", + "not-an-object", + "not-a-scalar" + ] + } + }, + "required": ["argument", "reason"], + "additionalProperties": false, + "if": { + "properties": { + "argument": { + "const": "headers" + }, + "reason": { + "enum": ["set-by-sdk", "not-declared", "malformed"] + } + }, + "required": ["argument", "reason"] + }, + "then": { + "required": ["header"] + }, + "else": { + "not": { + "required": ["header"] + } + } + }, + "case": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "schema": { + "type": "object", + "description": "The tool's served inputSchema." + }, + "arguments": { + "type": "object", + "description": "The model's arguments." + }, + "expected_arguments": { + "type": "object", + "description": "The arguments that reach tools/call." + }, + "expected_warnings": { + "type": "array", + "items": { + "$ref": "#/$defs/warning" + }, + "description": "In argument order." + } + }, + "required": [ + "id", + "description", + "schema", + "arguments", + "expected_arguments", + "expected_warnings" + ], + "additionalProperties": false + }, + "unresolved": { + "description": "A case the two SDKs disagree on today. Not graded: each SDK's observed value is recorded so the disagreement can be settled, and the case moved to `cases`.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "schema": { + "type": "object", + "description": "The tool's served inputSchema." + }, + "arguments": { + "type": "object", + "description": "The model's arguments." + }, + "python": { + "description": "What the Python SDK does today." + }, + "node": { + "description": "What the Node SDK does today." + } + }, + "required": ["id", "description", "schema", "arguments", "python", "node"], + "additionalProperties": false + } + } +} diff --git a/tests/vectors/schema/header-names.schema.json b/tests/vectors/schema/header-names.schema.json new file mode 100644 index 00000000..1e544b10 --- /dev/null +++ b/tests/vectors/schema/header-names.schema.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/header-names.schema.json", + "title": "Nested header name vectors", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bumped whenever a case's meaning or expected value changes; adding a case does not bump it." + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/case" + } + }, + "unresolved": { + "type": "array", + "items": { + "$ref": "#/$defs/unresolved" + } + } + }, + "required": ["description", "version", "cases"], + "additionalProperties": false, + "$defs": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "description": "Stable and unique within the file: an SDK test names its case by it." + }, + "description": { + "type": "string", + "minLength": 1 + }, + "case": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "name": { + "type": "string" + }, + "expected": { + "oneOf": [ + { + "type": "object", + "properties": { + "forwarded": { + "type": "string" + } + }, + "required": ["forwarded"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "dropped": { + "enum": ["set-by-sdk", "malformed"] + } + }, + "required": ["dropped"], + "additionalProperties": false + } + ] + } + }, + "required": ["id", "description", "name", "expected"], + "additionalProperties": false + }, + "unresolved": { + "description": "A case the two SDKs disagree on today. Not graded: each SDK's observed value is recorded so the disagreement can be settled, and the case moved to `cases`.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "name": { + "type": "string" + }, + "python": { + "description": "What the Python SDK does today." + }, + "node": { + "description": "What the Node SDK does today." + } + }, + "required": ["id", "description", "name", "python", "node"], + "additionalProperties": false + } + } +} diff --git a/tests/vectors/schema/header-values.schema.json b/tests/vectors/schema/header-values.schema.json new file mode 100644 index 00000000..6ec72ab5 --- /dev/null +++ b/tests/vectors/schema/header-values.schema.json @@ -0,0 +1,175 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/header-values.schema.json", + "title": "Nested header value vectors", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bumped whenever a case's meaning or expected value changes; adding a case does not bump it." + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/case" + } + }, + "unresolved": { + "type": "array", + "items": { + "$ref": "#/$defs/unresolved" + } + }, + "known_differences": { + "type": "array", + "items": { + "$ref": "#/$defs/known_difference" + }, + "description": "A case where the SDKs differ for a reason outside this rule (for example, their JSON parser already produced different values). Not graded." + } + }, + "required": ["description", "version", "cases"], + "additionalProperties": false, + "$defs": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "description": "Stable and unique within the file: an SDK test names its case by it." + }, + "description": { + "type": "string", + "minLength": 1 + }, + "value": { + "description": "Any JSON value, in which an object with a `$number` key is a marker for a number JSON cannot hold, and nothing else.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/value" + } + }, + { + "type": "object", + "properties": { + "$number": { + "enum": ["NaN", "Infinity", "-Infinity"] + } + }, + "required": ["$number"], + "additionalProperties": false + }, + { + "type": "object", + "not": { + "required": ["$number"] + }, + "additionalProperties": { + "$ref": "#/$defs/value" + } + } + ] + }, + "case": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "value": { + "$ref": "#/$defs/value" + }, + "expected": { + "oneOf": [ + { + "type": "string", + "description": "The header text forwarded." + }, + { + "type": "object", + "properties": { + "dropped": { + "const": true + } + }, + "required": ["dropped"], + "additionalProperties": false + } + ] + } + }, + "required": ["id", "description", "value", "expected"], + "additionalProperties": false + }, + "unresolved": { + "description": "A case the two SDKs disagree on today. Not graded: each SDK's observed value is recorded so the disagreement can be settled, and the case moved to `cases`.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "value": { + "$ref": "#/$defs/value" + }, + "python": { + "description": "What the Python SDK does today." + }, + "node": { + "description": "What the Node SDK does today." + } + }, + "required": ["id", "description", "value", "python", "node"], + "additionalProperties": false + }, + "known_difference": { + "description": "A case that is settled, but not identical across SDKs for a reason outside this rule. Not graded.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "value": { + "$ref": "#/$defs/value" + }, + "python": { + "description": "What the Python SDK does." + }, + "node": { + "description": "What the Node SDK does." + } + }, + "required": ["id", "description", "value", "python", "node"], + "additionalProperties": false + } + } +} diff --git a/tests/vectors/schema/messages.schema.json b/tests/vectors/schema/messages.schema.json new file mode 100644 index 00000000..29fd8ebc --- /dev/null +++ b/tests/vectors/schema/messages.schema.json @@ -0,0 +1,208 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/messages.schema.json", + "title": "Canonical SDK messages", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1 + }, + "prefixes": { + "type": "object", + "properties": { + "node": { + "type": "string" + }, + "python": { + "type": "string" + } + }, + "required": ["node", "python"], + "additionalProperties": false + }, + "formats": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "reasons": { + "type": "object", + "properties": { + "set-by-sdk": { + "type": "string" + }, + "not-declared": { + "type": "string" + }, + "malformed": { + "type": "string" + }, + "not-an-object": { + "type": "string" + }, + "not-a-scalar": { + "type": "string" + } + }, + "required": [ + "set-by-sdk", + "not-declared", + "malformed", + "not-an-object", + "not-a-scalar" + ], + "additionalProperties": false + }, + "messages": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/message" + } + }, + "unshared": { + "type": "array", + "items": { + "$ref": "#/$defs/unshared" + } + } + }, + "required": [ + "description", + "version", + "prefixes", + "formats", + "reasons", + "messages", + "unshared" + ], + "additionalProperties": false, + "$defs": { + "classes": { + "type": "object", + "properties": { + "python": { + "type": "string" + }, + "node": { + "type": "string" + } + }, + "required": ["python", "node"], + "additionalProperties": false + }, + "message": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "kind": { + "enum": ["warning", "error", "tool-result"] + }, + "error": { + "$ref": "#/$defs/classes", + "description": "The class each SDK raises it as." + }, + "template": { + "type": "string", + "description": "The canonical text. `{name}` is a placeholder; no template holds a literal brace." + }, + "placeholders": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "format": { + "enum": [ + "text", + "json", + "integer", + "seconds", + "seconds-2dp", + "json-type", + "reason" + ] + }, + "description": { + "type": "string" + } + }, + "required": ["format", "description"], + "additionalProperties": false + } + }, + "python": { + "type": "string", + "description": "The Python SDK's text today." + }, + "node": { + "type": "string", + "description": "The Node SDK's text today." + }, + "matches": { + "type": "object", + "properties": { + "python": { + "type": "boolean" + }, + "node": { + "type": "boolean" + } + }, + "required": ["python", "node"], + "additionalProperties": false + }, + "notes": { + "type": "string" + } + }, + "required": ["id", "kind", "template", "placeholders", "python", "node", "matches"], + "additionalProperties": false, + "if": { + "properties": { + "kind": { + "const": "error" + } + } + }, + "then": { + "required": ["error"] + }, + "else": { + "not": { + "required": ["error"] + } + } + }, + "unshared": { + "type": "object", + "properties": { + "sdk": { + "enum": ["python", "node"] + }, + "kind": { + "enum": ["warning", "error", "tool-result"] + }, + "text": { + "type": "string" + }, + "notes": { + "type": "string" + } + }, + "required": ["sdk", "kind", "text"], + "additionalProperties": false + } + } +} diff --git a/tests/vectors/schema/retry-after.schema.json b/tests/vectors/schema/retry-after.schema.json new file mode 100644 index 00000000..15f72899 --- /dev/null +++ b/tests/vectors/schema/retry-after.schema.json @@ -0,0 +1,128 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/StackOneHQ/sdk-conformance/vectors/schema/retry-after.schema.json", + "title": "Retry-After vectors", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "description": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bumped whenever a case's meaning or expected value changes; adding a case does not bump it." + }, + "max_delay_seconds": { + "type": "number", + "exclusiveMinimum": 0, + "description": "The cap the caller applies to the value afterwards." + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/case" + } + }, + "unresolved": { + "type": "array", + "items": { + "$ref": "#/$defs/unresolved" + } + } + }, + "required": ["description", "version", "cases", "max_delay_seconds"], + "additionalProperties": false, + "$defs": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "description": "Stable and unique within the file: an SDK test names its case by it." + }, + "description": { + "type": "string", + "minLength": 1 + }, + "number": { + "description": "A number JSON cannot hold, as a marker an SDK decodes before use.", + "type": "object", + "properties": { + "$number": { + "enum": ["NaN", "Infinity", "-Infinity"] + } + }, + "required": ["$number"], + "additionalProperties": false + }, + "case": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "header": { + "type": ["string", "null"], + "description": "The Retry-After value; null when the response carries no Retry-After header." + }, + "now": { + "type": "string", + "format": "date-time", + "description": "The current time the header is read at, RFC 3339." + }, + "expected_seconds": { + "description": "Seconds from `now`, before the cap; null when absent or unreadable.", + "oneOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "null" + }, + { + "$ref": "#/$defs/number" + } + ] + } + }, + "required": ["id", "description", "header", "now", "expected_seconds"], + "additionalProperties": false + }, + "unresolved": { + "description": "A case the two SDKs disagree on today. Not graded: each SDK's observed value is recorded so the disagreement can be settled, and the case moved to `cases`.", + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/id" + }, + "description": { + "$ref": "#/$defs/description" + }, + "header": { + "type": ["string", "null"], + "description": "The Retry-After value; null when the response carries no Retry-After header." + }, + "now": { + "type": "string", + "format": "date-time", + "description": "The current time the header is read at, RFC 3339." + }, + "python": { + "description": "What the Python SDK does today." + }, + "node": { + "description": "What the Node SDK does today." + } + }, + "required": ["id", "description", "header", "now", "python", "node"], + "additionalProperties": false + } + } +} diff --git a/uv.lock b/uv.lock index 28df1d50..705b9b39 100644 --- a/uv.lock +++ b/uv.lock @@ -2915,18 +2915,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] -[[package]] -name = "respx" -version = "0.22.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/7c/96bd0bc759cf009675ad1ee1f96535edcb11e9666b985717eb8c87192a95/respx-0.22.0.tar.gz", hash = "sha256:3c8924caa2a50bd71aefc07aa812f2466ff489f1848c96e954a5362d17095d91", size = 28439, upload-time = "2024-12-19T22:33:59.374Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/67/afbb0978d5399bc9ea200f1d4489a23c9a1dad4eee6376242b8182389c79/respx-0.22.0-py2.py3-none-any.whl", hash = "sha256:631128d4c9aba15e56903fb5f66fb1eff412ce28dd387ca3a81339e52dbd3ad0", size = 25127, upload-time = "2024-12-19T22:33:57.837Z" }, -] - [[package]] name = "rich" version = "14.2.0" @@ -3178,7 +3166,6 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, - { name = "respx" }, { name = "ruff" }, { name = "ty" }, ] @@ -3201,8 +3188,8 @@ requires-dist = [ { name = "httpx", specifier = ">=0.28.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.141.1" }, { name = "langchain", marker = "extra == 'examples'", specifier = ">=1.0.0" }, - { name = "langchain-core", marker = "extra == 'dev'", specifier = ">=0.1.0" }, - { name = "langchain-core", marker = "extra == 'langchain'", specifier = ">=0.3.0" }, + { name = "langchain-core", marker = "extra == 'dev'", specifier = ">=0.3.36" }, + { name = "langchain-core", marker = "extra == 'langchain'", specifier = ">=0.3.36" }, { name = "langchain-openai", marker = "extra == 'examples'", specifier = ">=1.1.14" }, { name = "mcp", specifier = ">=1.24.3,<2.0.0" }, { name = "openai", marker = "extra == 'examples'", specifier = ">=2.32.0" }, @@ -3213,7 +3200,6 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25.3" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0.0" }, { name = "python-dotenv", marker = "extra == 'examples'", specifier = ">=1.0.1" }, - { name = "respx", marker = "extra == 'dev'", specifier = ">=0.22.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.6" }, { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.3" }, ]