From 582ac401246defc3ca8f4561f4ef23e53a68b0d9 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Wed, 14 Oct 2026 23:54:17 -0700 Subject: [PATCH 1/3] feat(agents): requirement rows for the agent discovery extension (placeholder sep-9998) agents-wg PR 22 has no number in modelcontextprotocol/modelcontextprotocol yet, and both traceability filters are numeric, so the rows and check IDs carry the placeholder 9998. The header explains why and how to rename. 31 graded rows and 18 excluded. Every row quotes the proposal at 9b16840e, matched with line wrapping ignored, and points at the section it sits under. --- src/seps/sep-9998.yaml | 215 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 src/seps/sep-9998.yaml diff --git a/src/seps/sep-9998.yaml b/src/seps/sep-9998.yaml new file mode 100644 index 00000000..663d0539 --- /dev/null +++ b/src/seps/sep-9998.yaml @@ -0,0 +1,215 @@ +# spec_source: modelcontextprotocol/agents-wg@9b16840e2f958b3dd9a418bb321189a0d1ab3493 proposals/0000-mcp-agent-discovery-extension.md +# extracted: 2026-10-15 +# +# ############################################################################ +# # 9998 is a placeholder, not a SEP number. # +# ############################################################################ +# +# The MCP Agent Discovery Extension is agents-wg PR 22, a draft Extensions +# Track proposal in the Agents Working Group's repository +# (modelcontextprotocol/agents-wg). SEP numbers are PR numbers in +# modelcontextprotocol/modelcontextprotocol, and this proposal has no PR +# there yet, so it has no number. Both traceability filters are numeric (the +# file name /^sep-\d+\.yaml$/ and the check ID /^sep-\d+-/), and a file that +# does not match is silently dropped from the manifest, so an honest name +# such as sep-agents-* would vanish. 9999 is not used because the MCP Events +# suite carried it before SEP-3415 existed, and reports from that period +# still mention it. +# +# When the proposal gets a number, rename the file, every check ID and the +# two scenario names by script, as the Events suite did when it moved from +# 9999 to 3415. Nothing else in a row depends on the number. +# +# new-sep cannot scaffold this file: it looks the number up as a PR in +# modelcontextprotocol/modelcontextprotocol and derives the spec URL from a +# docs/specification/draft/*.mdx path, and the proposal has neither. The +# spec_url below points at the pinned proposal instead. +# +# quoting: every row's `text` is a span of the proposal at 9b16840e, and its +# `url` is the section that span sits under. The match ignores line +# wrapping (runs of whitespace compare equal), because the proposal wraps +# prose at a fixed width and a quote across a wrap point would otherwise need +# embedded newlines. Field-level rows quote the TypeScript definition they +# grade, since the proposal states field types only there. +# +# coverage: 31 graded rows across two server scenarios. +# sep-9998-agents-discovery emits the capability, roster, details, tool +# parity and error rows. +# sep-9998-agents-list-changed emits the four list-changed rows. +# Three rows need a diagnostic control on the server under test +# (agents_conformance_fixture names a hidden agent and a broken one), and +# two need agents_conformance_add_agent to change the roster mid-run. A +# server without them reports those rows untestable, naming the control. +# Excluded rows are client or host obligations, off-wire server behaviour, +# or rules that need two authorization contexts, which a harness holding one +# credential cannot compare. +sep: 9998 +spec_url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#specification +requirements: + # === Capability Negotiation === + - check: sep-9998-capability-declared + text: 'Clients and servers explicitly declare support for the extension using the MCP extension negotiation mechanism.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - check: sep-9998-capability-list-changed-boolean + text: 'The optional `listChanged` setting indicates that the server supports agent-roster change notifications.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - check: sep-9998-core-without-extension + text: 'A server **MUST NOT** require clients to support this extension in order to use otherwise compatible core MCP functionality.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + + # === Roster: agents/list === + - check: sep-9998-list-implemented + text: '`agents/list` returns the agent roster visible to the requesting client.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - check: sep-9998-list-result-shape + text: 'interface ListAgentsResult extends PaginatedResult, CacheableResult { resultType: "complete"; agents: AgentCard[]; }' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-card-name + text: "/** Unique identifier for this agent within the server's visible roster. */ name: string;" + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-card-description-type + text: "/** Optional human-readable summary of the agent's purpose. */ description?: string;" + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-card-capabilities + text: '/** Concise routing labels; these are not MCP tool names. */ capabilities: string[];' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + # No RFC 2119 keyword, so graded as a WARNING. The card interface does not + # forbid extra members, but a card carrying instructions or tool schemas + # defeats the progressive disclosure the two levels exist for. + - check: sep-9998-list-cards-only + text: 'The server returns compact cards only:' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#listing-agents + - check: sep-9998-name-unique + text: 'Each agent name **MUST** be unique within the roster visible to the requesting client.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-list-pagination + text: 'Clients omit `cursor` for the first page and continue with the returned `nextCursor` until the response omits it.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#listing-agents + - check: sep-9998-list-cacheable + text: 'Both `agents/list` and `agents/get` results follow the MCP `CacheableResult` model defined' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#caching-and-freshness + + # === Details: agents/get === + - check: sep-9998-get-implemented + text: '`agents/get` returns instructions and scoped MCP tool schemas for one agent.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - check: sep-9998-get-result-shape + text: 'interface GetAgentResult extends CacheableResult { resultType: "complete";' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-get-agent-name + text: '/** Name of the agent to which these details belong. */ agent: string;' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-get-cacheable + text: 'Both `agents/list` and `agents/get` results follow the MCP `CacheableResult` model defined' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#caching-and-freshness + - check: sep-9998-hint-role + text: 'type ModelRole = "small" | "medium" | "large";' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-hint-vision + text: '/** Whether a vision-capable model is preferred for this agent. */ vision: boolean;' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + + # === Details: tool parity with tools/list === + # The next three rows are graded by one probe (every tool from every + # agents/get, looked up in tools/list), since for a single caller they + # describe the same observation from three sides. + - check: sep-9998-get-tools-in-tools-list + text: 'Every returned tool **MUST** also be discoverable through `tools/list`' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-get-tools-intersection + text: "A server **MUST NOT** include a tool in `agents/get` that would be absent from that caller's `tools/list`." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-tools-list-not-omitted + text: 'A server **MUST NOT** omit a tool from `tools/list` solely because that tool is associated with one or more agents.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - check: sep-9998-get-tools-fields-match + text: 'When a tool appears in both responses for the same request context, all fields of the corresponding `Tool` definitions **MUST** match.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-shared-tool-same-definition + text: 'If the same tool name appears under multiple agents, it **MUST** identify the same server-wide callable tool.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + + # === Errors and visibility === + - check: sep-9998-agent-not-found + text: 'If the requested agent does not exist or is not visible to the requesting client, the server **MUST** return `-32602` (`Invalid params`) with `data.reason: "agent_not_found"`.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + - check: sep-9998-hidden-indistinguishable + text: "When a server policy requires hiding an agent's existence from unauthorized callers, the server **MUST** use the same `agent_not_found` response as for a nonexistent agent name." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - check: sep-9998-roster-filtered + text: "A server **MUST** filter both the roster and agent details according to the requesting client's effective permissions." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-invalid-agent-definition + text: 'If an agent''s configured membership references a tool name that does not exist in the server catalog, or the server would return a tool through `agents/get` that would be absent from that caller''s `tools/list` under the same authorization context, the server **MUST** return `-32603` (`Internal error`) with `data.reason: "invalid_agent_definition"`.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + + # === Roster changes === + - check: sep-9998-list-changed-not-declared + text: 'Omitting it or setting it to `false` indicates that such notifications are not supported.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - check: sep-9998-list-changed-acknowledged + text: 'Clients opt in by adding `agentsListChanged: true` to the `notifications` filter of a `subscriptions/listen` request. The server acknowledges and delivers the notification using the standard MCP subscription mechanism.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + - check: sep-9998-list-changed-notification + text: 'A server that advertises `listChanged: true` for this extension supports `notifications/agents/list_changed`.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + # No RFC 2119 keyword, so graded as a WARNING. + - check: sep-9998-list-changed-roster-updated + text: 'The notification indicates that the visible roster or one or more agent cards may have changed.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + + # === Excluded === + - text: 'Servers **SHOULD** validate at registration time that every tool name referenced by an agent exists in the server catalog.' + excluded: 'Registration happens off the wire. Its observable consequence, an agents/get for such an agent, is graded by sep-9998-invalid-agent-definition.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: 'remain callable through the existing MCP tool-calling mechanism, subject to the same authorization context and server policy as any other tool call.' + excluded: "Calling an arbitrary server's tools has side effects the harness cannot justify, and it has no argument values to call them with. Core tools/call behaviour is graded by the core tools scenarios." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: "For each `agents/get` request, the server **MUST** return only tools that would also appear in that caller's `tools/list` under the same authorization context (the intersection of agent membership with the authorized catalog)." + excluded: 'For the single authorization context a run has, this is the observation sep-9998-get-tools-intersection grades. Whether the intersection tracks the caller needs two credentials to compare, and the harness holds one.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: 'Servers **MUST NOT** use an agent card or agent membership to grant permissions that the requesting client does not otherwise possess.' + excluded: 'Needs a caller lacking a permission that membership could wrongly grant, compared against one holding it. The harness holds one credential.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - text: 'For both `agents/list` and `agents/get`, servers **SHOULD** use `private` unless they can guarantee that the particular response is identical and safe for every client.' + excluded: 'Whether a response is identical for every client is not observable from one client. A public scope is permitted when the guarantee holds, so grading the value alone would fail conformant servers.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#caching-and-freshness + - text: 'When a caller is permitted to know an agent exists but lacks authorization to retrieve it, the server **MAY** return an authorization error that enables' + excluded: 'Optional, and exercising it needs a credential that can be stepped up, which the harness does not hold.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - text: 'Servers **SHOULD** document which policy applies for their deployment.' + excluded: 'Documentation, not protocol behaviour.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - text: 'A server **MUST NOT** assume that a host honored the hint.' + excluded: 'Server-internal reasoning with no wire signature.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#retrieving-agent-details + - text: 'Servers **MAY** associate the same tool with more than one agent.' + excluded: 'A permission with nothing to grade. Its constraint is graded by sep-9998-shared-tool-same-definition.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: 'A client **MUST NOT** assume agent discovery is available unless the server advertises the extension.' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - text: 'A client receiving it **SHOULD** invalidate all cached `agents/list` pages and re-list before making a new routing decision.' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + - text: 'After re-listing, it **SHOULD** discard cached `agents/get` results for agents that are no longer visible.' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + - text: 'Clients **MAY** refresh `agents/list` and retry.' + excluded: 'Client permission; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + - text: 'Clients **SHOULD NOT** automatically retry the same request;' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + - text: 'The host **MAY** request the complete visible tool catalog through `tools/list` for existing discovery, indexing, or SDK operation.' + excluded: 'Host permission; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - text: 'Existing systems that already call `tools/list` **MAY** retain that behavior for indexing, schema validation, and other host operations.' + excluded: 'Host permission; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - text: "When the field is absent, unsupported, or cannot be satisfied, the host **MAY** use its default model or inherit the supervisor's model." + excluded: 'Host model selection is off the wire.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#retrieving-agent-details + - text: "Hosts **MAY** resolve the hint through local configuration, inherit the supervisor's model, or ignore unsupported values." + excluded: 'Host model selection is off the wire.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#retrieving-agent-details From 917bdd3c62d83b02a82ff43a15de427198a8d065 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Wed, 14 Oct 2026 23:54:17 -0700 Subject: [PATCH 2/3] feat(agents): sep-9998-agents-discovery and sep-9998-agents-list-changed scenarios Discovery grades the capability, the roster (shape, paging, cache hints, cards, unique names), the details for every listed agent, tool parity with tools/list, and the two error reasons. The hidden-agent and broken-agent rows read their names from the server's agents_conformance_fixture control and report untestable without it. List-changed opens a subscriptions/listen stream with agentsListChanged, grades the acknowledgement either way round, changes the roster through agents_conformance_add_agent, and waits for the notification and the re-listed roster. Registered as pending, since the everything-server does not implement the extension. --- src/scenarios/index.ts | 16 +- src/scenarios/server/agents/discovery.ts | 664 +++++++++++++++++++ src/scenarios/server/agents/helpers.ts | 309 +++++++++ src/scenarios/server/agents/list-changed.ts | 277 ++++++++ src/scenarios/server/agents/listen-stream.ts | 231 +++++++ src/types.ts | 3 +- 6 files changed, 1498 insertions(+), 2 deletions(-) create mode 100644 src/scenarios/server/agents/discovery.ts create mode 100644 src/scenarios/server/agents/helpers.ts create mode 100644 src/scenarios/server/agents/list-changed.ts create mode 100644 src/scenarios/server/agents/listen-stream.ts diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index ff377f54..c5885a97 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -63,6 +63,8 @@ import { import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; +import { AgentsDiscoveryScenario } from './server/agents/discovery'; +import { AgentsListChangedScenario } from './server/agents/list-changed'; import { PromptsListScenario, @@ -166,7 +168,14 @@ const pendingClientScenariosList: ClientScenario[] = [ // `npm start -- server --scenario sep-2640-skills-* --url `. new SkillsDirectoryReadScenario(), new SkillsEnumerationScenario(), - new SkillsManifestScenario() + new SkillsManifestScenario(), + + // MCP Agent Discovery Extension (agents-wg#22, placeholder number 9998). + // Pending because the everything-server does not implement + // io.modelcontextprotocol/agents; targeted runs point at a conformant + // fixture via `npm start -- server --scenario sep-9998-agents-* --url `. + new AgentsDiscoveryScenario(), + new AgentsListChangedScenario() ]; // All client scenarios @@ -224,6 +233,11 @@ const allClientScenariosList: ClientScenario[] = [ new SkillsEnumerationScenario(), new SkillsManifestScenario(), + // Agent discovery extension (agents-wg#22). Fixture-dependent; each + // scenario SKIPs cleanly when the extension is not declared. + new AgentsDiscoveryScenario(), + new AgentsListChangedScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), diff --git a/src/scenarios/server/agents/discovery.ts b/src/scenarios/server/agents/discovery.ts new file mode 100644 index 00000000..a555ea34 --- /dev/null +++ b/src/scenarios/server/agents/discovery.ts @@ -0,0 +1,664 @@ +/** + * agents-wg#22 (MCP Agent Discovery Extension): the capability, the roster + * (`agents/list`), the details (`agents/get`), tool parity with `tools/list`, + * and the two error reasons. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim quote of the proposal lives next to its check ID in + * src/seps/sep-9998.yaml. 9998 is a placeholder number; the yaml header says + * why. + * + * Discovery is dynamic. The scenario lists whatever roster the server serves + * and opens every agent in it, hardcoding no agent name. Two rows need agents + * the harness cannot find by listing: one hidden from this caller and one + * whose membership names a missing tool. It asks the server's + * `agents_conformance_fixture` control for their names, and reports those rows + * untestable when the control is absent. + * + * The capability gate is two-sided, as in the skills and events suites. A + * server that neither declares the extension nor answers `agents/list` skips + * the scenario, since the extension is optional. A server that answers + * `agents/list` while declaring nothing fails the capability row, because a + * client following the proposal would never call it. + */ + +import { isDeepStrictEqual } from 'util'; +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + AGENTS_CLIENT_CAPABILITIES, + AGENTS_EXTENSION_ID, + AGENTS_LIST_METHOD, + AGENTS_SPEC_REF, + CONTROL_FIXTURE, + JSONRPC_INTERNAL_ERROR, + JSONRPC_INVALID_PARAMS, + MODEL_ROLES, + REASON_AGENT_NOT_FOUND, + REASON_INVALID_AGENT_DEFINITION, + UNKNOWN_AGENT_NAME, + agentsCheck, + agentsGet, + agentsListAll, + cacheableResultErrors, + declaredAgentsCapability, + describeValue, + errorReason, + fixtureNames, + isObject, + joinErrs, + toolsListAll, + type RpcFailure +} from './helpers'; + +const CAPABILITY_IDS = [ + 'sep-9998-capability-declared', + 'sep-9998-capability-list-changed-boolean', + 'sep-9998-core-without-extension' +] as const; + +const LIST_IDS = [ + 'sep-9998-list-implemented', + 'sep-9998-list-result-shape', + 'sep-9998-list-pagination', + 'sep-9998-list-cacheable', + 'sep-9998-card-name', + 'sep-9998-card-description-type', + 'sep-9998-card-capabilities', + 'sep-9998-list-cards-only', + 'sep-9998-name-unique' +] as const; + +const GET_IDS = [ + 'sep-9998-get-implemented', + 'sep-9998-get-result-shape', + 'sep-9998-get-agent-name', + 'sep-9998-get-cacheable', + 'sep-9998-hint-role', + 'sep-9998-hint-vision' +] as const; + +const PARITY_IDS = [ + 'sep-9998-get-tools-in-tools-list', + 'sep-9998-get-tools-intersection', + 'sep-9998-tools-list-not-omitted', + 'sep-9998-get-tools-fields-match', + 'sep-9998-shared-tool-same-definition' +] as const; + +const ERROR_IDS = [ + 'sep-9998-agent-not-found', + 'sep-9998-hidden-indistinguishable', + 'sep-9998-roster-filtered', + 'sep-9998-invalid-agent-definition' +] as const; + +const ALL_IDS = [ + ...CAPABILITY_IDS, + ...LIST_IDS, + ...GET_IDS, + ...PARITY_IDS, + ...ERROR_IDS +]; + +/** One-line descriptions, keyed by check ID, used on every status. */ +const DESCRIPTIONS: Record<(typeof ALL_IDS)[number], string> = { + 'sep-9998-capability-declared': + 'The server declares io.modelcontextprotocol/agents under capabilities.extensions with a settings object.', + 'sep-9998-capability-list-changed-boolean': + 'The optional listChanged setting, when present, is a boolean.', + 'sep-9998-core-without-extension': + 'A client that does not declare the extension can still use core MCP functionality (tools/list).', + 'sep-9998-list-implemented': + 'agents/list returns the roster visible to the requesting client.', + 'sep-9998-list-result-shape': + 'Every agents/list page carries resultType "complete" and an agents array.', + 'sep-9998-list-pagination': + 'Following nextCursor from the first page reaches a page without one.', + 'sep-9998-list-cacheable': + 'Every agents/list page follows CacheableResult: ttlMs is a non-negative number and cacheScope is "public" or "private".', + 'sep-9998-card-name': 'Every agent card carries a string name.', + 'sep-9998-card-description-type': + 'An agent card description, when present, is a string.', + 'sep-9998-card-capabilities': + 'Every agent card carries capabilities as an array of strings.', + 'sep-9998-list-cards-only': + 'The roster carries compact cards only, without instructions, tools or modelRoutingHint.', + 'sep-9998-name-unique': + 'Agent names are unique within the roster visible to the caller, across every page.', + 'sep-9998-get-implemented': + 'agents/get returns details for every agent the roster lists.', + 'sep-9998-get-result-shape': + 'agents/get results carry resultType "complete", a tools array, and instructions as a string when present.', + 'sep-9998-get-agent-name': + 'agents/get result.agent names the agent that was requested.', + 'sep-9998-get-cacheable': + 'Every agents/get result follows CacheableResult: ttlMs is a non-negative number and cacheScope is "public" or "private".', + 'sep-9998-hint-role': + 'modelRoutingHint.role, when present, is "small", "medium" or "large".', + 'sep-9998-hint-vision': + 'modelRoutingHint, when present, is an object carrying a boolean vision.', + 'sep-9998-get-tools-in-tools-list': + 'Every tool agents/get returns is also listed by tools/list.', + 'sep-9998-get-tools-intersection': + "agents/get includes no tool absent from the caller's tools/list.", + 'sep-9998-tools-list-not-omitted': + 'tools/list does not omit a tool because an agent includes it.', + 'sep-9998-get-tools-fields-match': + 'A tool returned by both agents/get and tools/list has identical fields in both.', + 'sep-9998-shared-tool-same-definition': + 'A tool name that appears under several agents identifies the same tool in each.', + 'sep-9998-agent-not-found': + 'agents/get for an unknown agent fails with -32602 and data.reason "agent_not_found".', + 'sep-9998-hidden-indistinguishable': + 'agents/get for an agent hidden from the caller fails exactly as it does for a nonexistent name.', + 'sep-9998-roster-filtered': + 'An agent hidden from the caller is absent from its agents/list roster.', + 'sep-9998-invalid-agent-definition': + 'agents/get for an agent whose membership names a missing tool fails with -32603 and data.reason "invalid_agent_definition".' +}; + +/** Card members that belong to agents/get, never to the roster. */ +const DETAIL_MEMBERS = ['instructions', 'tools', 'modelRoutingHint', 'agent']; + +type CheckId = (typeof ALL_IDS)[number]; + +function check( + id: CheckId, + errs: string[], + details?: Record, + failStatus: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck { + return agentsCheck( + id, + DESCRIPTIONS[id], + errs.length === 0 ? 'SUCCESS' : failStatus, + errs.length === 0 + ? details + ? { details } + : {} + : { errorMessage: joinErrs(errs), details } + ); +} + +function skipped(id: CheckId, reason: string): ConformanceCheck { + return agentsCheck(id, DESCRIPTIONS[id], 'SKIPPED', { + errorMessage: reason + }); +} + +function untestable(id: CheckId, reason: string): ConformanceCheck { + return untestableCheck(id, id, DESCRIPTIONS[id], reason, [AGENTS_SPEC_REF]); +} + +function describeFailure(f: RpcFailure): string { + return `${f.code} ${JSON.stringify(f.message)}${f.data === undefined ? '' : ` data=${JSON.stringify(f.data)}`}`; +} + +export class AgentsDiscoveryScenario implements ClientScenario { + name = 'sep-9998-agents-discovery'; + readonly source = { extensionId: AGENTS_EXTENSION_ID } as const; + description = `agents-wg#22 MCP Agent Discovery Extension: the capability, \`agents/list\`, \`agents/get\`, tool parity with \`tools/list\`, and the error reasons. + +**Methods**: \`agents/list\`, \`agents/get\` (for a server declaring \`${AGENTS_EXTENSION_ID}\`) + +**Requirements covered** (each check carries a verbatim quote of the proposal in src/seps/sep-9998.yaml; 9998 is a placeholder number): +- \`sep-9998-capability-*\`, \`sep-9998-core-without-extension\`: the declaration, and that core functionality does not depend on it +- \`sep-9998-list-*\`, \`sep-9998-card-*\`, \`sep-9998-name-unique\`: the roster's shape, paging and cache hints +- \`sep-9998-get-*\`, \`sep-9998-hint-*\`: the flat details result and its optional routing hint +- \`sep-9998-get-tools-*\`, \`sep-9998-tools-list-not-omitted\`, \`sep-9998-shared-tool-same-definition\`: every returned tool matches \`tools/list\` +- \`sep-9998-agent-not-found\`, \`sep-9998-hidden-indistinguishable\`, \`sep-9998-roster-filtered\`, \`sep-9998-invalid-agent-definition\`: the error reasons and visibility + +The hidden-agent and broken-agent rows need the server's \`${CONTROL_FIXTURE}\` control; without it they report untestable.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect({ + capabilities: AGENTS_CLIENT_CAPABILITIES + }); + try { + const declaration = await declaredAgentsCapability(conn); + const listed = await agentsListAll(conn); + + if (!declaration.declared) { + if ('error' in listed) { + const reason = `The server did not declare ${AGENTS_EXTENSION_ID}, and ${AGENTS_LIST_METHOD} failed (${describeFailure(listed.error)}); the extension is optional.`; + return ALL_IDS.map((id) => skipped(id, reason)); + } + const reason = `The server answers ${AGENTS_LIST_METHOD} but does not declare ${AGENTS_EXTENSION_ID} under capabilities.extensions, so a client following the proposal never calls it.`; + return ALL_IDS.map((id) => + id === 'sep-9998-capability-declared' + ? check(id, [reason]) + : skipped( + id, + 'The extension is not declared; see sep-9998-capability-declared.' + ) + ); + } + + const checks: ConformanceCheck[] = []; + const settings = declaration.value; + + // === capability === + checks.push( + check( + 'sep-9998-capability-declared', + isObject(settings) + ? [] + : [ + `capabilities.extensions["${AGENTS_EXTENSION_ID}"] is ${describeValue(settings)}, not a settings object` + ], + { settings } + ) + ); + if (isObject(settings) && 'listChanged' in settings) { + checks.push( + check( + 'sep-9998-capability-list-changed-boolean', + typeof settings.listChanged === 'boolean' + ? [] + : [ + `listChanged is ${describeValue(settings.listChanged)}, not a boolean` + ], + { listChanged: settings.listChanged } + ) + ); + } else { + checks.push( + skipped( + 'sep-9998-capability-list-changed-boolean', + 'The optional listChanged setting is absent.' + ) + ); + } + + // A second connection that does not declare the extension. + const plain = await ctx.connect(); + let coreTools: Record[] | RpcFailure; + try { + coreTools = await toolsListAll(plain); + } finally { + await plain.close(); + } + checks.push( + check( + 'sep-9998-core-without-extension', + Array.isArray(coreTools) + ? [] + : [ + `tools/list failed for a client that did not declare the extension: ${describeFailure(coreTools)}` + ] + ) + ); + + // === roster === + if ('error' in listed) { + const reason = `${AGENTS_LIST_METHOD} failed: ${describeFailure(listed.error)}`; + checks.push(check('sep-9998-list-implemented', [reason])); + for (const id of ALL_IDS) { + if (!checks.some((c) => c.id === id)) { + checks.push(skipped(id, `${AGENTS_LIST_METHOD} is unavailable.`)); + } + } + return checks; + } + const { pages, truncated } = listed; + checks.push( + check('sep-9998-list-implemented', [], { pages: pages.length }) + ); + + const shapeErrs: string[] = []; + const cacheErrs: string[] = []; + pages.forEach((page, i) => { + if (page.result.resultType !== 'complete') { + shapeErrs.push( + `page ${i + 1}: resultType is ${JSON.stringify(page.result.resultType)}` + ); + } + if (!Array.isArray(page.result.agents)) { + shapeErrs.push( + `page ${i + 1}: agents is ${describeValue(page.result.agents)}` + ); + } + for (const e of cacheableResultErrors(page.result)) { + cacheErrs.push(`page ${i + 1}: ${e}`); + } + }); + checks.push(check('sep-9998-list-result-shape', shapeErrs)); + checks.push( + check( + 'sep-9998-list-pagination', + truncated + ? [ + `${AGENTS_LIST_METHOD} did not terminate: the server kept returning a nextCursor, or repeated one, across ${pages.length} pages` + ] + : [], + { pages: pages.length } + ) + ); + checks.push(check('sep-9998-list-cacheable', cacheErrs)); + + const cards = pages.flatMap((p) => p.agents); + const nameErrs: string[] = []; + const descErrs: string[] = []; + const capErrs: string[] = []; + const cardsOnlyErrs: string[] = []; + const names: string[] = []; + cards.forEach((card, i) => { + if (!isObject(card)) { + nameErrs.push(`card ${i} is ${describeValue(card)}`); + return; + } + const label = + typeof card.name === 'string' ? `"${card.name}"` : `card ${i}`; + if (typeof card.name !== 'string' || card.name === '') { + nameErrs.push(`${label}: name is ${describeValue(card.name)}`); + } else { + names.push(card.name); + } + if ('description' in card && typeof card.description !== 'string') { + descErrs.push( + `${label}: description is ${describeValue(card.description)}` + ); + } + if ( + !Array.isArray(card.capabilities) || + card.capabilities.some((c) => typeof c !== 'string') + ) { + capErrs.push( + `${label}: capabilities is ${Array.isArray(card.capabilities) ? 'an array with a non-string entry' : describeValue(card.capabilities)}` + ); + } + const leaked = DETAIL_MEMBERS.filter((m) => m in card); + if (leaked.length > 0) { + cardsOnlyErrs.push(`${label} carries ${leaked.join(', ')}`); + } + }); + checks.push( + check('sep-9998-card-name', nameErrs, { cards: cards.length }) + ); + checks.push(check('sep-9998-card-description-type', descErrs)); + checks.push(check('sep-9998-card-capabilities', capErrs)); + checks.push( + check('sep-9998-list-cards-only', cardsOnlyErrs, undefined, 'WARNING') + ); + const dupes = names.filter((n, i) => names.indexOf(n) !== i); + checks.push( + check( + 'sep-9998-name-unique', + dupes.length > 0 + ? [ + `name(s) listed more than once: ${[...new Set(dupes)].join(', ')}` + ] + : [], + { names } + ) + ); + + // === details === + const tools = await toolsListAll(conn); + const toolList = Array.isArray(tools) ? tools : []; + const fixture = await fixtureNames(conn, toolList); + const broken = fixture?.brokenAgent; + const toOpen = [...new Set(names)].filter((n) => n !== broken); + + const details: Array<{ name: string; result: Record }> = + []; + const getErrs: string[] = []; + for (const name of toOpen) { + const got = await agentsGet(conn, name); + if ('error' in got) { + getErrs.push(`"${name}": ${describeFailure(got.error)}`); + } else { + details.push({ name, result: got.result }); + } + } + + if (toOpen.length === 0) { + const reason = + 'The roster lists no agent to open, so there are no details to grade.'; + for (const id of [...GET_IDS, ...PARITY_IDS]) { + checks.push(skipped(id, reason)); + } + } else { + checks.push( + check('sep-9998-get-implemented', getErrs, { opened: toOpen.length }) + ); + const resultErrs: string[] = []; + const agentNameErrs: string[] = []; + const getCacheErrs: string[] = []; + const roleErrs: string[] = []; + const visionErrs: string[] = []; + let hinted = 0; + for (const { name, result } of details) { + if (result.resultType !== 'complete') { + resultErrs.push( + `"${name}": resultType is ${JSON.stringify(result.resultType)}` + ); + } + if (!Array.isArray(result.tools)) { + resultErrs.push( + `"${name}": tools is ${describeValue(result.tools)}` + ); + } + if ( + 'instructions' in result && + typeof result.instructions !== 'string' + ) { + resultErrs.push( + `"${name}": instructions is ${describeValue(result.instructions)}` + ); + } + if ('agent' in result === false || result.agent !== name) { + agentNameErrs.push( + `requested "${name}", result.agent is ${JSON.stringify(result.agent)}` + ); + } + for (const e of cacheableResultErrors(result)) { + getCacheErrs.push(`"${name}": ${e}`); + } + if ('modelRoutingHint' in result) { + hinted++; + const hint = result.modelRoutingHint; + if (!isObject(hint)) { + visionErrs.push( + `"${name}": modelRoutingHint is ${describeValue(hint)}` + ); + continue; + } + if (typeof hint.vision !== 'boolean') { + visionErrs.push( + `"${name}": vision is ${describeValue(hint.vision)}` + ); + } + if ( + 'role' in hint && + !(MODEL_ROLES as readonly unknown[]).includes(hint.role) + ) { + roleErrs.push(`"${name}": role is ${JSON.stringify(hint.role)}`); + } + } + } + checks.push(check('sep-9998-get-result-shape', resultErrs)); + checks.push(check('sep-9998-get-agent-name', agentNameErrs)); + checks.push(check('sep-9998-get-cacheable', getCacheErrs)); + if (hinted === 0) { + const reason = + 'No agent returned the optional modelRoutingHint, so there is no hint to grade.'; + checks.push(skipped('sep-9998-hint-role', reason)); + checks.push(skipped('sep-9998-hint-vision', reason)); + } else { + checks.push(check('sep-9998-hint-role', roleErrs, { hinted })); + checks.push(check('sep-9998-hint-vision', visionErrs, { hinted })); + } + + // === parity with tools/list === + if (!Array.isArray(tools)) { + const reason = `tools/list failed (${describeFailure(tools)}), so returned tools cannot be compared with it.`; + for (const id of PARITY_IDS) checks.push(untestable(id, reason)); + } else { + const listedByName = new Map( + tools.map((t) => [String(t.name), t] as const) + ); + const missing: string[] = []; + const differ: string[] = []; + const byToolName = new Map< + string, + { agent: string; def: unknown }[] + >(); + for (const { name, result } of details) { + for (const t of Array.isArray(result.tools) ? result.tools : []) { + const tn = isObject(t) ? String(t.name) : String(t); + const seen = byToolName.get(tn) ?? []; + seen.push({ agent: name, def: t }); + byToolName.set(tn, seen); + const listedDef = listedByName.get(tn); + if (!listedDef) { + missing.push(`"${name}" returns "${tn}"`); + } else if (!isDeepStrictEqual(listedDef, t)) { + differ.push(`"${tn}" under "${name}"`); + } + } + } + const missingErrs = missing.map( + (m) => `${m}, which tools/list does not list` + ); + checks.push(check('sep-9998-get-tools-in-tools-list', missingErrs)); + checks.push(check('sep-9998-get-tools-intersection', missingErrs)); + checks.push(check('sep-9998-tools-list-not-omitted', missingErrs)); + checks.push( + check( + 'sep-9998-get-tools-fields-match', + differ.map((d) => `${d} differs from its tools/list entry`) + ) + ); + const shared = [...byToolName.entries()].filter( + ([, uses]) => new Set(uses.map((u) => u.agent)).size > 1 + ); + if (shared.length === 0) { + checks.push( + skipped( + 'sep-9998-shared-tool-same-definition', + 'No tool name appears under more than one agent.' + ) + ); + } else { + const sharedErrs = shared + .filter(([, uses]) => + uses.some((u) => !isDeepStrictEqual(u.def, uses[0].def)) + ) + .map( + ([tn, uses]) => + `"${tn}" differs between ${uses.map((u) => `"${u.agent}"`).join(' and ')}` + ); + checks.push( + check('sep-9998-shared-tool-same-definition', sharedErrs, { + shared: shared.map(([tn]) => tn) + }) + ); + } + } + } + + // === errors and visibility === + const unknown = await agentsGet(conn, UNKNOWN_AGENT_NAME); + const unknownFailure = 'error' in unknown ? unknown.error : undefined; + checks.push( + check( + 'sep-9998-agent-not-found', + unknownFailure === undefined + ? [`agents/get "${UNKNOWN_AGENT_NAME}" succeeded`] + : unknownFailure.code !== JSONRPC_INVALID_PARAMS || + errorReason(unknownFailure) !== REASON_AGENT_NOT_FOUND + ? [ + `agents/get for an unknown name failed with ${describeFailure(unknownFailure)}` + ] + : [], + unknownFailure ? { error: unknownFailure } : undefined + ) + ); + + const noControl = `The server exposes no ${CONTROL_FIXTURE} control naming the agent this row needs, and an agent hidden from this caller cannot be found any other way.`; + const hidden = fixture?.hiddenAgent; + if (!hidden) { + checks.push(untestable('sep-9998-hidden-indistinguishable', noControl)); + checks.push(untestable('sep-9998-roster-filtered', noControl)); + } else { + const got = await agentsGet(conn, hidden); + const errs: string[] = []; + if (!('error' in got)) { + errs.push( + `agents/get "${hidden}" succeeded for an agent the server reports as hidden from this caller` + ); + } else if (unknownFailure === undefined) { + errs.push( + 'the unknown-name probe did not fail, so there is no not-found response to compare with' + ); + } else { + if (got.error.code !== unknownFailure.code) { + errs.push( + `code ${got.error.code} for the hidden agent, ${unknownFailure.code} for an unknown name` + ); + } + if (got.error.message !== unknownFailure.message) { + errs.push( + `message ${JSON.stringify(got.error.message)} for the hidden agent, ${JSON.stringify(unknownFailure.message)} for an unknown name` + ); + } + if (!isDeepStrictEqual(got.error.data, unknownFailure.data)) { + errs.push( + `data ${JSON.stringify(got.error.data)} for the hidden agent, ${JSON.stringify(unknownFailure.data)} for an unknown name` + ); + } + } + checks.push( + check('sep-9998-hidden-indistinguishable', errs, { hidden }) + ); + checks.push( + check( + 'sep-9998-roster-filtered', + names.includes(hidden) + ? [ + `agents/list lists "${hidden}", which the server reports as hidden from this caller` + ] + : [], + { hidden } + ) + ); + } + + if (!broken) { + checks.push( + untestable( + 'sep-9998-invalid-agent-definition', + `The server exposes no ${CONTROL_FIXTURE} control naming an agent whose membership references a missing tool.` + ) + ); + } else { + const got = await agentsGet(conn, broken); + checks.push( + check( + 'sep-9998-invalid-agent-definition', + !('error' in got) + ? [ + `agents/get "${broken}" succeeded for an agent whose membership names a missing tool` + ] + : got.error.code !== JSONRPC_INTERNAL_ERROR || + errorReason(got.error) !== REASON_INVALID_AGENT_DEFINITION + ? [ + `agents/get "${broken}" failed with ${describeFailure(got.error)}` + ] + : [], + { broken } + ) + ); + } + + return checks; + } finally { + await conn.close(); + } + } +} + +export const AGENTS_DISCOVERY_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/agents/helpers.ts b/src/scenarios/server/agents/helpers.ts new file mode 100644 index 00000000..f7b68860 --- /dev/null +++ b/src/scenarios/server/agents/helpers.ts @@ -0,0 +1,309 @@ +/** + * Shared helpers for the agents-wg#22 (MCP Agent Discovery Extension) server + * conformance scenarios under this directory. + * + * The scenarios treat the server under test as an arbitrary server declaring + * `io.modelcontextprotocol/agents`. Capability is read from discovery, never + * inferred from an error, and every agent is found through `agents/list`, so + * no fixture-specific agent name is hardcoded. The two rows that need an agent + * the harness cannot otherwise find (one hidden from this caller, one whose + * membership names a missing tool) ask the server's diagnostic control for + * their names, and report untestable when it is absent. + * + * The SEP has no number in modelcontextprotocol/modelcontextprotocol yet, so + * check IDs carry the placeholder `sep-9998-`. See src/seps/sep-9998.yaml. + */ + +import type { + CheckStatus, + ConformanceCheck, + SpecReference +} from '../../../types'; +import type { Connection } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; + +export const AGENTS_EXTENSION_ID = 'io.modelcontextprotocol/agents'; + +export const AGENTS_LIST_METHOD = 'agents/list'; +export const AGENTS_GET_METHOD = 'agents/get'; +export const AGENTS_LIST_CHANGED_NOTIFICATION = + 'notifications/agents/list_changed'; + +/** The `subscriptions/listen` filter key the SEP defines. */ +export const AGENTS_LIST_CHANGED_FILTER_KEY = 'agentsListChanged'; + +export const SUBSCRIPTIONS_LISTEN_METHOD = 'subscriptions/listen'; +export const SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION = + 'notifications/subscriptions/acknowledged'; + +export const REASON_AGENT_NOT_FOUND = 'agent_not_found'; +export const REASON_INVALID_AGENT_DEFINITION = 'invalid_agent_definition'; + +export const JSONRPC_METHOD_NOT_FOUND = -32601; +export const JSONRPC_INVALID_PARAMS = -32602; +export const JSONRPC_INTERNAL_ERROR = -32603; + +export const MODEL_ROLES = ['small', 'medium', 'large'] as const; + +/** + * Diagnostic controls the fixture exposes as tools. Both are optional: a + * server without them reports the rows that depend on them as untestable, + * naming the missing tool. + * + * `agents_conformance_fixture` takes no arguments and answers with a text + * item holding `{"hiddenAgent": "", "brokenAgent": ""}`: an agent + * hidden from this caller, and one whose membership names a tool missing from + * the catalog. `agents_conformance_add_agent` takes `{"name": string}` and + * adds a valid agent, which is how the harness changes the roster mid-run. + */ +export const CONTROL_FIXTURE = 'agents_conformance_fixture'; +export const CONTROL_ADD_AGENT = 'agents_conformance_add_agent'; + +const SEP_URL = + 'https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md'; + +export const AGENTS_SPEC_REF: SpecReference = { + id: 'agents-wg#22 (MCP Agent Discovery Extension)', + url: `${SEP_URL}#specification` +}; + +/** + * Build a check carrying the SEP reference. Per AGENTS.md the same `id` flips + * `status` + `errorMessage` between SUCCESS and FAILURE rather than branching + * into distinct slugs. + */ +export function agentsCheck( + id: string, + description: string, + status: CheckStatus, + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: [AGENTS_SPEC_REF], + ...extras + }; +} + +/** A JSON object, as opposed to an array, `null`, or a primitive. */ +export function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** How to name an observed value's type in an error message. */ +export function describeValue(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'an array'; + return `a ${typeof value}`; +} + +export function joinErrs(errs: string[], limit = 5): string { + const shown = errs.slice(0, limit).join('; '); + return errs.length > limit + ? `${shown} (+${errs.length - limit} more)` + : shown; +} + +/** The client capabilities the harness declares: the base set plus this extension. */ +export const AGENTS_CLIENT_CAPABILITIES = { + sampling: {}, + elicitation: {}, + roots: { listChanged: true }, + extensions: { [AGENTS_EXTENSION_ID]: {} } +}; + +/** + * What the server declared under `capabilities.extensions` for this + * extension. `declared` tells absent apart from malformed. + */ +export async function declaredAgentsCapability( + conn: Connection +): Promise<{ declared: boolean; value: unknown }> { + const discovered = await conn.discover(); + const caps = (discovered.capabilities as Record) ?? {}; + const extensions = caps.extensions as Record | undefined; + if (!isObject(extensions) || !(AGENTS_EXTENSION_ID in extensions)) { + return { declared: false, value: undefined }; + } + return { declared: true, value: extensions[AGENTS_EXTENSION_ID] }; +} + +/** A JSON-RPC error, as thrown by `conn.request`. */ +export interface RpcFailure { + code: number; + message: string; + data?: unknown; +} + +export function toRpcFailure(err: unknown): RpcFailure { + if (err instanceof JsonRpcError) { + return { code: err.code, message: err.message, data: err.data }; + } + return { code: 0, message: String(err) }; +} + +/** The `data.reason` of an error, when it carries one. */ +export function errorReason(failure: RpcFailure): string | undefined { + return isObject(failure.data) && typeof failure.data.reason === 'string' + ? failure.data.reason + : undefined; +} + +export interface ListPage { + result: Record; + agents: unknown[]; +} + +/** + * Walk `agents/list` from the first page, following `nextCursor` until the + * server stops sending one. `truncated` means the walk was cut off because + * the server kept answering with a cursor (or repeated one). + */ +export async function agentsListAll( + conn: Connection, + maxPages = 50 +): Promise< + | { pages: ListPage[]; truncated: boolean } + | { error: RpcFailure; pages: ListPage[] } +> { + const pages: ListPage[] = []; + const seen = new Set(); + let cursor: string | undefined; + for (let i = 0; i < maxPages; i++) { + let result: Record; + try { + result = await conn.request>( + AGENTS_LIST_METHOD, + cursor === undefined ? {} : { cursor } + ); + } catch (err) { + return { error: toRpcFailure(err), pages }; + } + const agents = Array.isArray(result?.agents) ? result.agents : []; + pages.push({ result: isObject(result) ? result : {}, agents }); + const next = result?.nextCursor; + if (typeof next !== 'string' || next === '') { + return { pages, truncated: false }; + } + if (seen.has(next)) return { pages, truncated: true }; + seen.add(next); + cursor = next; + } + return { pages, truncated: true }; +} + +/** `agents/get` for one name, as a result or a JSON-RPC failure. */ +export async function agentsGet( + conn: Connection, + name: string +): Promise<{ result: Record } | { error: RpcFailure }> { + try { + const result = await conn.request>( + AGENTS_GET_METHOD, + { name } + ); + return { result: isObject(result) ? result : {} }; + } catch (err) { + return { error: toRpcFailure(err) }; + } +} + +/** Every tool `tools/list` serves, following cursors. */ +export async function toolsListAll( + conn: Connection, + maxPages = 50 +): Promise[] | RpcFailure> { + const out: Record[] = []; + let cursor: string | undefined; + for (let i = 0; i < maxPages; i++) { + let res: { tools?: unknown; nextCursor?: unknown }; + try { + res = await conn.request('tools/list', cursor ? { cursor } : {}); + } catch (err) { + return toRpcFailure(err); + } + for (const t of Array.isArray(res.tools) ? res.tools : []) { + if (isObject(t)) out.push(t); + } + if (typeof res.nextCursor !== 'string' || res.nextCursor === '') break; + cursor = res.nextCursor; + } + return out; +} + +/** + * Call a diagnostic control and return its first text item, or undefined + * when the server rejected it or answered with an error result. + */ +export async function askControl( + conn: Connection, + tool: string, + args: Record +): Promise { + try { + const res = await conn.request<{ + content?: { type?: string; text?: string }[]; + isError?: boolean; + }>('tools/call', { name: tool, arguments: args }); + if (res.isError) return undefined; + const text = (res.content ?? []).find((c) => c?.type === 'text')?.text; + return typeof text === 'string' ? text : undefined; + } catch { + return undefined; + } +} + +/** + * Ask `agents_conformance_fixture` for the names of the hidden and the broken + * agent. Returns undefined when the control is missing or answers with + * something other than the documented JSON. + */ +export async function fixtureNames( + conn: Connection, + tools: Record[] +): Promise<{ hiddenAgent?: string; brokenAgent?: string } | undefined> { + if (!tools.some((t) => t.name === CONTROL_FIXTURE)) return undefined; + const text = await askControl(conn, CONTROL_FIXTURE, {}); + if (text === undefined) return undefined; + try { + const parsed: unknown = JSON.parse(text); + if (!isObject(parsed)) return undefined; + return { + hiddenAgent: + typeof parsed.hiddenAgent === 'string' ? parsed.hiddenAgent : undefined, + brokenAgent: + typeof parsed.brokenAgent === 'string' ? parsed.brokenAgent : undefined + }; + } catch { + return undefined; + } +} + +/** A name no conformant server should serve, for the not-found probe. */ +export const UNKNOWN_AGENT_NAME = 'mcp-conformance-nonexistent-agent-7c41d2'; + +/** A result's cache hints are a non-negative number and a known scope. */ +export function cacheableResultErrors( + result: Record +): string[] { + const errs: string[] = []; + const ttl = result.ttlMs; + if (ttl === undefined) { + errs.push('ttlMs is absent'); + } else if (typeof ttl !== 'number' || !Number.isFinite(ttl) || ttl < 0) { + errs.push(`ttlMs is ${JSON.stringify(ttl)}, not a non-negative number`); + } + const scope = result.cacheScope; + if (scope === undefined) { + errs.push('cacheScope is absent'); + } else if (scope !== 'public' && scope !== 'private') { + errs.push( + `cacheScope is ${JSON.stringify(scope)}, not "public" or "private"` + ); + } + return errs; +} diff --git a/src/scenarios/server/agents/list-changed.ts b/src/scenarios/server/agents/list-changed.ts new file mode 100644 index 00000000..c16b2273 --- /dev/null +++ b/src/scenarios/server/agents/list-changed.ts @@ -0,0 +1,277 @@ +/** + * agents-wg#22 (MCP Agent Discovery Extension): roster changes over + * `subscriptions/listen`. + * + * `notifications/agents/list_changed` reaches a client only on a listen + * stream it opened with `agentsListChanged: true`, and the server's + * acknowledgement says whether it will honor that. The rows hang off one + * stream: + * + * - a server that did not declare `listChanged: true` must not acknowledge + * the key, since the proposal says such notifications are not supported; + * - a server that did declare it acknowledges the key, and sends the + * notification when its roster changes. The harness cannot change a roster + * over the protocol, so it asks the server's `agents_conformance_add_agent` + * control to add one, waits for the notification, then re-lists. + * + * Check IDs carry the placeholder number 9998; see src/seps/sep-9998.yaml. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { openListenStream } from './listen-stream'; +import { + AGENTS_CLIENT_CAPABILITIES, + AGENTS_EXTENSION_ID, + AGENTS_LIST_CHANGED_FILTER_KEY, + AGENTS_LIST_CHANGED_NOTIFICATION, + AGENTS_SPEC_REF, + CONTROL_ADD_AGENT, + SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + SUBSCRIPTIONS_LISTEN_METHOD, + agentsCheck, + agentsListAll, + askControl, + declaredAgentsCapability, + isObject, + toolsListAll +} from './helpers'; + +const NOT_DECLARED_ID = 'sep-9998-list-changed-not-declared'; +const ACK_ID = 'sep-9998-list-changed-acknowledged'; +const NOTIFICATION_ID = 'sep-9998-list-changed-notification'; +const ROSTER_ID = 'sep-9998-list-changed-roster-updated'; + +const ALL_IDS = [NOT_DECLARED_ID, ACK_ID, NOTIFICATION_ID, ROSTER_ID] as const; +type CheckId = (typeof ALL_IDS)[number]; + +const DESCRIPTIONS: Record = { + [NOT_DECLARED_ID]: + 'A server that does not declare listChanged: true does not acknowledge agentsListChanged on subscriptions/listen.', + [ACK_ID]: + 'A server declaring listChanged: true acknowledges agentsListChanged on subscriptions/listen.', + [NOTIFICATION_ID]: + 'A server declaring listChanged: true sends notifications/agents/list_changed on the listen stream when its roster changes.', + [ROSTER_ID]: + 'After the notification, agents/list shows the change it announced.' +}; + +const LISTEN_ACK_MS = 3000; +const LIST_CHANGED_MS = 5000; + +function graded( + id: CheckId, + errs: string[], + details?: Record, + failStatus: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck { + return agentsCheck( + id, + DESCRIPTIONS[id], + errs.length === 0 ? 'SUCCESS' : failStatus, + errs.length === 0 + ? details + ? { details } + : {} + : { errorMessage: errs.join('; '), details } + ); +} + +function skipped(id: CheckId, reason: string): ConformanceCheck { + return agentsCheck(id, DESCRIPTIONS[id], 'SKIPPED', { errorMessage: reason }); +} + +function untestable( + id: CheckId, + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck { + return untestableCheck( + id, + id, + DESCRIPTIONS[id], + reason, + [AGENTS_SPEC_REF], + severity + ); +} + +export class AgentsListChangedScenario implements ClientScenario { + name = 'sep-9998-agents-list-changed'; + readonly source = { extensionId: AGENTS_EXTENSION_ID } as const; + description = `agents-wg#22 MCP Agent Discovery Extension: \`notifications/agents/list_changed\` over \`subscriptions/listen\`. + +**Requirements covered** (each check carries a verbatim quote of the proposal in src/seps/sep-9998.yaml; 9998 is a placeholder number): +- \`${NOT_DECLARED_ID}\`: without \`listChanged: true\`, the server does not acknowledge \`${AGENTS_LIST_CHANGED_FILTER_KEY}\` +- \`${ACK_ID}\`: with it, the server acknowledges the key +- \`${NOTIFICATION_ID}\`: a roster change, made through the \`${CONTROL_ADD_AGENT}\` control, sends the notification on the acknowledged stream +- \`${ROSTER_ID}\`: re-listing afterwards shows the added agent + +Without the \`${CONTROL_ADD_AGENT}\` control the last two rows report untestable.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect({ + capabilities: AGENTS_CLIENT_CAPABILITIES + }); + try { + const declaration = await declaredAgentsCapability(conn); + if (!declaration.declared) { + const reason = `The server did not declare ${AGENTS_EXTENSION_ID}; the extension is optional.`; + return ALL_IDS.map((id) => skipped(id, reason)); + } + const settings = isObject(declaration.value) ? declaration.value : {}; + const declaredTrue = settings.listChanged === true; + + const session = await openListenStream( + ctx.serverUrl, + ctx.specVersion, + { + notifications: { [AGENTS_LIST_CHANGED_FILTER_KEY]: true }, + _meta: { + 'io.modelcontextprotocol/clientCapabilities': + AGENTS_CLIENT_CAPABILITIES + } + }, + { method: SUBSCRIPTIONS_LISTEN_METHOD, openTimeoutMs: LISTEN_ACK_MS } + ); + try { + const ack = session.contentType?.includes('text/event-stream') + ? await session.waitFor( + (n) => n.method === SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + LISTEN_ACK_MS + ) + : undefined; + if (!ack) { + const why = session.error + ? `${SUBSCRIPTIONS_LISTEN_METHOD} failed: ${session.error.code} ${session.error.message}` + : `${SUBSCRIPTIONS_LISTEN_METHOD} did not confirm with ${SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION} within ${LISTEN_ACK_MS}ms (HTTP ${session.status}, ${session.contentType ?? 'no content type'})`; + if (!declaredTrue) { + return [ + untestable(NOT_DECLARED_ID, why), + skipped(ACK_ID, 'The server did not declare listChanged: true.'), + skipped( + NOTIFICATION_ID, + 'The server did not declare listChanged: true.' + ), + skipped( + ROSTER_ID, + 'The server did not declare listChanged: true.' + ) + ]; + } + return [ + skipped(NOT_DECLARED_ID, 'The server declared listChanged: true.'), + graded(ACK_ID, [why]), + untestable(NOTIFICATION_ID, why), + untestable(ROSTER_ID, why, 'WARNING') + ]; + } + + const honored = isObject(ack.params.notifications) + ? ack.params.notifications + : {}; + const echoed = honored[AGENTS_LIST_CHANGED_FILTER_KEY] === true; + + if (!declaredTrue) { + return [ + graded( + NOT_DECLARED_ID, + echoed + ? [ + `The server declared listChanged: ${JSON.stringify(settings.listChanged ?? false)} but acknowledged ${AGENTS_LIST_CHANGED_FILTER_KEY}, promising notifications it says it does not support.` + ] + : [], + { listChanged: settings.listChanged, honored } + ), + skipped(ACK_ID, 'The server did not declare listChanged: true.'), + skipped( + NOTIFICATION_ID, + 'The server did not declare listChanged: true.' + ), + skipped(ROSTER_ID, 'The server did not declare listChanged: true.') + ]; + } + + const checks: ConformanceCheck[] = [ + skipped( + NOT_DECLARED_ID, + 'The server declared listChanged: true; the rule covers servers that did not.' + ) + ]; + if (!echoed) { + const why = `The server declared listChanged: true but its acknowledgement does not honor ${AGENTS_LIST_CHANGED_FILTER_KEY}, which is the only channel the notification travels on.`; + checks.push(graded(ACK_ID, [why], { honored })); + checks.push(graded(NOTIFICATION_ID, [why], { honored })); + checks.push(skipped(ROSTER_ID, 'No notification can arrive.')); + return checks; + } + checks.push(graded(ACK_ID, [], { honored })); + + const tools = await toolsListAll(conn); + const hasControl = + Array.isArray(tools) && + tools.some((t) => t.name === CONTROL_ADD_AGENT); + const name = `mcp-conformance-added-${Math.random().toString(36).slice(2, 10)}`; + const added = hasControl + ? await askControl(conn, CONTROL_ADD_AGENT, { name }) + : undefined; + if (added === undefined) { + const why = hasControl + ? `The ${CONTROL_ADD_AGENT} control rejected {name: "${name}"}.` + : `The server exposes no ${CONTROL_ADD_AGENT} control, and the harness has no other way to change its roster mid-run.`; + checks.push(untestable(NOTIFICATION_ID, why)); + checks.push(untestable(ROSTER_ID, why, 'WARNING')); + return checks; + } + + const hit = await session.waitFor( + (n) => n.method === AGENTS_LIST_CHANGED_NOTIFICATION, + LIST_CHANGED_MS + ); + checks.push( + graded( + NOTIFICATION_ID, + hit + ? [] + : [ + `Added agent "${name}" through ${CONTROL_ADD_AGENT}, and no ${AGENTS_LIST_CHANGED_NOTIFICATION} arrived on the acknowledged listen stream within ${LIST_CHANGED_MS}ms.` + ], + hit ? { added: name, atMs: hit.atMs } : { added: name } + ) + ); + + const relisted = await agentsListAll(conn); + const names = + 'error' in relisted + ? undefined + : relisted.pages + .flatMap((p) => p.agents) + .map((c) => (isObject(c) ? c.name : undefined)); + checks.push( + graded( + ROSTER_ID, + names === undefined + ? [ + `agents/list failed after the change: ${'error' in relisted ? relisted.error.message : ''}` + ] + : names.includes(name) + ? [] + : [ + `agents/list does not list "${name}" after the server announced a roster change.` + ], + { added: name }, + 'WARNING' + ) + ); + return checks; + } finally { + await session.cancel(); + } + } finally { + await conn.close(); + } + } +} + +export const AGENTS_LIST_CHANGED_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/agents/listen-stream.ts b/src/scenarios/server/agents/listen-stream.ts new file mode 100644 index 00000000..cd5702b7 --- /dev/null +++ b/src/scenarios/server/agents/listen-stream.ts @@ -0,0 +1,231 @@ +/** + * A long-lived `subscriptions/listen` request, which the `Connection` + * abstraction cannot express. + * + * `conn.request()` resolves when the response for its JSON-RPC id arrives, and + * a listen stream withholds exactly that: the server acknowledges with a + * notification, delivers further notifications for as long as the stream is + * open, and sends a result only on close. So this opens the POST itself, + * hands back a session while the stream is still open, and appends frames as + * they arrive. The scenario waits for the frames it needs, grades them, then + * aborts. + * + * Adapted from the MCP Events suite's stream helper, which is not on upstream + * main yet. + */ + +import { + buildStandardHeaders, + withRequestMeta, + type JsonRpcResponse +} from '../../../connection'; +import type { SpecVersion } from '../../../types'; +import { isObject } from './helpers'; + +/** A JSON-RPC notification as it arrived on the stream. */ +export interface StreamNotification { + method: string; + params: Record; + /** Milliseconds since the stream was opened, for the heartbeat interval. */ + atMs: number; +} + +export interface StreamSession { + /** The JSON-RPC id of the listen request, echoed in `_meta`. */ + readonly requestId: number; + /** HTTP status of the POST that opened the stream. */ + readonly status: number; + readonly contentType?: string; + /** Set when the server answered with an immediate JSON-RPC error. */ + readonly error?: { code: number; message: string; data?: unknown }; + /** Set when a response for `requestId` arrived (the stream closed). */ + readonly finalResult?: JsonRpcResponse; + /** Notifications in arrival order. */ + readonly notifications: StreamNotification[]; + /** Frames that were neither a notification nor this request's response. */ + readonly foreignFrames: unknown[]; + /** SSE comment lines, kept for diagnostics. */ + readonly sseComments: string[]; + /** Whether the reader is still running. */ + readonly open: boolean; + /** Resolve once a notification matches, or undefined at the deadline. */ + waitFor( + predicate: (n: StreamNotification) => boolean, + timeoutMs: number + ): Promise; + /** Resolve after `ms`, or earlier if the stream closes. */ + settle(ms: number): Promise; + /** Abort the request stream. */ + cancel(): Promise; +} + +/** Open a long-lived request and return once the server has answered the POST. */ +export async function openListenStream( + serverUrl: string, + specVersion: SpecVersion, + params: Record, + options: { openTimeoutMs?: number; method?: string } = {} +): Promise { + const method = options.method ?? 'subscriptions/listen'; + const requestId = Math.floor(Math.random() * 1_000_000) + 1; + const headers = buildStandardHeaders(method, params, { + specVersion + }); + const body = JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method, + params: withRequestMeta(params, specVersion) + }); + + const controller = new AbortController(); + const openedAt = Date.now(); + const notifications: StreamNotification[] = []; + const foreignFrames: unknown[] = []; + const sseComments: string[] = []; + const state = { + open: true, + error: undefined as StreamSession['error'], + finalResult: undefined as JsonRpcResponse | undefined + }; + + const res = await fetch(serverUrl, { + method: 'POST', + headers, + body, + signal: controller.signal + }); + const contentType = res.headers.get('content-type') ?? undefined; + + const ingest = (frame: unknown): void => { + if (!isObject(frame)) { + foreignFrames.push(frame); + return; + } + if (typeof frame.method === 'string' && frame.id === undefined) { + notifications.push({ + method: frame.method, + params: isObject(frame.params) ? frame.params : {}, + atMs: Date.now() - openedAt + }); + return; + } + if (frame.id === requestId) { + state.finalResult = frame as unknown as JsonRpcResponse; + if (isObject(frame.error)) { + state.error = { + code: Number(frame.error.code), + message: String(frame.error.message ?? ''), + data: frame.error.data + }; + } + return; + } + foreignFrames.push(frame); + }; + + // A JSON body means the server answered rather than streamed, usually an + // immediate error for a request it would not hold open. + if (!contentType?.includes('text/event-stream')) { + state.open = false; + const text = await res.text(); + try { + ingest(JSON.parse(text)); + } catch { + foreignFrames.push(text); + } + } else { + void readFrames(res, ingest, sseComments).finally(() => { + state.open = false; + }); + } + + const session: StreamSession = { + requestId, + status: res.status, + contentType, + get error() { + return state.error; + }, + get finalResult() { + return state.finalResult; + }, + get open() { + return state.open; + }, + notifications, + foreignFrames, + sseComments, + async waitFor(predicate, timeoutMs) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const hit = notifications.find(predicate); + if (hit) return hit; + if (!state.open || Date.now() >= deadline) { + return notifications.find(predicate); + } + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async settle(ms) { + const deadline = Date.now() + ms; + while (state.open && Date.now() < deadline) { + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async cancel() { + controller.abort(); + state.open = false; + } + }; + + // Give a streaming server a moment to write its confirmation frame, so a + // caller that opens and immediately grades is not racing the first write. + if (state.open) { + await session.waitFor(() => true, options.openTimeoutMs ?? 2000); + } + return session; +} + +/** Read SSE `data:` frames until the stream ends or the request is aborted. */ +async function readFrames( + res: Response, + ingest: (frame: unknown) => void, + sseComments: string[] +): Promise { + if (!res.body) return; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + if (!value) continue; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() ?? ''; + for (const line of lines) { + if (line.startsWith(':')) { + sseComments.push(line); + continue; + } + if (!line.startsWith('data:')) continue; + const payload = line.slice(5).trim(); + if (!payload) continue; + try { + ingest(JSON.parse(payload)); + } catch { + ingest(payload); + } + } + } + } catch { + // Aborted by cancel(), or the connection dropped. The frames that arrived + // are what the scenario grades. + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/types.ts b/src/types.ts index ebe75a27..57d9793d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -103,7 +103,8 @@ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/auth/dpop', 'io.modelcontextprotocol/auth/wif', 'io.modelcontextprotocol/tasks', - 'io.modelcontextprotocol/skills' + 'io.modelcontextprotocol/skills', + 'io.modelcontextprotocol/agents' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number]; From 4661c430ca997f803cafffd18bef8e3f37ff2e8e Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Wed, 14 Oct 2026 23:54:17 -0700 Subject: [PATCH 3/3] test(agents): negative controls for the agent discovery scenarios A stateless fixture built per test, conformant by default, with one option per rule. 40 tests: a conformant run with every check passing, then one broken rule per test asserting its check flips. --- .../server/agents/negative-fixture.ts | 401 ++++++++++++++++ src/scenarios/server/agents/negative.test.ts | 433 ++++++++++++++++++ 2 files changed, 834 insertions(+) create mode 100644 src/scenarios/server/agents/negative-fixture.ts create mode 100644 src/scenarios/server/agents/negative.test.ts diff --git a/src/scenarios/server/agents/negative-fixture.ts b/src/scenarios/server/agents/negative-fixture.ts new file mode 100644 index 00000000..10335d47 --- /dev/null +++ b/src/scenarios/server/agents/negative-fixture.ts @@ -0,0 +1,401 @@ +/** + * The fixture the agent discovery negative controls are graded against. + * + * One server, two scenarios. Every option here exists to break exactly one + * rule while leaving the rest of the wire conformant, so a check that flips + * says which rule it caught. With no options it is a conformant server, which + * is what the false-positive test runs against. + * + * It speaks the SEP-2575 stateless wire (`server/discover` plus one POST per + * request), built per test rather than checked in as an example server, as the + * SEP-2640 negative tests do. `subscriptions/listen` is a real SSE response + * held open until the client aborts, so the list-changed rows see a + * notification arrive on a live stream. + */ + +import { createServer, type IncomingMessage, type ServerResponse } from 'http'; +import type { AddressInfo } from 'net'; +import { withRequiredDraftResultFields } from '../../../mock-server'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { + AGENTS_EXTENSION_ID, + AGENTS_LIST_CHANGED_FILTER_KEY, + AGENTS_LIST_CHANGED_NOTIFICATION, + CONTROL_ADD_AGENT, + CONTROL_FIXTURE, + SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + isObject +} from './helpers'; + +export interface FixtureAgent { + name: string; + description?: string; + capabilities?: string[]; + instructions?: string; + modelRoutingHint?: unknown; + tools: string[]; +} + +/** The catalog every fixture serves. `search` is shared by two agents. */ +export function catalog(): Record[] { + return [ + { + name: 'search', + description: 'Search the index.', + inputSchema: { type: 'object', properties: { q: { type: 'string' } } } + }, + { + name: 'summarize', + description: 'Summarize a document.', + inputSchema: { type: 'object', properties: {} } + }, + { + name: 'secret_tool', + description: 'Belongs to the hidden agent.', + inputSchema: { type: 'object', properties: {} } + } + ]; +} + +/** The roster every fixture serves, before any test overrides it. */ +export function roster(): FixtureAgent[] { + return [ + { + name: 'research-agent', + description: 'Researches things', + capabilities: ['research'], + instructions: 'Research the question.', + modelRoutingHint: { role: 'medium', vision: false }, + tools: ['search'] + }, + { + name: 'writer-agent', + capabilities: ['writing'], + tools: ['search', 'summarize'] + } + ]; +} + +export const HIDDEN_AGENT = 'hidden-agent'; +export const BROKEN_AGENT = 'broken-agent'; + +const NOT_FOUND = { + code: -32602, + message: 'agent not found', + data: { reason: 'agent_not_found' } +}; + +export interface AgentsFixtureOptions { + /** The value declared under extensions; `null` leaves it undeclared. */ + declared?: unknown; + agents?: FixtureAgent[]; + /** Agents per agents/list page. Default 1, so paging is exercised. */ + pageSize?: number; + /** Answer agents/list with -32601 instead of a roster. */ + listUnimplemented?: boolean; + /** Members merged into every agents/list page result. */ + listOverrides?: Record; + /** Hand out the same cursor forever. */ + loopingCursor?: boolean; + /** Members merged into the card for one agent name. */ + cardOverrides?: Record>; + /** Members merged into the agents/get result for one agent name. */ + getOverrides?: Record>; + /** Fail agents/get for these listed agents with -32603. */ + getFails?: string[]; + /** Tools served by agents/get for one agent, replacing the catalog lookup. */ + getTools?: Record[]>; + /** The error agents/get gives for a name it does not know. */ + unknownError?: Record; + /** The error agents/get gives for the hidden agent. */ + hiddenError?: Record; + /** List the hidden agent in agents/list anyway. */ + listHidden?: boolean; + /** The error agents/get gives for the broken agent. */ + brokenError?: Record; + /** Refuse tools/list from a client that did not declare the extension. */ + coreRequiresExtension?: boolean; + /** Leave out the named control tools. */ + withoutControls?: string[]; + /** Leave agentsListChanged out of the listen acknowledgement. */ + ackDropsKey?: boolean; + /** Acknowledge agentsListChanged even when listChanged is not declared. */ + ackAlways?: boolean; + /** Add agents without sending notifications/agents/list_changed. */ + silentAdd?: boolean; + /** Send the notification but leave the roster unchanged. */ + addWithoutChange?: boolean; +} + +export interface AgentsFixture { + url: string; + close(): Promise; +} + +async function readJsonBody( + req: IncomingMessage +): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; +} + +function clientDeclaredAgents(params: Record): boolean { + const meta = isObject(params._meta) ? params._meta : {}; + const caps = meta['io.modelcontextprotocol/clientCapabilities']; + return ( + isObject(caps) && + isObject(caps.extensions) && + AGENTS_EXTENSION_ID in caps.extensions + ); +} + +export async function startAgentsFixture( + opts: AgentsFixtureOptions = {} +): Promise { + const declared = + opts.declared === undefined ? { listChanged: true } : opts.declared; + const agents = (opts.agents ?? roster()).map((a) => ({ ...a })); + const tools = catalog(); + const pageSize = opts.pageSize ?? 1; + const listeners = new Set(); + const without = new Set(opts.withoutControls ?? []); + + const controls = [ + { + name: CONTROL_FIXTURE, + description: 'Names the hidden and broken agents.', + inputSchema: { type: 'object', properties: {} } + }, + { + name: CONTROL_ADD_AGENT, + description: 'Adds an agent to the roster.', + inputSchema: { + type: 'object', + properties: { name: { type: 'string' } }, + required: ['name'] + } + } + ].filter((c) => !without.has(c.name)); + + const visible = (): FixtureAgent[] => { + const out = [ + ...agents, + { name: BROKEN_AGENT, capabilities: [], tools: ['missing_tool'] } + ]; + if (opts.listHidden) { + out.push({ + name: HIDDEN_AGENT, + capabilities: [], + tools: ['secret_tool'] + }); + } + return out; + }; + + const card = (a: FixtureAgent): Record => { + const c: Record = { + name: a.name, + capabilities: a.capabilities ?? [] + }; + if (a.description !== undefined) c.description = a.description; + return { ...c, ...(opts.cardOverrides?.[a.name] ?? {}) }; + }; + + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + const body = await readJsonBody(req); + const method = body.method as string; + const id = body.id; + const params = isObject(body.params) ? body.params : {}; + const send = (result: object) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id, + result: withRequiredDraftResultFields(method, result) + }) + ); + }; + const fail = (error: Record) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ jsonrpc: '2.0', id, error })); + }; + + switch (method) { + case 'server/discover': { + const extensions = + declared === null ? {} : { [AGENTS_EXTENSION_ID]: declared }; + send({ + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: { tools: {}, extensions }, + serverInfo: { name: 'agents-negative', version: '1.0.0' } + }); + return; + } + case 'tools/list': { + if (opts.coreRequiresExtension && !clientDeclaredAgents(params)) { + fail({ code: -32600, message: 'declare the agents extension first' }); + return; + } + send({ tools: [...tools, ...controls] }); + return; + } + case 'tools/call': { + const name = params.name; + const args = isObject(params.arguments) ? params.arguments : {}; + if (name === CONTROL_FIXTURE && !without.has(CONTROL_FIXTURE)) { + send({ + content: [ + { + type: 'text', + text: JSON.stringify({ + hiddenAgent: HIDDEN_AGENT, + brokenAgent: BROKEN_AGENT + }) + } + ] + }); + return; + } + if (name === CONTROL_ADD_AGENT && !without.has(CONTROL_ADD_AGENT)) { + if (!opts.addWithoutChange) { + agents.push({ + name: String(args.name), + capabilities: [], + tools: ['search'] + }); + } + if (!opts.silentAdd) { + const frame = JSON.stringify({ + jsonrpc: '2.0', + method: AGENTS_LIST_CHANGED_NOTIFICATION, + params: {} + }); + for (const l of listeners) l.write(`data: ${frame}\n\n`); + } + send({ content: [{ type: 'text', text: 'ok' }] }); + return; + } + send({ content: [{ type: 'text', text: 'done' }] }); + return; + } + case 'agents/list': { + if (opts.listUnimplemented) { + fail({ code: -32601, message: 'method not found' }); + return; + } + const all = visible(); + const start = + typeof params.cursor === 'string' ? Number(params.cursor) : 0; + const page = all.slice(start, start + pageSize).map(card); + const next = + start + pageSize < all.length ? String(start + pageSize) : undefined; + const result: Record = { + agents: page, + ttlMs: 60000, + cacheScope: 'private', + ...(opts.loopingCursor + ? { nextCursor: '0' } + : next + ? { nextCursor: next } + : {}), + ...(opts.listOverrides ?? {}) + }; + send(result); + return; + } + case 'agents/get': { + const name = params.name; + if (name === HIDDEN_AGENT) { + fail(opts.hiddenError ?? NOT_FOUND); + return; + } + if (name === BROKEN_AGENT) { + fail( + opts.brokenError ?? { + code: -32603, + message: 'agent names a missing tool', + data: { reason: 'invalid_agent_definition' } + } + ); + return; + } + const agent = agents.find((a) => a.name === name); + if (!agent) { + fail(opts.unknownError ?? NOT_FOUND); + return; + } + if (opts.getFails?.includes(agent.name)) { + fail({ code: -32603, message: 'boom' }); + return; + } + const result: Record = { + agent: agent.name, + tools: + opts.getTools?.[agent.name] ?? + agent.tools.map((t) => tools.find((c) => c.name === t)), + ttlMs: 60000, + cacheScope: 'private' + }; + if (agent.instructions !== undefined) { + result.instructions = agent.instructions; + } + if (agent.modelRoutingHint !== undefined) { + result.modelRoutingHint = agent.modelRoutingHint; + } + send({ ...result, ...(opts.getOverrides?.[agent.name] ?? {}) }); + return; + } + case 'subscriptions/listen': { + const filter = isObject(params.notifications) + ? params.notifications + : {}; + const declaredTrue = + isObject(declared) && declared.listChanged === true; + const honor = + filter[AGENTS_LIST_CHANGED_FILTER_KEY] === true && + (declaredTrue || opts.ackAlways) && + !opts.ackDropsKey; + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + const ack = JSON.stringify({ + jsonrpc: '2.0', + method: SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + params: { + notifications: honor + ? { [AGENTS_LIST_CHANGED_FILTER_KEY]: true } + : {}, + _meta: { 'io.modelcontextprotocol/subscriptionId': id } + } + }); + res.write(`data: ${ack}\n\n`); + listeners.add(res); + req.on('close', () => listeners.delete(res)); + return; + } + default: + fail({ code: -32601, message: `method not found: ${method}` }); + } + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}/mcp`, + close: () => + new Promise((resolve) => { + for (const l of listeners) l.end(); + server.closeAllConnections?.(); + server.close(() => resolve()); + }) + }; +} diff --git a/src/scenarios/server/agents/negative.test.ts b/src/scenarios/server/agents/negative.test.ts new file mode 100644 index 00000000..f06d66e3 --- /dev/null +++ b/src/scenarios/server/agents/negative.test.ts @@ -0,0 +1,433 @@ +import { describe, test, expect } from 'vitest'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { AgentsDiscoveryScenario } from './discovery'; +import { AgentsListChangedScenario } from './list-changed'; +import { + BROKEN_AGENT, + HIDDEN_AGENT, + catalog, + startAgentsFixture, + type AgentsFixtureOptions +} from './negative-fixture'; + +/** + * Negative controls for the agent discovery scenarios. + * + * A passing run against a conformant fixture proves a check does not + * false-positive. It does not prove the check catches anything, which is what + * these tests are for: each one breaks the fixture in exactly one way and + * asserts the matching check flips. The fixture lives in negative-fixture.ts. + */ + +type Scenario = AgentsDiscoveryScenario | AgentsListChangedScenario; + +async function checksFor(scenario: Scenario, opts: AgentsFixtureOptions) { + const fixture = await startAgentsFixture(opts); + try { + const checks = await scenario.run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed response does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await fixture.close(); + } +} + +const discovery = () => new AgentsDiscoveryScenario(); +const listChanged = () => new AgentsListChangedScenario(); + +describe('agents discovery: a conformant fixture', () => { + test('every check passes, and only the optional ones skip', async () => { + const checks = await checksFor(discovery(), {}); + const notPassing = [...checks.values()] + .filter((c) => c.status !== 'SUCCESS') + .map((c) => `${c.id}: ${c.status} ${c.errorMessage ?? ''}`); + expect(notPassing).toEqual([]); + expect(checks.size).toBe(27); + }); + + test('each check ID is emitted exactly once', async () => { + const fixture = await startAgentsFixture({}); + try { + const checks = await discovery().run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + takeWireViolations(); + const ids = checks.map((c) => c.id); + expect(new Set(ids).size).toBe(ids.length); + } finally { + await fixture.close(); + } + }); +}); + +describe('agents discovery: capability', () => { + test('an undeclared extension the server does not serve skips everything', async () => { + const checks = await checksFor(discovery(), { + declared: null, + listUnimplemented: true + }); + expect([...checks.values()].every((c) => c.status === 'SKIPPED')).toBe( + true + ); + }); + + test('serving agents/list without declaring the extension fails', async () => { + const checks = await checksFor(discovery(), { declared: null }); + expect(checks.get('sep-9998-capability-declared')?.status).toBe('FAILURE'); + }); + + test('a declaration that is not a settings object fails', async () => { + const checks = await checksFor(discovery(), { declared: true }); + expect(checks.get('sep-9998-capability-declared')?.status).toBe('FAILURE'); + }); + + test('a non-boolean listChanged fails', async () => { + const checks = await checksFor(discovery(), { + declared: { listChanged: 'yes' } + }); + expect(checks.get('sep-9998-capability-list-changed-boolean')?.status).toBe( + 'FAILURE' + ); + }); + + test('refusing tools/list to a client without the extension fails', async () => { + const checks = await checksFor(discovery(), { + coreRequiresExtension: true + }); + expect(checks.get('sep-9998-core-without-extension')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe('agents discovery: roster', () => { + test('a declared extension without agents/list fails', async () => { + const checks = await checksFor(discovery(), { listUnimplemented: true }); + expect(checks.get('sep-9998-list-implemented')?.status).toBe('FAILURE'); + }); + + test('a page without resultType fails', async () => { + const checks = await checksFor(discovery(), { + listOverrides: { resultType: 'incomplete' } + }); + expect(checks.get('sep-9998-list-result-shape')?.status).toBe('FAILURE'); + }); + + test('a cursor that never ends fails', async () => { + const checks = await checksFor(discovery(), { loopingCursor: true }); + expect(checks.get('sep-9998-list-pagination')?.status).toBe('FAILURE'); + }); + + test('a page without ttlMs fails', async () => { + const checks = await checksFor(discovery(), { + listOverrides: { ttlMs: undefined } + }); + expect(checks.get('sep-9998-list-cacheable')?.status).toBe('FAILURE'); + }); + + test('a card whose name is not a string fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { name: 7 } } + }); + expect(checks.get('sep-9998-card-name')?.status).toBe('FAILURE'); + }); + + test('a description that is not a string fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { description: 3 } } + }); + expect(checks.get('sep-9998-card-description-type')?.status).toBe( + 'FAILURE' + ); + }); + + test('a card without capabilities fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { capabilities: undefined } } + }); + expect(checks.get('sep-9998-card-capabilities')?.status).toBe('FAILURE'); + }); + + test('a card carrying instructions warns', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { instructions: 'leaked' } } + }); + expect(checks.get('sep-9998-list-cards-only')?.status).toBe('WARNING'); + }); + + test('a name listed on two pages fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { name: 'research-agent' } } + }); + expect(checks.get('sep-9998-name-unique')?.status).toBe('FAILURE'); + }); +}); + +describe('agents discovery: details', () => { + test('agents/get failing for a listed agent fails', async () => { + const checks = await checksFor(discovery(), { getFails: ['writer-agent'] }); + expect(checks.get('sep-9998-get-implemented')?.status).toBe('FAILURE'); + }); + + test('a details result without resultType fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'writer-agent': { resultType: 'incomplete' } } + }); + expect(checks.get('sep-9998-get-result-shape')?.status).toBe('FAILURE'); + }); + + test('a details result naming another agent fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'writer-agent': { agent: 'research-agent' } } + }); + expect(checks.get('sep-9998-get-agent-name')?.status).toBe('FAILURE'); + }); + + test('a details result without cacheScope fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'writer-agent': { cacheScope: undefined } } + }); + expect(checks.get('sep-9998-get-cacheable')?.status).toBe('FAILURE'); + }); + + test('a role outside the enum fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { + 'research-agent': { modelRoutingHint: { role: 'huge', vision: false } } + } + }); + expect(checks.get('sep-9998-hint-role')?.status).toBe('FAILURE'); + }); + + test('a hint without vision fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { + 'research-agent': { modelRoutingHint: { role: 'small' } } + } + }); + expect(checks.get('sep-9998-hint-vision')?.status).toBe('FAILURE'); + }); + + test('no hint anywhere skips the hint rows', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'research-agent': { modelRoutingHint: undefined } } + }); + expect(checks.get('sep-9998-hint-role')?.status).toBe('SKIPPED'); + expect(checks.get('sep-9998-hint-vision')?.status).toBe('SKIPPED'); + }); +}); + +describe('agents discovery: tool parity', () => { + test('a returned tool missing from tools/list fails all three parity rows', async () => { + const checks = await checksFor(discovery(), { + getTools: { + 'writer-agent': [ + ...catalog().slice(0, 2), + { + name: 'unlisted_tool', + inputSchema: { type: 'object', properties: {} } + } + ] + } + }); + for (const id of [ + 'sep-9998-get-tools-in-tools-list', + 'sep-9998-get-tools-intersection', + 'sep-9998-tools-list-not-omitted' + ]) { + expect(checks.get(id)?.status, id).toBe('FAILURE'); + } + expect(checks.get('sep-9998-get-tools-fields-match')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a returned tool whose fields differ from tools/list fails', async () => { + const [search, summarize] = catalog(); + const checks = await checksFor(discovery(), { + getTools: { + 'writer-agent': [search, { ...summarize, description: 'Different.' }] + } + }); + expect(checks.get('sep-9998-get-tools-fields-match')?.status).toBe( + 'FAILURE' + ); + expect(checks.get('sep-9998-get-tools-in-tools-list')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a shared tool that differs between agents fails', async () => { + const [search, summarize] = catalog(); + const checks = await checksFor(discovery(), { + getTools: { + 'writer-agent': [ + { ...search, description: 'Another search.' }, + summarize + ] + } + }); + expect(checks.get('sep-9998-shared-tool-same-definition')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe('agents discovery: errors and visibility', () => { + test('an unknown name without the agent_not_found reason fails', async () => { + const checks = await checksFor(discovery(), { + unknownError: { code: -32602, message: 'agent not found' } + }); + expect(checks.get('sep-9998-agent-not-found')?.status).toBe('FAILURE'); + }); + + test('a hidden agent answering differently from an unknown one fails', async () => { + const checks = await checksFor(discovery(), { + hiddenError: { + code: -32602, + message: 'you may not see this agent', + data: { reason: 'agent_not_found' } + } + }); + expect(checks.get('sep-9998-hidden-indistinguishable')?.status).toBe( + 'FAILURE' + ); + expect(checks.get('sep-9998-roster-filtered')?.status).toBe('SUCCESS'); + }); + + test('a hidden agent listed in the roster fails', async () => { + const checks = await checksFor(discovery(), { listHidden: true }); + expect(checks.get('sep-9998-roster-filtered')?.status).toBe('FAILURE'); + expect(checks.get('sep-9998-hidden-indistinguishable')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a broken agent answering -32602 fails', async () => { + const checks = await checksFor(discovery(), { + brokenError: { + code: -32602, + message: 'missing tool', + data: { reason: 'invalid_agent_definition' } + } + }); + expect(checks.get('sep-9998-invalid-agent-definition')?.status).toBe( + 'FAILURE' + ); + }); + + test('a broken agent answering -32603 without the reason fails', async () => { + const checks = await checksFor(discovery(), { + brokenError: { code: -32603, message: 'missing tool' } + }); + expect(checks.get('sep-9998-invalid-agent-definition')?.status).toBe( + 'FAILURE' + ); + }); + + test('without the fixture control the three rows report untestable', async () => { + const checks = await checksFor(discovery(), { + withoutControls: ['agents_conformance_fixture'] + }); + for (const id of [ + 'sep-9998-hidden-indistinguishable', + 'sep-9998-roster-filtered', + 'sep-9998-invalid-agent-definition' + ]) { + const c = checks.get(id); + expect(c?.status, id).toBe('FAILURE'); + expect(c?.details?.untestable, id).toBe(true); + } + // Without the control the broken agent is opened like any other, and its + // -32603 is reported where it lands. + expect(checks.get('sep-9998-get-implemented')?.errorMessage).toContain( + BROKEN_AGENT + ); + expect(HIDDEN_AGENT).toBe('hidden-agent'); + }); +}); + +describe('agents list_changed', () => { + test('a conformant fixture passes', async () => { + const checks = await checksFor(listChanged(), {}); + expect(checks.get('sep-9998-list-changed-not-declared')?.status).toBe( + 'SKIPPED' + ); + for (const id of [ + 'sep-9998-list-changed-acknowledged', + 'sep-9998-list-changed-notification', + 'sep-9998-list-changed-roster-updated' + ]) { + expect( + checks.get(id)?.status, + `${id}: ${checks.get(id)?.errorMessage}` + ).toBe('SUCCESS'); + } + }); + + test('an undeclared extension skips everything', async () => { + const checks = await checksFor(listChanged(), { declared: null }); + expect([...checks.values()].every((c) => c.status === 'SKIPPED')).toBe( + true + ); + }); + + test('acknowledging the key without declaring listChanged fails', async () => { + const checks = await checksFor(listChanged(), { + declared: {}, + ackAlways: true + }); + expect(checks.get('sep-9998-list-changed-not-declared')?.status).toBe( + 'FAILURE' + ); + }); + + test('not acknowledging the key without declaring listChanged passes', async () => { + const checks = await checksFor(listChanged(), { declared: {} }); + expect(checks.get('sep-9998-list-changed-not-declared')?.status).toBe( + 'SUCCESS' + ); + }); + + test('declaring listChanged but dropping the key fails', async () => { + const checks = await checksFor(listChanged(), { ackDropsKey: true }); + expect(checks.get('sep-9998-list-changed-acknowledged')?.status).toBe( + 'FAILURE' + ); + }); + + test('a roster change without the notification fails', async () => { + const checks = await checksFor(listChanged(), { silentAdd: true }); + expect(checks.get('sep-9998-list-changed-notification')?.status).toBe( + 'FAILURE' + ); + }, 15000); + + test('a notification whose change never shows in agents/list warns', async () => { + const checks = await checksFor(listChanged(), { addWithoutChange: true }); + expect(checks.get('sep-9998-list-changed-notification')?.status).toBe( + 'SUCCESS' + ); + expect(checks.get('sep-9998-list-changed-roster-updated')?.status).toBe( + 'WARNING' + ); + }); + + test('without the add control the two rows report untestable', async () => { + const checks = await checksFor(listChanged(), { + withoutControls: ['agents_conformance_add_agent'] + }); + expect( + checks.get('sep-9998-list-changed-notification')?.details?.untestable + ).toBe(true); + expect( + checks.get('sep-9998-list-changed-roster-updated')?.details?.untestable + ).toBe(true); + }); +});