diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index ff377f54..c5885a97 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -63,6 +63,8 @@ import { import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; +import { AgentsDiscoveryScenario } from './server/agents/discovery'; +import { AgentsListChangedScenario } from './server/agents/list-changed'; import { PromptsListScenario, @@ -166,7 +168,14 @@ const pendingClientScenariosList: ClientScenario[] = [ // `npm start -- server --scenario sep-2640-skills-* --url `. new SkillsDirectoryReadScenario(), new SkillsEnumerationScenario(), - new SkillsManifestScenario() + new SkillsManifestScenario(), + + // MCP Agent Discovery Extension (agents-wg#22, placeholder number 9998). + // Pending because the everything-server does not implement + // io.modelcontextprotocol/agents; targeted runs point at a conformant + // fixture via `npm start -- server --scenario sep-9998-agents-* --url `. + new AgentsDiscoveryScenario(), + new AgentsListChangedScenario() ]; // All client scenarios @@ -224,6 +233,11 @@ const allClientScenariosList: ClientScenario[] = [ new SkillsEnumerationScenario(), new SkillsManifestScenario(), + // Agent discovery extension (agents-wg#22). Fixture-dependent; each + // scenario SKIPs cleanly when the extension is not declared. + new AgentsDiscoveryScenario(), + new AgentsListChangedScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), diff --git a/src/scenarios/server/agents/discovery.ts b/src/scenarios/server/agents/discovery.ts new file mode 100644 index 00000000..a555ea34 --- /dev/null +++ b/src/scenarios/server/agents/discovery.ts @@ -0,0 +1,664 @@ +/** + * agents-wg#22 (MCP Agent Discovery Extension): the capability, the roster + * (`agents/list`), the details (`agents/get`), tool parity with `tools/list`, + * and the two error reasons. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim quote of the proposal lives next to its check ID in + * src/seps/sep-9998.yaml. 9998 is a placeholder number; the yaml header says + * why. + * + * Discovery is dynamic. The scenario lists whatever roster the server serves + * and opens every agent in it, hardcoding no agent name. Two rows need agents + * the harness cannot find by listing: one hidden from this caller and one + * whose membership names a missing tool. It asks the server's + * `agents_conformance_fixture` control for their names, and reports those rows + * untestable when the control is absent. + * + * The capability gate is two-sided, as in the skills and events suites. A + * server that neither declares the extension nor answers `agents/list` skips + * the scenario, since the extension is optional. A server that answers + * `agents/list` while declaring nothing fails the capability row, because a + * client following the proposal would never call it. + */ + +import { isDeepStrictEqual } from 'util'; +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + AGENTS_CLIENT_CAPABILITIES, + AGENTS_EXTENSION_ID, + AGENTS_LIST_METHOD, + AGENTS_SPEC_REF, + CONTROL_FIXTURE, + JSONRPC_INTERNAL_ERROR, + JSONRPC_INVALID_PARAMS, + MODEL_ROLES, + REASON_AGENT_NOT_FOUND, + REASON_INVALID_AGENT_DEFINITION, + UNKNOWN_AGENT_NAME, + agentsCheck, + agentsGet, + agentsListAll, + cacheableResultErrors, + declaredAgentsCapability, + describeValue, + errorReason, + fixtureNames, + isObject, + joinErrs, + toolsListAll, + type RpcFailure +} from './helpers'; + +const CAPABILITY_IDS = [ + 'sep-9998-capability-declared', + 'sep-9998-capability-list-changed-boolean', + 'sep-9998-core-without-extension' +] as const; + +const LIST_IDS = [ + 'sep-9998-list-implemented', + 'sep-9998-list-result-shape', + 'sep-9998-list-pagination', + 'sep-9998-list-cacheable', + 'sep-9998-card-name', + 'sep-9998-card-description-type', + 'sep-9998-card-capabilities', + 'sep-9998-list-cards-only', + 'sep-9998-name-unique' +] as const; + +const GET_IDS = [ + 'sep-9998-get-implemented', + 'sep-9998-get-result-shape', + 'sep-9998-get-agent-name', + 'sep-9998-get-cacheable', + 'sep-9998-hint-role', + 'sep-9998-hint-vision' +] as const; + +const PARITY_IDS = [ + 'sep-9998-get-tools-in-tools-list', + 'sep-9998-get-tools-intersection', + 'sep-9998-tools-list-not-omitted', + 'sep-9998-get-tools-fields-match', + 'sep-9998-shared-tool-same-definition' +] as const; + +const ERROR_IDS = [ + 'sep-9998-agent-not-found', + 'sep-9998-hidden-indistinguishable', + 'sep-9998-roster-filtered', + 'sep-9998-invalid-agent-definition' +] as const; + +const ALL_IDS = [ + ...CAPABILITY_IDS, + ...LIST_IDS, + ...GET_IDS, + ...PARITY_IDS, + ...ERROR_IDS +]; + +/** One-line descriptions, keyed by check ID, used on every status. */ +const DESCRIPTIONS: Record<(typeof ALL_IDS)[number], string> = { + 'sep-9998-capability-declared': + 'The server declares io.modelcontextprotocol/agents under capabilities.extensions with a settings object.', + 'sep-9998-capability-list-changed-boolean': + 'The optional listChanged setting, when present, is a boolean.', + 'sep-9998-core-without-extension': + 'A client that does not declare the extension can still use core MCP functionality (tools/list).', + 'sep-9998-list-implemented': + 'agents/list returns the roster visible to the requesting client.', + 'sep-9998-list-result-shape': + 'Every agents/list page carries resultType "complete" and an agents array.', + 'sep-9998-list-pagination': + 'Following nextCursor from the first page reaches a page without one.', + 'sep-9998-list-cacheable': + 'Every agents/list page follows CacheableResult: ttlMs is a non-negative number and cacheScope is "public" or "private".', + 'sep-9998-card-name': 'Every agent card carries a string name.', + 'sep-9998-card-description-type': + 'An agent card description, when present, is a string.', + 'sep-9998-card-capabilities': + 'Every agent card carries capabilities as an array of strings.', + 'sep-9998-list-cards-only': + 'The roster carries compact cards only, without instructions, tools or modelRoutingHint.', + 'sep-9998-name-unique': + 'Agent names are unique within the roster visible to the caller, across every page.', + 'sep-9998-get-implemented': + 'agents/get returns details for every agent the roster lists.', + 'sep-9998-get-result-shape': + 'agents/get results carry resultType "complete", a tools array, and instructions as a string when present.', + 'sep-9998-get-agent-name': + 'agents/get result.agent names the agent that was requested.', + 'sep-9998-get-cacheable': + 'Every agents/get result follows CacheableResult: ttlMs is a non-negative number and cacheScope is "public" or "private".', + 'sep-9998-hint-role': + 'modelRoutingHint.role, when present, is "small", "medium" or "large".', + 'sep-9998-hint-vision': + 'modelRoutingHint, when present, is an object carrying a boolean vision.', + 'sep-9998-get-tools-in-tools-list': + 'Every tool agents/get returns is also listed by tools/list.', + 'sep-9998-get-tools-intersection': + "agents/get includes no tool absent from the caller's tools/list.", + 'sep-9998-tools-list-not-omitted': + 'tools/list does not omit a tool because an agent includes it.', + 'sep-9998-get-tools-fields-match': + 'A tool returned by both agents/get and tools/list has identical fields in both.', + 'sep-9998-shared-tool-same-definition': + 'A tool name that appears under several agents identifies the same tool in each.', + 'sep-9998-agent-not-found': + 'agents/get for an unknown agent fails with -32602 and data.reason "agent_not_found".', + 'sep-9998-hidden-indistinguishable': + 'agents/get for an agent hidden from the caller fails exactly as it does for a nonexistent name.', + 'sep-9998-roster-filtered': + 'An agent hidden from the caller is absent from its agents/list roster.', + 'sep-9998-invalid-agent-definition': + 'agents/get for an agent whose membership names a missing tool fails with -32603 and data.reason "invalid_agent_definition".' +}; + +/** Card members that belong to agents/get, never to the roster. */ +const DETAIL_MEMBERS = ['instructions', 'tools', 'modelRoutingHint', 'agent']; + +type CheckId = (typeof ALL_IDS)[number]; + +function check( + id: CheckId, + errs: string[], + details?: Record, + failStatus: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck { + return agentsCheck( + id, + DESCRIPTIONS[id], + errs.length === 0 ? 'SUCCESS' : failStatus, + errs.length === 0 + ? details + ? { details } + : {} + : { errorMessage: joinErrs(errs), details } + ); +} + +function skipped(id: CheckId, reason: string): ConformanceCheck { + return agentsCheck(id, DESCRIPTIONS[id], 'SKIPPED', { + errorMessage: reason + }); +} + +function untestable(id: CheckId, reason: string): ConformanceCheck { + return untestableCheck(id, id, DESCRIPTIONS[id], reason, [AGENTS_SPEC_REF]); +} + +function describeFailure(f: RpcFailure): string { + return `${f.code} ${JSON.stringify(f.message)}${f.data === undefined ? '' : ` data=${JSON.stringify(f.data)}`}`; +} + +export class AgentsDiscoveryScenario implements ClientScenario { + name = 'sep-9998-agents-discovery'; + readonly source = { extensionId: AGENTS_EXTENSION_ID } as const; + description = `agents-wg#22 MCP Agent Discovery Extension: the capability, \`agents/list\`, \`agents/get\`, tool parity with \`tools/list\`, and the error reasons. + +**Methods**: \`agents/list\`, \`agents/get\` (for a server declaring \`${AGENTS_EXTENSION_ID}\`) + +**Requirements covered** (each check carries a verbatim quote of the proposal in src/seps/sep-9998.yaml; 9998 is a placeholder number): +- \`sep-9998-capability-*\`, \`sep-9998-core-without-extension\`: the declaration, and that core functionality does not depend on it +- \`sep-9998-list-*\`, \`sep-9998-card-*\`, \`sep-9998-name-unique\`: the roster's shape, paging and cache hints +- \`sep-9998-get-*\`, \`sep-9998-hint-*\`: the flat details result and its optional routing hint +- \`sep-9998-get-tools-*\`, \`sep-9998-tools-list-not-omitted\`, \`sep-9998-shared-tool-same-definition\`: every returned tool matches \`tools/list\` +- \`sep-9998-agent-not-found\`, \`sep-9998-hidden-indistinguishable\`, \`sep-9998-roster-filtered\`, \`sep-9998-invalid-agent-definition\`: the error reasons and visibility + +The hidden-agent and broken-agent rows need the server's \`${CONTROL_FIXTURE}\` control; without it they report untestable.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect({ + capabilities: AGENTS_CLIENT_CAPABILITIES + }); + try { + const declaration = await declaredAgentsCapability(conn); + const listed = await agentsListAll(conn); + + if (!declaration.declared) { + if ('error' in listed) { + const reason = `The server did not declare ${AGENTS_EXTENSION_ID}, and ${AGENTS_LIST_METHOD} failed (${describeFailure(listed.error)}); the extension is optional.`; + return ALL_IDS.map((id) => skipped(id, reason)); + } + const reason = `The server answers ${AGENTS_LIST_METHOD} but does not declare ${AGENTS_EXTENSION_ID} under capabilities.extensions, so a client following the proposal never calls it.`; + return ALL_IDS.map((id) => + id === 'sep-9998-capability-declared' + ? check(id, [reason]) + : skipped( + id, + 'The extension is not declared; see sep-9998-capability-declared.' + ) + ); + } + + const checks: ConformanceCheck[] = []; + const settings = declaration.value; + + // === capability === + checks.push( + check( + 'sep-9998-capability-declared', + isObject(settings) + ? [] + : [ + `capabilities.extensions["${AGENTS_EXTENSION_ID}"] is ${describeValue(settings)}, not a settings object` + ], + { settings } + ) + ); + if (isObject(settings) && 'listChanged' in settings) { + checks.push( + check( + 'sep-9998-capability-list-changed-boolean', + typeof settings.listChanged === 'boolean' + ? [] + : [ + `listChanged is ${describeValue(settings.listChanged)}, not a boolean` + ], + { listChanged: settings.listChanged } + ) + ); + } else { + checks.push( + skipped( + 'sep-9998-capability-list-changed-boolean', + 'The optional listChanged setting is absent.' + ) + ); + } + + // A second connection that does not declare the extension. + const plain = await ctx.connect(); + let coreTools: Record[] | RpcFailure; + try { + coreTools = await toolsListAll(plain); + } finally { + await plain.close(); + } + checks.push( + check( + 'sep-9998-core-without-extension', + Array.isArray(coreTools) + ? [] + : [ + `tools/list failed for a client that did not declare the extension: ${describeFailure(coreTools)}` + ] + ) + ); + + // === roster === + if ('error' in listed) { + const reason = `${AGENTS_LIST_METHOD} failed: ${describeFailure(listed.error)}`; + checks.push(check('sep-9998-list-implemented', [reason])); + for (const id of ALL_IDS) { + if (!checks.some((c) => c.id === id)) { + checks.push(skipped(id, `${AGENTS_LIST_METHOD} is unavailable.`)); + } + } + return checks; + } + const { pages, truncated } = listed; + checks.push( + check('sep-9998-list-implemented', [], { pages: pages.length }) + ); + + const shapeErrs: string[] = []; + const cacheErrs: string[] = []; + pages.forEach((page, i) => { + if (page.result.resultType !== 'complete') { + shapeErrs.push( + `page ${i + 1}: resultType is ${JSON.stringify(page.result.resultType)}` + ); + } + if (!Array.isArray(page.result.agents)) { + shapeErrs.push( + `page ${i + 1}: agents is ${describeValue(page.result.agents)}` + ); + } + for (const e of cacheableResultErrors(page.result)) { + cacheErrs.push(`page ${i + 1}: ${e}`); + } + }); + checks.push(check('sep-9998-list-result-shape', shapeErrs)); + checks.push( + check( + 'sep-9998-list-pagination', + truncated + ? [ + `${AGENTS_LIST_METHOD} did not terminate: the server kept returning a nextCursor, or repeated one, across ${pages.length} pages` + ] + : [], + { pages: pages.length } + ) + ); + checks.push(check('sep-9998-list-cacheable', cacheErrs)); + + const cards = pages.flatMap((p) => p.agents); + const nameErrs: string[] = []; + const descErrs: string[] = []; + const capErrs: string[] = []; + const cardsOnlyErrs: string[] = []; + const names: string[] = []; + cards.forEach((card, i) => { + if (!isObject(card)) { + nameErrs.push(`card ${i} is ${describeValue(card)}`); + return; + } + const label = + typeof card.name === 'string' ? `"${card.name}"` : `card ${i}`; + if (typeof card.name !== 'string' || card.name === '') { + nameErrs.push(`${label}: name is ${describeValue(card.name)}`); + } else { + names.push(card.name); + } + if ('description' in card && typeof card.description !== 'string') { + descErrs.push( + `${label}: description is ${describeValue(card.description)}` + ); + } + if ( + !Array.isArray(card.capabilities) || + card.capabilities.some((c) => typeof c !== 'string') + ) { + capErrs.push( + `${label}: capabilities is ${Array.isArray(card.capabilities) ? 'an array with a non-string entry' : describeValue(card.capabilities)}` + ); + } + const leaked = DETAIL_MEMBERS.filter((m) => m in card); + if (leaked.length > 0) { + cardsOnlyErrs.push(`${label} carries ${leaked.join(', ')}`); + } + }); + checks.push( + check('sep-9998-card-name', nameErrs, { cards: cards.length }) + ); + checks.push(check('sep-9998-card-description-type', descErrs)); + checks.push(check('sep-9998-card-capabilities', capErrs)); + checks.push( + check('sep-9998-list-cards-only', cardsOnlyErrs, undefined, 'WARNING') + ); + const dupes = names.filter((n, i) => names.indexOf(n) !== i); + checks.push( + check( + 'sep-9998-name-unique', + dupes.length > 0 + ? [ + `name(s) listed more than once: ${[...new Set(dupes)].join(', ')}` + ] + : [], + { names } + ) + ); + + // === details === + const tools = await toolsListAll(conn); + const toolList = Array.isArray(tools) ? tools : []; + const fixture = await fixtureNames(conn, toolList); + const broken = fixture?.brokenAgent; + const toOpen = [...new Set(names)].filter((n) => n !== broken); + + const details: Array<{ name: string; result: Record }> = + []; + const getErrs: string[] = []; + for (const name of toOpen) { + const got = await agentsGet(conn, name); + if ('error' in got) { + getErrs.push(`"${name}": ${describeFailure(got.error)}`); + } else { + details.push({ name, result: got.result }); + } + } + + if (toOpen.length === 0) { + const reason = + 'The roster lists no agent to open, so there are no details to grade.'; + for (const id of [...GET_IDS, ...PARITY_IDS]) { + checks.push(skipped(id, reason)); + } + } else { + checks.push( + check('sep-9998-get-implemented', getErrs, { opened: toOpen.length }) + ); + const resultErrs: string[] = []; + const agentNameErrs: string[] = []; + const getCacheErrs: string[] = []; + const roleErrs: string[] = []; + const visionErrs: string[] = []; + let hinted = 0; + for (const { name, result } of details) { + if (result.resultType !== 'complete') { + resultErrs.push( + `"${name}": resultType is ${JSON.stringify(result.resultType)}` + ); + } + if (!Array.isArray(result.tools)) { + resultErrs.push( + `"${name}": tools is ${describeValue(result.tools)}` + ); + } + if ( + 'instructions' in result && + typeof result.instructions !== 'string' + ) { + resultErrs.push( + `"${name}": instructions is ${describeValue(result.instructions)}` + ); + } + if ('agent' in result === false || result.agent !== name) { + agentNameErrs.push( + `requested "${name}", result.agent is ${JSON.stringify(result.agent)}` + ); + } + for (const e of cacheableResultErrors(result)) { + getCacheErrs.push(`"${name}": ${e}`); + } + if ('modelRoutingHint' in result) { + hinted++; + const hint = result.modelRoutingHint; + if (!isObject(hint)) { + visionErrs.push( + `"${name}": modelRoutingHint is ${describeValue(hint)}` + ); + continue; + } + if (typeof hint.vision !== 'boolean') { + visionErrs.push( + `"${name}": vision is ${describeValue(hint.vision)}` + ); + } + if ( + 'role' in hint && + !(MODEL_ROLES as readonly unknown[]).includes(hint.role) + ) { + roleErrs.push(`"${name}": role is ${JSON.stringify(hint.role)}`); + } + } + } + checks.push(check('sep-9998-get-result-shape', resultErrs)); + checks.push(check('sep-9998-get-agent-name', agentNameErrs)); + checks.push(check('sep-9998-get-cacheable', getCacheErrs)); + if (hinted === 0) { + const reason = + 'No agent returned the optional modelRoutingHint, so there is no hint to grade.'; + checks.push(skipped('sep-9998-hint-role', reason)); + checks.push(skipped('sep-9998-hint-vision', reason)); + } else { + checks.push(check('sep-9998-hint-role', roleErrs, { hinted })); + checks.push(check('sep-9998-hint-vision', visionErrs, { hinted })); + } + + // === parity with tools/list === + if (!Array.isArray(tools)) { + const reason = `tools/list failed (${describeFailure(tools)}), so returned tools cannot be compared with it.`; + for (const id of PARITY_IDS) checks.push(untestable(id, reason)); + } else { + const listedByName = new Map( + tools.map((t) => [String(t.name), t] as const) + ); + const missing: string[] = []; + const differ: string[] = []; + const byToolName = new Map< + string, + { agent: string; def: unknown }[] + >(); + for (const { name, result } of details) { + for (const t of Array.isArray(result.tools) ? result.tools : []) { + const tn = isObject(t) ? String(t.name) : String(t); + const seen = byToolName.get(tn) ?? []; + seen.push({ agent: name, def: t }); + byToolName.set(tn, seen); + const listedDef = listedByName.get(tn); + if (!listedDef) { + missing.push(`"${name}" returns "${tn}"`); + } else if (!isDeepStrictEqual(listedDef, t)) { + differ.push(`"${tn}" under "${name}"`); + } + } + } + const missingErrs = missing.map( + (m) => `${m}, which tools/list does not list` + ); + checks.push(check('sep-9998-get-tools-in-tools-list', missingErrs)); + checks.push(check('sep-9998-get-tools-intersection', missingErrs)); + checks.push(check('sep-9998-tools-list-not-omitted', missingErrs)); + checks.push( + check( + 'sep-9998-get-tools-fields-match', + differ.map((d) => `${d} differs from its tools/list entry`) + ) + ); + const shared = [...byToolName.entries()].filter( + ([, uses]) => new Set(uses.map((u) => u.agent)).size > 1 + ); + if (shared.length === 0) { + checks.push( + skipped( + 'sep-9998-shared-tool-same-definition', + 'No tool name appears under more than one agent.' + ) + ); + } else { + const sharedErrs = shared + .filter(([, uses]) => + uses.some((u) => !isDeepStrictEqual(u.def, uses[0].def)) + ) + .map( + ([tn, uses]) => + `"${tn}" differs between ${uses.map((u) => `"${u.agent}"`).join(' and ')}` + ); + checks.push( + check('sep-9998-shared-tool-same-definition', sharedErrs, { + shared: shared.map(([tn]) => tn) + }) + ); + } + } + } + + // === errors and visibility === + const unknown = await agentsGet(conn, UNKNOWN_AGENT_NAME); + const unknownFailure = 'error' in unknown ? unknown.error : undefined; + checks.push( + check( + 'sep-9998-agent-not-found', + unknownFailure === undefined + ? [`agents/get "${UNKNOWN_AGENT_NAME}" succeeded`] + : unknownFailure.code !== JSONRPC_INVALID_PARAMS || + errorReason(unknownFailure) !== REASON_AGENT_NOT_FOUND + ? [ + `agents/get for an unknown name failed with ${describeFailure(unknownFailure)}` + ] + : [], + unknownFailure ? { error: unknownFailure } : undefined + ) + ); + + const noControl = `The server exposes no ${CONTROL_FIXTURE} control naming the agent this row needs, and an agent hidden from this caller cannot be found any other way.`; + const hidden = fixture?.hiddenAgent; + if (!hidden) { + checks.push(untestable('sep-9998-hidden-indistinguishable', noControl)); + checks.push(untestable('sep-9998-roster-filtered', noControl)); + } else { + const got = await agentsGet(conn, hidden); + const errs: string[] = []; + if (!('error' in got)) { + errs.push( + `agents/get "${hidden}" succeeded for an agent the server reports as hidden from this caller` + ); + } else if (unknownFailure === undefined) { + errs.push( + 'the unknown-name probe did not fail, so there is no not-found response to compare with' + ); + } else { + if (got.error.code !== unknownFailure.code) { + errs.push( + `code ${got.error.code} for the hidden agent, ${unknownFailure.code} for an unknown name` + ); + } + if (got.error.message !== unknownFailure.message) { + errs.push( + `message ${JSON.stringify(got.error.message)} for the hidden agent, ${JSON.stringify(unknownFailure.message)} for an unknown name` + ); + } + if (!isDeepStrictEqual(got.error.data, unknownFailure.data)) { + errs.push( + `data ${JSON.stringify(got.error.data)} for the hidden agent, ${JSON.stringify(unknownFailure.data)} for an unknown name` + ); + } + } + checks.push( + check('sep-9998-hidden-indistinguishable', errs, { hidden }) + ); + checks.push( + check( + 'sep-9998-roster-filtered', + names.includes(hidden) + ? [ + `agents/list lists "${hidden}", which the server reports as hidden from this caller` + ] + : [], + { hidden } + ) + ); + } + + if (!broken) { + checks.push( + untestable( + 'sep-9998-invalid-agent-definition', + `The server exposes no ${CONTROL_FIXTURE} control naming an agent whose membership references a missing tool.` + ) + ); + } else { + const got = await agentsGet(conn, broken); + checks.push( + check( + 'sep-9998-invalid-agent-definition', + !('error' in got) + ? [ + `agents/get "${broken}" succeeded for an agent whose membership names a missing tool` + ] + : got.error.code !== JSONRPC_INTERNAL_ERROR || + errorReason(got.error) !== REASON_INVALID_AGENT_DEFINITION + ? [ + `agents/get "${broken}" failed with ${describeFailure(got.error)}` + ] + : [], + { broken } + ) + ); + } + + return checks; + } finally { + await conn.close(); + } + } +} + +export const AGENTS_DISCOVERY_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/agents/helpers.ts b/src/scenarios/server/agents/helpers.ts new file mode 100644 index 00000000..f7b68860 --- /dev/null +++ b/src/scenarios/server/agents/helpers.ts @@ -0,0 +1,309 @@ +/** + * Shared helpers for the agents-wg#22 (MCP Agent Discovery Extension) server + * conformance scenarios under this directory. + * + * The scenarios treat the server under test as an arbitrary server declaring + * `io.modelcontextprotocol/agents`. Capability is read from discovery, never + * inferred from an error, and every agent is found through `agents/list`, so + * no fixture-specific agent name is hardcoded. The two rows that need an agent + * the harness cannot otherwise find (one hidden from this caller, one whose + * membership names a missing tool) ask the server's diagnostic control for + * their names, and report untestable when it is absent. + * + * The SEP has no number in modelcontextprotocol/modelcontextprotocol yet, so + * check IDs carry the placeholder `sep-9998-`. See src/seps/sep-9998.yaml. + */ + +import type { + CheckStatus, + ConformanceCheck, + SpecReference +} from '../../../types'; +import type { Connection } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; + +export const AGENTS_EXTENSION_ID = 'io.modelcontextprotocol/agents'; + +export const AGENTS_LIST_METHOD = 'agents/list'; +export const AGENTS_GET_METHOD = 'agents/get'; +export const AGENTS_LIST_CHANGED_NOTIFICATION = + 'notifications/agents/list_changed'; + +/** The `subscriptions/listen` filter key the SEP defines. */ +export const AGENTS_LIST_CHANGED_FILTER_KEY = 'agentsListChanged'; + +export const SUBSCRIPTIONS_LISTEN_METHOD = 'subscriptions/listen'; +export const SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION = + 'notifications/subscriptions/acknowledged'; + +export const REASON_AGENT_NOT_FOUND = 'agent_not_found'; +export const REASON_INVALID_AGENT_DEFINITION = 'invalid_agent_definition'; + +export const JSONRPC_METHOD_NOT_FOUND = -32601; +export const JSONRPC_INVALID_PARAMS = -32602; +export const JSONRPC_INTERNAL_ERROR = -32603; + +export const MODEL_ROLES = ['small', 'medium', 'large'] as const; + +/** + * Diagnostic controls the fixture exposes as tools. Both are optional: a + * server without them reports the rows that depend on them as untestable, + * naming the missing tool. + * + * `agents_conformance_fixture` takes no arguments and answers with a text + * item holding `{"hiddenAgent": "", "brokenAgent": ""}`: an agent + * hidden from this caller, and one whose membership names a tool missing from + * the catalog. `agents_conformance_add_agent` takes `{"name": string}` and + * adds a valid agent, which is how the harness changes the roster mid-run. + */ +export const CONTROL_FIXTURE = 'agents_conformance_fixture'; +export const CONTROL_ADD_AGENT = 'agents_conformance_add_agent'; + +const SEP_URL = + 'https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md'; + +export const AGENTS_SPEC_REF: SpecReference = { + id: 'agents-wg#22 (MCP Agent Discovery Extension)', + url: `${SEP_URL}#specification` +}; + +/** + * Build a check carrying the SEP reference. Per AGENTS.md the same `id` flips + * `status` + `errorMessage` between SUCCESS and FAILURE rather than branching + * into distinct slugs. + */ +export function agentsCheck( + id: string, + description: string, + status: CheckStatus, + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: [AGENTS_SPEC_REF], + ...extras + }; +} + +/** A JSON object, as opposed to an array, `null`, or a primitive. */ +export function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** How to name an observed value's type in an error message. */ +export function describeValue(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'an array'; + return `a ${typeof value}`; +} + +export function joinErrs(errs: string[], limit = 5): string { + const shown = errs.slice(0, limit).join('; '); + return errs.length > limit + ? `${shown} (+${errs.length - limit} more)` + : shown; +} + +/** The client capabilities the harness declares: the base set plus this extension. */ +export const AGENTS_CLIENT_CAPABILITIES = { + sampling: {}, + elicitation: {}, + roots: { listChanged: true }, + extensions: { [AGENTS_EXTENSION_ID]: {} } +}; + +/** + * What the server declared under `capabilities.extensions` for this + * extension. `declared` tells absent apart from malformed. + */ +export async function declaredAgentsCapability( + conn: Connection +): Promise<{ declared: boolean; value: unknown }> { + const discovered = await conn.discover(); + const caps = (discovered.capabilities as Record) ?? {}; + const extensions = caps.extensions as Record | undefined; + if (!isObject(extensions) || !(AGENTS_EXTENSION_ID in extensions)) { + return { declared: false, value: undefined }; + } + return { declared: true, value: extensions[AGENTS_EXTENSION_ID] }; +} + +/** A JSON-RPC error, as thrown by `conn.request`. */ +export interface RpcFailure { + code: number; + message: string; + data?: unknown; +} + +export function toRpcFailure(err: unknown): RpcFailure { + if (err instanceof JsonRpcError) { + return { code: err.code, message: err.message, data: err.data }; + } + return { code: 0, message: String(err) }; +} + +/** The `data.reason` of an error, when it carries one. */ +export function errorReason(failure: RpcFailure): string | undefined { + return isObject(failure.data) && typeof failure.data.reason === 'string' + ? failure.data.reason + : undefined; +} + +export interface ListPage { + result: Record; + agents: unknown[]; +} + +/** + * Walk `agents/list` from the first page, following `nextCursor` until the + * server stops sending one. `truncated` means the walk was cut off because + * the server kept answering with a cursor (or repeated one). + */ +export async function agentsListAll( + conn: Connection, + maxPages = 50 +): Promise< + | { pages: ListPage[]; truncated: boolean } + | { error: RpcFailure; pages: ListPage[] } +> { + const pages: ListPage[] = []; + const seen = new Set(); + let cursor: string | undefined; + for (let i = 0; i < maxPages; i++) { + let result: Record; + try { + result = await conn.request>( + AGENTS_LIST_METHOD, + cursor === undefined ? {} : { cursor } + ); + } catch (err) { + return { error: toRpcFailure(err), pages }; + } + const agents = Array.isArray(result?.agents) ? result.agents : []; + pages.push({ result: isObject(result) ? result : {}, agents }); + const next = result?.nextCursor; + if (typeof next !== 'string' || next === '') { + return { pages, truncated: false }; + } + if (seen.has(next)) return { pages, truncated: true }; + seen.add(next); + cursor = next; + } + return { pages, truncated: true }; +} + +/** `agents/get` for one name, as a result or a JSON-RPC failure. */ +export async function agentsGet( + conn: Connection, + name: string +): Promise<{ result: Record } | { error: RpcFailure }> { + try { + const result = await conn.request>( + AGENTS_GET_METHOD, + { name } + ); + return { result: isObject(result) ? result : {} }; + } catch (err) { + return { error: toRpcFailure(err) }; + } +} + +/** Every tool `tools/list` serves, following cursors. */ +export async function toolsListAll( + conn: Connection, + maxPages = 50 +): Promise[] | RpcFailure> { + const out: Record[] = []; + let cursor: string | undefined; + for (let i = 0; i < maxPages; i++) { + let res: { tools?: unknown; nextCursor?: unknown }; + try { + res = await conn.request('tools/list', cursor ? { cursor } : {}); + } catch (err) { + return toRpcFailure(err); + } + for (const t of Array.isArray(res.tools) ? res.tools : []) { + if (isObject(t)) out.push(t); + } + if (typeof res.nextCursor !== 'string' || res.nextCursor === '') break; + cursor = res.nextCursor; + } + return out; +} + +/** + * Call a diagnostic control and return its first text item, or undefined + * when the server rejected it or answered with an error result. + */ +export async function askControl( + conn: Connection, + tool: string, + args: Record +): Promise { + try { + const res = await conn.request<{ + content?: { type?: string; text?: string }[]; + isError?: boolean; + }>('tools/call', { name: tool, arguments: args }); + if (res.isError) return undefined; + const text = (res.content ?? []).find((c) => c?.type === 'text')?.text; + return typeof text === 'string' ? text : undefined; + } catch { + return undefined; + } +} + +/** + * Ask `agents_conformance_fixture` for the names of the hidden and the broken + * agent. Returns undefined when the control is missing or answers with + * something other than the documented JSON. + */ +export async function fixtureNames( + conn: Connection, + tools: Record[] +): Promise<{ hiddenAgent?: string; brokenAgent?: string } | undefined> { + if (!tools.some((t) => t.name === CONTROL_FIXTURE)) return undefined; + const text = await askControl(conn, CONTROL_FIXTURE, {}); + if (text === undefined) return undefined; + try { + const parsed: unknown = JSON.parse(text); + if (!isObject(parsed)) return undefined; + return { + hiddenAgent: + typeof parsed.hiddenAgent === 'string' ? parsed.hiddenAgent : undefined, + brokenAgent: + typeof parsed.brokenAgent === 'string' ? parsed.brokenAgent : undefined + }; + } catch { + return undefined; + } +} + +/** A name no conformant server should serve, for the not-found probe. */ +export const UNKNOWN_AGENT_NAME = 'mcp-conformance-nonexistent-agent-7c41d2'; + +/** A result's cache hints are a non-negative number and a known scope. */ +export function cacheableResultErrors( + result: Record +): string[] { + const errs: string[] = []; + const ttl = result.ttlMs; + if (ttl === undefined) { + errs.push('ttlMs is absent'); + } else if (typeof ttl !== 'number' || !Number.isFinite(ttl) || ttl < 0) { + errs.push(`ttlMs is ${JSON.stringify(ttl)}, not a non-negative number`); + } + const scope = result.cacheScope; + if (scope === undefined) { + errs.push('cacheScope is absent'); + } else if (scope !== 'public' && scope !== 'private') { + errs.push( + `cacheScope is ${JSON.stringify(scope)}, not "public" or "private"` + ); + } + return errs; +} diff --git a/src/scenarios/server/agents/list-changed.ts b/src/scenarios/server/agents/list-changed.ts new file mode 100644 index 00000000..c16b2273 --- /dev/null +++ b/src/scenarios/server/agents/list-changed.ts @@ -0,0 +1,277 @@ +/** + * agents-wg#22 (MCP Agent Discovery Extension): roster changes over + * `subscriptions/listen`. + * + * `notifications/agents/list_changed` reaches a client only on a listen + * stream it opened with `agentsListChanged: true`, and the server's + * acknowledgement says whether it will honor that. The rows hang off one + * stream: + * + * - a server that did not declare `listChanged: true` must not acknowledge + * the key, since the proposal says such notifications are not supported; + * - a server that did declare it acknowledges the key, and sends the + * notification when its roster changes. The harness cannot change a roster + * over the protocol, so it asks the server's `agents_conformance_add_agent` + * control to add one, waits for the notification, then re-lists. + * + * Check IDs carry the placeholder number 9998; see src/seps/sep-9998.yaml. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { openListenStream } from './listen-stream'; +import { + AGENTS_CLIENT_CAPABILITIES, + AGENTS_EXTENSION_ID, + AGENTS_LIST_CHANGED_FILTER_KEY, + AGENTS_LIST_CHANGED_NOTIFICATION, + AGENTS_SPEC_REF, + CONTROL_ADD_AGENT, + SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + SUBSCRIPTIONS_LISTEN_METHOD, + agentsCheck, + agentsListAll, + askControl, + declaredAgentsCapability, + isObject, + toolsListAll +} from './helpers'; + +const NOT_DECLARED_ID = 'sep-9998-list-changed-not-declared'; +const ACK_ID = 'sep-9998-list-changed-acknowledged'; +const NOTIFICATION_ID = 'sep-9998-list-changed-notification'; +const ROSTER_ID = 'sep-9998-list-changed-roster-updated'; + +const ALL_IDS = [NOT_DECLARED_ID, ACK_ID, NOTIFICATION_ID, ROSTER_ID] as const; +type CheckId = (typeof ALL_IDS)[number]; + +const DESCRIPTIONS: Record = { + [NOT_DECLARED_ID]: + 'A server that does not declare listChanged: true does not acknowledge agentsListChanged on subscriptions/listen.', + [ACK_ID]: + 'A server declaring listChanged: true acknowledges agentsListChanged on subscriptions/listen.', + [NOTIFICATION_ID]: + 'A server declaring listChanged: true sends notifications/agents/list_changed on the listen stream when its roster changes.', + [ROSTER_ID]: + 'After the notification, agents/list shows the change it announced.' +}; + +const LISTEN_ACK_MS = 3000; +const LIST_CHANGED_MS = 5000; + +function graded( + id: CheckId, + errs: string[], + details?: Record, + failStatus: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck { + return agentsCheck( + id, + DESCRIPTIONS[id], + errs.length === 0 ? 'SUCCESS' : failStatus, + errs.length === 0 + ? details + ? { details } + : {} + : { errorMessage: errs.join('; '), details } + ); +} + +function skipped(id: CheckId, reason: string): ConformanceCheck { + return agentsCheck(id, DESCRIPTIONS[id], 'SKIPPED', { errorMessage: reason }); +} + +function untestable( + id: CheckId, + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck { + return untestableCheck( + id, + id, + DESCRIPTIONS[id], + reason, + [AGENTS_SPEC_REF], + severity + ); +} + +export class AgentsListChangedScenario implements ClientScenario { + name = 'sep-9998-agents-list-changed'; + readonly source = { extensionId: AGENTS_EXTENSION_ID } as const; + description = `agents-wg#22 MCP Agent Discovery Extension: \`notifications/agents/list_changed\` over \`subscriptions/listen\`. + +**Requirements covered** (each check carries a verbatim quote of the proposal in src/seps/sep-9998.yaml; 9998 is a placeholder number): +- \`${NOT_DECLARED_ID}\`: without \`listChanged: true\`, the server does not acknowledge \`${AGENTS_LIST_CHANGED_FILTER_KEY}\` +- \`${ACK_ID}\`: with it, the server acknowledges the key +- \`${NOTIFICATION_ID}\`: a roster change, made through the \`${CONTROL_ADD_AGENT}\` control, sends the notification on the acknowledged stream +- \`${ROSTER_ID}\`: re-listing afterwards shows the added agent + +Without the \`${CONTROL_ADD_AGENT}\` control the last two rows report untestable.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect({ + capabilities: AGENTS_CLIENT_CAPABILITIES + }); + try { + const declaration = await declaredAgentsCapability(conn); + if (!declaration.declared) { + const reason = `The server did not declare ${AGENTS_EXTENSION_ID}; the extension is optional.`; + return ALL_IDS.map((id) => skipped(id, reason)); + } + const settings = isObject(declaration.value) ? declaration.value : {}; + const declaredTrue = settings.listChanged === true; + + const session = await openListenStream( + ctx.serverUrl, + ctx.specVersion, + { + notifications: { [AGENTS_LIST_CHANGED_FILTER_KEY]: true }, + _meta: { + 'io.modelcontextprotocol/clientCapabilities': + AGENTS_CLIENT_CAPABILITIES + } + }, + { method: SUBSCRIPTIONS_LISTEN_METHOD, openTimeoutMs: LISTEN_ACK_MS } + ); + try { + const ack = session.contentType?.includes('text/event-stream') + ? await session.waitFor( + (n) => n.method === SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + LISTEN_ACK_MS + ) + : undefined; + if (!ack) { + const why = session.error + ? `${SUBSCRIPTIONS_LISTEN_METHOD} failed: ${session.error.code} ${session.error.message}` + : `${SUBSCRIPTIONS_LISTEN_METHOD} did not confirm with ${SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION} within ${LISTEN_ACK_MS}ms (HTTP ${session.status}, ${session.contentType ?? 'no content type'})`; + if (!declaredTrue) { + return [ + untestable(NOT_DECLARED_ID, why), + skipped(ACK_ID, 'The server did not declare listChanged: true.'), + skipped( + NOTIFICATION_ID, + 'The server did not declare listChanged: true.' + ), + skipped( + ROSTER_ID, + 'The server did not declare listChanged: true.' + ) + ]; + } + return [ + skipped(NOT_DECLARED_ID, 'The server declared listChanged: true.'), + graded(ACK_ID, [why]), + untestable(NOTIFICATION_ID, why), + untestable(ROSTER_ID, why, 'WARNING') + ]; + } + + const honored = isObject(ack.params.notifications) + ? ack.params.notifications + : {}; + const echoed = honored[AGENTS_LIST_CHANGED_FILTER_KEY] === true; + + if (!declaredTrue) { + return [ + graded( + NOT_DECLARED_ID, + echoed + ? [ + `The server declared listChanged: ${JSON.stringify(settings.listChanged ?? false)} but acknowledged ${AGENTS_LIST_CHANGED_FILTER_KEY}, promising notifications it says it does not support.` + ] + : [], + { listChanged: settings.listChanged, honored } + ), + skipped(ACK_ID, 'The server did not declare listChanged: true.'), + skipped( + NOTIFICATION_ID, + 'The server did not declare listChanged: true.' + ), + skipped(ROSTER_ID, 'The server did not declare listChanged: true.') + ]; + } + + const checks: ConformanceCheck[] = [ + skipped( + NOT_DECLARED_ID, + 'The server declared listChanged: true; the rule covers servers that did not.' + ) + ]; + if (!echoed) { + const why = `The server declared listChanged: true but its acknowledgement does not honor ${AGENTS_LIST_CHANGED_FILTER_KEY}, which is the only channel the notification travels on.`; + checks.push(graded(ACK_ID, [why], { honored })); + checks.push(graded(NOTIFICATION_ID, [why], { honored })); + checks.push(skipped(ROSTER_ID, 'No notification can arrive.')); + return checks; + } + checks.push(graded(ACK_ID, [], { honored })); + + const tools = await toolsListAll(conn); + const hasControl = + Array.isArray(tools) && + tools.some((t) => t.name === CONTROL_ADD_AGENT); + const name = `mcp-conformance-added-${Math.random().toString(36).slice(2, 10)}`; + const added = hasControl + ? await askControl(conn, CONTROL_ADD_AGENT, { name }) + : undefined; + if (added === undefined) { + const why = hasControl + ? `The ${CONTROL_ADD_AGENT} control rejected {name: "${name}"}.` + : `The server exposes no ${CONTROL_ADD_AGENT} control, and the harness has no other way to change its roster mid-run.`; + checks.push(untestable(NOTIFICATION_ID, why)); + checks.push(untestable(ROSTER_ID, why, 'WARNING')); + return checks; + } + + const hit = await session.waitFor( + (n) => n.method === AGENTS_LIST_CHANGED_NOTIFICATION, + LIST_CHANGED_MS + ); + checks.push( + graded( + NOTIFICATION_ID, + hit + ? [] + : [ + `Added agent "${name}" through ${CONTROL_ADD_AGENT}, and no ${AGENTS_LIST_CHANGED_NOTIFICATION} arrived on the acknowledged listen stream within ${LIST_CHANGED_MS}ms.` + ], + hit ? { added: name, atMs: hit.atMs } : { added: name } + ) + ); + + const relisted = await agentsListAll(conn); + const names = + 'error' in relisted + ? undefined + : relisted.pages + .flatMap((p) => p.agents) + .map((c) => (isObject(c) ? c.name : undefined)); + checks.push( + graded( + ROSTER_ID, + names === undefined + ? [ + `agents/list failed after the change: ${'error' in relisted ? relisted.error.message : ''}` + ] + : names.includes(name) + ? [] + : [ + `agents/list does not list "${name}" after the server announced a roster change.` + ], + { added: name }, + 'WARNING' + ) + ); + return checks; + } finally { + await session.cancel(); + } + } finally { + await conn.close(); + } + } +} + +export const AGENTS_LIST_CHANGED_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/agents/listen-stream.ts b/src/scenarios/server/agents/listen-stream.ts new file mode 100644 index 00000000..cd5702b7 --- /dev/null +++ b/src/scenarios/server/agents/listen-stream.ts @@ -0,0 +1,231 @@ +/** + * A long-lived `subscriptions/listen` request, which the `Connection` + * abstraction cannot express. + * + * `conn.request()` resolves when the response for its JSON-RPC id arrives, and + * a listen stream withholds exactly that: the server acknowledges with a + * notification, delivers further notifications for as long as the stream is + * open, and sends a result only on close. So this opens the POST itself, + * hands back a session while the stream is still open, and appends frames as + * they arrive. The scenario waits for the frames it needs, grades them, then + * aborts. + * + * Adapted from the MCP Events suite's stream helper, which is not on upstream + * main yet. + */ + +import { + buildStandardHeaders, + withRequestMeta, + type JsonRpcResponse +} from '../../../connection'; +import type { SpecVersion } from '../../../types'; +import { isObject } from './helpers'; + +/** A JSON-RPC notification as it arrived on the stream. */ +export interface StreamNotification { + method: string; + params: Record; + /** Milliseconds since the stream was opened, for the heartbeat interval. */ + atMs: number; +} + +export interface StreamSession { + /** The JSON-RPC id of the listen request, echoed in `_meta`. */ + readonly requestId: number; + /** HTTP status of the POST that opened the stream. */ + readonly status: number; + readonly contentType?: string; + /** Set when the server answered with an immediate JSON-RPC error. */ + readonly error?: { code: number; message: string; data?: unknown }; + /** Set when a response for `requestId` arrived (the stream closed). */ + readonly finalResult?: JsonRpcResponse; + /** Notifications in arrival order. */ + readonly notifications: StreamNotification[]; + /** Frames that were neither a notification nor this request's response. */ + readonly foreignFrames: unknown[]; + /** SSE comment lines, kept for diagnostics. */ + readonly sseComments: string[]; + /** Whether the reader is still running. */ + readonly open: boolean; + /** Resolve once a notification matches, or undefined at the deadline. */ + waitFor( + predicate: (n: StreamNotification) => boolean, + timeoutMs: number + ): Promise; + /** Resolve after `ms`, or earlier if the stream closes. */ + settle(ms: number): Promise; + /** Abort the request stream. */ + cancel(): Promise; +} + +/** Open a long-lived request and return once the server has answered the POST. */ +export async function openListenStream( + serverUrl: string, + specVersion: SpecVersion, + params: Record, + options: { openTimeoutMs?: number; method?: string } = {} +): Promise { + const method = options.method ?? 'subscriptions/listen'; + const requestId = Math.floor(Math.random() * 1_000_000) + 1; + const headers = buildStandardHeaders(method, params, { + specVersion + }); + const body = JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method, + params: withRequestMeta(params, specVersion) + }); + + const controller = new AbortController(); + const openedAt = Date.now(); + const notifications: StreamNotification[] = []; + const foreignFrames: unknown[] = []; + const sseComments: string[] = []; + const state = { + open: true, + error: undefined as StreamSession['error'], + finalResult: undefined as JsonRpcResponse | undefined + }; + + const res = await fetch(serverUrl, { + method: 'POST', + headers, + body, + signal: controller.signal + }); + const contentType = res.headers.get('content-type') ?? undefined; + + const ingest = (frame: unknown): void => { + if (!isObject(frame)) { + foreignFrames.push(frame); + return; + } + if (typeof frame.method === 'string' && frame.id === undefined) { + notifications.push({ + method: frame.method, + params: isObject(frame.params) ? frame.params : {}, + atMs: Date.now() - openedAt + }); + return; + } + if (frame.id === requestId) { + state.finalResult = frame as unknown as JsonRpcResponse; + if (isObject(frame.error)) { + state.error = { + code: Number(frame.error.code), + message: String(frame.error.message ?? ''), + data: frame.error.data + }; + } + return; + } + foreignFrames.push(frame); + }; + + // A JSON body means the server answered rather than streamed, usually an + // immediate error for a request it would not hold open. + if (!contentType?.includes('text/event-stream')) { + state.open = false; + const text = await res.text(); + try { + ingest(JSON.parse(text)); + } catch { + foreignFrames.push(text); + } + } else { + void readFrames(res, ingest, sseComments).finally(() => { + state.open = false; + }); + } + + const session: StreamSession = { + requestId, + status: res.status, + contentType, + get error() { + return state.error; + }, + get finalResult() { + return state.finalResult; + }, + get open() { + return state.open; + }, + notifications, + foreignFrames, + sseComments, + async waitFor(predicate, timeoutMs) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const hit = notifications.find(predicate); + if (hit) return hit; + if (!state.open || Date.now() >= deadline) { + return notifications.find(predicate); + } + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async settle(ms) { + const deadline = Date.now() + ms; + while (state.open && Date.now() < deadline) { + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async cancel() { + controller.abort(); + state.open = false; + } + }; + + // Give a streaming server a moment to write its confirmation frame, so a + // caller that opens and immediately grades is not racing the first write. + if (state.open) { + await session.waitFor(() => true, options.openTimeoutMs ?? 2000); + } + return session; +} + +/** Read SSE `data:` frames until the stream ends or the request is aborted. */ +async function readFrames( + res: Response, + ingest: (frame: unknown) => void, + sseComments: string[] +): Promise { + if (!res.body) return; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + if (!value) continue; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() ?? ''; + for (const line of lines) { + if (line.startsWith(':')) { + sseComments.push(line); + continue; + } + if (!line.startsWith('data:')) continue; + const payload = line.slice(5).trim(); + if (!payload) continue; + try { + ingest(JSON.parse(payload)); + } catch { + ingest(payload); + } + } + } + } catch { + // Aborted by cancel(), or the connection dropped. The frames that arrived + // are what the scenario grades. + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/scenarios/server/agents/negative-fixture.ts b/src/scenarios/server/agents/negative-fixture.ts new file mode 100644 index 00000000..10335d47 --- /dev/null +++ b/src/scenarios/server/agents/negative-fixture.ts @@ -0,0 +1,401 @@ +/** + * The fixture the agent discovery negative controls are graded against. + * + * One server, two scenarios. Every option here exists to break exactly one + * rule while leaving the rest of the wire conformant, so a check that flips + * says which rule it caught. With no options it is a conformant server, which + * is what the false-positive test runs against. + * + * It speaks the SEP-2575 stateless wire (`server/discover` plus one POST per + * request), built per test rather than checked in as an example server, as the + * SEP-2640 negative tests do. `subscriptions/listen` is a real SSE response + * held open until the client aborts, so the list-changed rows see a + * notification arrive on a live stream. + */ + +import { createServer, type IncomingMessage, type ServerResponse } from 'http'; +import type { AddressInfo } from 'net'; +import { withRequiredDraftResultFields } from '../../../mock-server'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { + AGENTS_EXTENSION_ID, + AGENTS_LIST_CHANGED_FILTER_KEY, + AGENTS_LIST_CHANGED_NOTIFICATION, + CONTROL_ADD_AGENT, + CONTROL_FIXTURE, + SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + isObject +} from './helpers'; + +export interface FixtureAgent { + name: string; + description?: string; + capabilities?: string[]; + instructions?: string; + modelRoutingHint?: unknown; + tools: string[]; +} + +/** The catalog every fixture serves. `search` is shared by two agents. */ +export function catalog(): Record[] { + return [ + { + name: 'search', + description: 'Search the index.', + inputSchema: { type: 'object', properties: { q: { type: 'string' } } } + }, + { + name: 'summarize', + description: 'Summarize a document.', + inputSchema: { type: 'object', properties: {} } + }, + { + name: 'secret_tool', + description: 'Belongs to the hidden agent.', + inputSchema: { type: 'object', properties: {} } + } + ]; +} + +/** The roster every fixture serves, before any test overrides it. */ +export function roster(): FixtureAgent[] { + return [ + { + name: 'research-agent', + description: 'Researches things', + capabilities: ['research'], + instructions: 'Research the question.', + modelRoutingHint: { role: 'medium', vision: false }, + tools: ['search'] + }, + { + name: 'writer-agent', + capabilities: ['writing'], + tools: ['search', 'summarize'] + } + ]; +} + +export const HIDDEN_AGENT = 'hidden-agent'; +export const BROKEN_AGENT = 'broken-agent'; + +const NOT_FOUND = { + code: -32602, + message: 'agent not found', + data: { reason: 'agent_not_found' } +}; + +export interface AgentsFixtureOptions { + /** The value declared under extensions; `null` leaves it undeclared. */ + declared?: unknown; + agents?: FixtureAgent[]; + /** Agents per agents/list page. Default 1, so paging is exercised. */ + pageSize?: number; + /** Answer agents/list with -32601 instead of a roster. */ + listUnimplemented?: boolean; + /** Members merged into every agents/list page result. */ + listOverrides?: Record; + /** Hand out the same cursor forever. */ + loopingCursor?: boolean; + /** Members merged into the card for one agent name. */ + cardOverrides?: Record>; + /** Members merged into the agents/get result for one agent name. */ + getOverrides?: Record>; + /** Fail agents/get for these listed agents with -32603. */ + getFails?: string[]; + /** Tools served by agents/get for one agent, replacing the catalog lookup. */ + getTools?: Record[]>; + /** The error agents/get gives for a name it does not know. */ + unknownError?: Record; + /** The error agents/get gives for the hidden agent. */ + hiddenError?: Record; + /** List the hidden agent in agents/list anyway. */ + listHidden?: boolean; + /** The error agents/get gives for the broken agent. */ + brokenError?: Record; + /** Refuse tools/list from a client that did not declare the extension. */ + coreRequiresExtension?: boolean; + /** Leave out the named control tools. */ + withoutControls?: string[]; + /** Leave agentsListChanged out of the listen acknowledgement. */ + ackDropsKey?: boolean; + /** Acknowledge agentsListChanged even when listChanged is not declared. */ + ackAlways?: boolean; + /** Add agents without sending notifications/agents/list_changed. */ + silentAdd?: boolean; + /** Send the notification but leave the roster unchanged. */ + addWithoutChange?: boolean; +} + +export interface AgentsFixture { + url: string; + close(): Promise; +} + +async function readJsonBody( + req: IncomingMessage +): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; +} + +function clientDeclaredAgents(params: Record): boolean { + const meta = isObject(params._meta) ? params._meta : {}; + const caps = meta['io.modelcontextprotocol/clientCapabilities']; + return ( + isObject(caps) && + isObject(caps.extensions) && + AGENTS_EXTENSION_ID in caps.extensions + ); +} + +export async function startAgentsFixture( + opts: AgentsFixtureOptions = {} +): Promise { + const declared = + opts.declared === undefined ? { listChanged: true } : opts.declared; + const agents = (opts.agents ?? roster()).map((a) => ({ ...a })); + const tools = catalog(); + const pageSize = opts.pageSize ?? 1; + const listeners = new Set(); + const without = new Set(opts.withoutControls ?? []); + + const controls = [ + { + name: CONTROL_FIXTURE, + description: 'Names the hidden and broken agents.', + inputSchema: { type: 'object', properties: {} } + }, + { + name: CONTROL_ADD_AGENT, + description: 'Adds an agent to the roster.', + inputSchema: { + type: 'object', + properties: { name: { type: 'string' } }, + required: ['name'] + } + } + ].filter((c) => !without.has(c.name)); + + const visible = (): FixtureAgent[] => { + const out = [ + ...agents, + { name: BROKEN_AGENT, capabilities: [], tools: ['missing_tool'] } + ]; + if (opts.listHidden) { + out.push({ + name: HIDDEN_AGENT, + capabilities: [], + tools: ['secret_tool'] + }); + } + return out; + }; + + const card = (a: FixtureAgent): Record => { + const c: Record = { + name: a.name, + capabilities: a.capabilities ?? [] + }; + if (a.description !== undefined) c.description = a.description; + return { ...c, ...(opts.cardOverrides?.[a.name] ?? {}) }; + }; + + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + const body = await readJsonBody(req); + const method = body.method as string; + const id = body.id; + const params = isObject(body.params) ? body.params : {}; + const send = (result: object) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id, + result: withRequiredDraftResultFields(method, result) + }) + ); + }; + const fail = (error: Record) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ jsonrpc: '2.0', id, error })); + }; + + switch (method) { + case 'server/discover': { + const extensions = + declared === null ? {} : { [AGENTS_EXTENSION_ID]: declared }; + send({ + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: { tools: {}, extensions }, + serverInfo: { name: 'agents-negative', version: '1.0.0' } + }); + return; + } + case 'tools/list': { + if (opts.coreRequiresExtension && !clientDeclaredAgents(params)) { + fail({ code: -32600, message: 'declare the agents extension first' }); + return; + } + send({ tools: [...tools, ...controls] }); + return; + } + case 'tools/call': { + const name = params.name; + const args = isObject(params.arguments) ? params.arguments : {}; + if (name === CONTROL_FIXTURE && !without.has(CONTROL_FIXTURE)) { + send({ + content: [ + { + type: 'text', + text: JSON.stringify({ + hiddenAgent: HIDDEN_AGENT, + brokenAgent: BROKEN_AGENT + }) + } + ] + }); + return; + } + if (name === CONTROL_ADD_AGENT && !without.has(CONTROL_ADD_AGENT)) { + if (!opts.addWithoutChange) { + agents.push({ + name: String(args.name), + capabilities: [], + tools: ['search'] + }); + } + if (!opts.silentAdd) { + const frame = JSON.stringify({ + jsonrpc: '2.0', + method: AGENTS_LIST_CHANGED_NOTIFICATION, + params: {} + }); + for (const l of listeners) l.write(`data: ${frame}\n\n`); + } + send({ content: [{ type: 'text', text: 'ok' }] }); + return; + } + send({ content: [{ type: 'text', text: 'done' }] }); + return; + } + case 'agents/list': { + if (opts.listUnimplemented) { + fail({ code: -32601, message: 'method not found' }); + return; + } + const all = visible(); + const start = + typeof params.cursor === 'string' ? Number(params.cursor) : 0; + const page = all.slice(start, start + pageSize).map(card); + const next = + start + pageSize < all.length ? String(start + pageSize) : undefined; + const result: Record = { + agents: page, + ttlMs: 60000, + cacheScope: 'private', + ...(opts.loopingCursor + ? { nextCursor: '0' } + : next + ? { nextCursor: next } + : {}), + ...(opts.listOverrides ?? {}) + }; + send(result); + return; + } + case 'agents/get': { + const name = params.name; + if (name === HIDDEN_AGENT) { + fail(opts.hiddenError ?? NOT_FOUND); + return; + } + if (name === BROKEN_AGENT) { + fail( + opts.brokenError ?? { + code: -32603, + message: 'agent names a missing tool', + data: { reason: 'invalid_agent_definition' } + } + ); + return; + } + const agent = agents.find((a) => a.name === name); + if (!agent) { + fail(opts.unknownError ?? NOT_FOUND); + return; + } + if (opts.getFails?.includes(agent.name)) { + fail({ code: -32603, message: 'boom' }); + return; + } + const result: Record = { + agent: agent.name, + tools: + opts.getTools?.[agent.name] ?? + agent.tools.map((t) => tools.find((c) => c.name === t)), + ttlMs: 60000, + cacheScope: 'private' + }; + if (agent.instructions !== undefined) { + result.instructions = agent.instructions; + } + if (agent.modelRoutingHint !== undefined) { + result.modelRoutingHint = agent.modelRoutingHint; + } + send({ ...result, ...(opts.getOverrides?.[agent.name] ?? {}) }); + return; + } + case 'subscriptions/listen': { + const filter = isObject(params.notifications) + ? params.notifications + : {}; + const declaredTrue = + isObject(declared) && declared.listChanged === true; + const honor = + filter[AGENTS_LIST_CHANGED_FILTER_KEY] === true && + (declaredTrue || opts.ackAlways) && + !opts.ackDropsKey; + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + const ack = JSON.stringify({ + jsonrpc: '2.0', + method: SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + params: { + notifications: honor + ? { [AGENTS_LIST_CHANGED_FILTER_KEY]: true } + : {}, + _meta: { 'io.modelcontextprotocol/subscriptionId': id } + } + }); + res.write(`data: ${ack}\n\n`); + listeners.add(res); + req.on('close', () => listeners.delete(res)); + return; + } + default: + fail({ code: -32601, message: `method not found: ${method}` }); + } + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}/mcp`, + close: () => + new Promise((resolve) => { + for (const l of listeners) l.end(); + server.closeAllConnections?.(); + server.close(() => resolve()); + }) + }; +} diff --git a/src/scenarios/server/agents/negative.test.ts b/src/scenarios/server/agents/negative.test.ts new file mode 100644 index 00000000..f06d66e3 --- /dev/null +++ b/src/scenarios/server/agents/negative.test.ts @@ -0,0 +1,433 @@ +import { describe, test, expect } from 'vitest'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { AgentsDiscoveryScenario } from './discovery'; +import { AgentsListChangedScenario } from './list-changed'; +import { + BROKEN_AGENT, + HIDDEN_AGENT, + catalog, + startAgentsFixture, + type AgentsFixtureOptions +} from './negative-fixture'; + +/** + * Negative controls for the agent discovery scenarios. + * + * A passing run against a conformant fixture proves a check does not + * false-positive. It does not prove the check catches anything, which is what + * these tests are for: each one breaks the fixture in exactly one way and + * asserts the matching check flips. The fixture lives in negative-fixture.ts. + */ + +type Scenario = AgentsDiscoveryScenario | AgentsListChangedScenario; + +async function checksFor(scenario: Scenario, opts: AgentsFixtureOptions) { + const fixture = await startAgentsFixture(opts); + try { + const checks = await scenario.run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed response does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await fixture.close(); + } +} + +const discovery = () => new AgentsDiscoveryScenario(); +const listChanged = () => new AgentsListChangedScenario(); + +describe('agents discovery: a conformant fixture', () => { + test('every check passes, and only the optional ones skip', async () => { + const checks = await checksFor(discovery(), {}); + const notPassing = [...checks.values()] + .filter((c) => c.status !== 'SUCCESS') + .map((c) => `${c.id}: ${c.status} ${c.errorMessage ?? ''}`); + expect(notPassing).toEqual([]); + expect(checks.size).toBe(27); + }); + + test('each check ID is emitted exactly once', async () => { + const fixture = await startAgentsFixture({}); + try { + const checks = await discovery().run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + takeWireViolations(); + const ids = checks.map((c) => c.id); + expect(new Set(ids).size).toBe(ids.length); + } finally { + await fixture.close(); + } + }); +}); + +describe('agents discovery: capability', () => { + test('an undeclared extension the server does not serve skips everything', async () => { + const checks = await checksFor(discovery(), { + declared: null, + listUnimplemented: true + }); + expect([...checks.values()].every((c) => c.status === 'SKIPPED')).toBe( + true + ); + }); + + test('serving agents/list without declaring the extension fails', async () => { + const checks = await checksFor(discovery(), { declared: null }); + expect(checks.get('sep-9998-capability-declared')?.status).toBe('FAILURE'); + }); + + test('a declaration that is not a settings object fails', async () => { + const checks = await checksFor(discovery(), { declared: true }); + expect(checks.get('sep-9998-capability-declared')?.status).toBe('FAILURE'); + }); + + test('a non-boolean listChanged fails', async () => { + const checks = await checksFor(discovery(), { + declared: { listChanged: 'yes' } + }); + expect(checks.get('sep-9998-capability-list-changed-boolean')?.status).toBe( + 'FAILURE' + ); + }); + + test('refusing tools/list to a client without the extension fails', async () => { + const checks = await checksFor(discovery(), { + coreRequiresExtension: true + }); + expect(checks.get('sep-9998-core-without-extension')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe('agents discovery: roster', () => { + test('a declared extension without agents/list fails', async () => { + const checks = await checksFor(discovery(), { listUnimplemented: true }); + expect(checks.get('sep-9998-list-implemented')?.status).toBe('FAILURE'); + }); + + test('a page without resultType fails', async () => { + const checks = await checksFor(discovery(), { + listOverrides: { resultType: 'incomplete' } + }); + expect(checks.get('sep-9998-list-result-shape')?.status).toBe('FAILURE'); + }); + + test('a cursor that never ends fails', async () => { + const checks = await checksFor(discovery(), { loopingCursor: true }); + expect(checks.get('sep-9998-list-pagination')?.status).toBe('FAILURE'); + }); + + test('a page without ttlMs fails', async () => { + const checks = await checksFor(discovery(), { + listOverrides: { ttlMs: undefined } + }); + expect(checks.get('sep-9998-list-cacheable')?.status).toBe('FAILURE'); + }); + + test('a card whose name is not a string fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { name: 7 } } + }); + expect(checks.get('sep-9998-card-name')?.status).toBe('FAILURE'); + }); + + test('a description that is not a string fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { description: 3 } } + }); + expect(checks.get('sep-9998-card-description-type')?.status).toBe( + 'FAILURE' + ); + }); + + test('a card without capabilities fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { capabilities: undefined } } + }); + expect(checks.get('sep-9998-card-capabilities')?.status).toBe('FAILURE'); + }); + + test('a card carrying instructions warns', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { instructions: 'leaked' } } + }); + expect(checks.get('sep-9998-list-cards-only')?.status).toBe('WARNING'); + }); + + test('a name listed on two pages fails', async () => { + const checks = await checksFor(discovery(), { + cardOverrides: { 'writer-agent': { name: 'research-agent' } } + }); + expect(checks.get('sep-9998-name-unique')?.status).toBe('FAILURE'); + }); +}); + +describe('agents discovery: details', () => { + test('agents/get failing for a listed agent fails', async () => { + const checks = await checksFor(discovery(), { getFails: ['writer-agent'] }); + expect(checks.get('sep-9998-get-implemented')?.status).toBe('FAILURE'); + }); + + test('a details result without resultType fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'writer-agent': { resultType: 'incomplete' } } + }); + expect(checks.get('sep-9998-get-result-shape')?.status).toBe('FAILURE'); + }); + + test('a details result naming another agent fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'writer-agent': { agent: 'research-agent' } } + }); + expect(checks.get('sep-9998-get-agent-name')?.status).toBe('FAILURE'); + }); + + test('a details result without cacheScope fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'writer-agent': { cacheScope: undefined } } + }); + expect(checks.get('sep-9998-get-cacheable')?.status).toBe('FAILURE'); + }); + + test('a role outside the enum fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { + 'research-agent': { modelRoutingHint: { role: 'huge', vision: false } } + } + }); + expect(checks.get('sep-9998-hint-role')?.status).toBe('FAILURE'); + }); + + test('a hint without vision fails', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { + 'research-agent': { modelRoutingHint: { role: 'small' } } + } + }); + expect(checks.get('sep-9998-hint-vision')?.status).toBe('FAILURE'); + }); + + test('no hint anywhere skips the hint rows', async () => { + const checks = await checksFor(discovery(), { + getOverrides: { 'research-agent': { modelRoutingHint: undefined } } + }); + expect(checks.get('sep-9998-hint-role')?.status).toBe('SKIPPED'); + expect(checks.get('sep-9998-hint-vision')?.status).toBe('SKIPPED'); + }); +}); + +describe('agents discovery: tool parity', () => { + test('a returned tool missing from tools/list fails all three parity rows', async () => { + const checks = await checksFor(discovery(), { + getTools: { + 'writer-agent': [ + ...catalog().slice(0, 2), + { + name: 'unlisted_tool', + inputSchema: { type: 'object', properties: {} } + } + ] + } + }); + for (const id of [ + 'sep-9998-get-tools-in-tools-list', + 'sep-9998-get-tools-intersection', + 'sep-9998-tools-list-not-omitted' + ]) { + expect(checks.get(id)?.status, id).toBe('FAILURE'); + } + expect(checks.get('sep-9998-get-tools-fields-match')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a returned tool whose fields differ from tools/list fails', async () => { + const [search, summarize] = catalog(); + const checks = await checksFor(discovery(), { + getTools: { + 'writer-agent': [search, { ...summarize, description: 'Different.' }] + } + }); + expect(checks.get('sep-9998-get-tools-fields-match')?.status).toBe( + 'FAILURE' + ); + expect(checks.get('sep-9998-get-tools-in-tools-list')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a shared tool that differs between agents fails', async () => { + const [search, summarize] = catalog(); + const checks = await checksFor(discovery(), { + getTools: { + 'writer-agent': [ + { ...search, description: 'Another search.' }, + summarize + ] + } + }); + expect(checks.get('sep-9998-shared-tool-same-definition')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe('agents discovery: errors and visibility', () => { + test('an unknown name without the agent_not_found reason fails', async () => { + const checks = await checksFor(discovery(), { + unknownError: { code: -32602, message: 'agent not found' } + }); + expect(checks.get('sep-9998-agent-not-found')?.status).toBe('FAILURE'); + }); + + test('a hidden agent answering differently from an unknown one fails', async () => { + const checks = await checksFor(discovery(), { + hiddenError: { + code: -32602, + message: 'you may not see this agent', + data: { reason: 'agent_not_found' } + } + }); + expect(checks.get('sep-9998-hidden-indistinguishable')?.status).toBe( + 'FAILURE' + ); + expect(checks.get('sep-9998-roster-filtered')?.status).toBe('SUCCESS'); + }); + + test('a hidden agent listed in the roster fails', async () => { + const checks = await checksFor(discovery(), { listHidden: true }); + expect(checks.get('sep-9998-roster-filtered')?.status).toBe('FAILURE'); + expect(checks.get('sep-9998-hidden-indistinguishable')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a broken agent answering -32602 fails', async () => { + const checks = await checksFor(discovery(), { + brokenError: { + code: -32602, + message: 'missing tool', + data: { reason: 'invalid_agent_definition' } + } + }); + expect(checks.get('sep-9998-invalid-agent-definition')?.status).toBe( + 'FAILURE' + ); + }); + + test('a broken agent answering -32603 without the reason fails', async () => { + const checks = await checksFor(discovery(), { + brokenError: { code: -32603, message: 'missing tool' } + }); + expect(checks.get('sep-9998-invalid-agent-definition')?.status).toBe( + 'FAILURE' + ); + }); + + test('without the fixture control the three rows report untestable', async () => { + const checks = await checksFor(discovery(), { + withoutControls: ['agents_conformance_fixture'] + }); + for (const id of [ + 'sep-9998-hidden-indistinguishable', + 'sep-9998-roster-filtered', + 'sep-9998-invalid-agent-definition' + ]) { + const c = checks.get(id); + expect(c?.status, id).toBe('FAILURE'); + expect(c?.details?.untestable, id).toBe(true); + } + // Without the control the broken agent is opened like any other, and its + // -32603 is reported where it lands. + expect(checks.get('sep-9998-get-implemented')?.errorMessage).toContain( + BROKEN_AGENT + ); + expect(HIDDEN_AGENT).toBe('hidden-agent'); + }); +}); + +describe('agents list_changed', () => { + test('a conformant fixture passes', async () => { + const checks = await checksFor(listChanged(), {}); + expect(checks.get('sep-9998-list-changed-not-declared')?.status).toBe( + 'SKIPPED' + ); + for (const id of [ + 'sep-9998-list-changed-acknowledged', + 'sep-9998-list-changed-notification', + 'sep-9998-list-changed-roster-updated' + ]) { + expect( + checks.get(id)?.status, + `${id}: ${checks.get(id)?.errorMessage}` + ).toBe('SUCCESS'); + } + }); + + test('an undeclared extension skips everything', async () => { + const checks = await checksFor(listChanged(), { declared: null }); + expect([...checks.values()].every((c) => c.status === 'SKIPPED')).toBe( + true + ); + }); + + test('acknowledging the key without declaring listChanged fails', async () => { + const checks = await checksFor(listChanged(), { + declared: {}, + ackAlways: true + }); + expect(checks.get('sep-9998-list-changed-not-declared')?.status).toBe( + 'FAILURE' + ); + }); + + test('not acknowledging the key without declaring listChanged passes', async () => { + const checks = await checksFor(listChanged(), { declared: {} }); + expect(checks.get('sep-9998-list-changed-not-declared')?.status).toBe( + 'SUCCESS' + ); + }); + + test('declaring listChanged but dropping the key fails', async () => { + const checks = await checksFor(listChanged(), { ackDropsKey: true }); + expect(checks.get('sep-9998-list-changed-acknowledged')?.status).toBe( + 'FAILURE' + ); + }); + + test('a roster change without the notification fails', async () => { + const checks = await checksFor(listChanged(), { silentAdd: true }); + expect(checks.get('sep-9998-list-changed-notification')?.status).toBe( + 'FAILURE' + ); + }, 15000); + + test('a notification whose change never shows in agents/list warns', async () => { + const checks = await checksFor(listChanged(), { addWithoutChange: true }); + expect(checks.get('sep-9998-list-changed-notification')?.status).toBe( + 'SUCCESS' + ); + expect(checks.get('sep-9998-list-changed-roster-updated')?.status).toBe( + 'WARNING' + ); + }); + + test('without the add control the two rows report untestable', async () => { + const checks = await checksFor(listChanged(), { + withoutControls: ['agents_conformance_add_agent'] + }); + expect( + checks.get('sep-9998-list-changed-notification')?.details?.untestable + ).toBe(true); + expect( + checks.get('sep-9998-list-changed-roster-updated')?.details?.untestable + ).toBe(true); + }); +}); diff --git a/src/seps/sep-9998.yaml b/src/seps/sep-9998.yaml new file mode 100644 index 00000000..663d0539 --- /dev/null +++ b/src/seps/sep-9998.yaml @@ -0,0 +1,215 @@ +# spec_source: modelcontextprotocol/agents-wg@9b16840e2f958b3dd9a418bb321189a0d1ab3493 proposals/0000-mcp-agent-discovery-extension.md +# extracted: 2026-10-15 +# +# ############################################################################ +# # 9998 is a placeholder, not a SEP number. # +# ############################################################################ +# +# The MCP Agent Discovery Extension is agents-wg PR 22, a draft Extensions +# Track proposal in the Agents Working Group's repository +# (modelcontextprotocol/agents-wg). SEP numbers are PR numbers in +# modelcontextprotocol/modelcontextprotocol, and this proposal has no PR +# there yet, so it has no number. Both traceability filters are numeric (the +# file name /^sep-\d+\.yaml$/ and the check ID /^sep-\d+-/), and a file that +# does not match is silently dropped from the manifest, so an honest name +# such as sep-agents-* would vanish. 9999 is not used because the MCP Events +# suite carried it before SEP-3415 existed, and reports from that period +# still mention it. +# +# When the proposal gets a number, rename the file, every check ID and the +# two scenario names by script, as the Events suite did when it moved from +# 9999 to 3415. Nothing else in a row depends on the number. +# +# new-sep cannot scaffold this file: it looks the number up as a PR in +# modelcontextprotocol/modelcontextprotocol and derives the spec URL from a +# docs/specification/draft/*.mdx path, and the proposal has neither. The +# spec_url below points at the pinned proposal instead. +# +# quoting: every row's `text` is a span of the proposal at 9b16840e, and its +# `url` is the section that span sits under. The match ignores line +# wrapping (runs of whitespace compare equal), because the proposal wraps +# prose at a fixed width and a quote across a wrap point would otherwise need +# embedded newlines. Field-level rows quote the TypeScript definition they +# grade, since the proposal states field types only there. +# +# coverage: 31 graded rows across two server scenarios. +# sep-9998-agents-discovery emits the capability, roster, details, tool +# parity and error rows. +# sep-9998-agents-list-changed emits the four list-changed rows. +# Three rows need a diagnostic control on the server under test +# (agents_conformance_fixture names a hidden agent and a broken one), and +# two need agents_conformance_add_agent to change the roster mid-run. A +# server without them reports those rows untestable, naming the control. +# Excluded rows are client or host obligations, off-wire server behaviour, +# or rules that need two authorization contexts, which a harness holding one +# credential cannot compare. +sep: 9998 +spec_url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#specification +requirements: + # === Capability Negotiation === + - check: sep-9998-capability-declared + text: 'Clients and servers explicitly declare support for the extension using the MCP extension negotiation mechanism.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - check: sep-9998-capability-list-changed-boolean + text: 'The optional `listChanged` setting indicates that the server supports agent-roster change notifications.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - check: sep-9998-core-without-extension + text: 'A server **MUST NOT** require clients to support this extension in order to use otherwise compatible core MCP functionality.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + + # === Roster: agents/list === + - check: sep-9998-list-implemented + text: '`agents/list` returns the agent roster visible to the requesting client.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - check: sep-9998-list-result-shape + text: 'interface ListAgentsResult extends PaginatedResult, CacheableResult { resultType: "complete"; agents: AgentCard[]; }' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-card-name + text: "/** Unique identifier for this agent within the server's visible roster. */ name: string;" + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-card-description-type + text: "/** Optional human-readable summary of the agent's purpose. */ description?: string;" + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-card-capabilities + text: '/** Concise routing labels; these are not MCP tool names. */ capabilities: string[];' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + # No RFC 2119 keyword, so graded as a WARNING. The card interface does not + # forbid extra members, but a card carrying instructions or tool schemas + # defeats the progressive disclosure the two levels exist for. + - check: sep-9998-list-cards-only + text: 'The server returns compact cards only:' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#listing-agents + - check: sep-9998-name-unique + text: 'Each agent name **MUST** be unique within the roster visible to the requesting client.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-list-pagination + text: 'Clients omit `cursor` for the first page and continue with the returned `nextCursor` until the response omits it.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#listing-agents + - check: sep-9998-list-cacheable + text: 'Both `agents/list` and `agents/get` results follow the MCP `CacheableResult` model defined' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#caching-and-freshness + + # === Details: agents/get === + - check: sep-9998-get-implemented + text: '`agents/get` returns instructions and scoped MCP tool schemas for one agent.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - check: sep-9998-get-result-shape + text: 'interface GetAgentResult extends CacheableResult { resultType: "complete";' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-get-agent-name + text: '/** Name of the agent to which these details belong. */ agent: string;' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-get-cacheable + text: 'Both `agents/list` and `agents/get` results follow the MCP `CacheableResult` model defined' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#caching-and-freshness + - check: sep-9998-hint-role + text: 'type ModelRole = "small" | "medium" | "large";' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-hint-vision + text: '/** Whether a vision-capable model is preferred for this agent. */ vision: boolean;' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + + # === Details: tool parity with tools/list === + # The next three rows are graded by one probe (every tool from every + # agents/get, looked up in tools/list), since for a single caller they + # describe the same observation from three sides. + - check: sep-9998-get-tools-in-tools-list + text: 'Every returned tool **MUST** also be discoverable through `tools/list`' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-get-tools-intersection + text: "A server **MUST NOT** include a tool in `agents/get` that would be absent from that caller's `tools/list`." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-tools-list-not-omitted + text: 'A server **MUST NOT** omit a tool from `tools/list` solely because that tool is associated with one or more agents.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - check: sep-9998-get-tools-fields-match + text: 'When a tool appears in both responses for the same request context, all fields of the corresponding `Tool` definitions **MUST** match.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-shared-tool-same-definition + text: 'If the same tool name appears under multiple agents, it **MUST** identify the same server-wide callable tool.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + + # === Errors and visibility === + - check: sep-9998-agent-not-found + text: 'If the requested agent does not exist or is not visible to the requesting client, the server **MUST** return `-32602` (`Invalid params`) with `data.reason: "agent_not_found"`.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + - check: sep-9998-hidden-indistinguishable + text: "When a server policy requires hiding an agent's existence from unauthorized callers, the server **MUST** use the same `agent_not_found` response as for a nonexistent agent name." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - check: sep-9998-roster-filtered + text: "A server **MUST** filter both the roster and agent details according to the requesting client's effective permissions." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - check: sep-9998-invalid-agent-definition + text: 'If an agent''s configured membership references a tool name that does not exist in the server catalog, or the server would return a tool through `agents/get` that would be absent from that caller''s `tools/list` under the same authorization context, the server **MUST** return `-32603` (`Internal error`) with `data.reason: "invalid_agent_definition"`.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + + # === Roster changes === + - check: sep-9998-list-changed-not-declared + text: 'Omitting it or setting it to `false` indicates that such notifications are not supported.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - check: sep-9998-list-changed-acknowledged + text: 'Clients opt in by adding `agentsListChanged: true` to the `notifications` filter of a `subscriptions/listen` request. The server acknowledges and delivers the notification using the standard MCP subscription mechanism.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + - check: sep-9998-list-changed-notification + text: 'A server that advertises `listChanged: true` for this extension supports `notifications/agents/list_changed`.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + # No RFC 2119 keyword, so graded as a WARNING. + - check: sep-9998-list-changed-roster-updated + text: 'The notification indicates that the visible roster or one or more agent cards may have changed.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + + # === Excluded === + - text: 'Servers **SHOULD** validate at registration time that every tool name referenced by an agent exists in the server catalog.' + excluded: 'Registration happens off the wire. Its observable consequence, an agents/get for such an agent, is graded by sep-9998-invalid-agent-definition.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: 'remain callable through the existing MCP tool-calling mechanism, subject to the same authorization context and server policy as any other tool call.' + excluded: "Calling an arbitrary server's tools has side effects the harness cannot justify, and it has no argument values to call them with. Core tools/call behaviour is graded by the core tools scenarios." + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: "For each `agents/get` request, the server **MUST** return only tools that would also appear in that caller's `tools/list` under the same authorization context (the intersection of agent membership with the authorized catalog)." + excluded: 'For the single authorization context a run has, this is the observation sep-9998-get-tools-intersection grades. Whether the intersection tracks the caller needs two credentials to compare, and the harness holds one.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: 'Servers **MUST NOT** use an agent card or agent membership to grant permissions that the requesting client does not otherwise possess.' + excluded: 'Needs a caller lacking a permission that membership could wrongly grant, compared against one holding it. The harness holds one credential.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - text: 'For both `agents/list` and `agents/get`, servers **SHOULD** use `private` unless they can guarantee that the particular response is identical and safe for every client.' + excluded: 'Whether a response is identical for every client is not observable from one client. A public scope is permitted when the guarantee holds, so grading the value alone would fail conformant servers.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#caching-and-freshness + - text: 'When a caller is permitted to know an agent exists but lacks authorization to retrieve it, the server **MAY** return an authorization error that enables' + excluded: 'Optional, and exercising it needs a credential that can be stepped up, which the harness does not hold.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - text: 'Servers **SHOULD** document which policy applies for their deployment.' + excluded: 'Documentation, not protocol behaviour.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#security-implications + - text: 'A server **MUST NOT** assume that a host honored the hint.' + excluded: 'Server-internal reasoning with no wire signature.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#retrieving-agent-details + - text: 'Servers **MAY** associate the same tool with more than one agent.' + excluded: 'A permission with nothing to grade. Its constraint is graded by sep-9998-shared-tool-same-definition.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-discovery-model + - text: 'A client **MUST NOT** assume agent discovery is available unless the server advertises the extension.' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#capability-negotiation + - text: 'A client receiving it **SHOULD** invalidate all cached `agents/list` pages and re-list before making a new routing decision.' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + - text: 'After re-listing, it **SHOULD** discard cached `agents/get` results for agents that are no longer visible.' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#agent-roster-changes + - text: 'Clients **MAY** refresh `agents/list` and retry.' + excluded: 'Client permission; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + - text: 'Clients **SHOULD NOT** automatically retry the same request;' + excluded: 'Client obligation; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#error-handling + - text: 'The host **MAY** request the complete visible tool catalog through `tools/list` for existing discovery, indexing, or SDK operation.' + excluded: 'Host permission; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - text: 'Existing systems that already call `tools/list` **MAY** retain that behavior for indexing, schema validation, and other host operations.' + excluded: 'Host permission; these are server scenarios.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#discovery-and-invocation-flow + - text: "When the field is absent, unsupported, or cannot be satisfied, the host **MAY** use its default model or inherit the supervisor's model." + excluded: 'Host model selection is off the wire.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#retrieving-agent-details + - text: "Hosts **MAY** resolve the hint through local configuration, inherit the supervisor's model, or ignore unsupported values." + excluded: 'Host model selection is off the wire.' + url: https://github.com/modelcontextprotocol/agents-wg/blob/9b16840e2f958b3dd9a418bb321189a0d1ab3493/proposals/0000-mcp-agent-discovery-extension.md#retrieving-agent-details diff --git a/src/types.ts b/src/types.ts index ebe75a27..57d9793d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -103,7 +103,8 @@ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/auth/dpop', 'io.modelcontextprotocol/auth/wif', 'io.modelcontextprotocol/tasks', - 'io.modelcontextprotocol/skills' + 'io.modelcontextprotocol/skills', + 'io.modelcontextprotocol/agents' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number];