From 449dc0d6d161245122bc6e29f5b0f633af0e414d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 19:12:28 +0000 Subject: [PATCH] Add get-screenshot app tool to the map-server example The map view already captures scaled PNG screenshots of the globe for model-context updates, but only pushed them opportunistically (and only when the host supports images in updateModelContext). This factors that capture into a shared helper and registers a get-screenshot view tool, so the model can request the current view on demand and see exactly what the user sees. Takes an optional maxDimension (default 1024px) to trade detail against image token cost. The e2e test follows the pdf-get-current-pdf pattern: it plays the host, posting tools/list and tools/call JSON-RPC into the app iframe from the sandbox frame. --- examples/map-server/README.md | 8 ++ examples/map-server/src/mcp-app.ts | 112 ++++++++++++++----- tests/e2e/map-get-screenshot.spec.ts | 154 +++++++++++++++++++++++++++ 3 files changed, 248 insertions(+), 26 deletions(-) create mode 100644 tests/e2e/map-get-screenshot.spec.ts diff --git a/examples/map-server/README.md b/examples/map-server/README.md index e1e044a84..3931271c5 100644 --- a/examples/map-server/README.md +++ b/examples/map-server/README.md @@ -98,6 +98,14 @@ Display the 3D globe zoomed to a bounding box. Defaults to London if no coordinates provided. +### App tools + +The map view registers tools on the view itself (via `app.registerTool`), for hosts that list and call view tools directly: + +- `navigate-to` — fly the globe to a bounding box, with an optional label. +- `get-current-view` — return the camera position and the visible bounding box (also as `structuredContent`). +- `get-screenshot` — capture the current globe view as a PNG image in the tool result (optional `maxDimension`, default 1024px), so the model can see exactly what the user sees. + ## Architecture ### Server (`server.ts`) diff --git a/examples/map-server/src/mcp-app.ts b/examples/map-server/src/mcp-app.ts index d3090c7a4..8f37abcb3 100644 --- a/examples/map-server/src/mcp-app.ts +++ b/examples/map-server/src/mcp-app.ts @@ -17,6 +17,32 @@ const CESIUM_VERSION = "1.123"; const CESIUM_BASE_URL = `https://cesium.com/downloads/cesiumjs/releases/${CESIUM_VERSION}/Build/Cesium`; const MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION = 768; // Max width/height for screenshots in pixels for updateModelContext +const DEFAULT_SCREENSHOT_MAX_DIMENSION = 1024; // Default max width/height for the get-screenshot tool + +/** + * Capture the globe canvas as a base64 PNG, scaled down so neither side + * exceeds maxDimension (image token cost scales with dimensions). + */ +function captureScreenshotBase64( + sourceCanvas: HTMLCanvasElement, + maxDimension: number, +): { data: string; width: number; height: number } | null { + const scale = Math.min( + 1, + maxDimension / Math.max(sourceCanvas.width, sourceCanvas.height), + ); + const targetWidth = Math.round(sourceCanvas.width * scale); + const targetHeight = Math.round(sourceCanvas.height * scale); + const tempCanvas = document.createElement("canvas"); + tempCanvas.width = targetWidth; + tempCanvas.height = targetHeight; + const ctx = tempCanvas.getContext("2d"); + if (!ctx) return null; + ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight); + const base64Data = tempCanvas.toDataURL("image/png").split(",")[1]; + if (!base64Data) return null; + return { data: base64Data, width: targetWidth, height: targetHeight }; +} /** * Dynamically load CesiumJS from CDN @@ -300,33 +326,19 @@ function scheduleLocationUpdate(cesiumViewer: any): void { if (app.getHostCapabilities()?.updateModelContext?.image) { try { // Scale down to reduce token usage (tokens depend on dimensions) - const sourceCanvas = cesiumViewer.canvas; - const scale = Math.min( - 1, - MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION / - Math.max(sourceCanvas.width, sourceCanvas.height), + const shot = captureScreenshotBase64( + cesiumViewer.canvas, + MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION, ); - const targetWidth = Math.round(sourceCanvas.width * scale); - const targetHeight = Math.round(sourceCanvas.height * scale); - - const tempCanvas = document.createElement("canvas"); - tempCanvas.width = targetWidth; - tempCanvas.height = targetHeight; - const ctx = tempCanvas.getContext("2d"); - if (ctx) { - ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight); - const dataUrl = tempCanvas.toDataURL("image/png"); - const base64Data = dataUrl.split(",")[1]; - if (base64Data) { - content.push({ - type: "image", - data: base64Data, - mimeType: "image/png", - }); - log.info( - `Added screenshot to model context (${targetWidth}x${targetHeight})`, - ); - } + if (shot) { + content.push({ + type: "image", + data: shot.data, + mimeType: "image/png", + }); + log.info( + `Added screenshot to model context (${shot.width}x${shot.height})`, + ); } } catch (err) { log.warn("Failed to capture screenshot:", err); @@ -977,6 +989,54 @@ app.registerTool( }, ); +app.registerTool( + "get-screenshot", + { + title: "Get Screenshot", + description: + "Capture the current globe view as a PNG image for visual analysis", + inputSchema: z.object({ + maxDimension: z + .number() + .int() + .min(64) + .max(2048) + .optional() + .describe( + "Max width/height in pixels (default 1024). Larger images show more detail but cost more tokens.", + ), + }), + }, + async (args) => { + if (!viewer) { + return { + content: [ + { type: "text" as const, text: "Error: Viewer not initialized" }, + ], + isError: true, + }; + } + + const shot = captureScreenshotBase64( + viewer.canvas, + args.maxDimension ?? DEFAULT_SCREENSHOT_MAX_DIMENSION, + ); + if (!shot) { + return { + content: [ + { type: "text" as const, text: "Error: screenshot capture failed" }, + ], + isError: true, + }; + } + return { + content: [ + { type: "image" as const, data: shot.data, mimeType: "image/png" }, + ], + }; + }, +); + // Handle tool result - extract viewUUID and restore persisted view if available app.ontoolresult = async (result) => { viewUUID = result._meta?.viewUUID ? String(result._meta.viewUUID) : undefined; diff --git a/tests/e2e/map-get-screenshot.spec.ts b/tests/e2e/map-get-screenshot.spec.ts new file mode 100644 index 000000000..a954fbb2d --- /dev/null +++ b/tests/e2e/map-get-screenshot.spec.ts @@ -0,0 +1,154 @@ +/** + * Map Server — get-screenshot app tool + * + * The map view registers tools on the view itself (App.registerTool). A host + * calls them by sending `tools/call` JSON-RPC requests to the app iframe over + * its PostMessageTransport, which accepts messages whose source is + * window.parent. The basic-host test UI only drives server tools, so this + * test plays the host's role directly: it posts the JSON-RPC request from the + * sandbox frame (the app iframe's parent) and reads the response there. + */ + +import { test, expect, type Page, type Frame } from "@playwright/test"; + +/** Wait for the MCP App to load inside nested iframes. */ +async function waitForAppLoad(page: Page) { + const outerFrame = page.frameLocator("iframe").first(); + await expect(outerFrame.locator("iframe")).toBeVisible({ timeout: 30000 }); +} + +/** Get the app frame locator (nested: sandbox > app) */ +function getAppFrame(page: Page) { + return page.frameLocator("iframe").first().frameLocator("iframe").first(); +} + +/** Load the map server and call show-map with its default bounding box. */ +async function loadMapServer(page: Page) { + await page.goto("/?theme=hide"); + // The host fetches the server list once per page load, and the example + // servers rebuild on examples:start, so a server can come up after the + // first load — reload until it appears instead of waiting on a stale list. + await expect(async () => { + await page.reload(); + const serverSelect = page.locator("select").first(); + await expect(serverSelect).toBeEnabled({ timeout: 30000 }); + expect(await serverSelect.locator("option").allTextContents()).toContain( + "CesiumJS Map Server", + ); + }).toPass({ timeout: 90000, intervals: [2000] }); + await page + .locator("select") + .first() + .selectOption({ label: "CesiumJS Map Server" }); + await page.locator("select").nth(1).selectOption("show-map"); + await page.click('button:has-text("Call Tool")'); + await waitForAppLoad(page); +} + +/** Wait for the Cesium canvas to render and give tiles time to load. */ +async function waitForGlobeCanvas(page: Page) { + const appFrame = getAppFrame(page); + await expect(appFrame.locator("canvas").first()).toBeVisible({ + timeout: 60000, + }); + // CesiumJS needs time for tiles to load (same budget as servers.spec.ts) + await page.waitForTimeout(15000); +} + +/** Resolve the sandbox frame (the app iframe's parent). */ +function getSandboxFrame(page: Page): Frame { + const appFrame = page + .frames() + .find((f) => f.parentFrame() && f.parentFrame()!.parentFrame()); + if (!appFrame) throw new Error("App frame not found"); + return appFrame.parentFrame()!; +} + +/** + * Send one JSON-RPC request into the app iframe from the sandbox frame and + * await the matching response. + */ +function sendJsonRpcToApp( + sandboxFrame: Frame, + request: { id: number; method: string; params?: unknown }, + timeoutMs = 60_000, +): Promise { + return sandboxFrame.evaluate( + async ({ id, method, params, timeoutMs }) => { + const iframe = document.querySelector("iframe") as HTMLIFrameElement; + return await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`${method} timed out`)), + timeoutMs, + ); + const onMsg = (e: MessageEvent) => { + // Only accept a response (not an echoed request) to our id, from + // the app iframe itself. + if ( + e.source === iframe.contentWindow && + e.data?.jsonrpc === "2.0" && + e.data.id === id && + ("result" in e.data || "error" in e.data) + ) { + clearTimeout(timer); + window.removeEventListener("message", onMsg); + resolve(e.data); + } + }; + window.addEventListener("message", onMsg); + iframe.contentWindow!.postMessage( + { jsonrpc: "2.0", id, method, params }, + "*", + ); + }); + }, + { ...request, timeoutMs }, + ); +} + +test.describe("Map Server - get-screenshot", () => { + test("is listed and returns the current view as a PNG image", async ({ + page, + }) => { + // Server-list reload loop + CDN Cesium + tile loading add up. + test.setTimeout(240_000); + await loadMapServer(page); + await waitForGlobeCanvas(page); + + const sandboxFrame = getSandboxFrame(page); + + const listResponse = (await sendJsonRpcToApp(sandboxFrame, { + id: 9201, + method: "tools/list", + params: {}, + })) as { result: { tools: { name: string }[] } }; + expect(listResponse.result.tools.map((t) => t.name)).toContain( + "get-screenshot", + ); + + const callResponse = (await sendJsonRpcToApp(sandboxFrame, { + id: 9202, + method: "tools/call", + params: { name: "get-screenshot", arguments: { maxDimension: 512 } }, + })) as { + result: { + isError?: boolean; + content: { + type: string; + text?: string; + data?: string; + mimeType?: string; + }[]; + }; + }; + + const result = callResponse.result; + expect(result.isError).toBeFalsy(); + const image = result.content.find((c) => c.type === "image"); + expect(image).toBeTruthy(); + expect(image!.mimeType).toBe("image/png"); + // A real PNG: base64 of the \x89PNG magic is "iVBOR" + expect(image!.data!.slice(0, 5)).toBe("iVBOR"); + expect(image!.data!.length).toBeGreaterThan(1000); + }); +});