diff --git a/examples/map-server/README.md b/examples/map-server/README.md index e1e044a84..3931271c5 100644 --- a/examples/map-server/README.md +++ b/examples/map-server/README.md @@ -98,6 +98,14 @@ Display the 3D globe zoomed to a bounding box. Defaults to London if no coordinates provided. +### App tools + +The map view registers tools on the view itself (via `app.registerTool`), for hosts that list and call view tools directly: + +- `navigate-to` — fly the globe to a bounding box, with an optional label. +- `get-current-view` — return the camera position and the visible bounding box (also as `structuredContent`). +- `get-screenshot` — capture the current globe view as a PNG image in the tool result (optional `maxDimension`, default 1024px), so the model can see exactly what the user sees. + ## Architecture ### Server (`server.ts`) diff --git a/examples/map-server/src/mcp-app.ts b/examples/map-server/src/mcp-app.ts index d3090c7a4..8f37abcb3 100644 --- a/examples/map-server/src/mcp-app.ts +++ b/examples/map-server/src/mcp-app.ts @@ -17,6 +17,32 @@ const CESIUM_VERSION = "1.123"; const CESIUM_BASE_URL = `https://cesium.com/downloads/cesiumjs/releases/${CESIUM_VERSION}/Build/Cesium`; const MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION = 768; // Max width/height for screenshots in pixels for updateModelContext +const DEFAULT_SCREENSHOT_MAX_DIMENSION = 1024; // Default max width/height for the get-screenshot tool + +/** + * Capture the globe canvas as a base64 PNG, scaled down so neither side + * exceeds maxDimension (image token cost scales with dimensions). + */ +function captureScreenshotBase64( + sourceCanvas: HTMLCanvasElement, + maxDimension: number, +): { data: string; width: number; height: number } | null { + const scale = Math.min( + 1, + maxDimension / Math.max(sourceCanvas.width, sourceCanvas.height), + ); + const targetWidth = Math.round(sourceCanvas.width * scale); + const targetHeight = Math.round(sourceCanvas.height * scale); + const tempCanvas = document.createElement("canvas"); + tempCanvas.width = targetWidth; + tempCanvas.height = targetHeight; + const ctx = tempCanvas.getContext("2d"); + if (!ctx) return null; + ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight); + const base64Data = tempCanvas.toDataURL("image/png").split(",")[1]; + if (!base64Data) return null; + return { data: base64Data, width: targetWidth, height: targetHeight }; +} /** * Dynamically load CesiumJS from CDN @@ -300,33 +326,19 @@ function scheduleLocationUpdate(cesiumViewer: any): void { if (app.getHostCapabilities()?.updateModelContext?.image) { try { // Scale down to reduce token usage (tokens depend on dimensions) - const sourceCanvas = cesiumViewer.canvas; - const scale = Math.min( - 1, - MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION / - Math.max(sourceCanvas.width, sourceCanvas.height), + const shot = captureScreenshotBase64( + cesiumViewer.canvas, + MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION, ); - const targetWidth = Math.round(sourceCanvas.width * scale); - const targetHeight = Math.round(sourceCanvas.height * scale); - - const tempCanvas = document.createElement("canvas"); - tempCanvas.width = targetWidth; - tempCanvas.height = targetHeight; - const ctx = tempCanvas.getContext("2d"); - if (ctx) { - ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight); - const dataUrl = tempCanvas.toDataURL("image/png"); - const base64Data = dataUrl.split(",")[1]; - if (base64Data) { - content.push({ - type: "image", - data: base64Data, - mimeType: "image/png", - }); - log.info( - `Added screenshot to model context (${targetWidth}x${targetHeight})`, - ); - } + if (shot) { + content.push({ + type: "image", + data: shot.data, + mimeType: "image/png", + }); + log.info( + `Added screenshot to model context (${shot.width}x${shot.height})`, + ); } } catch (err) { log.warn("Failed to capture screenshot:", err); @@ -977,6 +989,54 @@ app.registerTool( }, ); +app.registerTool( + "get-screenshot", + { + title: "Get Screenshot", + description: + "Capture the current globe view as a PNG image for visual analysis", + inputSchema: z.object({ + maxDimension: z + .number() + .int() + .min(64) + .max(2048) + .optional() + .describe( + "Max width/height in pixels (default 1024). Larger images show more detail but cost more tokens.", + ), + }), + }, + async (args) => { + if (!viewer) { + return { + content: [ + { type: "text" as const, text: "Error: Viewer not initialized" }, + ], + isError: true, + }; + } + + const shot = captureScreenshotBase64( + viewer.canvas, + args.maxDimension ?? DEFAULT_SCREENSHOT_MAX_DIMENSION, + ); + if (!shot) { + return { + content: [ + { type: "text" as const, text: "Error: screenshot capture failed" }, + ], + isError: true, + }; + } + return { + content: [ + { type: "image" as const, data: shot.data, mimeType: "image/png" }, + ], + }; + }, +); + // Handle tool result - extract viewUUID and restore persisted view if available app.ontoolresult = async (result) => { viewUUID = result._meta?.viewUUID ? String(result._meta.viewUUID) : undefined; diff --git a/tests/e2e/map-get-screenshot.spec.ts b/tests/e2e/map-get-screenshot.spec.ts new file mode 100644 index 000000000..a954fbb2d --- /dev/null +++ b/tests/e2e/map-get-screenshot.spec.ts @@ -0,0 +1,154 @@ +/** + * Map Server — get-screenshot app tool + * + * The map view registers tools on the view itself (App.registerTool). A host + * calls them by sending `tools/call` JSON-RPC requests to the app iframe over + * its PostMessageTransport, which accepts messages whose source is + * window.parent. The basic-host test UI only drives server tools, so this + * test plays the host's role directly: it posts the JSON-RPC request from the + * sandbox frame (the app iframe's parent) and reads the response there. + */ + +import { test, expect, type Page, type Frame } from "@playwright/test"; + +/** Wait for the MCP App to load inside nested iframes. */ +async function waitForAppLoad(page: Page) { + const outerFrame = page.frameLocator("iframe").first(); + await expect(outerFrame.locator("iframe")).toBeVisible({ timeout: 30000 }); +} + +/** Get the app frame locator (nested: sandbox > app) */ +function getAppFrame(page: Page) { + return page.frameLocator("iframe").first().frameLocator("iframe").first(); +} + +/** Load the map server and call show-map with its default bounding box. */ +async function loadMapServer(page: Page) { + await page.goto("/?theme=hide"); + // The host fetches the server list once per page load, and the example + // servers rebuild on examples:start, so a server can come up after the + // first load — reload until it appears instead of waiting on a stale list. + await expect(async () => { + await page.reload(); + const serverSelect = page.locator("select").first(); + await expect(serverSelect).toBeEnabled({ timeout: 30000 }); + expect(await serverSelect.locator("option").allTextContents()).toContain( + "CesiumJS Map Server", + ); + }).toPass({ timeout: 90000, intervals: [2000] }); + await page + .locator("select") + .first() + .selectOption({ label: "CesiumJS Map Server" }); + await page.locator("select").nth(1).selectOption("show-map"); + await page.click('button:has-text("Call Tool")'); + await waitForAppLoad(page); +} + +/** Wait for the Cesium canvas to render and give tiles time to load. */ +async function waitForGlobeCanvas(page: Page) { + const appFrame = getAppFrame(page); + await expect(appFrame.locator("canvas").first()).toBeVisible({ + timeout: 60000, + }); + // CesiumJS needs time for tiles to load (same budget as servers.spec.ts) + await page.waitForTimeout(15000); +} + +/** Resolve the sandbox frame (the app iframe's parent). */ +function getSandboxFrame(page: Page): Frame { + const appFrame = page + .frames() + .find((f) => f.parentFrame() && f.parentFrame()!.parentFrame()); + if (!appFrame) throw new Error("App frame not found"); + return appFrame.parentFrame()!; +} + +/** + * Send one JSON-RPC request into the app iframe from the sandbox frame and + * await the matching response. + */ +function sendJsonRpcToApp( + sandboxFrame: Frame, + request: { id: number; method: string; params?: unknown }, + timeoutMs = 60_000, +): Promise { + return sandboxFrame.evaluate( + async ({ id, method, params, timeoutMs }) => { + const iframe = document.querySelector("iframe") as HTMLIFrameElement; + return await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`${method} timed out`)), + timeoutMs, + ); + const onMsg = (e: MessageEvent) => { + // Only accept a response (not an echoed request) to our id, from + // the app iframe itself. + if ( + e.source === iframe.contentWindow && + e.data?.jsonrpc === "2.0" && + e.data.id === id && + ("result" in e.data || "error" in e.data) + ) { + clearTimeout(timer); + window.removeEventListener("message", onMsg); + resolve(e.data); + } + }; + window.addEventListener("message", onMsg); + iframe.contentWindow!.postMessage( + { jsonrpc: "2.0", id, method, params }, + "*", + ); + }); + }, + { ...request, timeoutMs }, + ); +} + +test.describe("Map Server - get-screenshot", () => { + test("is listed and returns the current view as a PNG image", async ({ + page, + }) => { + // Server-list reload loop + CDN Cesium + tile loading add up. + test.setTimeout(240_000); + await loadMapServer(page); + await waitForGlobeCanvas(page); + + const sandboxFrame = getSandboxFrame(page); + + const listResponse = (await sendJsonRpcToApp(sandboxFrame, { + id: 9201, + method: "tools/list", + params: {}, + })) as { result: { tools: { name: string }[] } }; + expect(listResponse.result.tools.map((t) => t.name)).toContain( + "get-screenshot", + ); + + const callResponse = (await sendJsonRpcToApp(sandboxFrame, { + id: 9202, + method: "tools/call", + params: { name: "get-screenshot", arguments: { maxDimension: 512 } }, + })) as { + result: { + isError?: boolean; + content: { + type: string; + text?: string; + data?: string; + mimeType?: string; + }[]; + }; + }; + + const result = callResponse.result; + expect(result.isError).toBeFalsy(); + const image = result.content.find((c) => c.type === "image"); + expect(image).toBeTruthy(); + expect(image!.mimeType).toBe("image/png"); + // A real PNG: base64 of the \x89PNG magic is "iVBOR" + expect(image!.data!.slice(0, 5)).toBe("iVBOR"); + expect(image!.data!.length).toBeGreaterThan(1000); + }); +});