Add get-screenshot app tool to the map-server example

examples/map-server/README.md

@@ -98,6 +98,14 @@ Display the 3D globe zoomed to a bounding box.
 
 Defaults to London if no coordinates provided.
 
+### App tools
+
+The map view registers tools on the view itself (via `app.registerTool`), for hosts that list and call view tools directly:
+
+- `navigate-to` — fly the globe to a bounding box, with an optional label.
+- `get-current-view` — return the camera position and the visible bounding box (also as `structuredContent`).
+- `get-screenshot` — capture the current globe view as a PNG image in the tool result (optional `maxDimension`, default 1024px), so the model can see exactly what the user sees.
+
 ## Architecture
 
 ### Server (`server.ts`)

examples/map-server/src/mcp-app.ts

@@ -17,6 +17,32 @@ const CESIUM_VERSION = "1.123";
 const CESIUM_BASE_URL = `https://cesium.com/downloads/cesiumjs/releases/${CESIUM_VERSION}/Build/Cesium`;
 
 const MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION = 768; // Max width/height for screenshots in pixels for updateModelContext
+const DEFAULT_SCREENSHOT_MAX_DIMENSION = 1024; // Default max width/height for the get-screenshot tool
+
+/**
+ * Capture the globe canvas as a base64 PNG, scaled down so neither side
+ * exceeds maxDimension (image token cost scales with dimensions).
+ */
+function captureScreenshotBase64(
+  sourceCanvas: HTMLCanvasElement,
+  maxDimension: number,
+): { data: string; width: number; height: number } | null {
+  const scale = Math.min(
+    1,
+    maxDimension / Math.max(sourceCanvas.width, sourceCanvas.height),
+  );
+  const targetWidth = Math.round(sourceCanvas.width * scale);
+  const targetHeight = Math.round(sourceCanvas.height * scale);
+  const tempCanvas = document.createElement("canvas");
+  tempCanvas.width = targetWidth;
+  tempCanvas.height = targetHeight;
+  const ctx = tempCanvas.getContext("2d");
+  if (!ctx) return null;
+  ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight);
+  const base64Data = tempCanvas.toDataURL("image/png").split(",")[1];
+  if (!base64Data) return null;
+  return { data: base64Data, width: targetWidth, height: targetHeight };
+}
 
 /**
  * Dynamically load CesiumJS from CDN
@@ -300,33 +326,19 @@ function scheduleLocationUpdate(cesiumViewer: any): void {
     if (app.getHostCapabilities()?.updateModelContext?.image) {
       try {
         // Scale down to reduce token usage (tokens depend on dimensions)
-        const sourceCanvas = cesiumViewer.canvas;
-        const scale = Math.min(
-          1,
-          MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION /
-            Math.max(sourceCanvas.width, sourceCanvas.height),
+        const shot = captureScreenshotBase64(
+          cesiumViewer.canvas,
+          MAX_MODEL_CONTEXT_UPDATE_IMAGE_DIMENSION,
         );
-        const targetWidth = Math.round(sourceCanvas.width * scale);
-        const targetHeight = Math.round(sourceCanvas.height * scale);
-
-        const tempCanvas = document.createElement("canvas");
-        tempCanvas.width = targetWidth;
-        tempCanvas.height = targetHeight;
-        const ctx = tempCanvas.getContext("2d");
-        if (ctx) {
-          ctx.drawImage(sourceCanvas, 0, 0, targetWidth, targetHeight);
-          const dataUrl = tempCanvas.toDataURL("image/png");
-          const base64Data = dataUrl.split(",")[1];
-          if (base64Data) {
-            content.push({
-              type: "image",
-              data: base64Data,
-              mimeType: "image/png",
-            });
-            log.info(
-              `Added screenshot to model context (${targetWidth}x${targetHeight})`,
-            );
-          }
+        if (shot) {
+          content.push({
+            type: "image",
+            data: shot.data,
+            mimeType: "image/png",
+          });
+          log.info(
+            `Added screenshot to model context (${shot.width}x${shot.height})`,
+          );
         }
       } catch (err) {
         log.warn("Failed to capture screenshot:", err);
@@ -977,6 +989,54 @@ app.registerTool(
   },
 );
 
+app.registerTool(
+  "get-screenshot",
+  {
+    title: "Get Screenshot",
+    description:
+      "Capture the current globe view as a PNG image for visual analysis",
+    inputSchema: z.object({
+      maxDimension: z
+        .number()
+        .int()
+        .min(64)
+        .max(2048)
+        .optional()
+        .describe(
+          "Max width/height in pixels (default 1024). Larger images show more detail but cost more tokens.",
+        ),
+    }),
+  },
+  async (args) => {
+    if (!viewer) {
+      return {
+        content: [
+          { type: "text" as const, text: "Error: Viewer not initialized" },
+        ],
+        isError: true,
+      };
+    }
+
+    const shot = captureScreenshotBase64(
+      viewer.canvas,
+      args.maxDimension ?? DEFAULT_SCREENSHOT_MAX_DIMENSION,
+    );
+    if (!shot) {
+      return {
+        content: [
+          { type: "text" as const, text: "Error: screenshot capture failed" },
+        ],
+        isError: true,
+      };
+    }
+    return {
+      content: [
+        { type: "image" as const, data: shot.data, mimeType: "image/png" },
+      ],
+    };
+  },
+);
+
 // Handle tool result - extract viewUUID and restore persisted view if available
 app.ontoolresult = async (result) => {
   viewUUID = result._meta?.viewUUID ? String(result._meta.viewUUID) : undefined;

tests/e2e/map-get-screenshot.spec.ts

@@ -0,0 +1,154 @@
+/**
+ * Map Server — get-screenshot app tool
+ *
+ * The map view registers tools on the view itself (App.registerTool). A host
+ * calls them by sending `tools/call` JSON-RPC requests to the app iframe over
+ * its PostMessageTransport, which accepts messages whose source is
+ * window.parent. The basic-host test UI only drives server tools, so this
+ * test plays the host's role directly: it posts the JSON-RPC request from the
+ * sandbox frame (the app iframe's parent) and reads the response there.
+ */
+
+import { test, expect, type Page, type Frame } from "@playwright/test";
+
+/** Wait for the MCP App to load inside nested iframes. */
+async function waitForAppLoad(page: Page) {
+  const outerFrame = page.frameLocator("iframe").first();
+  await expect(outerFrame.locator("iframe")).toBeVisible({ timeout: 30000 });
+}
+
+/** Get the app frame locator (nested: sandbox > app) */
+function getAppFrame(page: Page) {
+  return page.frameLocator("iframe").first().frameLocator("iframe").first();
+}
+
+/** Load the map server and call show-map with its default bounding box. */
+async function loadMapServer(page: Page) {
+  await page.goto("/?theme=hide");
+  // The host fetches the server list once per page load, and the example
+  // servers rebuild on examples:start, so a server can come up after the
+  // first load — reload until it appears instead of waiting on a stale list.
+  await expect(async () => {
+    await page.reload();
+    const serverSelect = page.locator("select").first();
+    await expect(serverSelect).toBeEnabled({ timeout: 30000 });
+    expect(await serverSelect.locator("option").allTextContents()).toContain(
+      "CesiumJS Map Server",
+    );
+  }).toPass({ timeout: 90000, intervals: [2000] });
+  await page
+    .locator("select")
+    .first()
+    .selectOption({ label: "CesiumJS Map Server" });
+  await page.locator("select").nth(1).selectOption("show-map");
+  await page.click('button:has-text("Call Tool")');
+  await waitForAppLoad(page);
+}
+
+/** Wait for the Cesium canvas to render and give tiles time to load. */
+async function waitForGlobeCanvas(page: Page) {
+  const appFrame = getAppFrame(page);
+  await expect(appFrame.locator("canvas").first()).toBeVisible({
+    timeout: 60000,
+  });
+  // CesiumJS needs time for tiles to load (same budget as servers.spec.ts)
+  await page.waitForTimeout(15000);
+}
+
+/** Resolve the sandbox frame (the app iframe's parent). */
+function getSandboxFrame(page: Page): Frame {
+  const appFrame = page
+    .frames()
+    .find((f) => f.parentFrame() && f.parentFrame()!.parentFrame());
+  if (!appFrame) throw new Error("App frame not found");
+  return appFrame.parentFrame()!;
+}
+
+/**
+ * Send one JSON-RPC request into the app iframe from the sandbox frame and
+ * await the matching response.
+ */
+function sendJsonRpcToApp(
+  sandboxFrame: Frame,
+  request: { id: number; method: string; params?: unknown },
+  timeoutMs = 60_000,
+): Promise<unknown> {
+  return sandboxFrame.evaluate(
+    async ({ id, method, params, timeoutMs }) => {
+      const iframe = document.querySelector("iframe") as HTMLIFrameElement;
+      return await new Promise<unknown>((resolve, reject) => {
+        const timer = setTimeout(
+          () => reject(new Error(`${method} timed out`)),
+          timeoutMs,
+        );
+        const onMsg = (e: MessageEvent) => {
+          // Only accept a response (not an echoed request) to our id, from
+          // the app iframe itself.
+          if (
+            e.source === iframe.contentWindow &&
+            e.data?.jsonrpc === "2.0" &&
+            e.data.id === id &&
+            ("result" in e.data || "error" in e.data)
+          ) {
+            clearTimeout(timer);
+            window.removeEventListener("message", onMsg);
+            resolve(e.data);
+          }
+        };
+        window.addEventListener("message", onMsg);
+        iframe.contentWindow!.postMessage(
+          { jsonrpc: "2.0", id, method, params },
+          "*",
+        );
+      });
+    },
+    { ...request, timeoutMs },
+  );
+}
+
+test.describe("Map Server - get-screenshot", () => {
+  test("is listed and returns the current view as a PNG image", async ({
+    page,
+  }) => {
+    // Server-list reload loop + CDN Cesium + tile loading add up.
+    test.setTimeout(240_000);
+    await loadMapServer(page);
+    await waitForGlobeCanvas(page);
+
+    const sandboxFrame = getSandboxFrame(page);
+
+    const listResponse = (await sendJsonRpcToApp(sandboxFrame, {
+      id: 9201,
+      method: "tools/list",
+      params: {},
+    })) as { result: { tools: { name: string }[] } };
+    expect(listResponse.result.tools.map((t) => t.name)).toContain(
+      "get-screenshot",
+    );
+
+    const callResponse = (await sendJsonRpcToApp(sandboxFrame, {
+      id: 9202,
+      method: "tools/call",
+      params: { name: "get-screenshot", arguments: { maxDimension: 512 } },
+    })) as {
+      result: {
+        isError?: boolean;
+        content: {
+          type: string;
+          text?: string;
+          data?: string;
+          mimeType?: string;
+        }[];
+      };
+    };
+
+    const result = callResponse.result;
+    expect(result.isError).toBeFalsy();
+    const image = result.content.find((c) => c.type === "image");
+    expect(image).toBeTruthy();
+    expect(image!.mimeType).toBe("image/png");
+    // A real PNG: base64 of the \x89PNG magic is "iVBOR"
+    expect(image!.data!.slice(0, 5)).toBe("iVBOR");
+    expect(image!.data!.length).toBeGreaterThan(1000);
+  });
+});