From 0bbb1586c345c9acf6268d0fa7fe21a4a8f588d6 Mon Sep 17 00:00:00 2001 From: Yuruzuu Date: Wed, 4 Nov 2026 02:47:28 +0800 Subject: [PATCH] =?UTF-8?q?perf(server):=20text-only=20browser=20snapshots?= =?UTF-8?q?=20skip=20the=20PNG=20encode=20(66=20ms=20=E2=86=92=2020=20ms?= =?UTF-8?q?=20median)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/McpHttpServer.test.ts | 59 +++++++++++++++- apps/server/src/mcp/McpHttpServer.ts | 33 ++++++--- .../src/mcp/toolkits/preview/handlers.ts | 11 ++- apps/server/src/mcp/toolkits/preview/tools.ts | 3 +- apps/server/src/preview/ServerBrowser.ts | 9 +-- .../src/preview/ServerBrowserPage.test.ts | 35 ++++++++-- apps/server/src/preview/ServerBrowserPage.ts | 67 ++++++++++++++----- packages/contracts/src/previewAutomation.ts | 12 ++++ 8 files changed, 191 insertions(+), 38 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index c3ba3db1c028a..23e7a2f80f06a 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -20,6 +20,7 @@ import { import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import * as Schema from "effect/Schema"; @@ -233,6 +234,57 @@ it.effect.each([ ), ); +it.effect.each([ + { save: false, includeImage: false }, + { save: false, includeImage: true }, + { save: true, includeImage: false }, + { save: true, includeImage: true }, +])("preserves PNG output for snapshot options %#", (input) => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-png-options", snapshotResult); + const result = yield* callSnapshot(input); + expect(result.isError).toBe(false); + const image = result.content.find((part) => part.type === "image"); + expect(image?.type === "image" ? image.data : undefined).toEqual( + input.includeImage ? new Uint8Array(Buffer.from("png")) : undefined, + ); + const content = result.structuredContent; + const path = + content && typeof content === "object" && "screenshotPath" in content + ? content.screenshotPath + : undefined; + if (input.save) { + expect(typeof path).toBe("string"); + const fileSystem = yield* FileSystem.FileSystem; + expect(Array.from(yield* fileSystem.readFile(path as string))).toEqual( + Array.from(Buffer.from("png")), + ); + } else { + expect(path).toBeUndefined(); + } + }), + ).pipe(Effect.provide(layerTest)), +); + +it.effect.each([{ includeImage: true }, { save: true }, { save: true, includeImage: true }])( + "rejects a metadata-only response when pixels are requested %#", + (input) => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-missing-pixels", { + ...snapshotResult, + screenshot: { width: 10, height: 5 }, + }); + const result = yield* callSnapshot(input); + expect(result.isError).toBe(true); + expect(result.content).toEqual([ + { type: "text", text: expect.stringContaining("did not contain requested PNG pixels") }, + ]); + }), + ).pipe(Effect.provide(layerTest)), +); + const layerRecordingTools = (calls: Array) => { const RecordingToolkit = Toolkit.make( OrchestratorToolkit.tools.delegate_task, @@ -365,7 +417,9 @@ it.effect.each([ tabId: alternateTabId, threadId, }); - expect(event.request.input).toEqual({}); + expect(event.request.input).toEqual( + images && requests !== 7 ? {} : { pixelsRequired: false }, + ); return broker.respond({ clientId: "mcp-image-option-client", connectionId: event.connectionId, @@ -374,7 +428,8 @@ it.effect.each([ result: { ...page, title: `Snapshot ${requests}`, - screenshot: { ...screenshot, data: png }, + screenshot: + images || requests === 7 ? { ...screenshot, data: png } : { width: 1, height: 1 }, }, }); }).pipe(Effect.forkScoped); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 2ef080a7ef7e4..f5f10094e7470 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -479,7 +479,10 @@ const previewSnapshotFailure = Effect.fn("previewSnapshotFailure")(function ( ? firstFailure._tag : "PreviewSnapshotError"; // Preview errors contain server-authored recovery guidance, never page output. - const message = isPreviewAutomationError(firstFailure) ? firstFailure.message : undefined; + const message = + isPreviewAutomationError(firstFailure) || isOrchestratorMcpFailure(firstFailure) + ? firstFailure.message + : undefined; const result = new McpSchema.CallToolResult({ isError: true, structuredContent: { @@ -544,18 +547,30 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot const snapshot = encodedResult as SnapshotMetadata & { readonly url: string; readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; + readonly mimeType?: "image/png"; + readonly data?: string; readonly width: number; readonly height: number; }; }; const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); + const includeImage = payload?.includeImage === true; + const needsPixels = payload?.save === true || includeImage; + if (needsPixels && (screenshot.mimeType !== "image/png" || !screenshot.data)) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "Preview snapshot did not contain requested PNG pixels.", + }); + } + const png = + needsPixels && screenshot.data + ? new Uint8Array(Buffer.from(screenshot.data, "base64")) + : undefined; const screenshotPath = - payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + payload?.save === true && png + ? yield* saveScreenshot(snapshot.url, png) + : undefined; // Providers replay images on later requests, and some reject them; include images only when requested. - const includeImage = payload?.includeImage === true; if (screenshotPath !== undefined && !includeImage) { const saved = { url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), @@ -570,7 +585,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot const metadata = { ...page, screenshot: { - mimeType: screenshot.mimeType, + mimeType: "image/png" as const, width: screenshot.width, height: screenshot.height, }, @@ -599,8 +614,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, }, ]), - ...(includeImage - ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }] + ...(includeImage && png + ? [{ type: "image" as const, data: png, mimeType: "image/png" as const }] : []), ], }); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 94df4a13fec48..eae6d12edc594 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -18,6 +18,7 @@ import { type PreviewAutomationResizeResult, type PreviewAutomationSetColorSchemeResult, type PreviewAutomationSnapshot, + type PreviewAutomationSnapshotMetadata, type PreviewAutomationStatus, type PreviewTabId, } from "@t3tools/contracts"; @@ -221,8 +222,14 @@ const handlers = { invokeTargeted("setColorScheme", input), ), preview_snapshot: McpToolAccess.readsAsCaller((input) => { - const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {}; - return invokeTargeted("snapshot", operationInput); + const { includeImage, save, ...operationInput } = input ?? {}; + return invokeTargeted( + "snapshot", + { + ...operationInput, + ...(includeImage === true || save === true ? {} : { pixelsRequired: false }), + }, + ); }), preview_act: McpToolAccess.actsAsCaller(({ action, ...input }) => decodeActInput[action](input).pipe( diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 33c3443a1c26e..7308281acd439 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -16,6 +16,7 @@ import { PreviewAutomationSetColorSchemeInput, PreviewAutomationSetColorSchemeResult, PreviewAutomationSnapshot, + PreviewAutomationSnapshotMetadata, PreviewAutomationStatus, PreviewAutomationTabTargetInput, PreviewAutomationWaitForInput, @@ -151,7 +152,7 @@ export const PreviewSnapshotTool = safeBrowserTool( }), ), }), - success: PreviewAutomationSnapshot, + success: Schema.Union([PreviewAutomationSnapshot, PreviewAutomationSnapshotMetadata]), failure: PreviewToolFailure, dependencies, }).annotate(Tool.Title, "Inspect browser page"), diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 7444b84f906e3..bed84cdb4bb38 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -30,6 +30,7 @@ import { type PreviewAutomationScrollInput, type PreviewAutomationSelectInput, type PreviewAutomationSetColorSchemeInput, + type PreviewAutomationSnapshotInput, type PreviewAutomationStatus, type PreviewAutomationTypeInput, type PreviewAutomationUploadInput, @@ -2163,11 +2164,11 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { + const rendering = { ...tab, renderScale: tab.desktop ? "display" : RENDER_SCALE } as const; return withScreencastsPaused(tab, () => - ServerBrowserPage.snapshot({ - ...tab, - renderScale: tab.desktop ? "display" : RENDER_SCALE, - }), + (input as PreviewAutomationSnapshotInput).pixelsRequired === false + ? ServerBrowserPage.snapshotMetadata(rendering) + : ServerBrowserPage.snapshot(rendering), ); } case "click": { diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 2c6d3a01bcf74..c091cfa2f7594 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -9,6 +9,24 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from import * as ServerBrowserPage from "./ServerBrowserPage.ts"; +describe("JPEG screenshot dimensions", () => { + const encode = (bytes: number[]) => Buffer.from(bytes).toString("base64"); + const frame = [0xff, 0xd8, 0xff, 0xc0, 0, 7, 8, 0, 24, 0, 32]; + + it("reads valid dimensions and rejects invalid or incomplete frames", () => { + expect(ServerBrowserPage.jpegSize(encode(frame))).toEqual({ width: 32, height: 24 }); + for (const bytes of [ + [], + [0xff, 0xd8, 0xff, 0xe1, 0, 20, 1], + frame.slice(0, -1), + [...frame.slice(0, 7), 0, 0, 0, 32], + [...frame.slice(0, 9), 0, 0], + ]) { + expect(() => ServerBrowserPage.jpegSize(encode(bytes))).toThrow("Invalid JPEG screenshot."); + } + }); +}); + describe("server browser element refs", () => { let browser: Browser; let context: BrowserContext; @@ -309,14 +327,15 @@ describe("server browser snapshot screenshot", () => { const screenshotOf = async (html: string) => { const page = await context.newPage(); await page.setContent(html); - const { screenshot } = await ServerBrowserPage.snapshot({ + const input = { page, cdp: await context.newCDPSession(page), renderScale: 2, consoleEntries: [], networkEntries: [], actionTimeline: [], - }); + }; + const { screenshot } = await ServerBrowserPage.snapshot(input); // The server has no DOM types, so the decoder runs as page source. const points = [ [0.15, 0.75], @@ -335,7 +354,7 @@ describe("server browser snapshot screenshot", () => { Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data), ); })()`)) as Array<[number, number, number, number]>; - return { screenshot, pixels }; + return { screenshot, pixels, input }; }; const colorOf = ([red, green, blue]: readonly [number, number, number, number]) => blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`; @@ -350,12 +369,18 @@ describe("server browser snapshot screenshot", () => { }); it("captures a plain page at the shrunk size with its content in place", async () => { - const { screenshot, pixels } = await screenshotOf(halves("")); + const { screenshot, pixels, input } = await screenshotOf(halves("")); + const metadata = await ServerBrowserPage.snapshotMetadata(input); expect({ size: [screenshot.width, screenshot.height], + metadataSize: metadata.screenshot, colors: pixels.map(colorOf), - }).toEqual({ size: [1280, 800], colors: ["blue", "red", "red"] }); + }).toEqual({ + size: [1280, 800], + metadataSize: { width: 1280, height: 800 }, + colors: ["blue", "red", "red"], + }); }); it("shrinks the screenshot of an XML document, whose createElement makes no canvas", async () => { diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index f128bf445e603..d664ca4858d7c 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -14,6 +14,7 @@ import { type PreviewAutomationSelectInput, type PreviewAutomationSelectResult, type PreviewAutomationSnapshot, + type PreviewAutomationSnapshotMetadata, type PreviewAutomationTypeInput, type PreviewAutomationUploadInput, type PreviewAutomationWaitForInput, @@ -412,6 +413,37 @@ const pngSize = (base64: string) => { return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; }; +export const jpegSize = (base64: string) => { + const bytes = Buffer.from(base64, "base64"); + if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) + throw new Error("Invalid JPEG screenshot."); + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot."); + while (bytes[offset] === 0xff) offset++; + const marker = bytes[offset++]; + if (marker === undefined || marker === 0xd9 || marker === 0xda) break; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) break; + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) break; + if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { + if (length < 7) break; + const height = bytes.readUInt16BE(offset + 3); + const width = bytes.readUInt16BE(offset + 5); + if (width && height) return { width, height }; + break; + } + offset += length; + } + throw new Error("Invalid JPEG screenshot."); +}; + +const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) => + scale < 1 + ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) } + : size; + /** * A page's display ratio and CSS viewport, read in an isolated world since page script can * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom. @@ -523,10 +555,7 @@ const renderingOf = async (input: SnapshotInput) => { }; }; -const capture = async ( - input: SnapshotInput, - deadline: number, -): Promise => { +const capture = async (input: SnapshotInput, deadline: number, metadataOnly: boolean) => { const { renderScale, viewport } = await renderingOf(input); // A late metrics reply must not begin a capture after viewers have resumed. if (Date.now() >= deadline) throw tooBusy(); @@ -544,9 +573,12 @@ const capture = async ( "url" | "title" | "loading" | "visibleText" | "interactiveElements" > >, - captureViewport(input.cdp, { format: "png" }).then((png) => - scale < 1 ? shrinkPng(input.cdp, png, scale) : png, - ), + // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG. + metadataOnly + ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }) + : captureViewport(input.cdp, { format: "png" }).then((png) => + scale < 1 ? shrinkPng(input.cdp, png, scale) : png, + ), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. @@ -592,18 +624,17 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - ...pngSize(data), - }, + // Reports the size the PNG path would have shrunk to. + screenshot: metadataOnly + ? scaledSize(jpegSize(data), scale) + : { mimeType: "image/png" as const, data, ...pngSize(data) }, }; }; /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */ const snapshotTurns = new WeakMap(); -export const snapshot = async (input: SnapshotInput): Promise => { +const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => { const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS; let timer: ReturnType | undefined; const expired = new Promise((_, reject) => { @@ -612,11 +643,11 @@ export const snapshot = async (input: SnapshotInput): Promise + const captured = capture(input, deadline, metadataOnly).catch((cause: unknown) => Date.now() < deadline && snapshotTurns.get(input.page) === turn && !(cause instanceof Error && cause.message === VALUE_TOO_LONG) - ? capture(input, deadline) + ? capture(input, deadline, metadataOnly) : Promise.reject(cause), ); try { @@ -628,6 +659,12 @@ export const snapshot = async (input: SnapshotInput): Promise + runSnapshot(input, false) as Promise; + +export const snapshotMetadata = (input: SnapshotInput) => + runSnapshot(input, true) as Promise; + /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */ export const click = async ( page: Page, diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index bfeb5cf916672..9460b90d35972 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -555,6 +555,18 @@ export const PreviewAutomationSnapshot = Schema.Struct({ }); export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type; +// False when the caller discards the screenshot; hosts that ignore it still return a PNG. +export const PreviewAutomationSnapshotInput = Schema.Struct({ + pixelsRequired: Schema.optional(Schema.Boolean), +}); +export type PreviewAutomationSnapshotInput = typeof PreviewAutomationSnapshotInput.Type; + +export const PreviewAutomationSnapshotMetadata = Schema.Struct({ + ...PreviewAutomationSnapshot.fields, + screenshot: Schema.Struct({ width: Schema.Int, height: Schema.Int }), +}); +export type PreviewAutomationSnapshotMetadata = typeof PreviewAutomationSnapshotMetadata.Type; + export const PreviewAutomationRecordingStatus = Schema.Struct({ tabId: PreviewTabId, recording: Schema.Boolean,