diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 3802a5c2e4023..e0dff59237eb7 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -273,6 +273,57 @@ it.effect.each([ ), ); +it.effect.each([ + { save: false, includeImage: false }, + { save: false, includeImage: true }, + { save: true, includeImage: false }, + { save: true, includeImage: true }, +])("preserves PNG output for snapshot options %#", (input) => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-png-options", snapshotResult); + const result = yield* callSnapshot(input); + expect(result.isError).toBe(false); + const image = result.content.find((part) => part.type === "image"); + expect(image?.type === "image" ? image.data : undefined).toEqual( + input.includeImage ? new Uint8Array(Buffer.from("png")) : undefined, + ); + const content = result.structuredContent; + const path = + content && typeof content === "object" && "screenshotPath" in content + ? content.screenshotPath + : undefined; + if (input.save) { + expect(typeof path).toBe("string"); + const fileSystem = yield* FileSystem.FileSystem; + expect(Array.from(yield* fileSystem.readFile(path as string))).toEqual( + Array.from(Buffer.from("png")), + ); + } else { + expect(path).toBeUndefined(); + } + }), + ).pipe(Effect.provide(layerTest)), +); + +it.effect.each([{ includeImage: true }, { save: true }, { save: true, includeImage: true }])( + "rejects a metadata-only response when pixels are requested %#", + (input) => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-missing-pixels", { + ...snapshotResult, + screenshot: { width: 10, height: 5 }, + }); + const result = yield* callSnapshot(input); + expect(result.isError).toBe(true); + expect(result.content).toEqual([ + { type: "text", text: expect.stringContaining("did not contain requested PNG pixels") }, + ]); + }), + ).pipe(Effect.provide(layerTest)), +); + const layerRecordingTools = (calls: Array) => { const RecordingToolkit = Toolkit.make( OrchestratorToolkit.tools.delegate_task, @@ -418,7 +469,9 @@ it.effect.each([ tabId: alternateTabId, threadId, }); - expect(event.request.input).toEqual({}); + expect(event.request.input).toEqual( + images && requests !== 7 ? {} : { pixelsRequired: false }, + ); return broker.respond({ clientId: "mcp-image-option-client", connectionId: event.connectionId, @@ -427,7 +480,8 @@ it.effect.each([ result: { ...page, title: `Snapshot ${requests}`, - screenshot: { ...screenshot, data: png }, + screenshot: + images || requests === 7 ? { ...screenshot, data: png } : { width: 1, height: 1 }, }, }); }).pipe(Effect.forkScoped); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 20f1ccf94aab7..ecbd1ba39e3c3 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -574,7 +574,9 @@ const previewSnapshotFailure = Effect.fn("previewSnapshotFailure")(function ( : "PreviewSnapshotError"; // Preview errors contain server-authored recovery guidance, never page output. const message = - isPreviewAutomationError(firstFailure) || isPreviewScreenshotSaveError(firstFailure) + isPreviewAutomationError(firstFailure) || + isPreviewScreenshotSaveError(firstFailure) || + isOrchestratorMcpFailure(firstFailure) ? firstFailure.message : parameterValidationMessage(firstFailure); const result = new McpSchema.CallToolResult({ @@ -641,17 +643,28 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot const snapshot = encodedResult as SnapshotMetadata & { readonly url: string; readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; + readonly mimeType?: "image/png"; + readonly data?: string; readonly width: number; readonly height: number; }; readonly fullScreenshot?: { readonly data: string }; }; const { screenshot, fullScreenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); + const includeImage = payload?.includeImage === true; + const needsPixels = payload?.save === true || includeImage; + if (needsPixels && (screenshot.mimeType !== "image/png" || !screenshot.data)) { + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: "Preview snapshot did not contain requested PNG pixels.", + }); + } + const png = + needsPixels && screenshot.data + ? new Uint8Array(Buffer.from(screenshot.data, "base64")) + : undefined; const screenshotPath = - payload?.save === true + payload?.save === true && png ? yield* saveScreenshot( snapshot.url, fullScreenshot @@ -660,7 +673,6 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot ) : undefined; // Providers replay images on later requests, and some reject them; include images only when requested. - const includeImage = payload?.includeImage === true; if (screenshotPath !== undefined && !includeImage) { const saved = { url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), @@ -675,7 +687,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot const metadata = { ...page, screenshot: { - mimeType: screenshot.mimeType, + mimeType: "image/png" as const, width: screenshot.width, height: screenshot.height, }, @@ -704,8 +716,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, }, ]), - ...(includeImage - ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }] + ...(includeImage && png + ? [{ type: "image" as const, data: png, mimeType: "image/png" as const }] : []), ], }); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index c541ad1664a96..421f9be597379 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -18,6 +18,7 @@ import { type PreviewAutomationResizeResult, type PreviewAutomationSetColorSchemeResult, type PreviewAutomationSnapshot, + type PreviewAutomationSnapshotMetadata, type PreviewAutomationStatus, type PreviewTabId, } from "@t3tools/contracts"; @@ -221,12 +222,16 @@ const handlers = { invokeTargeted("setColorScheme", input), ), preview_snapshot: McpToolAccess.readsAsCaller((input) => { - const { includeImage: _includeImage, save, ...operationInput } = input ?? {}; + const { includeImage, save, ...operationInput } = input ?? {}; // The saved file is for people, so it keeps the resolution the agent's image is capped below. - return invokeTargeted("snapshot", { - ...operationInput, - ...(save === true ? { fullScreenshot: true } : {}), - }); + return invokeTargeted( + "snapshot", + { + ...operationInput, + ...(save === true ? { fullScreenshot: true } : {}), + ...(includeImage === true || save === true ? {} : { pixelsRequired: false }), + }, + ); }), preview_act: McpToolAccess.actsAsCaller(({ action, ...input }) => decodeActInput[action](input).pipe( diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index b2c9afa11bb2a..18c000d12bdf5 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -16,6 +16,7 @@ import { PreviewAutomationSetColorSchemeInput, PreviewAutomationSetColorSchemeResult, PreviewAutomationSnapshot, + PreviewAutomationSnapshotMetadata, PreviewAutomationStatus, PreviewAutomationTabTargetInput, PreviewAutomationWaitForInput, @@ -151,7 +152,7 @@ export const PreviewSnapshotTool = safeBrowserTool( }), ), }), - success: PreviewAutomationSnapshot, + success: Schema.Union([PreviewAutomationSnapshot, PreviewAutomationSnapshotMetadata]), failure: PreviewToolFailure, dependencies, }).annotate(Tool.Title, "Inspect browser page"), diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 7aa74a9d172b7..b8078edac0e0a 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -2779,12 +2779,16 @@ const make = Effect.gen(function* () { return { tabId: tab.tabId, colorScheme }; } case "snapshot": { + const snapshotInput = input as PreviewAutomationSnapshotInput | undefined; + const rendering = { + ...tab, + renderScale: tab.desktop ? "display" : RENDER_SCALE, + fullScreenshot: snapshotInput?.fullScreenshot, + } as const; const snapshot = await withScreencastsPaused(tab, () => - ServerBrowserPage.snapshot({ - ...tab, - renderScale: tab.desktop ? "display" : RENDER_SCALE, - fullScreenshot: (input as PreviewAutomationSnapshotInput | undefined)?.fullScreenshot, - }), + snapshotInput?.pixelsRequired === false + ? ServerBrowserPage.snapshotMetadata(rendering) + : ServerBrowserPage.snapshot(rendering), ); const loadError = loadErrorOf(tab); return loadError ? { ...snapshot, url: loadError.url, loadError } : snapshot; diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 36b133cf185d8..35f8e6a79507e 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -23,6 +23,24 @@ import { SessionControl } from "./SessionControl.ts"; const decodeEvaluateInput = Schema.decodeUnknownSync(PreviewAutomationEvaluateInput); +describe("JPEG screenshot dimensions", () => { + const encode = (bytes: number[]) => Buffer.from(bytes).toString("base64"); + const frame = [0xff, 0xd8, 0xff, 0xc0, 0, 7, 8, 0, 24, 0, 32]; + + it("reads valid dimensions and rejects invalid or incomplete frames", () => { + expect(ServerBrowserPage.jpegSize(encode(frame))).toEqual({ width: 32, height: 24 }); + for (const bytes of [ + [], + [0xff, 0xd8, 0xff, 0xe1, 0, 20, 1], + frame.slice(0, -1), + [...frame.slice(0, 7), 0, 0, 0, 32], + [...frame.slice(0, 9), 0, 0], + ]) { + expect(() => ServerBrowserPage.jpegSize(encode(bytes))).toThrow("Invalid JPEG screenshot."); + } + }); +}); + describe("snapshot deadline diagnostics", () => { const never = () => new Promise(() => {}); const stalledSnapshot = async (options: { @@ -591,7 +609,7 @@ describe("server browser snapshot screenshot", () => { const screenshotOf = async (html: string) => { const page = await context.newPage(); await page.setContent(html); - const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({ + const input = { page, cdp: await context.newCDPSession(page), renderScale: 2, @@ -599,7 +617,8 @@ describe("server browser snapshot screenshot", () => { consoleEntries: [], networkEntries: [], actionTimeline: [], - }); + }; + const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot(input); // The server has no DOM types, so the decoder runs as page source. const points = [ [0.15, 0.75], @@ -618,7 +637,7 @@ describe("server browser snapshot screenshot", () => { Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data), ); })()`)) as Array<[number, number, number, number]>; - return { screenshot, fullScreenshot, pixels }; + return { screenshot, fullScreenshot, pixels, input }; }; const colorOf = ([red, green, blue]: readonly [number, number, number, number]) => blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`; @@ -633,13 +652,20 @@ describe("server browser snapshot screenshot", () => { }); it("captures a plain page at the shrunk size with its content in place", async () => { - const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves("")); + const { screenshot, fullScreenshot, pixels, input } = await screenshotOf(halves("")); + const metadata = await ServerBrowserPage.snapshotMetadata(input); expect({ size: [screenshot.width, screenshot.height], fullSize: [fullScreenshot?.width, fullScreenshot?.height], + metadataSize: metadata.screenshot, colors: pixels.map(colorOf), - }).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] }); + }).toEqual({ + size: [1280, 800], + fullSize: [2560, 1600], + metadataSize: { width: 1280, height: 800 }, + colors: ["blue", "red", "red"], + }); }); it("records the destination of a finite entrance animation without waiting on infinite ones", async () => { diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index bcfbc296a2cb0..3616ca64f9db9 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -14,6 +14,7 @@ import { type PreviewAutomationSelectInput, type PreviewAutomationSelectResult, type PreviewAutomationSnapshot, + type PreviewAutomationSnapshotMetadata, type PreviewAutomationSnapshotStage, type PreviewAutomationTypeInput, type PreviewAutomationUploadInput, @@ -483,6 +484,37 @@ const pngSize = (base64: string) => { return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; }; +export const jpegSize = (base64: string) => { + const bytes = Buffer.from(base64, "base64"); + if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) + throw new Error("Invalid JPEG screenshot."); + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot."); + while (bytes[offset] === 0xff) offset++; + const marker = bytes[offset++]; + if (marker === undefined || marker === 0xd9 || marker === 0xda) break; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) break; + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) break; + if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { + if (length < 7) break; + const height = bytes.readUInt16BE(offset + 3); + const width = bytes.readUInt16BE(offset + 5); + if (width && height) return { width, height }; + break; + } + offset += length; + } + throw new Error("Invalid JPEG screenshot."); +}; + +const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) => + scale < 1 + ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) } + : size; + /** * A page's display ratio and CSS viewport, read in an isolated world since page script can * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom. @@ -634,7 +666,8 @@ const capture = async ( input: SnapshotInput, deadline: number, stages: SnapshotStages, -): Promise => { + metadataOnly: boolean, +) => { const { running } = stages; const track = async (stage: SnapshotStage, operation: () => Promise) => { running.set(stage, (running.get(stage) ?? 0) + 1); @@ -672,15 +705,21 @@ const capture = async ( > >, ), + // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG. track("screenshot", () => - captureViewport(input.cdp, { format: "png" }).then(async (full) => - scale < 1 - ? { - data: await shrinkPng(input.cdp, full, scale), - full: input.fullScreenshot ? full : undefined, - } - : { data: full, full: undefined }, - ), + metadataOnly + ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({ + data, + full: undefined, + })) + : captureViewport(input.cdp, { format: "png" }).then(async (full) => + scale < 1 + ? { + data: await shrinkPng(input.cdp, full, scale), + full: input.fullScreenshot ? full : undefined, + } + : { data: full, full: undefined }, + ), ), ]); void pageAndScreenshot.catch(constVoid); @@ -731,11 +770,10 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - ...pngSize(data), - }, + // Reports the size the PNG path would have shrunk to. + screenshot: metadataOnly + ? scaledSize(jpegSize(data), scale) + : { mimeType: "image/png" as const, data, ...pngSize(data) }, ...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}), }; }; @@ -743,7 +781,7 @@ const capture = async ( /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */ const snapshotTurns = new WeakMap(); -export const snapshot = async (input: SnapshotInput): Promise => { +const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => { const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS; const stages: SnapshotStages = { running: new Map(), timedOut: new Set() }; let timer: ReturnType | undefined; @@ -753,11 +791,11 @@ export const snapshot = async (input: SnapshotInput): Promise + const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) => Date.now() < deadline && snapshotTurns.get(input.page) === turn && !(cause instanceof Error && cause.message === VALUE_TOO_LONG) - ? capture(input, deadline, stages) + ? capture(input, deadline, stages, metadataOnly) : Promise.reject(cause), ); try { @@ -769,6 +807,12 @@ export const snapshot = async (input: SnapshotInput): Promise + runSnapshot(input, false) as Promise; + +export const snapshotMetadata = (input: SnapshotInput) => + runSnapshot(input, true) as Promise; + /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */ export const click = async ( page: Page, diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 228297667d318..eba5b37e6034f 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -569,11 +569,19 @@ export const PreviewAutomationSnapshot = Schema.Struct({ }); export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type; +// pixelsRequired is false when the caller discards the screenshot; hosts that ignore it still return a PNG. export const PreviewAutomationSnapshotInput = Schema.Struct({ fullScreenshot: Schema.optional(Schema.Boolean), + pixelsRequired: Schema.optional(Schema.Boolean), }); export type PreviewAutomationSnapshotInput = typeof PreviewAutomationSnapshotInput.Type; +export const PreviewAutomationSnapshotMetadata = Schema.Struct({ + ...PreviewAutomationSnapshot.fields, + screenshot: Schema.Struct({ width: Schema.Int, height: Schema.Int }), +}); +export type PreviewAutomationSnapshotMetadata = typeof PreviewAutomationSnapshotMetadata.Type; + export const PreviewAutomationRecordingStatus = Schema.Struct({ tabId: PreviewTabId, recording: Schema.Boolean,