From 2f5e1ba0646de907e70e6e8bf122093515809180 Mon Sep 17 00:00:00 2001 From: Max schwenk Date: Sun, 8 Nov 2026 19:40:23 -0500 Subject: [PATCH] test(computer-use): bake-off subcommand for background Chrome node scripts/computer-use-dev.ts bakeoff serves a small page that logs every DOM event with isTrusted, opens it in a background Google Chrome with its own profile, runs 11 tasks through computer_action over MCP, and prints a pass/fail table with response sizes, delivery paths, and whether the user's pointer or front app moved. --- docs/operations/development.md | 13 +- scripts/computer-use-bakeoff.html | 167 +++++++++++ scripts/computer-use-dev.ts | 450 +++++++++++++++++++++++++++++- 3 files changed, 623 insertions(+), 7 deletions(-) create mode 100644 scripts/computer-use-bakeoff.html diff --git a/docs/operations/development.md b/docs/operations/development.md index 8aaa3c97f6a7..b3dcdda2e15d 100644 --- a/docs/operations/development.md +++ b/docs/operations/development.md @@ -166,6 +166,7 @@ backend's computer tools as the "Dev harness" agent: ```sh node scripts/computer-use-dev.ts smoke # Calculator computes 7 + 8 = 15 +node scripts/computer-use-dev.ts bakeoff # 11 tasks on a background Chrome page node scripts/computer-use-dev.ts snapshot Calculator node scripts/computer-use-dev.ts action '{"steps":[{"ref":12,"action":{"kind":"press"}}]}' node scripts/computer-use-dev.ts snapshot TextEdit --image /tmp/textedit.png @@ -175,7 +176,17 @@ It reads the server origin from the main checkout's `.t3` (override with `--home day-long admin session for itself, and keeps the token in the system temp directory. `action` defaults to the latest snapshot the harness took. Only development servers accept the harness, and only over direct loopback requests. Computer access must be enabled in the -app's settings, with the target app allowed. +app's settings, with the target app allowed. A `vp run dev:server` started from a T3 Code +terminal also has the grants, since macOS credits them to the app that started the shell; pass +it `--home .t3`. + +`bakeoff` serves [computer-use-bakeoff.html](../../scripts/computer-use-bakeoff.html), opens it +in Google Chrome with the harness's own profile and in the background, then runs button, canvas +point, hover, typing, Return, scroll, drag and select tasks through `computer_action`. Quit any +other Chrome first, since computer tools find apps by name. The page logs every DOM event with +`isTrusted`. A task passes when the page logged the expected trusted events, shows the result +at the end, and the user's pointer and front app never moved. The table also lists each +`computer_action` response's size and delivery path. ## Desktop artifacts diff --git a/scripts/computer-use-bakeoff.html b/scripts/computer-use-bakeoff.html new file mode 100644 index 000000000000..81506216461e --- /dev/null +++ b/scripts/computer-use-bakeoff.html @@ -0,0 +1,167 @@ + + + + + Computer use bake-off + + + +

Computer use bake-off

+
count: 0
+
+ + no canvas clicks +
+
+ Hover here to reveal: +
+ +
+ +
+
+ +
+
+
+ scrollTop 0 +
+
+
drag
+
drop zone
+ not dropped +
+ + + diff --git a/scripts/computer-use-dev.ts b/scripts/computer-use-dev.ts index 3f3073b1cf17..f068c0ca0ec2 100644 --- a/scripts/computer-use-dev.ts +++ b/scripts/computer-use-dev.ts @@ -1,10 +1,11 @@ -// @effect-diagnostics nodeBuiltinImport:off - Host-side dev tool; hashes the home path to name its token file. +// @effect-diagnostics nodeBuiltinImport:off - Host-side dev tool; hashes the home path to name its token file and serves the bake-off page. // Drives the running dev app's computer-use tools from a shell, for agents // whose own process lacks the macOS Accessibility and Screen Recording grants. // See docs/operations/development.md#computer-use. import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeCrypto from "node:crypto"; +import * as NodeHttp from "node:http"; import * as NodeOS from "node:os"; import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; @@ -57,19 +58,36 @@ export interface SnapshotElement { readonly ref: number; readonly role: string; readonly name: string | undefined; + readonly value: string | undefined; + /** Screen points. */ + readonly bounds: { x: number; y: number; width: number; height: number } | undefined; } -/** Elements of a snapshot's text, one per `[ref] role "name" ...` line. */ +const unquote = (quoted: string | undefined) => + quoted === undefined ? undefined : (JSON.parse(quoted) as string); + +/** Elements of a snapshot's text, one per `[ref] role "name" = "value" ... @x,y wxh` line. */ export const parseElements = (text: string): SnapshotElement[] => text.split("\n").flatMap((line) => { - const match = /^\s*\[(\d+)\] (\S+)(?: ("(?:[^"\\]|\\.)*"))?/.exec(line); + const match = /^\s*\[(\d+)\] (\S+)(?: ("(?:[^"\\]|\\.)*"))?(?: = ("(?:[^"\\]|\\.)*"))?/.exec( + line, + ); if (!match) return []; - const quoted = match[3]; + const bounds = / @(-?\d+),(-?\d+) (\d+)x(\d+)(?: in |$)/.exec(line); return [ { ref: Number(match[1]), role: match[2]!, - name: quoted === undefined ? undefined : (JSON.parse(quoted) as string), + name: unquote(match[3]), + value: unquote(match[4]), + bounds: bounds + ? { + x: Number(bounds[1]), + y: Number(bounds[2]), + width: Number(bounds[3]), + height: Number(bounds[4]), + } + : undefined, }, ]; }); @@ -383,9 +401,429 @@ const smoke = Command.make( }), ); +interface PageEvent { + readonly type: string; + readonly isTrusted?: boolean; + readonly cx?: number; + readonly cy?: number; + readonly key?: string; + readonly value?: string; + readonly inside?: boolean; +} + +/** Serves the bake-off page on a free loopback port and keeps the events it logs. */ +const servePage = Effect.acquireRelease( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const html = yield* fs.readFileString( + path.join(import.meta.dirname, "computer-use-bakeoff.html"), + ); + const events: PageEvent[] = []; + const server = NodeHttp.createServer((request, response) => { + if (request.method !== "POST") { + response.setHeader("content-type", "text/html"); + response.end(html); + return; + } + let body = ""; + request.on("data", (chunk: Buffer) => (body += chunk.toString())); + request.on("end", () => { + events.push(JSON.parse(body) as PageEvent); + response.end(); + }); + }); + const port = yield* Effect.callback((resume) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resume(Effect.succeed(typeof address === "object" && address ? address.port : 0)); + }); + }); + return { server, events, url: `http://127.0.0.1:${port}/` }; + }), + ({ server }) => + Effect.sync(() => { + server.closeAllConnections(); + server.close(); + }), +); + +const waitFor = Effect.fn("computerUseDev.waitFor")(function* ( + done: Effect.Effect, + message: string, + timeoutMs = 15_000, +) { + for (let waited = 0; waited < timeoutMs; waited += 200) { + if (yield* done) return; + yield* Effect.sleep(200); + } + return yield* fail(message); +}); + +const UserState = Schema.Struct({ mouse: Schema.String, front: Schema.String, chrome: Schema.Int }); +/** Where the user's real pointer is, which app is in front, and how many Chromes run. */ +const probeUser = run("osascript", [ + "-l", + "JavaScript", + "-e", + `ObjC.import("AppKit"); +const mouse = $.NSEvent.mouseLocation; +JSON.stringify({ + mouse: Math.round(mouse.x) + "," + Math.round(mouse.y), + front: ObjC.unwrap($.NSWorkspace.sharedWorkspace.frontmostApplication.bundleIdentifier), + chrome: Number($.NSRunningApplication.runningApplicationsWithBundleIdentifier("com.google.Chrome").count), +})`, +]).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(UserState))), + Effect.mapError( + () => new DevHarnessError({ message: "Could not read the pointer and front app." }), + ), +); + +/** + * Opens `url` in the bake-off's own background Chrome profile, quit by the pid + * that profile names. The profile is kept so managed extensions open their + * welcome tab only on its first run. + */ +const launchChrome = Effect.fn("computerUseDev.launchChrome")(function* (url: string) { + const fs = yield* FileSystem.FileSystem; + const profile = (yield* Path.Path).join(NodeOS.tmpdir(), "t3-computer-use-bakeoff-chrome"); + yield* fs.makeDirectory(profile, { recursive: true }); + const ours = (command: string) => + /\/Contents\/MacOS\/Google Chrome /.test(command) && + command.split(" ").includes(`--user-data-dir=${profile}`); + yield* run("open", [ + "-g", + "-na", + "Google Chrome", + "--args", + `--user-data-dir=${profile}`, + "--no-first-run", + "--no-default-browser-check", + // A covered Chrome stops painting without these two. + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--window-size=900,1000", + "--window-position=40,40", + url, + ]); + const find = run("ps", ["-axo", "pid=,command="]).pipe( + Effect.map((lines) => lines.split("\n").find(ours)), + ); + yield* waitFor(find.pipe(Effect.map((line) => line !== undefined)), "Chrome did not start."); + const pid = Number.parseInt((yield* find) ?? "", 10); + const running = run("ps", ["-o", "command=", "-p", String(pid)]).pipe( + Effect.map(ours), + Effect.orElseSucceed(() => false), + ); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + // Only while that pid's command line still names this profile. + if (!(yield* running)) return; + process.kill(pid, "SIGTERM"); + yield* waitFor(running.pipe(Effect.map((up) => !up)), "Chrome did not quit."); + }).pipe(Effect.ignore), + ); +}); + +type Harness = Effect.Success>; +const CHROME = "Google Chrome"; +const PAGE_TITLE = "Computer use bake-off"; + +interface Page { + readonly text: string; + readonly elements: readonly SnapshotElement[]; + readonly snapshotId: string | undefined; + /** The element with this role whose name or value is `label`. */ + readonly find: (role: string, label: string) => SnapshotElement | undefined; + /** The window image pixel at an offset in screen points from an element's top left. */ + readonly pixel: ( + element: SnapshotElement | undefined, + dx: number, + dy: number, + ) => { x: number; y: number } | undefined; +} + +const pageOf = (result: ToolResult): Page => { + const text = textOf(result); + const elements = parseElements(text); + const window = elements.find((element) => element.role === "window")?.bounds; + const png = result.content.find((part) => part.type === "image"); + // A PNG's IHDR chunk holds its width at bytes 16-19. + const width = + png?.type === "image" ? Buffer.from(png.data.slice(0, 32), "base64").readUInt32BE(16) : 0; + const scale = window && width ? width / window.width : undefined; + return { + text, + elements, + snapshotId: snapshotIdOf(text), + find: (role, label) => + elements.find( + (element) => element.role === role && (element.name === label || element.value === label), + ), + pixel: (element, dx, dy) => + element?.bounds && window && scale + ? { + x: Math.round((element.bounds.x + dx - window.x) * scale), + y: Math.round((element.bounds.y + dy - window.y) * scale), + } + : undefined, + }; +}; + +/** + * The page with a window image. Closes any other tab first, such as the + * welcome tab a managed extension opens some seconds after launch. + */ +const snapshotPage = Effect.fn("computerUseDev.snapshotPage")(function* (harness: Harness) { + for (let attempt = 0; attempt < 10; attempt++) { + const page = pageOf( + yield* harness.call("computer_snapshot", { app: CHROME, includeImage: true }), + ); + const steps = page.elements.flatMap((element, index) => { + if (element.role !== "tab" || element.name?.startsWith(PAGE_TITLE)) return []; + const close = page.elements + .slice(index + 1) + .find((next) => next.role === "button" && next.name === "Close"); + return close ? [{ ref: close.ref, action: { kind: "press" } }] : []; + }); + if (steps.length === 0 && page.find("web_area", PAGE_TITLE)) return page; + if (steps.length > 0) + yield* harness.call("computer_action", { snapshotId: page.snapshotId, steps }); + yield* Effect.sleep(500); + } + return yield* fail("Chrome does not show the bake-off page."); +}); + +interface BakeoffTask { + readonly name: string; + /** computer_action steps, one call per entry against a fresh snapshot; undefined when a target is missing. */ + readonly actions: ReadonlyArray<(page: Page) => ReadonlyArray | undefined>; + /** Whether the page's DOM event log shows the task done. */ + readonly logged: (events: readonly PageEvent[]) => boolean; + /** Whether the page shows the task's result at the end of the run. */ + readonly shows?: (page: Page) => boolean; +} + +const trusted = (events: readonly PageEvent[], type: string) => + events.filter((event) => event.type === type && event.isTrusted); +const on = (element: SnapshotElement | undefined, action: object) => + element && [{ ref: element.ref, action }]; +const at = (pixel: { x: number; y: number } | undefined, action: object) => + pixel && [{ action: { ...action, ...pixel } }]; +const leftClick = { kind: "click", button: "left", count: 1 }; + +const BAKEOFF: readonly BakeoffTask[] = [ + { + name: "button", + actions: [(page) => on(page.find("button", "Increment"), leftClick)], + logged: (events) => trusted(events, "btn.click").length === 1, + shows: (page) => page.find("static_text", "count: 1") !== undefined, + }, + { + name: "canvas click at 101,60", + actions: [ + (page) => + at(page.pixel(page.find("image", "drawing canvas"), 101, 60), { + ...leftClick, + kind: "click_at", + }), + ], + logged: (events) => + trusted(events, "canvas.click") + .map((event) => `${event.cx},${event.cy}`) + .join() === "101,60", + shows: (page) => page.find("static_text", "canvas click @ 101,60") !== undefined, + }, + { + name: "hover by ref", + actions: [(page) => on(page.find("static_text", "Hover here to reveal:"), { kind: "move" })], + logged: (events) => trusted(events, "hover.enter").length > 0, + }, + { + name: "move away by point", + actions: [ + (page) => at(page.pixel(page.find("heading", PAGE_TITLE), 10, 10), { kind: "move_at" }), + ], + logged: (events) => + trusted(events, "mousemove").length > 0 && trusted(events, "mousedown").length === 0, + }, + { + name: "hover by point", + actions: [ + (page) => { + const label = page.find("static_text", "Hover here to reveal:"); + return at(page.pixel(label, (label?.bounds?.width ?? 0) + 200, 5), { kind: "move_at" }); + }, + ], + logged: (events) => trusted(events, "hover.enter").length > 0, + }, + { + name: "revealed button", + actions: [(page) => on(page.find("button", "Secret action"), leftClick)], + logged: (events) => trusted(events, "secret.click").length === 1, + shows: (page) => page.find("static_text", "secret clicked") !== undefined, + }, + { + name: "typing", + actions: [ + (page) => + on(page.find("text_field", "Name"), { kind: "type", text: "Max Bakeoff", replace: false }), + ], + logged: (events) => + trusted(events, "keydown").length === 11 && + trusted(events, "input").some((event) => event.value === "Max Bakeoff"), + shows: (page) => page.find("static_text", "typed: Max Bakeoff") !== undefined, + }, + { + name: "Return", + actions: [(page) => on(page.find("text_field", "Name"), { kind: "key", key: "Return" })], + logged: (events) => + trusted(events, "keydown") + .map((event) => event.key) + .join() === "Enter", + }, + { + name: "list scroll", + actions: [(page) => on(page.find("static_text", "item 0"), { kind: "scroll", dx: 0, dy: 100 })], + logged: (events) => trusted(events, "wheel").length > 0, + shows: (page) => /"scrollTop [1-9]/.test(page.text), + }, + { + name: "drag", + actions: [ + (page) => { + const zone = page.find("static_text", "drop zone"); + return ( + zone && on(page.find("static_text", "drag"), { kind: "drag", to: { ref: zone.ref } }) + ); + }, + ], + logged: (events) => + trusted(events, "drag.down").length === 1 && + trusted(events, "drag.up").some((event) => event.inside), + shows: (page) => page.find("static_text", "DROPPED in zone") !== undefined, + }, + { + name: "select", + actions: [ + (page) => on(page.find("combo_box", "Flavor"), { kind: "press" }), + (page) => on(page.find("menu_item", "chocolate"), { kind: "press" }), + ], + logged: (events) => trusted(events, "select").some((event) => event.value === "chocolate"), + shows: (page) => page.find("combo_box", "Flavor")?.value === "chocolate", + }, +]; + +/** Event types with their counts, plus the detail a task checks. */ +const describeEvents = (events: readonly PageEvent[]) => { + const counts = new Map(); + for (const event of events) { + const label = [ + event.type, + event.cx === undefined ? "" : `${event.cx},${event.cy}`, + event.type === "select" ? event.value : "", + event.inside === undefined ? "" : event.inside ? "inside" : "outside", + event.key !== undefined && event.key.length > 1 ? event.key : "", + event.isTrusted ? "" : "(untrusted)", + ] + .filter(Boolean) + .join(" "); + counts.set(label, (counts.get(label) ?? 0) + 1); + } + return ( + [...counts].map(([label, count]) => (count > 1 ? `${label} ×${count}` : label)).join(", ") || + "none" + ); +}; + +const bakeoff = Command.make( + "bakeoff", + { home }, + Effect.fn(function* ({ home }) { + const harness = yield* makeHarness(home); + const user = yield* probeUser; + if (user.chrome > 0) + return yield* fail( + "Quit Google Chrome first: computer tools find Chrome by app name, so the bake-off's must be the only one.", + ); + const page = yield* servePage; + yield* launchChrome(page.url); + yield* waitFor( + Effect.sync(() => page.events.some((event) => event.type === "ready")), + "The bake-off page did not load.", + ); + // A first launch can take the front despite -g. + const restoreFront = Effect.gen(function* () { + if ((yield* probeUser).front !== user.front) yield* run("open", ["-b", user.front]); + }); + yield* restoreFront; + yield* snapshotPage(harness); + yield* restoreFront; + const baseline = yield* probeUser; + + const untouched = (state: typeof UserState.Type) => + state.mouse === baseline.mouse && state.front === baseline.front; + let snapshotSize = 0; + const runs = []; + for (const task of BAKEOFF) { + const start = page.events.length; + const responses: string[] = []; + let error: string | undefined; + let moved = false; + for (const build of task.actions) { + const view = yield* snapshotPage(harness); + snapshotSize ||= view.text.length; + const steps = build(view); + if (!steps) { + error = "target not in the snapshot"; + break; + } + const before = yield* probeUser; + const acted = textOf( + yield* harness.call("computer_action", { snapshotId: view.snapshotId, steps }), + ); + const via = /^Completed 1 of 1 steps \((\w+)\)/.exec(acted)?.[1]; + responses.push(`${acted.length}${via ? ` ${via}` : ""}`); + yield* Effect.sleep(300); + moved ||= !untouched(before) || !untouched(yield* probeUser); + if (!via) { + error = acted.split("\n").slice(0, 2).join(" "); + break; + } + } + // Chrome handles posted input, and the page posts its log, asynchronously. + yield* waitFor( + Effect.sync(() => task.logged(page.events.slice(start))), + "", + 3000, + ).pipe(Effect.ignore); + runs.push({ task, events: page.events.slice(start), responses, error, moved }); + } + const final = pageOf(yield* harness.call("computer_snapshot", { app: CHROME })); + const rows = runs.map(({ task, events, responses, error, moved }) => { + const ok = !error && !moved && task.logged(events) && (task.shows?.(final) ?? true); + const result = ok ? "pass" : `FAIL${error ? `: ${error}` : ""}`; + return { + ok, + row: `| ${task.name} | ${describeEvents(events)} | ${responses.join(" + ")} | ${moved ? "moved" : "unchanged"} | ${result} |`, + }; + }); + yield* Console.log( + [ + "| task | page events | computer_action chars, via | pointer, front app | result |", + "| --- | --- | --- | --- | --- |", + ...rows.map(({ row }) => row), + "", + `${rows.filter(({ ok }) => ok).length}/${BAKEOFF.length} passed. First snapshot ${snapshotSize} chars. Pointer ${baseline.mouse}, front ${baseline.front}.`, + ].join("\n"), + ); + }, Effect.scoped), +); + const main = Command.make("computer-use-dev").pipe( Command.withDescription("Call the running dev app's computer-use tools as the Dev harness."), - Command.withSubcommands([status, open, snapshot, action, smoke]), + Command.withSubcommands([status, open, snapshot, action, smoke, bakeoff]), ); if (import.meta.main) {