diff --git a/docs/operations/development.md b/docs/operations/development.md
index 8aaa3c97f6a7..b3dcdda2e15d 100644
--- a/docs/operations/development.md
+++ b/docs/operations/development.md
@@ -166,6 +166,7 @@ backend's computer tools as the "Dev harness" agent:
```sh
node scripts/computer-use-dev.ts smoke # Calculator computes 7 + 8 = 15
+node scripts/computer-use-dev.ts bakeoff # 11 tasks on a background Chrome page
node scripts/computer-use-dev.ts snapshot Calculator
node scripts/computer-use-dev.ts action '{"steps":[{"ref":12,"action":{"kind":"press"}}]}'
node scripts/computer-use-dev.ts snapshot TextEdit --image /tmp/textedit.png
@@ -175,7 +176,17 @@ It reads the server origin from the main checkout's `.t3` (override with `--home
day-long admin session for itself, and keeps the token in the system temp directory.
`action` defaults to the latest snapshot the harness took. Only development servers accept
the harness, and only over direct loopback requests. Computer access must be enabled in the
-app's settings, with the target app allowed.
+app's settings, with the target app allowed. A `vp run dev:server` started from a T3 Code
+terminal also has the grants, since macOS credits them to the app that started the shell; pass
+it `--home .t3`.
+
+`bakeoff` serves [computer-use-bakeoff.html](../../scripts/computer-use-bakeoff.html), opens it
+in Google Chrome with the harness's own profile and in the background, then runs button, canvas
+point, hover, typing, Return, scroll, drag and select tasks through `computer_action`. Quit any
+other Chrome first, since computer tools find apps by name. The page logs every DOM event with
+`isTrusted`. A task passes when the page logged the expected trusted events, shows the result
+at the end, and the user's pointer and front app never moved. The table also lists each
+`computer_action` response's size and delivery path.
## Desktop artifacts
diff --git a/scripts/computer-use-bakeoff.html b/scripts/computer-use-bakeoff.html
new file mode 100644
index 000000000000..81506216461e
--- /dev/null
+++ b/scripts/computer-use-bakeoff.html
@@ -0,0 +1,167 @@
+
+
+
+
+ Computer use bake-off
+
+
+
+ Computer use bake-off
+
+
+
+ Hover here to reveal: Secret action
+
+
+
+
+ Flavor
+
+ vanilla
+ chocolate
+ pistachio
+
+
+
+
+ drag
+ drop zone
+ not dropped
+
+
+
+
diff --git a/scripts/computer-use-dev.ts b/scripts/computer-use-dev.ts
index 3f3073b1cf17..f068c0ca0ec2 100644
--- a/scripts/computer-use-dev.ts
+++ b/scripts/computer-use-dev.ts
@@ -1,10 +1,11 @@
-// @effect-diagnostics nodeBuiltinImport:off - Host-side dev tool; hashes the home path to name its token file.
+// @effect-diagnostics nodeBuiltinImport:off - Host-side dev tool; hashes the home path to name its token file and serves the bake-off page.
// Drives the running dev app's computer-use tools from a shell, for agents
// whose own process lacks the macOS Accessibility and Screen Recording grants.
// See docs/operations/development.md#computer-use.
import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
import * as NodeServices from "@effect/platform-node/NodeServices";
import * as NodeCrypto from "node:crypto";
+import * as NodeHttp from "node:http";
import * as NodeOS from "node:os";
import * as Console from "effect/Console";
import * as Effect from "effect/Effect";
@@ -57,19 +58,36 @@ export interface SnapshotElement {
readonly ref: number;
readonly role: string;
readonly name: string | undefined;
+ readonly value: string | undefined;
+ /** Screen points. */
+ readonly bounds: { x: number; y: number; width: number; height: number } | undefined;
}
-/** Elements of a snapshot's text, one per `[ref] role "name" ...` line. */
+const unquote = (quoted: string | undefined) =>
+ quoted === undefined ? undefined : (JSON.parse(quoted) as string);
+
+/** Elements of a snapshot's text, one per `[ref] role "name" = "value" ... @x,y wxh` line. */
export const parseElements = (text: string): SnapshotElement[] =>
text.split("\n").flatMap((line) => {
- const match = /^\s*\[(\d+)\] (\S+)(?: ("(?:[^"\\]|\\.)*"))?/.exec(line);
+ const match = /^\s*\[(\d+)\] (\S+)(?: ("(?:[^"\\]|\\.)*"))?(?: = ("(?:[^"\\]|\\.)*"))?/.exec(
+ line,
+ );
if (!match) return [];
- const quoted = match[3];
+ const bounds = / @(-?\d+),(-?\d+) (\d+)x(\d+)(?: in |$)/.exec(line);
return [
{
ref: Number(match[1]),
role: match[2]!,
- name: quoted === undefined ? undefined : (JSON.parse(quoted) as string),
+ name: unquote(match[3]),
+ value: unquote(match[4]),
+ bounds: bounds
+ ? {
+ x: Number(bounds[1]),
+ y: Number(bounds[2]),
+ width: Number(bounds[3]),
+ height: Number(bounds[4]),
+ }
+ : undefined,
},
];
});
@@ -383,9 +401,429 @@ const smoke = Command.make(
}),
);
+interface PageEvent {
+ readonly type: string;
+ readonly isTrusted?: boolean;
+ readonly cx?: number;
+ readonly cy?: number;
+ readonly key?: string;
+ readonly value?: string;
+ readonly inside?: boolean;
+}
+
+/** Serves the bake-off page on a free loopback port and keeps the events it logs. */
+const servePage = Effect.acquireRelease(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const html = yield* fs.readFileString(
+ path.join(import.meta.dirname, "computer-use-bakeoff.html"),
+ );
+ const events: PageEvent[] = [];
+ const server = NodeHttp.createServer((request, response) => {
+ if (request.method !== "POST") {
+ response.setHeader("content-type", "text/html");
+ response.end(html);
+ return;
+ }
+ let body = "";
+ request.on("data", (chunk: Buffer) => (body += chunk.toString()));
+ request.on("end", () => {
+ events.push(JSON.parse(body) as PageEvent);
+ response.end();
+ });
+ });
+ const port = yield* Effect.callback((resume) => {
+ server.listen(0, "127.0.0.1", () => {
+ const address = server.address();
+ resume(Effect.succeed(typeof address === "object" && address ? address.port : 0));
+ });
+ });
+ return { server, events, url: `http://127.0.0.1:${port}/` };
+ }),
+ ({ server }) =>
+ Effect.sync(() => {
+ server.closeAllConnections();
+ server.close();
+ }),
+);
+
+const waitFor = Effect.fn("computerUseDev.waitFor")(function* (
+ done: Effect.Effect,
+ message: string,
+ timeoutMs = 15_000,
+) {
+ for (let waited = 0; waited < timeoutMs; waited += 200) {
+ if (yield* done) return;
+ yield* Effect.sleep(200);
+ }
+ return yield* fail(message);
+});
+
+const UserState = Schema.Struct({ mouse: Schema.String, front: Schema.String, chrome: Schema.Int });
+/** Where the user's real pointer is, which app is in front, and how many Chromes run. */
+const probeUser = run("osascript", [
+ "-l",
+ "JavaScript",
+ "-e",
+ `ObjC.import("AppKit");
+const mouse = $.NSEvent.mouseLocation;
+JSON.stringify({
+ mouse: Math.round(mouse.x) + "," + Math.round(mouse.y),
+ front: ObjC.unwrap($.NSWorkspace.sharedWorkspace.frontmostApplication.bundleIdentifier),
+ chrome: Number($.NSRunningApplication.runningApplicationsWithBundleIdentifier("com.google.Chrome").count),
+})`,
+]).pipe(
+ Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(UserState))),
+ Effect.mapError(
+ () => new DevHarnessError({ message: "Could not read the pointer and front app." }),
+ ),
+);
+
+/**
+ * Opens `url` in the bake-off's own background Chrome profile, quit by the pid
+ * that profile names. The profile is kept so managed extensions open their
+ * welcome tab only on its first run.
+ */
+const launchChrome = Effect.fn("computerUseDev.launchChrome")(function* (url: string) {
+ const fs = yield* FileSystem.FileSystem;
+ const profile = (yield* Path.Path).join(NodeOS.tmpdir(), "t3-computer-use-bakeoff-chrome");
+ yield* fs.makeDirectory(profile, { recursive: true });
+ const ours = (command: string) =>
+ /\/Contents\/MacOS\/Google Chrome /.test(command) &&
+ command.split(" ").includes(`--user-data-dir=${profile}`);
+ yield* run("open", [
+ "-g",
+ "-na",
+ "Google Chrome",
+ "--args",
+ `--user-data-dir=${profile}`,
+ "--no-first-run",
+ "--no-default-browser-check",
+ // A covered Chrome stops painting without these two.
+ "--disable-backgrounding-occluded-windows",
+ "--disable-renderer-backgrounding",
+ "--window-size=900,1000",
+ "--window-position=40,40",
+ url,
+ ]);
+ const find = run("ps", ["-axo", "pid=,command="]).pipe(
+ Effect.map((lines) => lines.split("\n").find(ours)),
+ );
+ yield* waitFor(find.pipe(Effect.map((line) => line !== undefined)), "Chrome did not start.");
+ const pid = Number.parseInt((yield* find) ?? "", 10);
+ const running = run("ps", ["-o", "command=", "-p", String(pid)]).pipe(
+ Effect.map(ours),
+ Effect.orElseSucceed(() => false),
+ );
+ yield* Effect.addFinalizer(() =>
+ Effect.gen(function* () {
+ // Only while that pid's command line still names this profile.
+ if (!(yield* running)) return;
+ process.kill(pid, "SIGTERM");
+ yield* waitFor(running.pipe(Effect.map((up) => !up)), "Chrome did not quit.");
+ }).pipe(Effect.ignore),
+ );
+});
+
+type Harness = Effect.Success>;
+const CHROME = "Google Chrome";
+const PAGE_TITLE = "Computer use bake-off";
+
+interface Page {
+ readonly text: string;
+ readonly elements: readonly SnapshotElement[];
+ readonly snapshotId: string | undefined;
+ /** The element with this role whose name or value is `label`. */
+ readonly find: (role: string, label: string) => SnapshotElement | undefined;
+ /** The window image pixel at an offset in screen points from an element's top left. */
+ readonly pixel: (
+ element: SnapshotElement | undefined,
+ dx: number,
+ dy: number,
+ ) => { x: number; y: number } | undefined;
+}
+
+const pageOf = (result: ToolResult): Page => {
+ const text = textOf(result);
+ const elements = parseElements(text);
+ const window = elements.find((element) => element.role === "window")?.bounds;
+ const png = result.content.find((part) => part.type === "image");
+ // A PNG's IHDR chunk holds its width at bytes 16-19.
+ const width =
+ png?.type === "image" ? Buffer.from(png.data.slice(0, 32), "base64").readUInt32BE(16) : 0;
+ const scale = window && width ? width / window.width : undefined;
+ return {
+ text,
+ elements,
+ snapshotId: snapshotIdOf(text),
+ find: (role, label) =>
+ elements.find(
+ (element) => element.role === role && (element.name === label || element.value === label),
+ ),
+ pixel: (element, dx, dy) =>
+ element?.bounds && window && scale
+ ? {
+ x: Math.round((element.bounds.x + dx - window.x) * scale),
+ y: Math.round((element.bounds.y + dy - window.y) * scale),
+ }
+ : undefined,
+ };
+};
+
+/**
+ * The page with a window image. Closes any other tab first, such as the
+ * welcome tab a managed extension opens some seconds after launch.
+ */
+const snapshotPage = Effect.fn("computerUseDev.snapshotPage")(function* (harness: Harness) {
+ for (let attempt = 0; attempt < 10; attempt++) {
+ const page = pageOf(
+ yield* harness.call("computer_snapshot", { app: CHROME, includeImage: true }),
+ );
+ const steps = page.elements.flatMap((element, index) => {
+ if (element.role !== "tab" || element.name?.startsWith(PAGE_TITLE)) return [];
+ const close = page.elements
+ .slice(index + 1)
+ .find((next) => next.role === "button" && next.name === "Close");
+ return close ? [{ ref: close.ref, action: { kind: "press" } }] : [];
+ });
+ if (steps.length === 0 && page.find("web_area", PAGE_TITLE)) return page;
+ if (steps.length > 0)
+ yield* harness.call("computer_action", { snapshotId: page.snapshotId, steps });
+ yield* Effect.sleep(500);
+ }
+ return yield* fail("Chrome does not show the bake-off page.");
+});
+
+interface BakeoffTask {
+ readonly name: string;
+ /** computer_action steps, one call per entry against a fresh snapshot; undefined when a target is missing. */
+ readonly actions: ReadonlyArray<(page: Page) => ReadonlyArray | undefined>;
+ /** Whether the page's DOM event log shows the task done. */
+ readonly logged: (events: readonly PageEvent[]) => boolean;
+ /** Whether the page shows the task's result at the end of the run. */
+ readonly shows?: (page: Page) => boolean;
+}
+
+const trusted = (events: readonly PageEvent[], type: string) =>
+ events.filter((event) => event.type === type && event.isTrusted);
+const on = (element: SnapshotElement | undefined, action: object) =>
+ element && [{ ref: element.ref, action }];
+const at = (pixel: { x: number; y: number } | undefined, action: object) =>
+ pixel && [{ action: { ...action, ...pixel } }];
+const leftClick = { kind: "click", button: "left", count: 1 };
+
+const BAKEOFF: readonly BakeoffTask[] = [
+ {
+ name: "button",
+ actions: [(page) => on(page.find("button", "Increment"), leftClick)],
+ logged: (events) => trusted(events, "btn.click").length === 1,
+ shows: (page) => page.find("static_text", "count: 1") !== undefined,
+ },
+ {
+ name: "canvas click at 101,60",
+ actions: [
+ (page) =>
+ at(page.pixel(page.find("image", "drawing canvas"), 101, 60), {
+ ...leftClick,
+ kind: "click_at",
+ }),
+ ],
+ logged: (events) =>
+ trusted(events, "canvas.click")
+ .map((event) => `${event.cx},${event.cy}`)
+ .join() === "101,60",
+ shows: (page) => page.find("static_text", "canvas click @ 101,60") !== undefined,
+ },
+ {
+ name: "hover by ref",
+ actions: [(page) => on(page.find("static_text", "Hover here to reveal:"), { kind: "move" })],
+ logged: (events) => trusted(events, "hover.enter").length > 0,
+ },
+ {
+ name: "move away by point",
+ actions: [
+ (page) => at(page.pixel(page.find("heading", PAGE_TITLE), 10, 10), { kind: "move_at" }),
+ ],
+ logged: (events) =>
+ trusted(events, "mousemove").length > 0 && trusted(events, "mousedown").length === 0,
+ },
+ {
+ name: "hover by point",
+ actions: [
+ (page) => {
+ const label = page.find("static_text", "Hover here to reveal:");
+ return at(page.pixel(label, (label?.bounds?.width ?? 0) + 200, 5), { kind: "move_at" });
+ },
+ ],
+ logged: (events) => trusted(events, "hover.enter").length > 0,
+ },
+ {
+ name: "revealed button",
+ actions: [(page) => on(page.find("button", "Secret action"), leftClick)],
+ logged: (events) => trusted(events, "secret.click").length === 1,
+ shows: (page) => page.find("static_text", "secret clicked") !== undefined,
+ },
+ {
+ name: "typing",
+ actions: [
+ (page) =>
+ on(page.find("text_field", "Name"), { kind: "type", text: "Max Bakeoff", replace: false }),
+ ],
+ logged: (events) =>
+ trusted(events, "keydown").length === 11 &&
+ trusted(events, "input").some((event) => event.value === "Max Bakeoff"),
+ shows: (page) => page.find("static_text", "typed: Max Bakeoff") !== undefined,
+ },
+ {
+ name: "Return",
+ actions: [(page) => on(page.find("text_field", "Name"), { kind: "key", key: "Return" })],
+ logged: (events) =>
+ trusted(events, "keydown")
+ .map((event) => event.key)
+ .join() === "Enter",
+ },
+ {
+ name: "list scroll",
+ actions: [(page) => on(page.find("static_text", "item 0"), { kind: "scroll", dx: 0, dy: 100 })],
+ logged: (events) => trusted(events, "wheel").length > 0,
+ shows: (page) => /"scrollTop [1-9]/.test(page.text),
+ },
+ {
+ name: "drag",
+ actions: [
+ (page) => {
+ const zone = page.find("static_text", "drop zone");
+ return (
+ zone && on(page.find("static_text", "drag"), { kind: "drag", to: { ref: zone.ref } })
+ );
+ },
+ ],
+ logged: (events) =>
+ trusted(events, "drag.down").length === 1 &&
+ trusted(events, "drag.up").some((event) => event.inside),
+ shows: (page) => page.find("static_text", "DROPPED in zone") !== undefined,
+ },
+ {
+ name: "select",
+ actions: [
+ (page) => on(page.find("combo_box", "Flavor"), { kind: "press" }),
+ (page) => on(page.find("menu_item", "chocolate"), { kind: "press" }),
+ ],
+ logged: (events) => trusted(events, "select").some((event) => event.value === "chocolate"),
+ shows: (page) => page.find("combo_box", "Flavor")?.value === "chocolate",
+ },
+];
+
+/** Event types with their counts, plus the detail a task checks. */
+const describeEvents = (events: readonly PageEvent[]) => {
+ const counts = new Map();
+ for (const event of events) {
+ const label = [
+ event.type,
+ event.cx === undefined ? "" : `${event.cx},${event.cy}`,
+ event.type === "select" ? event.value : "",
+ event.inside === undefined ? "" : event.inside ? "inside" : "outside",
+ event.key !== undefined && event.key.length > 1 ? event.key : "",
+ event.isTrusted ? "" : "(untrusted)",
+ ]
+ .filter(Boolean)
+ .join(" ");
+ counts.set(label, (counts.get(label) ?? 0) + 1);
+ }
+ return (
+ [...counts].map(([label, count]) => (count > 1 ? `${label} ×${count}` : label)).join(", ") ||
+ "none"
+ );
+};
+
+const bakeoff = Command.make(
+ "bakeoff",
+ { home },
+ Effect.fn(function* ({ home }) {
+ const harness = yield* makeHarness(home);
+ const user = yield* probeUser;
+ if (user.chrome > 0)
+ return yield* fail(
+ "Quit Google Chrome first: computer tools find Chrome by app name, so the bake-off's must be the only one.",
+ );
+ const page = yield* servePage;
+ yield* launchChrome(page.url);
+ yield* waitFor(
+ Effect.sync(() => page.events.some((event) => event.type === "ready")),
+ "The bake-off page did not load.",
+ );
+ // A first launch can take the front despite -g.
+ const restoreFront = Effect.gen(function* () {
+ if ((yield* probeUser).front !== user.front) yield* run("open", ["-b", user.front]);
+ });
+ yield* restoreFront;
+ yield* snapshotPage(harness);
+ yield* restoreFront;
+ const baseline = yield* probeUser;
+
+ const untouched = (state: typeof UserState.Type) =>
+ state.mouse === baseline.mouse && state.front === baseline.front;
+ let snapshotSize = 0;
+ const runs = [];
+ for (const task of BAKEOFF) {
+ const start = page.events.length;
+ const responses: string[] = [];
+ let error: string | undefined;
+ let moved = false;
+ for (const build of task.actions) {
+ const view = yield* snapshotPage(harness);
+ snapshotSize ||= view.text.length;
+ const steps = build(view);
+ if (!steps) {
+ error = "target not in the snapshot";
+ break;
+ }
+ const before = yield* probeUser;
+ const acted = textOf(
+ yield* harness.call("computer_action", { snapshotId: view.snapshotId, steps }),
+ );
+ const via = /^Completed 1 of 1 steps \((\w+)\)/.exec(acted)?.[1];
+ responses.push(`${acted.length}${via ? ` ${via}` : ""}`);
+ yield* Effect.sleep(300);
+ moved ||= !untouched(before) || !untouched(yield* probeUser);
+ if (!via) {
+ error = acted.split("\n").slice(0, 2).join(" ");
+ break;
+ }
+ }
+ // Chrome handles posted input, and the page posts its log, asynchronously.
+ yield* waitFor(
+ Effect.sync(() => task.logged(page.events.slice(start))),
+ "",
+ 3000,
+ ).pipe(Effect.ignore);
+ runs.push({ task, events: page.events.slice(start), responses, error, moved });
+ }
+ const final = pageOf(yield* harness.call("computer_snapshot", { app: CHROME }));
+ const rows = runs.map(({ task, events, responses, error, moved }) => {
+ const ok = !error && !moved && task.logged(events) && (task.shows?.(final) ?? true);
+ const result = ok ? "pass" : `FAIL${error ? `: ${error}` : ""}`;
+ return {
+ ok,
+ row: `| ${task.name} | ${describeEvents(events)} | ${responses.join(" + ")} | ${moved ? "moved" : "unchanged"} | ${result} |`,
+ };
+ });
+ yield* Console.log(
+ [
+ "| task | page events | computer_action chars, via | pointer, front app | result |",
+ "| --- | --- | --- | --- | --- |",
+ ...rows.map(({ row }) => row),
+ "",
+ `${rows.filter(({ ok }) => ok).length}/${BAKEOFF.length} passed. First snapshot ${snapshotSize} chars. Pointer ${baseline.mouse}, front ${baseline.front}.`,
+ ].join("\n"),
+ );
+ }, Effect.scoped),
+);
+
const main = Command.make("computer-use-dev").pipe(
Command.withDescription("Call the running dev app's computer-use tools as the Dev harness."),
- Command.withSubcommands([status, open, snapshot, action, smoke]),
+ Command.withSubcommands([status, open, snapshot, action, smoke, bakeoff]),
);
if (import.meta.main) {