Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion docs/operations/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ backend's computer tools as the "Dev harness" agent:

```sh
node scripts/computer-use-dev.ts smoke # Calculator computes 7 + 8 = 15
node scripts/computer-use-dev.ts bakeoff # 11 tasks on a background Chrome page
node scripts/computer-use-dev.ts snapshot Calculator
node scripts/computer-use-dev.ts action '{"steps":[{"ref":12,"action":{"kind":"press"}}]}'
node scripts/computer-use-dev.ts snapshot TextEdit --image /tmp/textedit.png
Expand All @@ -175,7 +176,17 @@ It reads the server origin from the main checkout's `.t3` (override with `--home
day-long admin session for itself, and keeps the token in the system temp directory.
`action` defaults to the latest snapshot the harness took. Only development servers accept
the harness, and only over direct loopback requests. Computer access must be enabled in the
app's settings, with the target app allowed.
app's settings, with the target app allowed. A `vp run dev:server` started from a T3 Code
terminal also has the grants, since macOS credits them to the app that started the shell; pass
it `--home .t3`.

`bakeoff` serves [computer-use-bakeoff.html](../../scripts/computer-use-bakeoff.html), opens it
in Google Chrome with the harness's own profile and in the background, then runs button, canvas
point, hover, typing, Return, scroll, drag and select tasks through `computer_action`. Quit any
other Chrome first, since computer tools find apps by name. The page logs every DOM event with
`isTrusted`. A task passes when the page logged the expected trusted events, shows the result
at the end, and the user's pointer and front app never moved. The table also lists each
`computer_action` response's size and delivery path.

## Desktop artifacts

Expand Down
167 changes: 167 additions & 0 deletions scripts/computer-use-bakeoff.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Computer use bake-off</title>
<style>
body {
font: 15px system-ui;
margin: 20px;
}
section {
margin: 10px 0;
padding: 8px;
border: 1px solid #ccc;
}
#canvas {
border: 1px solid #333;
}
#hoverzone {
padding: 16px;
background: #eef;
}
#secret {
display: none;
}
#hoverzone:hover #secret {
display: inline-block;
}
#list {
height: 120px;
overflow: auto;
border: 1px solid #999;
}
#drag {
width: 60px;
height: 40px;
background: #f96;
position: relative;
touch-action: none;
}
#drop {
width: 160px;
height: 60px;
border: 2px dashed #69f;
margin-top: 8px;
}
</style>
</head>
<body>
<h1>Computer use bake-off</h1>
<section><button id="btn">Increment</button> <span id="count">count: 0</span></section>
<section>
<canvas id="canvas" width="300" height="120" aria-label="drawing canvas"></canvas>
<span id="canvaslog">no canvas clicks</span>
</section>
<section id="hoverzone">
Hover here to reveal: <button id="secret">Secret action</button> <span id="secretlog"></span>
</section>
<!-- Never id="name": a bare `name` is window.name and the script below would break. -->
<section>
<label>Name <input id="nameField" /></label> <span id="echo"></span>
</section>
<section>
<label
>Flavor
<select id="sel">
<option>vanilla</option>
<option>chocolate</option>
<option>pistachio</option>
</select></label
>
</section>
<section>
<div id="list"></div>
<span id="scrolllog">scrollTop 0</span>
</section>
<section>
<div id="drag">drag</div>
<div id="drop">drop zone</div>
<span id="draglog">not dropped</span>
</section>
<script>
const log = (type, e, extra = {}) =>
fetch("/log", {
method: "POST",
body: JSON.stringify({
type,
isTrusted: e?.isTrusted,
x: e?.clientX,
y: e?.clientY,
...extra,
}),
});
for (let i = 0; i < 50; i++)
list.append(Object.assign(document.createElement("div"), { textContent: `item ${i}` }));
let n = 0;
btn.onclick = (e) => {
count.textContent = `count: ${++n}`;
log("btn.click", e);
};
const ctx = canvas.getContext("2d");
canvas.addEventListener("click", (e) => {
const r = canvas.getBoundingClientRect();
const cx = Math.round(e.clientX - r.left),
cy = Math.round(e.clientY - r.top);
ctx.fillStyle = "red";
ctx.beginPath();
ctx.arc(cx, cy, 5, 0, 7);
ctx.fill();
canvaslog.textContent = `canvas click @ ${cx},${cy}`;
log("canvas.click", e, { cx, cy });
});
hoverzone.addEventListener("mouseenter", (e) => log("hover.enter", e));
secret.onclick = (e) => {
secretlog.textContent = "secret clicked";
log("secret.click", e);
};
nameField.addEventListener("input", (e) => {
echo.textContent = `typed: ${nameField.value}`;
log("input", e, { value: nameField.value });
});
nameField.addEventListener("keydown", (e) => log("keydown", e, { key: e.key }));
sel.onchange = (e) => log("select", e, { value: sel.value });
list.onscroll = () => {
scrolllog.textContent = `scrollTop ${list.scrollTop}`;
};
list.addEventListener("wheel", (e) => log("wheel", e, { dy: e.deltaY }), { passive: true });
let ox = 0,
oy = 0,
dragging = false;
drag.addEventListener("pointerdown", (e) => {
dragging = true;
ox = e.clientX;
oy = e.clientY;
drag.setPointerCapture(e.pointerId);
log("drag.down", e);
});
drag.addEventListener("pointermove", (e) => {
if (dragging) {
drag.style.left = `${e.clientX - ox}px`;
drag.style.top = `${e.clientY - oy}px`;
}
});
drag.addEventListener("pointerup", (e) => {
dragging = false;
const d = drop.getBoundingClientRect();
const inside =
e.clientX >= d.left &&
e.clientX <= d.right &&
e.clientY >= d.top &&
e.clientY <= d.bottom;
draglog.textContent = inside ? "DROPPED in zone" : "released outside";
log("drag.up", e, { inside });
});
let moved = false;
document.addEventListener("mousedown", (e) => log("mousedown", e));
document.addEventListener("mousemove", (e) => {
if (!moved) {
moved = true;
setTimeout(() => (moved = false), 300);
log("mousemove", e);
}
});
log("ready");
</script>
</body>
</html>
Loading
Loading