diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index c32947b30b995..61af124efb55c 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -2525,3 +2525,237 @@ it.live("a full server takes back the slot of the longest idle agent tab nobody }), ).pipe(Effect.provide(layer)), ); + +/** Makes the tab's next color scheme change hang in Playwright; resolves once it has started. */ +const stallColorScheme = (page: ReturnType["page"]) => { + const started = Promise.withResolvers(); + page.emulateMedia.mockImplementationOnce(() => { + started.resolve(); + return new Promise(() => {}); + }); + return Effect.promise(() => started.promise); +}; + +it.effect("a stuck agent action times out on the server and later calls on its tab fail fast", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const stalled = stallColorScheme(contexts[0]!.page); + const stuck = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* stalled; + // The broker evicts a host that has not answered by timeoutMs plus its one second grace. + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + // The server answered in time, so the host every tab shares stays connected. + const status = yield* broker + .invoke({ scope, tabId, operation: "status", input: {} }) + .pipe(Effect.exit); + expect(status).toMatchObject({ _tag: "Success" }); + const busy = yield* broker + .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) + .pipe(Effect.flip); + expect(busy.message).toMatch(/busy|still running/i); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("a tab stuck in an agent action does not hold back setting changes for other tabs", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const other = yield* broker.invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + const stalled = stallColorScheme(contexts[0]!.page); + const applied = Promise.withResolvers(); + contexts[1]!.page.emulateMedia.mockImplementationOnce(async () => applied.resolve()); + yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.ignore, Effect.forkScoped); + // The stuck action has already published its own tab's resized event. + yield* stalled; + yield* manager.adjust({ + threadId: scope.thread.threadId, + tabId: PreviewTabId.make(other.tabId!), + colorScheme: "dark", + }); + yield* Effect.promise(() => applied.promise); + expect(contexts[1]!.page.emulateMedia).toHaveBeenCalledWith({ colorScheme: "dark" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("a dialog on a stuck tab is reported before the tab is busy", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const page = contexts[0]!.page; + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const stalled = stallColorScheme(page); + const stuck = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* stalled; + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + page.emit("dialog", { + type: () => "alert", + message: () => "Saved", + defaultValue: () => "", + accept: vi.fn(async () => {}), + dismiss: vi.fn(async () => {}), + }); + const blocked = yield* broker + .invoke({ + scope, + tabId, + operation: "click", + input: { ref: "e1" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip); + expect(blocked).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "dialogPending", + }); + const stop = yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.flip); + expect(stop).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "dialogPending", + }); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("an action that fails just after its timeout still reports its own failure", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const failing = Promise.withResolvers(); + const started = Promise.withResolvers(); + contexts[0]!.page.emulateMedia.mockImplementationOnce(() => { + started.resolve(); + return failing.promise; + }); + const failed = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 1_000, + }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => started.promise); + // Operations time themselves out a little after the request arrives; their reason wins. + yield* TestClock.adjust("1100 millis"); + failing.reject(new Error("Target page crashed")); + expect(yield* Fiber.join(failed)).toMatchObject({ _tag: "PreviewAutomationExecutionError" }); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("a recording stop that never finishes leaves its tab busy", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const context = contexts[0]!; + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const started = Promise.withResolvers(); + context.sessions.at(-1)!.detach.mockImplementationOnce(() => { + started.resolve(); + return new Promise(() => {}); + }); + const stuck = yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Effect.promise(() => started.promise); + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(stuck)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + const busy = yield* broker + .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) + .pipe(Effect.flip, Effect.forkScoped); + // The host answers in arrival order, so a refused click has its answer before this one. + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + // A click that queued instead of being refused would time out here. + yield* TestClock.adjust("2 seconds"); + expect(yield* Fiber.join(busy)).toMatchObject({ + _tag: "PreviewAutomationControlInterruptedError", + reason: "busy", + }); + }), + ).pipe(Effect.provide(layer)), +); + +it.effect("a queued action whose timeout passes while it waits never runs", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const page = contexts[0]!.page; + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} }); + const stopRan = Promise.withResolvers(); + contexts[0]!.sessions.at(-1)!.detach.mockImplementationOnce(() => { + stopRan.resolve(); + return new Promise(() => {}); + }); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + page.emulateMedia.mockImplementationOnce(() => { + started.resolve(); + return finish.promise; + }); + const slow = yield* broker + .invoke({ + scope, + tabId, + operation: "setColorScheme", + input: { colorScheme: "dark" }, + timeoutMs: 3_000, + }) + .pipe(Effect.forkScoped); + yield* Effect.promise(() => started.promise); + const queued = yield* broker + .invoke({ scope, tabId, operation: "click", input: { ref: "e1" }, timeoutMs: 1_000 }) + .pipe(Effect.flip, Effect.forkScoped); + // The host answers in arrival order, so the click is queued once this answers. + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + yield* broker + .invoke({ scope, tabId, operation: "recordingStop", input: {}, timeoutMs: 1_000 }) + .pipe(Effect.ignore, Effect.forkScoped); + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + // Past the click's timeout but before the server answers it. + yield* TestClock.adjust("1200 millis"); + finish.resolve(); + yield* Fiber.join(slow); + expect(yield* Fiber.join(queued)).toMatchObject({ _tag: "PreviewAutomationTimeoutError" }); + expect(page.locator).not.toHaveBeenCalled(); + // A late stop still runs, or capture would keep going with no recording to stop. + yield* Effect.promise(() => stopRan.promise); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index ee31b17cc5889..7071a0d40c3d2 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -127,6 +127,8 @@ const EDITABLE_AT_POINT_SCRIPT = `(x, y) => { }`; const UNATTACHED_FILL_VIEWPORT = { width: 1280, height: 800 } as const; const NAVIGATION_TIMEOUT_MS = 15_000; +// Must stay under the broker's 1 s host response grace. +const HOST_ANSWER_MARGIN_MS = 500; const VIEWER_NAVIGATION_OPTIONS = { waitUntil: "commit", timeout: NAVIGATION_TIMEOUT_MS } as const; const ACTION_TIMELINE_LIMIT = 50; @@ -1928,6 +1930,15 @@ const make = Effect.gen(function* () { ); }; + // Work blocked by a dialog outlives its request; point the agent at the dialog, not at `busy`. + const requireNoOverdueDialog = (tab: ServerTab, agentSessionId: string) => { + if (tab.dialog && tab.control.overdue && tab.control.agentCanActNow(agentSessionId)) + throw new BrowserControlInterrupted({ + message: "A browser dialog is pending. Read preview_status and use preview_dialog first.", + reason: "dialogPending", + }); + }; + const requireTab = async (request: PreviewAutomationRequest) => { if (!request.agentSessionId) throw new BrowserControlInterrupted({ @@ -1976,7 +1987,7 @@ const make = Effect.gen(function* () { const runOperation = async ( request: PreviewAutomationRequest, - deadlineMs: number, + signal: AbortSignal, ): Promise => { const input = request.input; switch (request.operation) { @@ -2101,7 +2112,7 @@ const make = Effect.gen(function* () { } return statusWithTitle(tab, request.agentSessionId); }, - deadlineMs, + signal, ); } case "recordingStop": { @@ -2135,9 +2146,12 @@ const make = Effect.gen(function* () { message: "No recording is active for this thread.", }); } + requireNoOverdueDialog(tab, request.agentSessionId ?? ""); // A stop cannot act twice (a retry finds no recording); // dropping a late one would leave capture running. - return tab.control.agent(request.agentSessionId ?? "", () => stopRecording(tab)); + return tab.control.agent(request.agentSessionId ?? "", () => stopRecording(tab), signal, { + runIfExpired: true, + }); } } const tab = await requireTab(request); @@ -2184,6 +2198,7 @@ const make = Effect.gen(function* () { return executeTabOperation(tab, request); }); } + requireNoOverdueDialog(tab, agentSessionId); return tab.control.agent( agentSessionId, async () => { @@ -2204,7 +2219,7 @@ const make = Effect.gen(function* () { if (generation !== tab.control.generation) ServerBrowserPage.invalidateRefs(tab.page); } }, - deadlineMs, + signal, ); }; @@ -2302,14 +2317,35 @@ const make = Effect.gen(function* () { } }; - const handleRequest = (connectionId: string, request: PreviewAutomationRequest) => - Effect.tryPromise({ - try: () => - runOperation(request, performance.now() + request.timeoutMs).finally(() => - markUsed(request), - ), + const handleRequest = Effect.fn("ServerBrowser.handleRequest")(function* ( + connectionId: string, + request: PreviewAutomationRequest, + ) { + // Queued work expires at the request's timeout; a started run past it marks its tab busy. + const expiry = yield* Scope.make(); + const signal = yield* Effect.abortSignal.pipe(Scope.provide(expiry)); + yield* Effect.sleep(request.timeoutMs).pipe( + Effect.andThen(Scope.close(expiry, Exit.void)), + Effect.forkChild, + ); + const outcome = yield* Effect.tryPromise({ + try: () => runOperation(request, signal).finally(() => markUsed(request)), catch: ServerBrowserPage.toOperationError, }).pipe( + // Lets the operation report its own timeout first, yet answers before the broker's + // response grace evicts this host. The abandoned run still holds its tab's queue. + Effect.timeoutOrElse({ + duration: request.timeoutMs + HOST_ANSWER_MARGIN_MS, + orElse: () => + Effect.fail( + new ServerBrowserPage.ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `The browser did not finish ${request.operation} in ${request.timeoutMs}ms.`, + detail: { timeoutMs: request.timeoutMs }, + }), + ), + }), + Effect.ensuring(Scope.close(expiry, Exit.void)), Effect.match({ onSuccess: (result) => ({ ok: true as const, result }), onFailure: (error) => ({ @@ -2321,16 +2357,16 @@ const make = Effect.gen(function* () { }, }), }), - Effect.flatMap((outcome) => - broker.respond({ - clientId: SERVER_HOST_CLIENT_ID, - connectionId, - requestId: request.requestId, - ...outcome, - }), - ), - Effect.ignore, ); + yield* broker + .respond({ + clientId: SERVER_HOST_CLIENT_ID, + connectionId, + requestId: request.requestId, + ...outcome, + }) + .pipe(Effect.ignore); + }); const mirrorManagerEvent = (event: PreviewEvent) => Effect.promise(async () => { @@ -2350,8 +2386,9 @@ const make = Effect.gen(function* () { closedSessions.delete(key); return; } + // Queued per tab at call time, so one stuck tab cannot hold back the others. if (event.type === "resized") { - await tab.control + void tab.control .system(async () => { if ( !viewportSettingsEqual(event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, tab.setting) diff --git a/apps/server/src/preview/SessionControl.test.ts b/apps/server/src/preview/SessionControl.test.ts index f47be6319cb81..c92d31b1c7702 100644 --- a/apps/server/src/preview/SessionControl.test.ts +++ b/apps/server/src/preview/SessionControl.test.ts @@ -344,6 +344,17 @@ describe("SessionControl", () => { expect(control.generation).toBe(1); }); + it("rejects system work on a closed tab instead of throwing", async () => { + const control = new SessionControl("agent"); + await control.close(); + // Callers handle failures on the returned promise; a synchronous throw escapes them. + let applied: Promise | undefined; + expect(() => { + applied = control.system(async () => {}); + }).not.toThrow(); + await expect(applied).rejects.toThrow("closed"); + }); + it("lets any agent act on a tab no agent opened, but only while no human controls it", async () => { const control = new SessionControl(null); await expect(control.agent("agent-a", async () => "acted")).resolves.toBe("acted"); diff --git a/apps/server/src/preview/SessionControl.ts b/apps/server/src/preview/SessionControl.ts index f19f566ed30bb..8f6cc0aebf6f6 100644 --- a/apps/server/src/preview/SessionControl.ts +++ b/apps/server/src/preview/SessionControl.ts @@ -34,6 +34,7 @@ export class SessionControl { private pending: Promise = Promise.resolve(); private readonly handoff = new Set>(); private owner: string | null = null; + private running: AbortSignal | undefined; private epoch = 0; private closed = false; @@ -88,7 +89,17 @@ export class SessionControl { this.onGenerationChange(); } - private async action(allowed: () => boolean, run: () => Promise, deadlineMs?: number) { + /** An agent run outlived its request; it keeps the queue until it settles. */ + get overdue() { + return this.running?.aborted === true; + } + + private async action( + allowed: () => boolean, + run: () => Promise, + signal?: AbortSignal, + runIfExpired = false, + ) { this.assertOpen(); if (!allowed()) throw new BrowserControlInterrupted({ message: "You do not control this browser tab." }); @@ -96,13 +107,18 @@ export class SessionControl { return this.enqueue(async () => { this.assertOpen(); if (epoch !== this.epoch || !allowed()) throw new BrowserControlInterrupted(); - if (deadlineMs !== undefined && performance.now() >= deadlineMs) { + if (signal?.aborted && !runIfExpired) { throw new DOMException( "This browser request expired before it could start.", "TimeoutError", ); } - return run(); + this.running = signal; + try { + return await run(); + } finally { + this.running = undefined; + } }); } @@ -110,7 +126,13 @@ export class SessionControl { return this.agentId === null || this.agentId === agentId; } - agent(agentId: string, run: () => Promise, deadlineMs?: number) { + /** `runIfExpired` still starts work whose request expired while queued, for work that must not be dropped. */ + agent( + agentId: string, + run: () => Promise, + signal?: AbortSignal, + { runIfExpired = false }: { readonly runIfExpired?: boolean } = {}, + ) { if (!this.agentMayAct(agentId)) return Promise.reject( new BrowserControlInterrupted({ @@ -125,7 +147,19 @@ export class SessionControl { reason: "humanControl", }), ); - return this.action(() => this.agentMayAct(agentId) && this.owner === null, run, deadlineMs); + if (this.overdue) + return Promise.reject( + new BrowserControlInterrupted({ + message: "An earlier action on this tab timed out and is still running.", + reason: "busy", + }), + ); + return this.action( + () => this.agentMayAct(agentId) && this.owner === null, + run, + signal, + runIfExpired, + ); } agentCanActNow(agentId: string) { @@ -138,7 +172,7 @@ export class SessionControl { return run(); } - system(run: () => Promise) { + async system(run: () => Promise) { this.assertOpen(); return this.enqueue(async () => { this.assertOpen(); diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index c5802a35af63b..ec3ab35266114 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -27,8 +27,9 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work. - `experience`: the real mistake, in a sentence. `source`: where it happened. - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. +- `setup.priorTurns[].toolCalls[].result`: a T3 preview error (`{"_tag": "PreviewAutomationControlInterruptedError", ...}`, every field the contracts class needs) replays as T3 sends it: an error result with the class's own message. - `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt. -- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). +- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `firstCall` (the first call of any of its `tools` matches one of its `calls`), `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). Check that the case builds with `--dry-run --case `. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass. diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson index edd66f26966e9..33854939c12cb 100644 --- a/evals/mcp-tool-use/cases/visual-preview.ndjson +++ b/evals/mcp-tool-use/cases/visual-preview.ndjson @@ -34,3 +34,5 @@ {"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} {"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return ; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} {"id": "revise-page-resends-html", "area": "visual-preview", "kind": "bad", "pattern": "revising a page on disk resends its full HTML to html_preview and html_render", "experience": "The pages already existed as generated files on disk, but the agent retyped each one into html_preview and again into html_render, about four minutes per call. To save time it published fixes without previewing them again.", "source": {"discussion": "https://github.com/pingdotgg/t3code/discussions/23303"}, "rule": "edit the .html file and pass its path to html_preview and html_render instead of resending its html", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Here are four options.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "html_preview", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "{\"width\":728,\"contentHeight\":1840,\"capturedHeight\":1840,\"consoleMessages\":[]}"}, {"tool": "html_render", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n", "title": "New worktree: four options", "height": 1840}, "result": "{\"htmlRender\":{\"attachmentId\":\"att-1\",\"title\":\"New worktree: four options\",\"height\":1840},\"message\":\"Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.\"}"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}}}, "prompt": "Make the Send in new worktree button in option C blue, and show me the page again.", "expect": {"calls": [{"tool": "html_preview", "args": {"path": "/tmp/design/index.html", "html": null}}, {"tool": "html_render", "args": {"path": "/tmp/design/index.html", "html": null}}], "callsMustNot": [{"tool": "html_preview", "args": {"html": "~Ask anything"}}, {"tool": "html_render", "args": {"html": "~Ask anything"}}], "order": ["html_preview", "html_render"]}, "notes": "On main neither tool has a path parameter."} +{"id": "preview-busy-tab-opens-new-tab", "area": "visual-preview", "kind": "bad", "pattern": "preview tab stuck behind a timed-out action; the next preview call retries the same tab", "experience": "A server-browser action never finished, so every later call on that tab timed out with the same generic error and the agent kept retrying the stuck tab.", "rule": "On a busy tab, continue in a new tab (preview_open with reuseExistingTab=false) or close it with t3_preview_close; do not retry it.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849: a browser action that never finishes wedges its tab; the agent only sees a generic timeout and keeps retrying the same tab"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "setColorScheme", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-2", "tabId": "tab_1", "timeoutMs": 15000}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12", "timeoutMs": 3000}, "result": {"_tag": "PreviewAutomationControlInterruptedError", "operation": "click", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-3", "tabId": "tab_1", "timeoutMs": 3000, "remoteTag": "BrowserControlInterrupted", "remoteMessageLength": 0, "cause": null, "reason": "busy"}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]"}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}, {"tool": "t3_preview_close", "args": {"tabId": "tab_1"}}]}, "callsMustNot": [{"tool": "preview_act|preview_resize|preview_navigate|preview_snapshot|preview_wait_for|preview_evaluate|preview_set_appearance", "args": {"tabId": "tab_1"}}]}, "maxSteps": 6, "notes": "The prior preview_act result is the contracts busy error, rendered with its real message. preview_status and t3_preview_list are reads, so they may come first."} +{"id": "preview-wait-timeout-keeps-tab", "area": "visual-preview", "kind": "control", "pattern": "an ordinary wait_for timeout; the agent keeps working in the same tab", "experience": "Guards #32849's busy-tab recovery: a plain timeout on a responsive tab is a page result, not a reason to abandon the tab.", "rule": "After an ordinary preview timeout, inspect or retry in the same tab.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#32849 (control)"}, "setup": {"role": "top", "context": "The dev server is healthy at http://localhost:5173. The agent opened preview tab tab_1 (owner agent) earlier in this turn.", "priorTurns": [{"role": "user", "text": "Open http://localhost:5173/settings in the browser, switch it to dark mode, click Save, and check that the settings-saved toast still reads well in dark mode."}, {"role": "assistant", "toolCalls": [{"tool": "preview_open", "input": {"url": "http://localhost:5173/settings"}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}}, {"tool": "preview_set_appearance", "input": {"tabId": "tab_1", "colorScheme": "dark"}, "result": {"available": true, "tabId": "tab_1", "colorScheme": "dark"}}, {"tool": "preview_act", "input": {"tabId": "tab_1", "action": "click", "locator": "aria-ref=e12"}, "result": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings"}}, {"tool": "preview_wait_for", "input": {"tabId": "tab_1", "text": "Settings saved", "timeoutMs": 5000}, "result": {"_tag": "PreviewAutomationTimeoutError", "operation": "waitFor", "environmentId": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", "threadId": "thread-settings-check", "providerSessionId": "session-1", "providerInstanceId": "claudeAgent", "clientId": "server-browser", "connectionId": "conn-1", "requestId": "req-4", "tabId": "tab_1", "timeoutMs": 5000}}]}], "mocks": {"preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false}, "preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "loading": false, "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 1}}, "t3_preview_list": {"tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/settings", "owner": "agent"}]}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/settings", "title": "Settings - Acme", "snapshot": "- heading \"Settings\" [level=1]\n- switch \"Dark mode\" [checked] [ref=e7]\n- button \"Save\" [ref=e12]\n- status: \"Saved\""}}}, "prompt": "", "expect": {"firstCall": {"tools": ["preview_act", "preview_resize", "preview_navigate", "preview_snapshot", "preview_wait_for", "preview_evaluate", "preview_set_appearance", "preview_open", "t3_preview_close"], "calls": [{"tool": "preview_act|preview_resize|preview_navigate|preview_snapshot|preview_wait_for|preview_evaluate|preview_set_appearance", "args": {"tabId": "tab_1"}}, {"tool": "preview_act|preview_resize|preview_navigate|preview_snapshot|preview_wait_for|preview_evaluate|preview_set_appearance", "args": {"tabId": null}}]}}, "maxSteps": 6} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 6e9413ed9141e..e48ccb329ee2b 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -7,7 +7,12 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { FetchHttpClient } from "effect/http"; import { assert, expect, it } from "@effect/vitest"; -import { ComposerContextId, EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { + ComposerContextId, + EnvironmentId, + PreviewAutomationControlInterruptedError, + ThreadId, +} from "@t3tools/contracts"; import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import { @@ -244,6 +249,10 @@ it("passes a good run and names every rule a bad run breaks", () => { callsMustNot: [{ tool: "delegate_task", args: { runtimeMode: "approval-required" } }], notCalls: ["t3_thread_send"], order: ["orchestrator_capabilities", "delegate_task"], + firstCall: { + tools: ["delegate_task", "t3_thread_send"], + calls: [{ tool: "t3_thread_send" }, { tool: "delegate_task", args: { task: "~FINDING" } }], + }, minCalls: { orchestrator_capabilities: 1 }, maxCalls: { task_status: 0 }, endsTurnAfter: "delegate_task", @@ -287,6 +296,7 @@ it("passes a good run and names every rule a bad run breaks", () => { 'callsMustNot delegate_task {"runtimeMode":"approval-required"}', "notCalls t3_thread_send", "order orchestrator_capabilities > delegate_task", + 'firstCall t3_thread_send or delegate_task {"task":"~FINDING"}', "minCalls orchestrator_capabilities >= 1", "maxCalls task_status <= 0", "endsTurnAfter delegate_task", @@ -578,6 +588,24 @@ it("lists link tools only while a case turns the beta on, and sends chips as T3 assert.include(prompt?.text, "environmentId: environment-vps"); }); +const decodeInterrupted = Schema.decodeUnknownSync(PreviewAutomationControlInterruptedError); +it("replays an earlier preview error as T3's error result, with its contracts message", () => { + const base = buildBase(template, { model: "claude-opus-5-5", keepUserInstructions: false }); + const busyCase = NodeFS.readFileSync( + new URL("./cases/visual-preview.ndjson", import.meta.url), + "utf8", + ) + .trim() + .split("\n") + .map((line) => decodeDeviceCase(line)) + .find((c) => c.id === "preview-busy-tab-opens-new-tab")!; + const prior = busyCase.setup.priorTurns!.at(-1)!.toolCalls!.at(-1)!; + const error = decodeInterrupted(prior.result); + assert.equal(error.reason, "busy"); + const replayed = buildMessages(makeSample(busyCase, base)).at(-1)?.content.at(-1); + assert.deepInclude(replayed, { content: error.message, is_error: true }); +}); + it("replays earlier capability results with the model under test as the caller", () => { const base = buildBase(template, { model: "claude-opus-5-5", keepUserInstructions: false }); const capabilities = { diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 7a7a8400fe612..0673c960552c1 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -18,6 +18,7 @@ import { DeviceOperationError, DeviceSummary, OrchestratorMcpTaskRole, + PreviewAutomationError, ProviderInstanceId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -193,6 +194,10 @@ const McpToolUseExpect = Schema.Struct({ notCalls: Schema.optional(Schema.Array(Schema.String)), /** First calls of these tools happen in this order. */ order: Schema.optional(Schema.Array(Schema.String)), + /** The first call of any tool in `tools` matches one of `calls`. */ + firstCall: Schema.optional( + Schema.Struct({ tools: Schema.Array(Schema.String), calls: Schema.Array(McpToolUseCall) }), + ), minCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)), maxCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)), /** @@ -393,6 +398,10 @@ const toolResult = (id: string, content: unknown): Block => ({ tool_use_id: id, content: valueText(content), }); +const decodePreviewError = Schema.decodeUnknownOption(PreviewAutomationError); +/** A prior result that is a T3 preview error, `{"_tag": "PreviewAutomationTimeoutError", ...}`. */ +const previewFailure = (value: unknown) => + Option.getOrUndefined(Option.map(decodePreviewError(value), (error) => error.message)); const texts = (content: typeof Content.Type) => typeof content === "string" ? [content] : content.flatMap((block) => block.text ?? []); const isT3Name = (name: string) => name.startsWith(T3_PREFIX) || name.startsWith("mcp__t3_code__"); @@ -1013,7 +1022,14 @@ export const buildMessages = (sample: Sample) => { normalizeToolName(tool.name) === "orchestrator_capabilities" ? capabilities(sample, result) : (sample.priorResults.get(prior) ?? result); - push("user", toolResult(id, replayed)); + // T3 sends a preview tool's failure as an error result with only its message. + const failure = previewFailure(replayed); + push( + "user", + failure === undefined + ? toolResult(id, replayed) + : { ...toolResult(id, failure), is_error: true }, + ); } if (turn.text) push(turn.role, text(turn.text)); } @@ -1796,6 +1812,12 @@ export const gradeSample = ( const inOrder = firsts.every((at, i) => at > (firsts[i - 1] ?? -1)); add(`order ${expect.order.join(" > ")}`, inOrder); } + if (expect.firstCall !== undefined) { + const { tools, calls: allowed } = expect.firstCall; + const first = calls.find((call) => tools.includes(call.tool)); + const ok = first !== undefined && allowed.some((call) => found(call, [first])); + add(`firstCall ${allowed.map(label).join(" or ")}`, ok); + } for (const [t, n] of Object.entries(expect.minCalls ?? {})) add(`minCalls ${t} >= ${n}`, count(t) >= n); for (const [t, n] of Object.entries(expect.maxCalls ?? {})) diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 8c038f4fe81cc..5ef8ff6e7926e 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -792,6 +792,7 @@ export const PreviewAutomationControlReason = Schema.Literals([ "interrupted", "dialogPending", "tabLimit", + "busy", ]); export type PreviewAutomationControlReason = typeof PreviewAutomationControlReason.Type; @@ -814,6 +815,8 @@ export class PreviewAutomationControlInterruptedError extends Schema.TaggedError return "A browser dialog is pending. Read preview_status and resolve it with preview_dialog."; if (this.reason === "tabLimit") return "Too many server browser tabs are open. Close one with t3_preview_close, or reuse a tabId from preview_status tabs."; + if (this.reason === "busy") + return "This browser tab is busy with an earlier action that timed out. Open a new tab with preview_open and reuseExistingTab=false, or close this one with t3_preview_close."; if (this.reason === "closed") return "This browser tab is closed. Call preview_open."; if (this.reason === "interrupted") return "Browser control changed. Take a fresh snapshot before trying again.";