diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 0a354c1e7d560..e5465f6709809 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5930, "codexAdditionalContextBytes": 7324, - "mcpToolListJsonBytes": 49806 + "mcpToolListJsonBytes": 49994 } diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson index edd66f26966e9..9a4e406d71657 100644 --- a/evals/mcp-tool-use/cases/visual-preview.ndjson +++ b/evals/mcp-tool-use/cases/visual-preview.ndjson @@ -32,5 +32,7 @@ {"id": "requested-after-video-for-edit", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "agent refuses requested browser recording of its own edit", "experience": "A user asked for an after video of a UI fix. The agent declined because the injected instructions prohibited preview for its own edits.", "rule": "Use preview when the user asks for browser work, including recording an edit.", "setup": {"role": "top", "context": "The UI fix is complete and a development server is already running at http://localhost:3000. Browser verification and recording are authorized.", "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_recording_start": {"recording": true, "tabId": "tab_1", "startedAt": "2026-01-01T00:00:00.000Z"}, "preview_recording_stop": {"id": "recording_1", "tabId": "tab_1", "path": "/tmp/after.webm", "mimeType": "video/webm", "sizeBytes": 1024, "createdAt": "2026-01-01T00:00:01.000Z"}}}, "prompt": "Use the T3 browser to verify the UI fix you just made and record an after video.", "expect": {"calls": [{"tool": "preview_recording_start"}, {"tool": "preview_recording_stop"}], "order": ["preview_recording_start", "preview_recording_stop"]}, "source": {"report": "A user requested an after video of a UI fix; the agent declined because preview was prohibited for its own edits."}} {"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}} {"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} -{"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return ; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."} +{"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return ; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}], "callsMustNot": [{"tool": "preview_open", "args": {"open": true}}, {"tool": "preview_open", "args": {"show": true}}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation. The callsMustNot entries guard the fix for asked-to-see-page-passes-open-true: an agent's own verification leaves the user's auto-show setting in charge."} {"id": "revise-page-resends-html", "area": "visual-preview", "kind": "bad", "pattern": "revising a page on disk resends its full HTML to html_preview and html_render", "experience": "The pages already existed as generated files on disk, but the agent retyped each one into html_preview and again into html_render, about four minutes per call. To save time it published fixes without previewing them again.", "source": {"discussion": "https://github.com/pingdotgg/t3code/discussions/23303"}, "rule": "edit the .html file and pass its path to html_preview and html_render instead of resending its html", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Here are four options.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "html_preview", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}, "result": "{\"width\":728,\"contentHeight\":1840,\"capturedHeight\":1840,\"consoleMessages\":[]}"}, {"tool": "html_render", "input": {"html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n", "title": "New worktree: four options", "height": 1840}, "result": "{\"htmlRender\":{\"attachmentId\":\"att-1\",\"title\":\"New worktree: four options\",\"height\":1840},\"message\":\"Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.\"}"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "\n

A. Checkbox under the composer

Ask anything
\n

B. Toggle in the toolbar

Ask anything
\n

C. Split send button

Ask anything
Send in new worktree
\n

D. Two visible choices

Ask anything
\n\n"}}}, "prompt": "Make the Send in new worktree button in option C blue, and show me the page again.", "expect": {"calls": [{"tool": "html_preview", "args": {"path": "/tmp/design/index.html", "html": null}}, {"tool": "html_render", "args": {"path": "/tmp/design/index.html", "html": null}}], "callsMustNot": [{"tool": "html_preview", "args": {"html": "~Ask anything"}}, {"tool": "html_render", "args": {"html": "~Ask anything"}}], "order": ["html_preview", "html_render"]}, "notes": "On main neither tool has a path parameter."} +{"id": "asked-to-see-page-passes-open-true", "area": "visual-preview", "kind": "bad", "pattern": "'open it so I can see it' -> preview_open with open: true", "experience": "With Auto-show floating preview off, the user asked to see a page in the T3 browser and nothing appeared. The agent left `open` out, which follows that setting; the parameter text called true the default.", "rule": "preview_open `open`: leave out for your own checks, so the user's auto-show setting decides; true only when the user asks to see or use the page", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "https://acme.test/pricing", "title": "Pricing - Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "Open https://acme.test/pricing in the T3 browser so I can see it, I want to look at the page myself.", "expect": {"calls": [{"tool": "preview_open", "args": {"url": "~acme.test/pricing", "open": true}}], "notCalls": ["t3_preview_close"]}, "maxSteps": 6, "source": {"report": "Contributor report, 2026-11-14: with Auto-show floating preview off, asked an agent to open a page to look at and nothing appeared"}, "notes": "Reproduces on Sonnet 5.5 (0/3 on the old text). Opus 5.5 passed 2/3 on the old text."} +{"id": "guard-page-check-leaves-open-out", "area": "visual-preview", "kind": "guard", "pattern": "asked to check a page, not to be shown it -> preview_open without open: true", "experience": "Asked what a page said, the agent read it in the T3 browser and left `open` out, so the user's auto-show setting decided whether it floated.", "rule": "preview_open `open`: leave out for your own checks, so the user's auto-show setting decides; true only when the user asks to see or use the page", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "https://acme.test/pricing", "title": "Pricing - Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_snapshot": {"url": "https://acme.test/pricing", "title": "Pricing - Acme", "loading": false, "visibleText": "Pricing\nFree\n$0\nTeam\n$12 per seat / month\nEnterprise\nContact us"}, "t3_preview_close": {"closed": true}}}, "prompt": "Can you check the pricing page in the T3 browser again? https://acme.test/pricing. I want to know if the Team plan still says $12 per seat.", "expect": {"calls": [{"tool": "preview_open", "args": {"url": "~acme.test/pricing"}}], "callsMustNot": [{"tool": "preview_open", "args": {"open": true}}, {"tool": "preview_open", "args": {"show": true}}]}, "maxSteps": 8, "notes": "Guards the fix for asked-to-see-page-passes-open-true. open: false passes too; only a forced reveal fails.", "source": {"report": "Contributor report, 2026-11-14: with Auto-show floating preview off, asked an agent to open a page to look at and nothing appeared"}} diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 228297667d318..f50a9f1c6c384 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -155,7 +155,8 @@ export const PreviewAutomationOpenInput = Schema.Struct({ }), open: Schema.optional( Schema.Boolean.annotate({ - description: "Show the preview to the user. Default true. False for background only.", + description: + "Leave out for your own checks and screenshots: the user's auto-show setting then decides, and preview tools work on a hidden tab. True only when the user asks to see or use the page; it shows the preview even with that setting off. False for background only.", }), ), show: Schema.optional(