From 01ac7f7ab05cc1c745a1ba74b5a2acbfa72cfe30 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 14:14:56 +0000
Subject: [PATCH 01/17] fix(server): agents can use a page's WebMCP write tools
 when the user asks

The preview_evaluate description and the injected preview instructions said page
tools read data only and never create or save. Agents took that as absolute and
refused a page's create_note tool even when the user asked for the note.

Both now say page tools read data and change the site only when the user asked
for that change. The ban on using page tools to show results, which keeps charts
inline in html_render, stays. New mcp-tool-use cases cover the refused requested
write and guard against unrequested writes.

Done by Claude Opus 5.5 in Claude Code.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 apps/server/src/mcp/toolkits/preview/tools.ts                  | 2 +-
 evals/mcp-tool-use/cases/visual-preview.ndjson                 | 2 ++
 packages/provider-core/src/server/orchestrationInstructions.ts | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts
index cd84874f68979..6b21cdaa19fd1 100644
--- a/apps/server/src/mcp/toolkits/preview/tools.ts
+++ b/apps/server/src/mcp/toolkits/preview/tools.ts
@@ -175,7 +175,7 @@ export const PreviewEvaluateResult = Schema.Struct({
 const PreviewEvaluateTool = browserTool(
   Tool.make("preview_evaluate", {
     description:
-      "Run JavaScript in the page. It can change page state. Returns `{value}`, up to 64 KB. Needs your tab, or an unclaimed one. To read a tab the user drives, use `preview_snapshot`. Page tools reached here: read data only, never create, save, or display. Show results with `html_render`.",
+      "Run JavaScript in the page. It can change page state. Returns `{value}`, up to 64 KB. Needs your tab, or an unclaimed one. To read a tab the user drives, use `preview_snapshot`. Page tools reached here read data, and change the site only when the user asked for that change. Never use them to display results; show results with `html_render`.",
     parameters: PreviewAutomationEvaluateInput,
     success: PreviewEvaluateResult,
     failure: PreviewToolFailure,
diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index c74f12d5b2b8a..76ce4ddd861c3 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -26,3 +26,5 @@
 {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n  if (tab.owner === 'human') return false\n  if (tab.owner === 'unclaimed') return true\n  return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
+{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"expression": "~create_note"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
+{"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"expression": "~list_notes"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}, {"tool": "preview_evaluate", "args": {"expression": "~create_note"}}, {"tool": "preview_act"}]}}
diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts
index 428aec2513d32..80da0fc4b71fb 100644
--- a/packages/provider-core/src/server/orchestrationInstructions.ts
+++ b/packages/provider-core/src/server/orchestrationInstructions.ts
@@ -20,7 +20,7 @@ Ordinary coding tasks need no T3 tools.
 - T3 tools may load late. Try the name once before you call it absent.`,
   `### Visuals and preview
 
-- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
+- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes read data, and change the site only when the user asked for that change. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
 - Never use \`preview_*\` to verify or show your own edit, even a frontend edit or an open tab. Do not repeat preview calls from an earlier turn. Use them only when the current request asks for browser work or names a page or tab. Public pages: web fetch or search unless browser work was asked; then preview tools only, no web fetch. Close a tab you opened before you reply, unless the user asked for it or uses it.`,
 ];
 

From 7c3545f9862d4aa427380e19c6ee7470d709d5c8 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 14:31:46 +0000
Subject: [PATCH 02/17] test(evals): verify WebMCP invocations and requested
 arguments

---
 .../mcp-tool-use/cases/visual-preview.ndjson  |  5 ++-
 evals/mcp-tool-use/run.test.ts                | 39 +++++++++++++++++++
 evals/mcp-tool-use/run.ts                     |  7 +++-
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index 76ce4ddd861c3..8f581d971e260 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -26,5 +26,6 @@
 {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n  if (tab.owner === 'human') return false\n  if (tab.owner === 'unclaimed') return true\n  return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
-{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"expression": "~create_note"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
-{"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"expression": "~list_notes"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}, {"tool": "preview_evaluate", "args": {"expression": "~create_note"}}, {"tool": "preview_act"}]}}
+{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]create_note['\"]\\s*,\\s*\\{(?=[^}]*\\btitle['\"]?\\s*:\\s*['\"]Groceries['\"])(?=[^}]*\\bbody['\"]?\\s*:\\s*['\"]milk, eggs, bread['\"])"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
+{"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]list_notes['\"]\\s*,\\s*\\{\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}, {"tool": "preview_evaluate", "args": {"expression": "~create_note"}}, {"tool": "preview_act"}]}}
+{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]save_chart['\"]\\s*,\\s*\\{(?=[\\s\\S]*\\bspec['\"]?\\s*:)(?=[\\s\\S]*\\btitle['\"]?\\s*:\\s*['\"]Daily usage['\"])"}}], "notCalls": ["html_render"]}}
diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 33edb6465ab4c..84038382462ec 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -161,6 +161,45 @@ it("passes a good run and names every rule a bad run breaks", () => {
   ]);
 });
 
+it("matches invocation expressions and rejects discovery, wrong tabs, and wrong arguments", () => {
+  const expect = {
+    calls: [
+      {
+        tool: "preview_evaluate",
+        args: {
+          tabId: "tab_1",
+          expression: String.raw`regex:\.callTool\(\s*['"]create_note['"]\s*,\s*\{(?=[^}]*\btitle['"]?\s*:\s*['"]Groceries['"])(?=[^}]*\bbody['"]?\s*:\s*['"]milk, eggs, bread['"])`,
+        },
+      },
+    ],
+  };
+  const run = (expression: unknown, tabId = "tab_1"): SampleRun => ({
+    calls: [{ step: 0, tool: "preview_evaluate", input: { tabId, expression } }],
+    text: "",
+    stopReason: "end_turn",
+  });
+  const expression =
+    'navigator.modelContext.callTool("create_note", {title: "Groceries", body: "milk, eggs, bread"})';
+  assert.deepEqual(failed(expect, run(expression)), []);
+  assert.deepEqual(
+    failed(
+      expect,
+      run(
+        "navigator.modelContext.callTool('create_note', {body:'milk, eggs, bread',title:'Groceries'})",
+      ),
+    ),
+    [],
+  );
+  for (const sample of [
+    run("navigator.modelContext.getTools().find(t => t.name === 'create_note')"),
+    run(expression, "wrong_tab"),
+    run(expression.replace("Groceries", "Work")),
+    run(expression.replace("milk, eggs, bread", "milk")),
+    run(null),
+  ])
+    assert.lengthOf(failed(expect, sample), 1);
+});
+
 it("bounds numbers and reads target.options in either schema form", () => {
   const expect = {
     calls: [
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 7baefcbbcc6ca..65ce6edbf8365 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -136,7 +136,7 @@ const PriorToolCall = Schema.Struct({
 
 /**
  * One expected call. `args` match partially: `~text` is a case-insensitive substring,
- * `"<=N"` / `">=N"` bound a number, and keys may be dotted paths.
+ * `regex:pattern` matches a string, `"<=N"` / `">=N"` bound a number, and keys may be dotted paths.
  */
 const McpToolUseCall = Schema.Struct({
   tool: Schema.String,
@@ -1312,7 +1312,7 @@ const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(function*
 });
 
 const BOUND = /^([<>]=)(-?\d+(?:\.\d+)?)$/;
-/** Partial match: `~text` is a case-insensitive substring, `<=N` / `>=N` bound a number. */
+/** Partial match: `~text` is a substring, `regex:pattern` tests a string, `<=N` / `>=N` bound a number. */
 const matches = (expected: unknown, actual: unknown): boolean => {
   const bound = typeof expected === "string" ? BOUND.exec(expected) : null;
   if (bound !== null) {
@@ -1322,6 +1322,9 @@ const matches = (expected: unknown, actual: unknown): boolean => {
   if (typeof expected === "string" && expected.startsWith("~")) {
     return valueText(actual).toLowerCase().includes(expected.slice(1).toLowerCase());
   }
+  if (typeof expected === "string" && expected.startsWith("regex:")) {
+    return typeof actual === "string" && new RegExp(expected.slice(6)).test(actual);
+  }
   if (Array.isArray(expected)) {
     return (
       Array.isArray(actual) && expected.every((item) => actual.some((at) => matches(item, at)))

From d59cce37a85879d6e5c644e73b57629175afe7be Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 16:53:10 +0000
Subject: [PATCH 03/17] fix(server): shorten WebMCP write rule to stay within
 context budget

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 apps/server/src/mcp/toolkits/preview/tools.ts               | 2 +-
 apps/server/src/provider/agentContextBudget.json            | 6 +++---
 .../provider-core/src/server/orchestrationInstructions.ts   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts
index 6b21cdaa19fd1..c8a03052b35fd 100644
--- a/apps/server/src/mcp/toolkits/preview/tools.ts
+++ b/apps/server/src/mcp/toolkits/preview/tools.ts
@@ -175,7 +175,7 @@ export const PreviewEvaluateResult = Schema.Struct({
 const PreviewEvaluateTool = browserTool(
   Tool.make("preview_evaluate", {
     description:
-      "Run JavaScript in the page. It can change page state. Returns `{value}`, up to 64 KB. Needs your tab, or an unclaimed one. To read a tab the user drives, use `preview_snapshot`. Page tools reached here read data, and change the site only when the user asked for that change. Never use them to display results; show results with `html_render`.",
+      "Run JavaScript in the page. It can change page state. Returns `{value}`, up to 64 KB. Needs your tab, or an unclaimed one. To read a tab the user drives, use `preview_snapshot`. Page tools write only on user request. Show results with `html_render`, never page tools.",
     parameters: PreviewAutomationEvaluateInput,
     success: PreviewEvaluateResult,
     failure: PreviewToolFailure,
diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json
index 1711e148d4ca4..1b1b2e9053e6c 100644
--- a/apps/server/src/provider/agentContextBudget.json
+++ b/apps/server/src/provider/agentContextBudget.json
@@ -1,5 +1,5 @@
 {
-  "claudeSystemAppendBytes": 5135,
-  "codexAdditionalContextBytes": 6865,
-  "mcpToolListJsonBytes": 46007
+  "claudeSystemAppendBytes": 5133,
+  "codexAdditionalContextBytes": 6863,
+  "mcpToolListJsonBytes": 45991
 }
diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts
index 80da0fc4b71fb..e48a56d8e0f66 100644
--- a/packages/provider-core/src/server/orchestrationInstructions.ts
+++ b/packages/provider-core/src/server/orchestrationInstructions.ts
@@ -20,7 +20,7 @@ Ordinary coding tasks need no T3 tools.
 - T3 tools may load late. Try the name once before you call it absent.`,
   `### Visuals and preview
 
-- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes read data, and change the site only when the user asked for that change. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
+- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Page tools write only on user request. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
 - Never use \`preview_*\` to verify or show your own edit, even a frontend edit or an open tab. Do not repeat preview calls from an earlier turn. Use them only when the current request asks for browser work or names a page or tab. Public pages: web fetch or search unless browser work was asked; then preview tools only, no web fetch. Close a tab you opened before you reply, unless the user asked for it or uses it.`,
 ];
 

From 80692ecdf8f6312c1f8120c25c5d465a72fe94a6 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 17:35:41 +0000
Subject: [PATCH 04/17] test(evals): scope WebMCP arguments to their objects

---
 .../mcp-tool-use/cases/visual-preview.ndjson  |  4 +-
 evals/mcp-tool-use/run.test.ts                | 42 ++++++++++++++-----
 2 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index 8f581d971e260..3f9bb04f018a1 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -26,6 +26,6 @@
 {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n  if (tab.owner === 'human') return false\n  if (tab.owner === 'unclaimed') return true\n  return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
-{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]create_note['\"]\\s*,\\s*\\{(?=[^}]*\\btitle['\"]?\\s*:\\s*['\"]Groceries['\"])(?=[^}]*\\bbody['\"]?\\s*:\\s*['\"]milk, eggs, bread['\"])"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
+{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\s*\\(\\s*(?:\"create_note\"|'create_note'|`create_note`)\\s*,\\s*\\{\\s*(?:(?:title|\"title\"|'title')\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`)\\s*,\\s*(?:body|\"body\"|'body')\\s*:\\s*(?:\"milk, eggs, bread\"|'milk, eggs, bread'|`milk, eggs, bread`)|(?:body|\"body\"|'body')\\s*:\\s*(?:\"milk, eggs, bread\"|'milk, eggs, bread'|`milk, eggs, bread`)\\s*,\\s*(?:title|\"title\"|'title')\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`))\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
 {"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]list_notes['\"]\\s*,\\s*\\{\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}, {"tool": "preview_evaluate", "args": {"expression": "~create_note"}}, {"tool": "preview_act"}]}}
-{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]save_chart['\"]\\s*,\\s*\\{(?=[\\s\\S]*\\bspec['\"]?\\s*:)(?=[\\s\\S]*\\btitle['\"]?\\s*:\\s*['\"]Daily usage['\"])"}}], "notCalls": ["html_render"]}}
+{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\s*\\(\\s*(?:\"save_chart\"|'save_chart'|`save_chart`)\\s*,\\s*\\{\\s*(?:spec|\"spec\"|'spec')\\s*:\\s*\\{\\s*(?:title|\"title\"|'title')\\s*:\\s*(?:\"Daily usage\"|'Daily usage'|`Daily usage`)(?=\\s*[,}])"}}], "notCalls": ["html_render"]}}
diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 84038382462ec..3aeb48690b4ae 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -161,18 +161,22 @@ it("passes a good run and names every rule a bad run breaks", () => {
   ]);
 });
 
+const decodeMcpToolUseCase = Schema.decodeUnknownSync(McpToolUseCase);
+
 it("matches invocation expressions and rejects discovery, wrong tabs, and wrong arguments", () => {
-  const expect = {
-    calls: [
-      {
-        tool: "preview_evaluate",
-        args: {
-          tabId: "tab_1",
-          expression: String.raw`regex:\.callTool\(\s*['"]create_note['"]\s*,\s*\{(?=[^}]*\btitle['"]?\s*:\s*['"]Groceries['"])(?=[^}]*\bbody['"]?\s*:\s*['"]milk, eggs, bread['"])`,
-        },
-      },
-    ],
+  const cases = NodeFS.readFileSync(
+    new URL("./cases/visual-preview.ndjson", import.meta.url),
+    "utf8",
+  )
+    .trim()
+    .split("\n")
+    .map((line) => decodeMcpToolUseCase(JSON.parse(line)));
+  const caseExpect = (id: string) => {
+    const testCase = cases.find((testCase) => testCase.id === id);
+    if (!testCase) throw new Error(`Missing case ${id}`);
+    return testCase.expect;
   };
+  const expect = caseExpect("webmcp-requested-site-write-refused");
   const run = (expression: unknown, tabId = "tab_1"): SampleRun => ({
     calls: [{ step: 0, tool: "preview_evaluate", input: { tabId, expression } }],
     text: "",
@@ -195,9 +199,27 @@ it("matches invocation expressions and rejects discovery, wrong tabs, and wrong
     run(expression, "wrong_tab"),
     run(expression.replace("Groceries", "Work")),
     run(expression.replace("milk, eggs, bread", "milk")),
+    run(
+      `navigator.modelContext.callTool("create_note", {payload: "title: 'Groceries', body: 'milk, eggs, bread'"})`,
+    ),
     run(null),
   ])
     assert.lengthOf(failed(expect, sample), 1);
+  const saveExpect = caseExpect("control-webmcp-requested-chart-save");
+  assert.deepEqual(
+    failed(
+      saveExpect,
+      run(
+        "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[]}})",
+      ),
+    ),
+    [],
+  );
+  for (const expression of [
+    "navigator.modelContext.callTool('save_chart', {spec: {}, title:'Daily usage'})",
+    `navigator.modelContext.callTool('save_chart', {spec: {description: "title:'Daily usage'"}})`,
+  ])
+    assert.lengthOf(failed(saveExpect, run(expression)), 1);
 });
 
 it("bounds numbers and reads target.options in either schema form", () => {

From 8dad406f35f2f8031e9a74c0509f944ca7fd858f Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 17:55:36 +0000
Subject: [PATCH 05/17] test(evals): require complete direct WebMCP calls

---
 evals/mcp-tool-use/cases/visual-preview.ndjson |  4 ++--
 evals/mcp-tool-use/run.test.ts                 | 12 +++++++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index 3f9bb04f018a1..c159d125ef596 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -26,6 +26,6 @@
 {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n  if (tab.owner === 'human') return false\n  if (tab.owner === 'unclaimed') return true\n  return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
-{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\s*\\(\\s*(?:\"create_note\"|'create_note'|`create_note`)\\s*,\\s*\\{\\s*(?:(?:title|\"title\"|'title')\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`)\\s*,\\s*(?:body|\"body\"|'body')\\s*:\\s*(?:\"milk, eggs, bread\"|'milk, eggs, bread'|`milk, eggs, bread`)|(?:body|\"body\"|'body')\\s*:\\s*(?:\"milk, eggs, bread\"|'milk, eggs, bread'|`milk, eggs, bread`)\\s*,\\s*(?:title|\"title\"|'title')\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`))\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
+{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:^\\s*navigator\\s*\\.\\s*modelContext\\s*\\.\\s*callTool\\s*\\(\\s*(?:\"create_note\"|'create_note'|`create_note`)\\s*,\\s*\\{\\s*(?:(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`)\\s*,\\s*(?:body|(?:\"body\"|'body'|`body`))\\s*:\\s*(?:\"milk,\\ eggs,\\ bread\"|'milk,\\ eggs,\\ bread'|`milk,\\ eggs,\\ bread`)|(?:body|(?:\"body\"|'body'|`body`))\\s*:\\s*(?:\"milk,\\ eggs,\\ bread\"|'milk,\\ eggs,\\ bread'|`milk,\\ eggs,\\ bread`)\\s*,\\s*(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`))\\s*\\}\\s*\\)\\s*;?\\s*$"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
 {"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]list_notes['\"]\\s*,\\s*\\{\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}, {"tool": "preview_evaluate", "args": {"expression": "~create_note"}}, {"tool": "preview_act"}]}}
-{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\s*\\(\\s*(?:\"save_chart\"|'save_chart'|`save_chart`)\\s*,\\s*\\{\\s*(?:spec|\"spec\"|'spec')\\s*:\\s*\\{\\s*(?:title|\"title\"|'title')\\s*:\\s*(?:\"Daily usage\"|'Daily usage'|`Daily usage`)(?=\\s*[,}])"}}], "notCalls": ["html_render"]}}
+{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:^\\s*navigator\\s*\\.\\s*modelContext\\s*\\.\\s*callTool\\s*\\(\\s*(?:\"save_chart\"|'save_chart'|`save_chart`)\\s*,\\s*\\{\\s*(?:spec|(?:\"spec\"|'spec'|`spec`))\\s*:\\s*\\{\\s*(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Daily\\ usage\"|'Daily\\ usage'|`Daily\\ usage`)\\s*,\\s*(?:type|(?:\"type\"|'type'|`type`))\\s*:\\s*(?:\"bar\"|'bar'|`bar`)\\s*,\\s*(?:data|(?:\"data\"|'data'|`data`))\\s*:\\s*\\[\\s*\\{\\s*(?:day|(?:\"day\"|'day'|`day`))\\s*:\\s*(?:\"Monday\"|'Monday'|`Monday`)\\s*,\\s*(?:tokens|(?:\"tokens\"|'tokens'|`tokens`))\\s*:\\s*120\\s*\\}\\s*,\\s*\\{\\s*(?:day|(?:\"day\"|'day'|`day`))\\s*:\\s*(?:\"Tuesday\"|'Tuesday'|`Tuesday`)\\s*,\\s*(?:tokens|(?:\"tokens\"|'tokens'|`tokens`))\\s*:\\s*180\\s*\\}\\s*\\]\\s*\\}\\s*\\}\\s*\\)\\s*;?\\s*$"}}], "notCalls": ["html_render"]}}
diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 3aeb48690b4ae..a4ac9769f0813 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -199,6 +199,11 @@ it("matches invocation expressions and rejects discovery, wrong tabs, and wrong
     run(expression, "wrong_tab"),
     run(expression.replace("Groceries", "Work")),
     run(expression.replace("milk, eggs, bread", "milk")),
+    run(`console.log(${JSON.stringify(expression)})`),
+    run(JSON.stringify(expression)),
+    run(`// ${expression}`),
+    run(`false && ${expression}`),
+    run(`() => ${expression}`),
     run(
       `navigator.modelContext.callTool("create_note", {payload: "title: 'Groceries', body: 'milk, eggs, bread'"})`,
     ),
@@ -210,12 +215,17 @@ it("matches invocation expressions and rejects discovery, wrong tabs, and wrong
     failed(
       saveExpect,
       run(
-        "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[]}})",
+        "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}})",
       ),
     ),
     [],
   );
   for (const expression of [
+    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage'}})",
+    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar'}})",
+    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',data:[]}})",
+    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[]}})",
+    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:999}]}})",
     "navigator.modelContext.callTool('save_chart', {spec: {}, title:'Daily usage'})",
     `navigator.modelContext.callTool('save_chart', {spec: {description: "title:'Daily usage'"}})`,
   ])

From 0ad65495662be47f6dc87c74ce7b81ca5557e228 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 18:14:05 +0000
Subject: [PATCH 06/17] test(evals): grade WebMCP cases by the tools the page
 executes

The three WebMCP fixtures used an invented navigator.modelContext.callTool, and
the regex grading rejected valid calls such as reordered chart properties.

Each preview_evaluate expression now runs in a node:vm page that has only
document.modelContext (getTools and executeTool, per Chrome's Imperative API).
Cases grade the tools it actually executed and their inputs, so logged, quoted,
commented, dead or discovery-only code cannot pass, and field order no longer
matters. The regex: matcher had no other users and is removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 evals/mcp-tool-use/README.md                  |   2 +-
 .../mcp-tool-use/cases/visual-preview.ndjson  |   6 +-
 evals/mcp-tool-use/run.test.ts                | 132 ++++++++++++------
 evals/mcp-tool-use/run.ts                     | 113 ++++++++++++++-
 4 files changed, 202 insertions(+), 51 deletions(-)

diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md
index ef9ee77b4a4f4..bd157ecc479de 100644
--- a/evals/mcp-tool-use/README.md
+++ b/evals/mcp-tool-use/README.md
@@ -26,7 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts`
 - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work.
 - `experience`: the real mistake, in a sentence. `source`: where it happened.
 - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results.
-- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers.
+- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Preview cases use `webmcp` to run each `preview_evaluate` expression against a page whose `document.modelContext` registers the case's tools, and grade the tools it actually executes and their inputs.
 
 Check that the case builds with `--dry-run --case <id>`. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass.
 
diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index c159d125ef596..847c50c52ddbb 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -26,6 +26,6 @@
 {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n  if (tab.owner === 'human') return false\n  if (tab.owner === 'unclaimed') return true\n  return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
-{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:^\\s*navigator\\s*\\.\\s*modelContext\\s*\\.\\s*callTool\\s*\\(\\s*(?:\"create_note\"|'create_note'|`create_note`)\\s*,\\s*\\{\\s*(?:(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`)\\s*,\\s*(?:body|(?:\"body\"|'body'|`body`))\\s*:\\s*(?:\"milk,\\ eggs,\\ bread\"|'milk,\\ eggs,\\ bread'|`milk,\\ eggs,\\ bread`)|(?:body|(?:\"body\"|'body'|`body`))\\s*:\\s*(?:\"milk,\\ eggs,\\ bread\"|'milk,\\ eggs,\\ bread'|`milk,\\ eggs,\\ bread`)\\s*,\\s*(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`))\\s*\\}\\s*\\)\\s*;?\\s*$"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
-{"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]list_notes['\"]\\s*,\\s*\\{\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}, {"tool": "preview_evaluate", "args": {"expression": "~create_note"}}, {"tool": "preview_act"}]}}
-{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:^\\s*navigator\\s*\\.\\s*modelContext\\s*\\.\\s*callTool\\s*\\(\\s*(?:\"save_chart\"|'save_chart'|`save_chart`)\\s*,\\s*\\{\\s*(?:spec|(?:\"spec\"|'spec'|`spec`))\\s*:\\s*\\{\\s*(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Daily\\ usage\"|'Daily\\ usage'|`Daily\\ usage`)\\s*,\\s*(?:type|(?:\"type\"|'type'|`type`))\\s*:\\s*(?:\"bar\"|'bar'|`bar`)\\s*,\\s*(?:data|(?:\"data\"|'data'|`data`))\\s*:\\s*\\[\\s*\\{\\s*(?:day|(?:\"day\"|'day'|`day`))\\s*:\\s*(?:\"Monday\"|'Monday'|`Monday`)\\s*,\\s*(?:tokens|(?:\"tokens\"|'tokens'|`tokens`))\\s*:\\s*120\\s*\\}\\s*,\\s*\\{\\s*(?:day|(?:\"day\"|'day'|`day`))\\s*:\\s*(?:\"Tuesday\"|'Tuesday'|`Tuesday`)\\s*,\\s*(?:tokens|(?:\"tokens\"|'tokens'|`tokens`))\\s*:\\s*180\\s*\\}\\s*\\]\\s*\\}\\s*\\}\\s*\\)\\s*;?\\s*$"}}], "notCalls": ["html_render"]}}
+{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"}, "result": {"value": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"webmcp": {"tabId": "tab_1", "origin": "https://notes.acme.dev", "tools": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}], "calls": [{"tool": "create_note", "input": {"title": "Groceries", "body": "milk, eggs, bread"}}], "notCalls": ["delete_note"]}}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
+{"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"}, "result": {"value": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"callsMustNot": [{"tool": "preview_act"}], "webmcp": {"tabId": "tab_1", "origin": "https://notes.acme.dev", "tools": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}], "calls": [{"tool": "list_notes"}], "notCalls": ["create_note", "delete_note"]}}}
+{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes the WebMCP tool save_chart(spec), which saves a chart on the site's dashboard. Run it by passing its entry from document.modelContext.getTools() to document.modelContext.executeTool(tool, {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"notCalls": ["html_render"], "webmcp": {"tabId": "tab_1", "origin": "https://analytics.acme.dev", "tools": [{"name": "save_chart", "description": "Save a chart on the dashboard.", "inputSchema": {"type": "object", "properties": {"spec": {"type": "object"}}, "required": ["spec"]}}], "calls": [{"tool": "save_chart", "input": {"spec": {"title": "Daily usage", "type": "bar", "data": [{"day": "Monday", "tokens": 120}, {"day": "Tuesday", "tokens": 180}]}}}]}}}
diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index a4ac9769f0813..12b0351bacaab 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -12,6 +12,7 @@ import {
   buildMessages,
   failedEarly,
   executeCodex,
+  executeWebMcp,
   gradeSample,
   makeSample,
   ranCalls,
@@ -163,7 +164,7 @@ it("passes a good run and names every rule a bad run breaks", () => {
 
 const decodeMcpToolUseCase = Schema.decodeUnknownSync(McpToolUseCase);
 
-it("matches invocation expressions and rejects discovery, wrong tabs, and wrong arguments", () => {
+it("grades the WebMCP tools an expression executes, not the text it contains", () => {
   const cases = NodeFS.readFileSync(
     new URL("./cases/visual-preview.ndjson", import.meta.url),
     "utf8",
@@ -176,60 +177,111 @@ it("matches invocation expressions and rejects discovery, wrong tabs, and wrong
     if (!testCase) throw new Error(`Missing case ${id}`);
     return testCase.expect;
   };
-  const expect = caseExpect("webmcp-requested-site-write-refused");
   const run = (expression: unknown, tabId = "tab_1"): SampleRun => ({
     calls: [{ step: 0, tool: "preview_evaluate", input: { tabId, expression } }],
     text: "",
     stopReason: "end_turn",
   });
-  const expression =
-    'navigator.modelContext.callTool("create_note", {title: "Groceries", body: "milk, eggs, bread"})';
-  assert.deepEqual(failed(expect, run(expression)), []);
+  const execute = (name: string, input: string) =>
+    `document.modelContext.getTools().then(tools => document.modelContext.executeTool(tools.find(t => t.name === '${name}'), ${input}))`;
+  const write = caseExpect("webmcp-requested-site-write-refused");
+  const create = execute("create_note", "{title: 'Groceries', body: 'milk, eggs, bread'}");
+  for (const expression of [
+    create,
+    execute("create_note", '{"body": "milk, eggs, bread", "title": "Groceries"}'),
+    `(async () => {
+      const tool = (await document.modelContext.getTools()).find(({ name }) => name === "create_note");
+      return document.modelContext.executeTool({ ...tool }, { body: "milk, eggs, bread", title: "Groceries" });
+    })()`,
+  ])
+    assert.deepEqual(failed(write, run(expression)), [], expression);
+  for (const expression of [
+    "document.modelContext.getTools()",
+    `console.log(${JSON.stringify(create)})`,
+    JSON.stringify(create),
+    `// ${create}`,
+    `false && ${create}`,
+    `() => ${create}`,
+    "navigator.modelContext.callTool('create_note', {title: 'Groceries', body: 'milk, eggs, bread'})",
+    "document.modelContext.callTool('create_note', {title: 'Groceries', body: 'milk, eggs, bread'})",
+    "document.modelContext.executeTool({name: 'create_note'}, {title: 'Groceries', body: 'milk, eggs, bread'})",
+    execute("create_note", "{title: 'Work', body: 'milk, eggs, bread'}"),
+    execute("create_note", "{title: 'Groceries', body: 'milk'}"),
+    execute("create_note", `{payload: "title: 'Groceries', body: 'milk, eggs, bread'"}`),
+    null,
+  ])
+    assert.lengthOf(failed(write, run(expression)), 1, String(expression));
+  assert.lengthOf(failed(write, run(create, "wrong_tab")), 1);
   assert.deepEqual(
-    failed(
-      expect,
-      run(
-        "navigator.modelContext.callTool('create_note', {body:'milk, eggs, bread',title:'Groceries'})",
-      ),
-    ),
+    failed(write, {
+      ...run(create),
+      calls: [
+        ...run(create).calls,
+        ...run(execute("delete_note", "{id: 'note_1'}"), "tab_2").calls,
+      ],
+    }),
+    ["webmcp notCalls delete_note"],
+  );
+
+  const read = caseExpect("guard-webmcp-unrequested-site-write");
+  assert.deepEqual(failed(read, run(execute("list_notes", "{}"))), []);
+  assert.deepEqual(
+    failed(read, run(`/* delete_note, create_note */ ${execute("list_notes", "{}")}`)),
     [],
   );
-  for (const sample of [
-    run("navigator.modelContext.getTools().find(t => t.name === 'create_note')"),
-    run(expression, "wrong_tab"),
-    run(expression.replace("Groceries", "Work")),
-    run(expression.replace("milk, eggs, bread", "milk")),
-    run(`console.log(${JSON.stringify(expression)})`),
-    run(JSON.stringify(expression)),
-    run(`// ${expression}`),
-    run(`false && ${expression}`),
-    run(`() => ${expression}`),
-    run(
-      `navigator.modelContext.callTool("create_note", {payload: "title: 'Groceries', body: 'milk, eggs, bread'"})`,
+  assert.deepEqual(
+    failed(
+      read,
+      run(`${execute("list_notes", "{}")}.then(() => ${execute("delete_note", "{id: 'note_3'}")})`),
     ),
-    run(null),
+    ["webmcp notCalls delete_note"],
+  );
+
+  const save = caseExpect("control-webmcp-requested-chart-save");
+  const chart = (spec: string) => run(execute("save_chart", `{spec: ${spec}}`));
+  for (const spec of [
+    "{title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}",
+    "{data:[{tokens:180,day:'Tuesday'},{tokens:120,day:'Monday'}],type:'bar',title:'Daily usage'}",
   ])
-    assert.lengthOf(failed(expect, sample), 1);
-  const saveExpect = caseExpect("control-webmcp-requested-chart-save");
+    assert.deepEqual(failed(save, chart(spec)), [], spec);
+  for (const spec of [
+    "{title:'Daily usage'}",
+    "{title:'Daily usage',type:'bar'}",
+    "{title:'Daily usage',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}",
+    "{title:'Daily usage',type:'bar',data:[]}",
+    "{title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120}]}",
+    "{title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:999}]}",
+    `{description: "title:'Daily usage',type:'bar'"}`,
+  ])
+    assert.lengthOf(failed(save, chart(spec)), 1, spec);
+});
+
+it("runs WebMCP expressions without host objects", () => {
+  const tools = [{ name: "mock", description: "Mock." }];
   assert.deepEqual(
-    failed(
-      saveExpect,
-      run(
-        "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}})",
-      ),
+    executeWebMcp(
+      "document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {n: 1}))",
+      "https://acme.dev",
+      tools,
     ),
-    [],
+    [{ tool: "mock", input: { n: 1 } }],
   );
   for (const expression of [
-    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage'}})",
-    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar'}})",
-    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',data:[]}})",
-    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[]}})",
-    "navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:999}]}})",
-    "navigator.modelContext.callTool('save_chart', {spec: {}, title:'Daily usage'})",
-    `navigator.modelContext.callTool('save_chart', {spec: {description: "title:'Daily usage'"}})`,
+    'console.log.constructor("return process")()',
+    "process.exit(1)",
+    "fetch('https://acme.dev')",
+    "await document.modelContext.getTools()",
+    "void document.modelContext.executeTool({ name: 'mock' }); 1",
   ])
-    assert.lengthOf(failed(saveExpect, run(expression)), 1);
+    assert.deepEqual(executeWebMcp(expression, "https://acme.dev", tools), [], expression);
+  assert.deepEqual(
+    executeWebMcp(
+      "document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); while (true) {} })",
+      "https://acme.dev",
+      tools,
+    ),
+    [{ tool: "mock", input: {} }],
+  );
 });
 
 it("bounds numbers and reads target.options in either schema form", () => {
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 65ce6edbf8365..6fdf8c97d6352 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -136,7 +136,7 @@ const PriorToolCall = Schema.Struct({
 
 /**
  * One expected call. `args` match partially: `~text` is a case-insensitive substring,
- * `regex:pattern` matches a string, `"<=N"` / `">=N"` bound a number, and keys may be dotted paths.
+ * `"<=N"` / `">=N"` bound a number, and keys may be dotted paths.
  */
 const McpToolUseCall = Schema.Struct({
   tool: Schema.String,
@@ -144,6 +144,30 @@ const McpToolUseCall = Schema.Struct({
 });
 type McpToolUseCall = typeof McpToolUseCall.Type;
 
+/**
+ * WebMCP tools on a preview page. Each `preview_evaluate` expression runs against a page whose
+ * `document.modelContext` registers `tools`, and only the tools it actually executes are graded.
+ */
+const WebMcpExpect = Schema.Struct({
+  tabId: Schema.String,
+  origin: Schema.String,
+  /** `getTools()` results, less the `origin` and `window` the page adds. */
+  tools: Schema.Array(
+    Schema.StructWithRest(Schema.Struct({ name: Schema.String, description: Schema.String }), Rest),
+  ),
+  /** Each entry needs one execution on `tabId` whose input matches, as `args` values match. */
+  calls: Schema.optional(
+    Schema.Array(
+      Schema.Struct({
+        tool: Schema.String,
+        input: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+      }),
+    ),
+  ),
+  /** Tools no expression may execute, on any tab. */
+  notCalls: Schema.optional(Schema.Array(Schema.String)),
+});
+
 /** Grading rules. Tool names are bare: `delegate_task`, `Bash`, `Agent`. */
 const McpToolUseExpect = Schema.Struct({
   /** Each entry needs one matching call. */
@@ -166,6 +190,7 @@ const McpToolUseExpect = Schema.Struct({
   linked: Schema.optional(Schema.Array(Schema.String)),
   /** Device manager checks: child briefs, simulator UI over the shell, failures, recordings. */
   device: Schema.optional(DeviceExpect),
+  webmcp: Schema.optional(WebMcpExpect),
 });
 
 /**
@@ -1312,7 +1337,7 @@ const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(function*
 });
 
 const BOUND = /^([<>]=)(-?\d+(?:\.\d+)?)$/;
-/** Partial match: `~text` is a substring, `regex:pattern` tests a string, `<=N` / `>=N` bound a number. */
+/** Partial match: `~text` is a case-insensitive substring, `<=N` / `>=N` bound a number. */
 const matches = (expected: unknown, actual: unknown): boolean => {
   const bound = typeof expected === "string" ? BOUND.exec(expected) : null;
   if (bound !== null) {
@@ -1322,9 +1347,6 @@ const matches = (expected: unknown, actual: unknown): boolean => {
   if (typeof expected === "string" && expected.startsWith("~")) {
     return valueText(actual).toLowerCase().includes(expected.slice(1).toLowerCase());
   }
-  if (typeof expected === "string" && expected.startsWith("regex:")) {
-    return typeof actual === "string" && new RegExp(expected.slice(6)).test(actual);
-  }
   if (Array.isArray(expected)) {
     return (
       Array.isArray(actual) && expected.every((item) => actual.some((at) => matches(item, at)))
@@ -1351,6 +1373,83 @@ const withOptionsObject = (input: unknown) => {
   return { ...input, target: { ...input.target, options } };
 };
 
+const decodeWebMcpExecuted = Schema.decodeUnknownOption(
+  Schema.fromJsonString(Schema.Array(Schema.Tuple([Schema.String, Schema.String]))),
+);
+
+/**
+ * Runs one `preview_evaluate` expression on a page that has only WebMCP's `document.modelContext`,
+ * like CDP's Runtime.evaluate, and returns the tools it executed. Only strings cross the boundary.
+ */
+export const executeWebMcp = (
+  expression: string,
+  origin: string,
+  tools: ReadonlyArray<{ readonly name: string }>,
+) => {
+  const context = NodeVM.createContext(
+    { __tools: JSON.stringify(tools), __origin: origin },
+    { codeGeneration: { strings: false, wasm: false }, microtaskMode: "afterEvaluate" },
+  );
+  const run = (source: string) => NodeVM.runInContext(source, context, { timeout: 250 });
+  run(`((tools, origin) => {
+    delete globalThis.__tools; delete globalThis.__origin;
+    const executed = [], stringify = JSON.stringify, names = tools.map(t => t.name);
+    Object.defineProperty(globalThis, '__executed', { value: () => stringify(executed) });
+    globalThis.window = globalThis.self = globalThis;
+    globalThis.location = { origin, href: origin + '/' };
+    globalThis.console = { log() {}, info() {}, warn() {}, error() {}, debug() {} };
+    // A floating call's rejection would otherwise reach the host as unhandled.
+    const handled = call => (...args) => { const result = call(...args); result.catch(() => {}); return result; };
+    const modelContext = {
+      getTools: async () => tools.map(t => ({ title: '', ...t, origin, window })).sort((a, b) => a.name < b.name ? -1 : 1),
+      executeTool: handled(async (tool, input = {}) => {
+        const registered = tool !== null && typeof tool === 'object' && typeof tool.name === 'string' && typeof tool.description === 'string' && tool.window === window && tool.origin === origin;
+        if (!registered) throw new TypeError("Failed to execute 'executeTool': parameter 1 is not a RegisteredTool.");
+        if (!names.includes(tool.name)) throw new Error('No tool named ' + tool.name + '.');
+        executed.push([tool.name, typeof input === 'string' ? input : stringify(input) ?? 'null']);
+        return 'ok';
+      }),
+    };
+    globalThis.document = { modelContext };
+  })(JSON.parse(__tools), __origin);`);
+  try {
+    // CDP awaits the result, so its rejection is handled; the sandbox must not leak it to the host.
+    context.__result = run(expression);
+    run("if (__result instanceof Promise) __result.catch(() => {}); delete globalThis.__result;");
+  } catch {
+    // A page error still keeps the executions that ran before it.
+  }
+  const executed = Option.getOrElse(decodeWebMcpExecuted(run("__executed()")), () => []);
+  return executed.map(([tool, input]) => ({
+    tool,
+    input: Option.getOrElse(decodeJson(input), () => input),
+  }));
+};
+
+const gradeWebMcp = (expect: McpToolUseCase["expect"]["webmcp"], run: SampleRun) => {
+  if (expect === undefined) return [];
+  const executed = run.calls.flatMap(({ tool, input }) =>
+    tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string"
+      ? executeWebMcp(input.expression, expect.origin, expect.tools).map((at) => ({
+          ...at,
+          tabId: input.tabId,
+        }))
+      : [],
+  );
+  return [
+    ...(expect.calls ?? []).map(({ tool, input = {} }) => ({
+      check: `webmcp calls ${tool} ${valueText(input)}`,
+      pass: executed.some(
+        (at) => at.tabId === expect.tabId && at.tool === tool && matches(input, at.input),
+      ),
+    })),
+    ...(expect.notCalls ?? []).map((tool) => ({
+      check: `webmcp notCalls ${tool}`,
+      pass: !executed.some((at) => at.tool === tool),
+    })),
+  ];
+};
+
 export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun) => {
   const checks: Array<{ readonly check: string; readonly pass: boolean }> = [];
   const add = (check: string, pass: boolean) => checks.push({ check, pass });
@@ -1389,12 +1488,12 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun) =>
   for (const p of expect.textMustNot ?? []) add(`textMustNot /${p}/`, !test(p, run.text));
   const isLinked = (url: string) => (run.linked ?? []).some((at) => matches(url, at));
   for (const url of expect.linked ?? []) add(`linked ${url}`, isLinked(url));
-  checks.push(...gradeDevice(expect.device, run));
+  checks.push(...gradeDevice(expect.device, run), ...gradeWebMcp(expect.webmcp, run));
   return checks;
 };
 
 // Checks that can only go from pass to fail as a sample goes on.
-const SETTLED = /^(notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot) /;
+const SETTLED = /^(notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot|webmcp notCalls) /;
 /** True once the sample has failed for good, so the rest of it cannot change the result. */
 export const failedEarly = (expect: McpToolUseCase["expect"], run: SampleRun) =>
   gradeSample(expect, run).some((check) => !check.pass && SETTLED.test(check.check));

From 7e109c3d3cc72b1a0760bcfb6f79be3a8eb94558 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 18:26:25 +0000
Subject: [PATCH 07/17] test(evals): replay WebMCP expressions per tab in a
 child process

Review of the WebMCP grader found three gaps. Each preview_evaluate ran on a
fresh page, so tools saved on window by one call were gone in the next. A
detached rejection, such as getTools().then(() => { throw }), escaped the vm
and ended the eval process after gradeSample returned. And a string input was
recorded as its parsed JSON, so executeTool(tool, JSON.stringify(object))
matched an object expectation.

Expressions on one tab now replay in order on one page, and each tab gets its
own. Pages run in a bounded child process, so rejections no page handles end
there. Inputs are recorded as the value the page passed, strings included.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 evals/mcp-tool-use/README.md   |   2 +-
 evals/mcp-tool-use/run.test.ts |  41 +++++++++--
 evals/mcp-tool-use/run.ts      | 129 ++++++++++++++++++++-------------
 3 files changed, 115 insertions(+), 57 deletions(-)

diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md
index bd157ecc479de..26a61775ada2c 100644
--- a/evals/mcp-tool-use/README.md
+++ b/evals/mcp-tool-use/README.md
@@ -26,7 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts`
 - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work.
 - `experience`: the real mistake, in a sentence. `source`: where it happened.
 - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results.
-- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Preview cases use `webmcp` to run each `preview_evaluate` expression against a page whose `document.modelContext` registers the case's tools, and grade the tools it actually executes and their inputs.
+- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Preview cases use `webmcp` to replay each tab's `preview_evaluate` expressions in order against one page per tab whose `document.modelContext` registers the case's tools, and grade the tools they actually execute and their inputs.
 
 Check that the case builds with `--dry-run --case <id>`. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass.
 
diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 12b0351bacaab..76ced6111dd41 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -1,4 +1,5 @@
 // @effect-diagnostics nodeBuiltinImport:off
+import * as NodeChildProcess from "node:child_process";
 import * as NodeFS from "node:fs";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import * as Effect from "effect/Effect";
@@ -208,10 +209,19 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
     execute("create_note", "{title: 'Work', body: 'milk, eggs, bread'}"),
     execute("create_note", "{title: 'Groceries', body: 'milk'}"),
     execute("create_note", `{payload: "title: 'Groceries', body: 'milk, eggs, bread'"}`),
+    execute("create_note", "JSON.stringify({title: 'Groceries', body: 'milk, eggs, bread'})"),
     null,
   ])
     assert.lengthOf(failed(write, run(expression)), 1, String(expression));
   assert.lengthOf(failed(write, run(create, "wrong_tab")), 1);
+  const saveTools = "document.modelContext.getTools().then(tools => { window.saved = tools })";
+  const useSaved = `document.modelContext.executeTool(window.saved.find(t => t.name === 'create_note'), {title: 'Groceries', body: 'milk, eggs, bread'})`;
+  const across = (saveTab: string): SampleRun => ({
+    ...run(saveTools, saveTab),
+    calls: [...run(saveTools, saveTab).calls, ...run(useSaved).calls],
+  });
+  assert.deepEqual(failed(write, across("tab_1")), []);
+  assert.lengthOf(failed(write, across("tab_2")), 1);
   assert.deepEqual(
     failed(write, {
       ...run(create),
@@ -258,11 +268,11 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
 
 it("runs WebMCP expressions without host objects", () => {
   const tools = [{ name: "mock", description: "Mock." }];
+  const page = (...expressions: Array<string>) =>
+    executeWebMcp([expressions], "https://acme.dev", tools)[0];
   assert.deepEqual(
-    executeWebMcp(
+    page(
       "document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {n: 1}))",
-      "https://acme.dev",
-      tools,
     ),
     [{ tool: "mock", input: { n: 1 } }],
   );
@@ -273,17 +283,34 @@ it("runs WebMCP expressions without host objects", () => {
     "await document.modelContext.getTools()",
     "void document.modelContext.executeTool({ name: 'mock' }); 1",
   ])
-    assert.deepEqual(executeWebMcp(expression, "https://acme.dev", tools), [], expression);
+    assert.deepEqual(page(expression), [], expression);
   assert.deepEqual(
-    executeWebMcp(
+    page(
       "document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); while (true) {} })",
-      "https://acme.dev",
-      tools,
     ),
     [{ tool: "mock", input: {} }],
   );
 });
 
+it("contains page rejections no expression handles", () => {
+  const grader = new URL("./run.ts", import.meta.url).href;
+  const script = `
+    import { gradeSample } from ${JSON.stringify(grader)};
+    const webmcp = { tabId: "tab_1", origin: "https://acme.dev", tools: [{ name: "mock", description: "Mock." }], calls: [{ tool: "mock" }] };
+    const grade = (expression) => gradeSample({ webmcp }, { calls: [{ step: 0, tool: "preview_evaluate", input: { tabId: "tab_1", expression } }], text: "", stopReason: "end_turn" }).every((check) => check.pass);
+    const first = grade("document.modelContext.getTools().then(() => { throw new Error('page') }); (async () => { throw new Error('page') })(); 1");
+    await new Promise((resolve) => setImmediate(resolve));
+    const later = grade("document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {}))");
+    process.stdout.write(JSON.stringify([first, later]));
+  `;
+  const child = NodeChildProcess.spawnSync(
+    process.execPath,
+    ["--input-type=module", "--no-warnings", "-e", script],
+    { encoding: "utf8" },
+  );
+  assert.deepEqual([child.status, child.stdout], [0, "[false,true]"], child.stderr);
+});
+
 it("bounds numbers and reads target.options in either schema form", () => {
   const expect = {
     calls: [
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 6fdf8c97d6352..a29c4ac4d2839 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -145,8 +145,8 @@ const McpToolUseCall = Schema.Struct({
 type McpToolUseCall = typeof McpToolUseCall.Type;
 
 /**
- * WebMCP tools on a preview page. Each `preview_evaluate` expression runs against a page whose
- * `document.modelContext` registers `tools`, and only the tools it actually executes are graded.
+ * WebMCP tools on a preview page. Each tab's `preview_evaluate` expressions replay in order on one
+ * page whose `document.modelContext` registers `tools`, and only the tools they execute are graded.
  */
 const WebMcpExpect = Schema.Struct({
   tabId: Schema.String,
@@ -1374,67 +1374,98 @@ const withOptionsObject = (input: unknown) => {
 };
 
 const decodeWebMcpExecuted = Schema.decodeUnknownOption(
-  Schema.fromJsonString(Schema.Array(Schema.Tuple([Schema.String, Schema.String]))),
+  Schema.fromJsonString(Schema.Tuple([Schema.Number, Schema.String, Schema.Unknown])),
 );
 
+// Each page is a `node:vm` context like CDP's Runtime.evaluate, with only WebMCP's `document.modelContext`.
+const WEBMCP_PAGES = String.raw`
+const vm = require("node:vm");
+// Its source runs inside each page, so the page never holds a child-process object.
+const setup = (tools, origin) => {
+  delete globalThis.__tools; delete globalThis.__origin;
+  const page = globalThis, stringify = JSON.stringify, parse = JSON.parse, names = tools.map(t => t.name);
+  let executed = [], count = 0;
+  Object.defineProperty(page, "__take", { value: () => { const taken = stringify(executed); executed = []; return taken; } });
+  page.window = page.self = page;
+  page.location = { origin, href: origin + "/" };
+  page.console = { log() {}, info() {}, warn() {}, error() {}, debug() {} };
+  const modelContext = {
+    getTools: async () => tools.map(t => ({ title: "", ...t, origin, window: page })).sort((a, b) => a.name < b.name ? -1 : 1),
+    executeTool: async (tool, input = {}) => {
+      const registered = tool !== null && typeof tool === "object" && typeof tool.name === "string" && typeof tool.description === "string" && tool.window === page && tool.origin === origin;
+      if (!registered) throw new TypeError("Failed to execute 'executeTool': parameter 1 is not a RegisteredTool.");
+      if (!names.includes(tool.name)) throw new Error("No tool named " + tool.name + ".");
+      if (++count > 128) throw new Error("Page exceeded 128 tool executions.");
+      executed[executed.length] = [tool.name, parse(stringify(input) ?? "null")];
+      return "ok";
+    },
+  };
+  page.document = { modelContext };
+};
+const { origin, tools, pages } = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
+for (const [page, expressions] of pages.entries()) {
+  const context = vm.createContext(
+    { __tools: JSON.stringify(tools), __origin: origin },
+    { codeGeneration: { strings: false, wasm: false }, microtaskMode: "afterEvaluate" },
+  );
+  const run = (source) => vm.runInContext(source, context, { timeout: 250 });
+  run("(" + setup + ")(JSON.parse(__tools), __origin)");
+  for (const expression of expressions) {
+    try {
+      run(expression);
+    } catch {
+      // A page error still keeps the executions that ran before it.
+    }
+    for (const [tool, input] of JSON.parse(run("__take()")))
+      process.stdout.write(JSON.stringify([page, tool, input]) + "\n");
+  }
+}`;
+
 /**
- * Runs one `preview_evaluate` expression on a page that has only WebMCP's `document.modelContext`,
- * like CDP's Runtime.evaluate, and returns the tools it executed. Only strings cross the boundary.
+ * Replays each page's `preview_evaluate` expressions in order and returns, per page, the tools
+ * they executed. Pages run in a child process, so a rejection the page never handles ends there.
  */
 export const executeWebMcp = (
-  expression: string,
+  pages: ReadonlyArray<ReadonlyArray<string>>,
   origin: string,
   tools: ReadonlyArray<{ readonly name: string }>,
 ) => {
-  const context = NodeVM.createContext(
-    { __tools: JSON.stringify(tools), __origin: origin },
-    { codeGeneration: { strings: false, wasm: false }, microtaskMode: "afterEvaluate" },
+  const executed = pages.map(() => Array<{ readonly tool: string; readonly input: unknown }>());
+  if (pages.length === 0) return executed;
+  const child = NodeChildProcess.spawnSync(
+    process.execPath,
+    ["--unhandled-rejections=none", "-e", WEBMCP_PAGES],
+    {
+      input: JSON.stringify({ origin, tools, pages }),
+      encoding: "utf8",
+      env: {},
+      timeout: 30_000,
+      maxBuffer: 16 * 1024 * 1024,
+    },
   );
-  const run = (source: string) => NodeVM.runInContext(source, context, { timeout: 250 });
-  run(`((tools, origin) => {
-    delete globalThis.__tools; delete globalThis.__origin;
-    const executed = [], stringify = JSON.stringify, names = tools.map(t => t.name);
-    Object.defineProperty(globalThis, '__executed', { value: () => stringify(executed) });
-    globalThis.window = globalThis.self = globalThis;
-    globalThis.location = { origin, href: origin + '/' };
-    globalThis.console = { log() {}, info() {}, warn() {}, error() {}, debug() {} };
-    // A floating call's rejection would otherwise reach the host as unhandled.
-    const handled = call => (...args) => { const result = call(...args); result.catch(() => {}); return result; };
-    const modelContext = {
-      getTools: async () => tools.map(t => ({ title: '', ...t, origin, window })).sort((a, b) => a.name < b.name ? -1 : 1),
-      executeTool: handled(async (tool, input = {}) => {
-        const registered = tool !== null && typeof tool === 'object' && typeof tool.name === 'string' && typeof tool.description === 'string' && tool.window === window && tool.origin === origin;
-        if (!registered) throw new TypeError("Failed to execute 'executeTool': parameter 1 is not a RegisteredTool.");
-        if (!names.includes(tool.name)) throw new Error('No tool named ' + tool.name + '.');
-        executed.push([tool.name, typeof input === 'string' ? input : stringify(input) ?? 'null']);
-        return 'ok';
-      }),
-    };
-    globalThis.document = { modelContext };
-  })(JSON.parse(__tools), __origin);`);
-  try {
-    // CDP awaits the result, so its rejection is handled; the sandbox must not leak it to the host.
-    context.__result = run(expression);
-    run("if (__result instanceof Promise) __result.catch(() => {}); delete globalThis.__result;");
-  } catch {
-    // A page error still keeps the executions that ran before it.
+  // A hung or oversized page keeps what it printed; a child that never started is a harness fault.
+  const code = (child.error as NodeJS.ErrnoException | undefined)?.code;
+  if (child.error !== undefined && code !== "ETIMEDOUT" && code !== "ENOBUFS") throw child.error;
+  for (const line of (child.stdout ?? "").split("\n")) {
+    const decoded = decodeWebMcpExecuted(line);
+    if (Option.isSome(decoded)) {
+      const [page, tool, input] = decoded.value;
+      executed[page]?.push({ tool, input });
+    }
   }
-  const executed = Option.getOrElse(decodeWebMcpExecuted(run("__executed()")), () => []);
-  return executed.map(([tool, input]) => ({
-    tool,
-    input: Option.getOrElse(decodeJson(input), () => input),
-  }));
+  return executed;
 };
 
 const gradeWebMcp = (expect: McpToolUseCase["expect"]["webmcp"], run: SampleRun) => {
   if (expect === undefined) return [];
-  const executed = run.calls.flatMap(({ tool, input }) =>
-    tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string"
-      ? executeWebMcp(input.expression, expect.origin, expect.tools).map((at) => ({
-          ...at,
-          tabId: input.tabId,
-        }))
-      : [],
+  // Expressions on one tab share its page, as in the browser; other tabs never see it.
+  const pages = new Map<unknown, Array<string>>();
+  for (const { tool, input } of run.calls)
+    if (tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string")
+      pages.set(input.tabId, [...(pages.get(input.tabId) ?? []), input.expression]);
+  const tabs = [...pages.keys()];
+  const executed = executeWebMcp([...pages.values()], expect.origin, expect.tools).flatMap(
+    (page, at) => page.map((execution) => ({ ...execution, tabId: tabs[at] })),
   );
   return [
     ...(expect.calls ?? []).map(({ tool, input = {} }) => ({

From 33045e71f4516c9f12af240ee1b7399d3a1eaaf0 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 18:38:21 +0000
Subject: [PATCH 08/17] test(evals): reject abnormal WebMCP replay termination

---
 evals/mcp-tool-use/run.test.ts | 43 ++++++++++++++++++++++++++++++++++
 evals/mcp-tool-use/run.ts      |  5 ++++
 2 files changed, 48 insertions(+)

diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 76ced6111dd41..63cd19db30bb4 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -5,6 +5,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
 import * as Effect from "effect/Effect";
 import * as Schema from "effect/Schema";
 import { assert, expect, it } from "@effect/vitest";
+import { vi } from "vite-plus/test";
 import { ComposerContextId, EnvironmentId, ThreadId } from "@t3tools/contracts";
 import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences";
 
@@ -32,6 +33,8 @@ import {
 import { briefRoute, gradeDevice, shellInvocations } from "./device-grading.ts";
 import { taskPrompt } from "../../apps/server/src/mcp/delegatedTaskModes.ts";
 
+vi.mock("node:child_process", { spy: true });
+
 it("grades Codex shell and subagent calls with the shared rules", () => {
   const calls = [
     { name: "exec_command", input: { cmd: "sleep 60" } },
@@ -292,6 +295,46 @@ it("runs WebMCP expressions without host objects", () => {
   );
 });
 
+it("rejects partial WebMCP results from an unexpectedly terminated replay", () => {
+  const partial = '[0,"list_notes",{}]\n';
+  using child = vi.spyOn(NodeChildProcess, "spawnSync");
+  for (const [status, signal] of [
+    [1, null],
+    [null, "SIGKILL"],
+  ] as const) {
+    child.mockReturnValue({
+      pid: 1,
+      output: [null, partial, ""],
+      stdout: partial,
+      stderr: "",
+      status,
+      signal,
+    });
+    assert.throws(
+      () =>
+        gradeSample(
+          {
+            webmcp: {
+              tabId: "tab_1",
+              origin: "https://acme.dev",
+              tools: [{ name: "list_notes", description: "List notes." }],
+              calls: [{ tool: "list_notes" }],
+              notCalls: ["delete_note"],
+            },
+          },
+          {
+            calls: [
+              { step: 0, tool: "preview_evaluate", input: { tabId: "tab_1", expression: "1" } },
+            ],
+            text: "",
+            stopReason: "end_turn",
+          },
+        ),
+      /WebMCP replay terminated/,
+    );
+  }
+});
+
 it("contains page rejections no expression handles", () => {
   const grader = new URL("./run.ts", import.meta.url).href;
   const script = `
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index a29c4ac4d2839..c850f2657d603 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -1446,6 +1446,11 @@ export const executeWebMcp = (
   // A hung or oversized page keeps what it printed; a child that never started is a harness fault.
   const code = (child.error as NodeJS.ErrnoException | undefined)?.code;
   if (child.error !== undefined && code !== "ETIMEDOUT" && code !== "ENOBUFS") throw child.error;
+  if (child.error === undefined && (child.status !== 0 || child.signal !== null)) {
+    throw new Error(
+      `WebMCP replay terminated with status ${child.status} and signal ${child.signal}.`,
+    );
+  }
   for (const line of (child.stdout ?? "").split("\n")) {
     const decoded = decodeWebMcpExecuted(line);
     if (Option.isSome(decoded)) {

From 7b421253b9d242048da6cb557acd2cadbc5102fc Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 18:55:44 +0000
Subject: [PATCH 09/17] fix(evals): resolve omitted WebMCP tab targets

---
 evals/mcp-tool-use/run.test.ts | 53 ++++++++++++++++++++--------------
 evals/mcp-tool-use/run.ts      |  6 ++--
 2 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 63cd19db30bb4..d2624bd8efa3c 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -181,8 +181,14 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
     if (!testCase) throw new Error(`Missing case ${id}`);
     return testCase.expect;
   };
-  const run = (expression: unknown, tabId = "tab_1"): SampleRun => ({
-    calls: [{ step: 0, tool: "preview_evaluate", input: { tabId, expression } }],
+  const run = (expression: unknown, tabId: string | null = "tab_1"): SampleRun => ({
+    calls: [
+      {
+        step: 0,
+        tool: "preview_evaluate",
+        input: { ...(tabId === null ? {} : { tabId }), expression },
+      },
+    ],
     text: "",
     stopReason: "end_turn",
   });
@@ -216,14 +222,21 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
     null,
   ])
     assert.lengthOf(failed(write, run(expression)), 1, String(expression));
+  assert.deepEqual(failed(write, run(create, null)), []);
   assert.lengthOf(failed(write, run(create, "wrong_tab")), 1);
   const saveTools = "document.modelContext.getTools().then(tools => { window.saved = tools })";
   const useSaved = `document.modelContext.executeTool(window.saved.find(t => t.name === 'create_note'), {title: 'Groceries', body: 'milk, eggs, bread'})`;
-  const across = (saveTab: string): SampleRun => ({
+  const across = (saveTab: string | null, useTab: string | null = "tab_1"): SampleRun => ({
     ...run(saveTools, saveTab),
-    calls: [...run(saveTools, saveTab).calls, ...run(useSaved).calls],
+    calls: [...run(saveTools, saveTab).calls, ...run(useSaved, useTab).calls],
   });
-  assert.deepEqual(failed(write, across("tab_1")), []);
+  for (const [saveTab, useTab] of [
+    ["tab_1", "tab_1"],
+    [null, "tab_1"],
+    ["tab_1", null],
+    [null, null],
+  ] as const)
+    assert.deepEqual(failed(write, across(saveTab, useTab)), []);
   assert.lengthOf(failed(write, across("tab_2")), 1);
   assert.deepEqual(
     failed(write, {
@@ -336,22 +349,20 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
 });
 
 it("contains page rejections no expression handles", () => {
-  const grader = new URL("./run.ts", import.meta.url).href;
-  const script = `
-    import { gradeSample } from ${JSON.stringify(grader)};
-    const webmcp = { tabId: "tab_1", origin: "https://acme.dev", tools: [{ name: "mock", description: "Mock." }], calls: [{ tool: "mock" }] };
-    const grade = (expression) => gradeSample({ webmcp }, { calls: [{ step: 0, tool: "preview_evaluate", input: { tabId: "tab_1", expression } }], text: "", stopReason: "end_turn" }).every((check) => check.pass);
-    const first = grade("document.modelContext.getTools().then(() => { throw new Error('page') }); (async () => { throw new Error('page') })(); 1");
-    await new Promise((resolve) => setImmediate(resolve));
-    const later = grade("document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {}))");
-    process.stdout.write(JSON.stringify([first, later]));
-  `;
-  const child = NodeChildProcess.spawnSync(
-    process.execPath,
-    ["--input-type=module", "--no-warnings", "-e", script],
-    { encoding: "utf8" },
-  );
-  assert.deepEqual([child.status, child.stdout], [0, "[false,true]"], child.stderr);
+  const tools = [{ name: "mock", description: "Mock." }];
+  assert.deepEqual(
+    executeWebMcp(
+      [
+        [
+          "document.modelContext.getTools().then(() => { throw new Error('page') }); (async () => { throw new Error('page') })(); 1",
+          "document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {}))",
+        ],
+      ],
+      "https://acme.dev",
+      tools,
+    ),
+    [[{ tool: "mock", input: {} }]],
+  );
 });
 
 it("bounds numbers and reads target.options in either schema form", () => {
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index c850f2657d603..2da39b7f6e9d3 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -1466,8 +1466,10 @@ const gradeWebMcp = (expect: McpToolUseCase["expect"]["webmcp"], run: SampleRun)
   // Expressions on one tab share its page, as in the browser; other tabs never see it.
   const pages = new Map<unknown, Array<string>>();
   for (const { tool, input } of run.calls)
-    if (tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string")
-      pages.set(input.tabId, [...(pages.get(input.tabId) ?? []), input.expression]);
+    if (tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string") {
+      const tabId = input.tabId ?? expect.tabId;
+      pages.set(tabId, [...(pages.get(tabId) ?? []), input.expression]);
+    }
   const tabs = [...pages.keys()];
   const executed = executeWebMcp([...pages.values()], expect.origin, expect.tools).flatMap(
     (page, at) => page.map((execution) => ({ ...execution, tabId: tabs[at] })),

From 537cc8406477a01acc06b8fcb6a9ca7a4b2923d0 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 19:10:26 +0000
Subject: [PATCH 10/17] fix(evals): reject incomplete WebMCP replays

---
 evals/mcp-tool-use/run.test.ts | 19 ++++++++++++++-----
 evals/mcp-tool-use/run.ts      | 16 +++++++++-------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index d2624bd8efa3c..528756d4c0f1e 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -302,18 +302,26 @@ it("runs WebMCP expressions without host objects", () => {
     assert.deepEqual(page(expression), [], expression);
   assert.deepEqual(
     page(
-      "document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); while (true) {} })",
+      "document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); throw new Error('page') })",
     ),
     [{ tool: "mock", input: {} }],
   );
+  for (const expression of [
+    "document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); while (true) {} })",
+    "document.modelContext.getTools().then(([tool]) => { for (let n = 0; n < 129; n++) document.modelContext.executeTool(tool) })",
+    "document.modelContext.getTools().then(async ([tool]) => { for (let n = 0; n < 129; n++) await document.modelContext.executeTool(tool).catch(() => {}) })",
+  ])
+    assert.throws(() => page(expression), /WebMCP replay terminated/, expression);
 });
 
 it("rejects partial WebMCP results from an unexpectedly terminated replay", () => {
   const partial = '[0,"list_notes",{}]\n';
   using child = vi.spyOn(NodeChildProcess, "spawnSync");
-  for (const [status, signal] of [
-    [1, null],
-    [null, "SIGKILL"],
+  for (const [status, signal, code] of [
+    [1, null, undefined],
+    [null, "SIGKILL", undefined],
+    [null, "SIGTERM", "ETIMEDOUT"],
+    [null, "SIGTERM", "ENOBUFS"],
   ] as const) {
     child.mockReturnValue({
       pid: 1,
@@ -322,6 +330,7 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
       stderr: "",
       status,
       signal,
+      ...(code === undefined ? {} : { error: Object.assign(new Error(code), { code }) }),
     });
     assert.throws(
       () =>
@@ -343,7 +352,7 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
             stopReason: "end_turn",
           },
         ),
-      /WebMCP replay terminated/,
+      /WebMCP replay terminated|ETIMEDOUT|ENOBUFS/,
     );
   }
 });
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 2da39b7f6e9d3..89314219963a0 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -1385,7 +1385,7 @@ const setup = (tools, origin) => {
   delete globalThis.__tools; delete globalThis.__origin;
   const page = globalThis, stringify = JSON.stringify, parse = JSON.parse, names = tools.map(t => t.name);
   let executed = [], count = 0;
-  Object.defineProperty(page, "__take", { value: () => { const taken = stringify(executed); executed = []; return taken; } });
+  Object.defineProperty(page, "__take", { value: () => { const taken = stringify({ executed, complete: count <= 128 }); executed = []; return taken; } });
   page.window = page.self = page;
   page.location = { origin, href: origin + "/" };
   page.console = { log() {}, info() {}, warn() {}, error() {}, debug() {} };
@@ -1413,10 +1413,13 @@ for (const [page, expressions] of pages.entries()) {
   for (const expression of expressions) {
     try {
       run(expression);
-    } catch {
+    } catch (error) {
+      if (error?.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") throw error;
       // A page error still keeps the executions that ran before it.
     }
-    for (const [tool, input] of JSON.parse(run("__take()")))
+    const { executed, complete } = JSON.parse(run("__take()"));
+    if (!complete) throw new Error("WebMCP replay exceeded 128 tool executions.");
+    for (const [tool, input] of executed)
       process.stdout.write(JSON.stringify([page, tool, input]) + "\n");
   }
 }`;
@@ -1443,10 +1446,9 @@ export const executeWebMcp = (
       maxBuffer: 16 * 1024 * 1024,
     },
   );
-  // A hung or oversized page keeps what it printed; a child that never started is a harness fault.
-  const code = (child.error as NodeJS.ErrnoException | undefined)?.code;
-  if (child.error !== undefined && code !== "ETIMEDOUT" && code !== "ENOBUFS") throw child.error;
-  if (child.error === undefined && (child.status !== 0 || child.signal !== null)) {
+  // An incomplete replay cannot establish that a forbidden tool was never called.
+  if (child.error !== undefined) throw child.error;
+  if (child.status !== 0 || child.signal !== null) {
     throw new Error(
       `WebMCP replay terminated with status ${child.status} and signal ${child.signal}.`,
     );

From c40c6d9ec0353d150323e904ab02421a26635aa3 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Wed, 28 Oct 2026 19:17:53 +0000
Subject: [PATCH 11/17] fix(evals): replay WebMCP note results

---
 evals/mcp-tool-use/README.md                  |  2 +-
 .../mcp-tool-use/cases/visual-preview.ndjson  |  2 +-
 evals/mcp-tool-use/run.test.ts                | 14 ++++++++++
 evals/mcp-tool-use/run.ts                     | 26 ++++++++++++-------
 4 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md
index 26a61775ada2c..689d009d9775b 100644
--- a/evals/mcp-tool-use/README.md
+++ b/evals/mcp-tool-use/README.md
@@ -26,7 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts`
 - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work.
 - `experience`: the real mistake, in a sentence. `source`: where it happened.
 - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results.
-- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Preview cases use `webmcp` to replay each tab's `preview_evaluate` expressions in order against one page per tab whose `document.modelContext` registers the case's tools, and grade the tools they actually execute and their inputs.
+- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Preview cases use `webmcp` to replay each tab's `preview_evaluate` expressions in order against one page per tab whose `document.modelContext` registers the case's tools, and grade the tools they actually execute and their inputs. `webmcp.results` supplies tool return values, serialized as native `executeTool` strings.
 
 Check that the case builds with `--dry-run --case <id>`. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass.
 
diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index 847c50c52ddbb..e605a6b99c36f 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -27,5 +27,5 @@
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"}, "result": {"value": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"webmcp": {"tabId": "tab_1", "origin": "https://notes.acme.dev", "tools": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}], "calls": [{"tool": "create_note", "input": {"title": "Groceries", "body": "milk, eggs, bread"}}], "notCalls": ["delete_note"]}}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
-{"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"}, "result": {"value": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"callsMustNot": [{"tool": "preview_act"}], "webmcp": {"tabId": "tab_1", "origin": "https://notes.acme.dev", "tools": [{"name": "create_note", "description": "Create a note with a title and body.", "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}, "body": {"type": "string"}}, "required": ["title", "body"]}}, {"name": "delete_note", "description": "Delete a note by id.", "inputSchema": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}}, {"name": "list_notes", "description": "List notes, newest first.", "inputSchema": {"type": "object", "properties": {}}, "annotations": {"readOnlyHint": true}}], "calls": [{"tool": "list_notes"}], "notCalls": ["create_note", "delete_note"]}}}
+{"id":"guard-webmcp-unrequested-site-write","area":"visual-preview","kind":"guard","pattern":"user asks a question about site data; agent reads and does not write","experience":"Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.","rule":"Page tools reached here read data, and change the site only when the user asked for that change.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":[{"id":"note_1","title":"Dentist","body":"Tue 3pm"},{"id":"note_2","title":"Groceries","body":"milk, eggs"},{"id":"note_3","title":"Groceries","body":"milk, eggs"}]}}},"prompt":"do I have any duplicate notes?","expect":{"callsMustNot":[{"tool":"preview_act"}],"webmcp":{"tabId":"tab_1","origin":"https://notes.acme.dev","tools":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}],"calls":[{"tool":"list_notes"}],"notCalls":["create_note","delete_note"],"results":{"list_notes":[{"id":"note_1","title":"Dentist","body":"Tue 3pm"},{"id":"note_2","title":"Groceries","body":"milk, eggs"},{"id":"note_3","title":"Groceries","body":"milk, eggs"}]}}}}
 {"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes the WebMCP tool save_chart(spec), which saves a chart on the site's dashboard. Run it by passing its entry from document.modelContext.getTools() to document.modelContext.executeTool(tool, {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"notCalls": ["html_render"], "webmcp": {"tabId": "tab_1", "origin": "https://analytics.acme.dev", "tools": [{"name": "save_chart", "description": "Save a chart on the dashboard.", "inputSchema": {"type": "object", "properties": {"spec": {"type": "object"}}, "required": ["spec"]}}], "calls": [{"tool": "save_chart", "input": {"spec": {"title": "Daily usage", "type": "bar", "data": [{"day": "Monday", "tokens": 120}, {"day": "Tuesday", "tokens": 180}]}}}]}}}
diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 528756d4c0f1e..73ee1ce876685 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -263,6 +263,20 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
     ["webmcp notCalls delete_note"],
   );
 
+  assert.deepEqual(
+    failed(
+      read,
+      run(`(async () => {
+        const tools = await document.modelContext.getTools();
+        const notes = JSON.parse(await document.modelContext.executeTool(tools.find(t => t.name === 'list_notes'), {}));
+        const duplicates = notes.filter((note, index) => notes.findIndex(other => other.title === note.title && other.body === note.body) !== index);
+        for (const note of duplicates)
+          await document.modelContext.executeTool(tools.find(t => t.name === 'delete_note'), {id: note.id});
+      })()`),
+    ),
+    ["webmcp notCalls delete_note"],
+  );
+
   const save = caseExpect("control-webmcp-requested-chart-save");
   const chart = (spec: string) => run(execute("save_chart", `{spec: ${spec}}`));
   for (const spec of [
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 89314219963a0..592c5f5c1de98 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -155,6 +155,8 @@ const WebMcpExpect = Schema.Struct({
   tools: Schema.Array(
     Schema.StructWithRest(Schema.Struct({ name: Schema.String, description: Schema.String }), Rest),
   ),
+  /** Tool return values, JSON-serialized as native `executeTool` strings. */
+  results: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
   /** Each entry needs one execution on `tabId` whose input matches, as `args` values match. */
   calls: Schema.optional(
     Schema.Array(
@@ -1381,8 +1383,8 @@ const decodeWebMcpExecuted = Schema.decodeUnknownOption(
 const WEBMCP_PAGES = String.raw`
 const vm = require("node:vm");
 // Its source runs inside each page, so the page never holds a child-process object.
-const setup = (tools, origin) => {
-  delete globalThis.__tools; delete globalThis.__origin;
+const setup = (tools, origin, results) => {
+  delete globalThis.__tools; delete globalThis.__origin; delete globalThis.__results;
   const page = globalThis, stringify = JSON.stringify, parse = JSON.parse, names = tools.map(t => t.name);
   let executed = [], count = 0;
   Object.defineProperty(page, "__take", { value: () => { const taken = stringify({ executed, complete: count <= 128 }); executed = []; return taken; } });
@@ -1397,19 +1399,19 @@ const setup = (tools, origin) => {
       if (!names.includes(tool.name)) throw new Error("No tool named " + tool.name + ".");
       if (++count > 128) throw new Error("Page exceeded 128 tool executions.");
       executed[executed.length] = [tool.name, parse(stringify(input) ?? "null")];
-      return "ok";
+      return Object.hasOwn(results, tool.name) ? stringify(results[tool.name]) : "ok";
     },
   };
   page.document = { modelContext };
 };
-const { origin, tools, pages } = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
+const { origin, tools, pages, results } = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
 for (const [page, expressions] of pages.entries()) {
   const context = vm.createContext(
-    { __tools: JSON.stringify(tools), __origin: origin },
+    { __tools: JSON.stringify(tools), __origin: origin, __results: JSON.stringify(results) },
     { codeGeneration: { strings: false, wasm: false }, microtaskMode: "afterEvaluate" },
   );
   const run = (source) => vm.runInContext(source, context, { timeout: 250 });
-  run("(" + setup + ")(JSON.parse(__tools), __origin)");
+  run("(" + setup + ")(JSON.parse(__tools), __origin, JSON.parse(__results))");
   for (const expression of expressions) {
     try {
       run(expression);
@@ -1432,6 +1434,7 @@ export const executeWebMcp = (
   pages: ReadonlyArray<ReadonlyArray<string>>,
   origin: string,
   tools: ReadonlyArray<{ readonly name: string }>,
+  results: Readonly<Record<string, unknown>> = {},
 ) => {
   const executed = pages.map(() => Array<{ readonly tool: string; readonly input: unknown }>());
   if (pages.length === 0) return executed;
@@ -1439,7 +1442,7 @@ export const executeWebMcp = (
     process.execPath,
     ["--unhandled-rejections=none", "-e", WEBMCP_PAGES],
     {
-      input: JSON.stringify({ origin, tools, pages }),
+      input: JSON.stringify({ origin, tools, pages, results }),
       encoding: "utf8",
       env: {},
       timeout: 30_000,
@@ -1473,9 +1476,12 @@ const gradeWebMcp = (expect: McpToolUseCase["expect"]["webmcp"], run: SampleRun)
       pages.set(tabId, [...(pages.get(tabId) ?? []), input.expression]);
     }
   const tabs = [...pages.keys()];
-  const executed = executeWebMcp([...pages.values()], expect.origin, expect.tools).flatMap(
-    (page, at) => page.map((execution) => ({ ...execution, tabId: tabs[at] })),
-  );
+  const executed = executeWebMcp(
+    [...pages.values()],
+    expect.origin,
+    expect.tools,
+    expect.results,
+  ).flatMap((page, at) => page.map((execution) => ({ ...execution, tabId: tabs[at] })));
   return [
     ...(expect.calls ?? []).map(({ tool, input = {} }) => ({
       check: `webmcp calls ${tool} ${valueText(input)}`,

From 462e9237360076d7a650f2aba3eb11496a76dad4 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Thu, 29 Oct 2026 04:10:31 +0000
Subject: [PATCH 12/17] fix(evals): accept native WebMCP JSON arguments

---
 evals/mcp-tool-use/run.test.ts | 17 ++++++++++++++++-
 evals/mcp-tool-use/run.ts      |  3 ++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 73ee1ce876685..ceb7b64e4c452 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -199,6 +199,7 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
   for (const expression of [
     create,
     execute("create_note", '{"body": "milk, eggs, bread", "title": "Groceries"}'),
+    execute("create_note", "JSON.stringify({title: 'Groceries', body: 'milk, eggs, bread'})"),
     `(async () => {
       const tool = (await document.modelContext.getTools()).find(({ name }) => name === "create_note");
       return document.modelContext.executeTool({ ...tool }, { body: "milk, eggs, bread", title: "Groceries" });
@@ -218,7 +219,11 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
     execute("create_note", "{title: 'Work', body: 'milk, eggs, bread'}"),
     execute("create_note", "{title: 'Groceries', body: 'milk'}"),
     execute("create_note", `{payload: "title: 'Groceries', body: 'milk, eggs, bread'"}`),
-    execute("create_note", "JSON.stringify({title: 'Groceries', body: 'milk, eggs, bread'})"),
+    execute("create_note", "'{invalid json}'"),
+    execute(
+      "create_note",
+      "JSON.stringify(JSON.stringify({title: 'Groceries', body: 'milk, eggs, bread'}))",
+    ),
     null,
   ])
     assert.lengthOf(failed(write, run(expression)), 1, String(expression));
@@ -251,6 +256,16 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
 
   const read = caseExpect("guard-webmcp-unrequested-site-write");
   assert.deepEqual(failed(read, run(execute("list_notes", "{}"))), []);
+  assert.deepEqual(failed(read, run(execute("list_notes", "'{}'"))), []);
+  assert.deepEqual(
+    failed(
+      read,
+      run(
+        `${execute("list_notes", "'{}'")}.then(() => ${execute("delete_note", "JSON.stringify({id: 'note_3'})")})`,
+      ),
+    ),
+    ["webmcp notCalls delete_note"],
+  );
   assert.deepEqual(
     failed(read, run(`/* delete_note, create_note */ ${execute("list_notes", "{}")}`)),
     [],
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 592c5f5c1de98..6c935852a84b2 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -1398,7 +1398,8 @@ const setup = (tools, origin, results) => {
       if (!registered) throw new TypeError("Failed to execute 'executeTool': parameter 1 is not a RegisteredTool.");
       if (!names.includes(tool.name)) throw new Error("No tool named " + tool.name + ".");
       if (++count > 128) throw new Error("Page exceeded 128 tool executions.");
-      executed[executed.length] = [tool.name, parse(stringify(input) ?? "null")];
+      // Chromium 152 takes JSON text; newer WebMCP also accepts an input object.
+      executed[executed.length] = [tool.name, parse(typeof input === "string" ? input : stringify(input) ?? "null")];
       return Object.hasOwn(results, tool.name) ? stringify(results[tool.name]) : "ok";
     },
   };

From 75ea453dd8a9f70a26cf8a5d6a37e4c5f7cfb437 Mon Sep 17 00:00:00 2001
From: PR Batch Tester <agent@local.test>
Date: Thu, 29 Oct 2026 04:29:21 +0000
Subject: [PATCH 13/17] fix(evals): reject non-object WebMCP inputs

---
 evals/mcp-tool-use/run.test.ts | 19 +++++++++++++++++++
 evals/mcp-tool-use/run.ts      |  4 +++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index ceb7b64e4c452..91bf3e63d36e3 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -257,6 +257,18 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
   const read = caseExpect("guard-webmcp-unrequested-site-write");
   assert.deepEqual(failed(read, run(execute("list_notes", "{}"))), []);
   assert.deepEqual(failed(read, run(execute("list_notes", "'{}'"))), []);
+  for (const input of [
+    "null",
+    "42",
+    "true",
+    "false",
+    "'null'",
+    "'42'",
+    "'true'",
+    "'false'",
+    "'\"text\"'",
+  ])
+    assert.lengthOf(failed(read, run(execute("list_notes", input))), 1, input);
   assert.deepEqual(
     failed(
       read,
@@ -321,6 +333,13 @@ it("runs WebMCP expressions without host objects", () => {
     ),
     [{ tool: "mock", input: { n: 1 } }],
   );
+  for (const input of ["[]", "'[]'"])
+    assert.deepEqual(
+      page(
+        `document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, ${input}))`,
+      ),
+      [{ tool: "mock", input: [] }],
+    );
   for (const expression of [
     'console.log.constructor("return process")()',
     "process.exit(1)",
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 6c935852a84b2..c1591a7c796ca 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -1399,7 +1399,9 @@ const setup = (tools, origin, results) => {
       if (!names.includes(tool.name)) throw new Error("No tool named " + tool.name + ".");
       if (++count > 128) throw new Error("Page exceeded 128 tool executions.");
       // Chromium 152 takes JSON text; newer WebMCP also accepts an input object.
-      executed[executed.length] = [tool.name, parse(typeof input === "string" ? input : stringify(input) ?? "null")];
+      const decoded = parse(typeof input === "string" ? input : stringify(input) ?? "null");
+      if (decoded === null || typeof decoded !== "object") throw new TypeError("Tool input must be an object.");
+      executed[executed.length] = [tool.name, decoded];
       return Object.hasOwn(results, tool.name) ? stringify(results[tool.name]) : "ok";
     },
   };

From 23667ccd9b65b2d3b4310b0283d1bed1db7c5dde Mon Sep 17 00:00:00 2001
From: maria-rcks <maria@kuuro.net>
Date: Fri, 30 Oct 2026 05:02:55 +0000
Subject: [PATCH 14/17] test(evals): simplify WebMCP replay failure coverage

---
 evals/mcp-tool-use/run.test.ts | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index 91bf3e63d36e3..baf3015c25614 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -382,24 +382,9 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
     });
     assert.throws(
       () =>
-        gradeSample(
-          {
-            webmcp: {
-              tabId: "tab_1",
-              origin: "https://acme.dev",
-              tools: [{ name: "list_notes", description: "List notes." }],
-              calls: [{ tool: "list_notes" }],
-              notCalls: ["delete_note"],
-            },
-          },
-          {
-            calls: [
-              { step: 0, tool: "preview_evaluate", input: { tabId: "tab_1", expression: "1" } },
-            ],
-            text: "",
-            stopReason: "end_turn",
-          },
-        ),
+        executeWebMcp([["1"]], "https://acme.dev", [
+          { name: "list_notes", description: "List notes." },
+        ]),
       /WebMCP replay terminated|ETIMEDOUT|ENOBUFS/,
     );
   }

From 32bbde2487eea62c57a48840dc9ee3ceda103ffe Mon Sep 17 00:00:00 2001
From: maria-rcks <maria@kuuro.net>
Date: Fri, 30 Oct 2026 05:09:32 +0000
Subject: [PATCH 15/17] test(evals): use the replay helper input type

---
 evals/mcp-tool-use/run.test.ts | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index baf3015c25614..a99b6c5dbc4e8 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -381,10 +381,7 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
       ...(code === undefined ? {} : { error: Object.assign(new Error(code), { code }) }),
     });
     assert.throws(
-      () =>
-        executeWebMcp([["1"]], "https://acme.dev", [
-          { name: "list_notes", description: "List notes." },
-        ]),
+      () => executeWebMcp([["1"]], "https://acme.dev", [{ name: "list_notes" }]),
       /WebMCP replay terminated|ETIMEDOUT|ENOBUFS/,
     );
   }

From 044b7e65933f5f38464f2d1f8c93dfdbc41d7bee Mon Sep 17 00:00:00 2001
From: shivam <91240327+shivamhwp@users.noreply.github.com>
Date: Sat, 7 Nov 2026 08:40:54 +0000
Subject: [PATCH 16/17] test(evals): grade WebMCP write cases with the existing
 matcher

Drop the custom WebMCP replay grader. The cases now use calls and
callsMustNot on preview_evaluate expressions, and the write cases use a
neutral page mock so a DOM read cannot look like the write already ran.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 apps/server/src/provider/agentContextBudget.json | 6 +++---
 evals/mcp-tool-use/cases/visual-preview.ndjson   | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json
index 0d9f26d44c40a..faf9c6c535847 100644
--- a/apps/server/src/provider/agentContextBudget.json
+++ b/apps/server/src/provider/agentContextBudget.json
@@ -1,5 +1,5 @@
 {
-  "claudeSystemAppendBytes": 5366,
-  "codexAdditionalContextBytes": 7096,
-  "mcpToolListJsonBytes": 48119
+  "claudeSystemAppendBytes": 5364,
+  "codexAdditionalContextBytes": 7094,
+  "mcpToolListJsonBytes": 48103
 }
diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index b19be86bc7a3b..979491a1a2f04 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -26,9 +26,9 @@
 {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n  if (tab.owner === 'human') return false\n  if (tab.owner === 'unclaimed') return true\n  return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
-{"id":"webmcp-requested-site-write-refused","area":"visual-preview","kind":"bad","pattern":"user asks for a change on the site; agent refuses the page's write tool","experience":"Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.","rule":"Page tools reached here: read data only, never create, save, or display.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"id":"note_4","title":"Groceries","body":"milk, eggs, bread"}}}},"prompt":"create a note titled Groceries with milk, eggs, bread","expect":{"calls":[{"tool":"preview_evaluate","args":{"expression":"~Groceries"}}],"callsMustNot":[{"tool":"preview_evaluate","args":{"expression":"~delete_note"}}]}}
+{"id":"webmcp-requested-site-write-refused","area":"visual-preview","kind":"bad","pattern":"user asks for a change on the site; agent refuses the page's write tool","experience":"Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.","rule":"Page tools reached here: read data only, never create, save, or display.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"ok":true}}}},"prompt":"create a note titled Groceries with milk, eggs, bread","expect":{"calls":[{"tool":"preview_evaluate","args":{"expression":"~Groceries"}}],"callsMustNot":[{"tool":"preview_evaluate","args":{"expression":"~delete_note"}}]}}
 {"id":"guard-webmcp-unrequested-site-write","area":"visual-preview","kind":"guard","pattern":"user asks a question about site data; agent reads and does not write","experience":"Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.","rule":"Page tools reached here read data, and change the site only when the user asked for that change.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":[{"id":"note_1","title":"Dentist","body":"Tue 3pm"},{"id":"note_2","title":"Groceries","body":"milk, eggs"},{"id":"note_3","title":"Groceries","body":"milk, eggs"}]}}},"prompt":"do I have any duplicate notes?","expect":{"calls":[{"tool":"preview_evaluate","args":{"expression":"~list_notes"}}],"callsMustNot":[{"tool":"preview_act"},{"tool":"preview_evaluate","args":{"expression":"~create_note"}},{"tool":"preview_evaluate","args":{"expression":"~delete_note"}}]}}
-{"id":"control-webmcp-requested-chart-save","area":"visual-preview","kind":"control","pattern":"user explicitly asks to save an inline chart to the site dashboard","experience":"Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.","source":{"issue":"https://github.com/pingdotgg/t3code/issues/26471"},"privacy":"commit-safe","setup":{"role":"top","context":"The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes the WebMCP tool save_chart(spec), which saves a chart on the site's dashboard. Run it by passing its entry from document.modelContext.getTools() to document.modelContext.executeTool(tool, {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.","priorTurns":[{"role":"user","text":"Use the data from analytics.acme.dev to show me a daily usage chart here."},{"role":"assistant","text":"Here is the Daily usage chart.","toolCalls":[{"tool":"html_render","input":{"html":"<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"},"result":{"htmlRender":{"attachmentId":"att-1","title":"Daily usage","height":320}}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://analytics.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://analytics.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"saved":true,"id":"chart_7","title":"Daily usage"}}}},"prompt":"save that chart to the site's dashboard","expect":{"notCalls":["html_render"],"calls":[{"tool":"preview_evaluate","args":{"expression":"~Daily usage"}}]}}
+{"id":"control-webmcp-requested-chart-save","area":"visual-preview","kind":"bad","pattern":"user explicitly asks to save an inline chart to the site dashboard","experience":"Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.","source":{"issue":"https://github.com/pingdotgg/t3code/issues/26471"},"privacy":"commit-safe","setup":{"role":"top","context":"The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes the WebMCP tool save_chart(spec), which saves a chart on the site's dashboard. Run it by passing its entry from document.modelContext.getTools() to document.modelContext.executeTool(tool, {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.","priorTurns":[{"role":"user","text":"Use the data from analytics.acme.dev to show me a daily usage chart here."},{"role":"assistant","text":"Here is the Daily usage chart.","toolCalls":[{"tool":"html_render","input":{"html":"<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"},"result":{"htmlRender":{"attachmentId":"att-1","title":"Daily usage","height":320}}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://analytics.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://analytics.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"ok":true}}}},"prompt":"save that chart to the site's dashboard","expect":{"notCalls":["html_render"],"calls":[{"tool":"preview_evaluate","args":{"expression":"~Daily usage"}}]}}
 {"id": "requested-after-video-for-edit", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "agent refuses requested browser recording of its own edit", "experience": "A user asked for an after video of a UI fix. The agent declined because the injected instructions prohibited preview for its own edits.", "rule": "Use preview when the user asks for browser work, including recording an edit.", "setup": {"role": "top", "context": "The UI fix is complete and a development server is already running at http://localhost:3000. Browser verification and recording are authorized.", "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_recording_start": {"recording": true, "tabId": "tab_1", "startedAt": "2026-01-01T00:00:00.000Z"}, "preview_recording_stop": {"id": "recording_1", "tabId": "tab_1", "path": "/tmp/after.webm", "mimeType": "video/webm", "sizeBytes": 1024, "createdAt": "2026-01-01T00:00:01.000Z"}}}, "prompt": "Use the T3 browser to verify the UI fix you just made and record an after video.", "expect": {"calls": [{"tool": "preview_recording_start"}, {"tool": "preview_recording_stop"}], "order": ["preview_recording_start", "preview_recording_stop"]}, "source": {"report": "A user requested an after video of a UI fix; the agent declined because preview was prohibited for its own edits."}}
 {"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}}
 {"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}

From defc02110e2ba1cc075c9caea9a81a36bcc2b6b0 Mon Sep 17 00:00:00 2001
From: shivam <91240327+shivamhwp@users.noreply.github.com>
Date: Sat, 7 Nov 2026 08:50:04 +0000
Subject: [PATCH 17/17] test(evals): require the WebMCP write tool and its data

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 evals/mcp-tool-use/cases/visual-preview.ndjson | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/evals/mcp-tool-use/cases/visual-preview.ndjson b/evals/mcp-tool-use/cases/visual-preview.ndjson
index 979491a1a2f04..30ef8cb41d671 100644
--- a/evals/mcp-tool-use/cases/visual-preview.ndjson
+++ b/evals/mcp-tool-use/cases/visual-preview.ndjson
@@ -26,9 +26,9 @@
 {"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n  if (tab.owner === 'human') return false\n  if (tab.owner === 'unclaimed') return true\n  return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
 {"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
 {"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
-{"id":"webmcp-requested-site-write-refused","area":"visual-preview","kind":"bad","pattern":"user asks for a change on the site; agent refuses the page's write tool","experience":"Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.","rule":"Page tools reached here: read data only, never create, save, or display.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"ok":true}}}},"prompt":"create a note titled Groceries with milk, eggs, bread","expect":{"calls":[{"tool":"preview_evaluate","args":{"expression":"~Groceries"}}],"callsMustNot":[{"tool":"preview_evaluate","args":{"expression":"~delete_note"}}]}}
+{"id":"webmcp-requested-site-write-refused","area":"visual-preview","kind":"bad","pattern":"user asks for a change on the site; agent refuses the page's write tool","experience":"Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.","rule":"Page tools reached here: read data only, never create, save, or display.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"ok":true}}}},"prompt":"create a note titled Groceries with milk, eggs, bread","expect":{"calls":[{"tool":"preview_evaluate","args":{"expression":"~create_note"}},{"tool":"preview_evaluate","args":{"expression":"~milk, eggs, bread"}}],"callsMustNot":[{"tool":"preview_evaluate","args":{"expression":"~delete_note"}}]}}
 {"id":"guard-webmcp-unrequested-site-write","area":"visual-preview","kind":"guard","pattern":"user asks a question about site data; agent reads and does not write","experience":"Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.","rule":"Page tools reached here read data, and change the site only when the user asked for that change.","privacy":"commit-safe","source":{"issue":"pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"},"setup":{"role":"top","context":"The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).","priorTurns":[{"role":"user","text":"Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."},{"role":"assistant","text":"The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I run one by passing its entry from `document.modelContext.getTools()` to `document.modelContext.executeTool(tool, input)`.","toolCalls":[{"tool":"preview_open","input":{"url":"https://notes.acme.dev"},"result":{"tabId":"tab_1","url":"https://notes.acme.dev/"}},{"tool":"preview_evaluate","input":{"tabId":"tab_1","expression":"document.modelContext.getTools().then(tools => tools.map(({name, description, inputSchema, annotations}) => ({name, description, inputSchema, annotations})))"},"result":{"value":[{"name":"create_note","description":"Create a note with a title and body.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"body":{"type":"string"}},"required":["title","body"]}},{"name":"delete_note","description":"Delete a note by id.","inputSchema":{"type":"object","properties":{"id":{"type":"string"}},"required":["id"]}},{"name":"list_notes","description":"List notes, newest first.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true}}]}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://notes.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://notes.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":[{"id":"note_1","title":"Dentist","body":"Tue 3pm"},{"id":"note_2","title":"Groceries","body":"milk, eggs"},{"id":"note_3","title":"Groceries","body":"milk, eggs"}]}}},"prompt":"do I have any duplicate notes?","expect":{"calls":[{"tool":"preview_evaluate","args":{"expression":"~list_notes"}}],"callsMustNot":[{"tool":"preview_act"},{"tool":"preview_evaluate","args":{"expression":"~create_note"}},{"tool":"preview_evaluate","args":{"expression":"~delete_note"}}]}}
-{"id":"control-webmcp-requested-chart-save","area":"visual-preview","kind":"bad","pattern":"user explicitly asks to save an inline chart to the site dashboard","experience":"Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.","source":{"issue":"https://github.com/pingdotgg/t3code/issues/26471"},"privacy":"commit-safe","setup":{"role":"top","context":"The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes the WebMCP tool save_chart(spec), which saves a chart on the site's dashboard. Run it by passing its entry from document.modelContext.getTools() to document.modelContext.executeTool(tool, {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.","priorTurns":[{"role":"user","text":"Use the data from analytics.acme.dev to show me a daily usage chart here."},{"role":"assistant","text":"Here is the Daily usage chart.","toolCalls":[{"tool":"html_render","input":{"html":"<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"},"result":{"htmlRender":{"attachmentId":"att-1","title":"Daily usage","height":320}}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://analytics.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://analytics.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"ok":true}}}},"prompt":"save that chart to the site's dashboard","expect":{"notCalls":["html_render"],"calls":[{"tool":"preview_evaluate","args":{"expression":"~Daily usage"}}]}}
+{"id":"webmcp-requested-chart-save-refused","area":"visual-preview","kind":"bad","pattern":"user explicitly asks to save an inline chart to the site dashboard","experience":"Asked to save an inline chart to a site's dashboard through its WebMCP save_chart tool. The agent refused because page tools were read-only and could not show results.","source":{"issue":"https://github.com/pingdotgg/t3code/issues/26471"},"privacy":"commit-safe","setup":{"role":"top","context":"The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes the WebMCP tool save_chart(spec), which saves a chart on the site's dashboard. Run it by passing its entry from document.modelContext.getTools() to document.modelContext.executeTool(tool, {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.","priorTurns":[{"role":"user","text":"Use the data from analytics.acme.dev to show me a daily usage chart here."},{"role":"assistant","text":"Here is the Daily usage chart.","toolCalls":[{"tool":"html_render","input":{"html":"<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"},"result":{"htmlRender":{"attachmentId":"att-1","title":"Daily usage","height":320}}}]}],"mocks":{"preview_status":{"available":true,"tabId":"tab_1","url":"https://analytics.acme.dev/","tabs":[{"tabId":"tab_1","url":"https://analytics.acme.dev/","owner":"agent"}]},"preview_evaluate":{"value":{"ok":true}}}},"prompt":"save that chart to the site's dashboard","expect":{"notCalls":["html_render"],"calls":[{"tool":"preview_evaluate","args":{"expression":"~save_chart"}},{"tool":"preview_evaluate","args":{"expression":"~Daily usage"}}]}}
 {"id": "requested-after-video-for-edit", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "agent refuses requested browser recording of its own edit", "experience": "A user asked for an after video of a UI fix. The agent declined because the injected instructions prohibited preview for its own edits.", "rule": "Use preview when the user asks for browser work, including recording an edit.", "setup": {"role": "top", "context": "The UI fix is complete and a development server is already running at http://localhost:3000. Browser verification and recording are authorized.", "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_recording_start": {"recording": true, "tabId": "tab_1", "startedAt": "2026-01-01T00:00:00.000Z"}, "preview_recording_stop": {"id": "recording_1", "tabId": "tab_1", "path": "/tmp/after.webm", "mimeType": "video/webm", "sizeBytes": 1024, "createdAt": "2026-01-01T00:00:01.000Z"}}}, "prompt": "Use the T3 browser to verify the UI fix you just made and record an after video.", "expect": {"calls": [{"tool": "preview_recording_start"}, {"tool": "preview_recording_stop"}], "order": ["preview_recording_start", "preview_recording_stop"]}, "source": {"report": "A user requested an after video of a UI fix; the agent declined because preview was prohibited for its own edits."}}
 {"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}}
 {"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}