From 6d939a9496eeee3d60072ff3213c808cdd5ca5b2 Mon Sep 17 00:00:00 2001 From: Roger Vilaseca <12006627+rgrvlsk@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:21:24 +0100 Subject: [PATCH 1/5] fix(server): request_secret asks for one value per card The secret card has one masked, single-line field, but the tool description did not say so. Agents sometimes asked for several values in one card, for example three API keys as three lines, and the user could not enter them. The description now states the limit and asks for one call per secret. A new eval case reproduces the bundled request. The eval mock now gives each card its own file, as T3 does. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/orchestrator/tools.ts | 2 +- apps/server/src/provider/agentContextBudget.json | 2 +- evals/mcp-tool-use/cases/pull-requests.ndjson | 1 + evals/mcp-tool-use/run.ts | 3 ++- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 7c05f5c8d8e25..ccfef93b0b414 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -197,7 +197,7 @@ const DeleteScheduledTaskTool = Tool.make("delete_scheduled_task", { const RequestSecretTool = Tool.make("request_secret", { description: - 'Ask the user to enter a secret in a private card and wait. You never see the value. Returns a one-use `secretRef` for a tool that accepts it, or with `delivery: "file"` a private file path to read inside a shell command. Asked how to provide a secret: call this now, do not explain instead.', + 'Ask the user to enter one secret in a private card and wait. The card has one masked, single-line field. Call once per secret. You never see the value. Returns a one-use `secretRef` for a tool that accepts it, or with `delivery: "file"` a private file path to read inside a shell command. Asked how to provide a secret: call this now, do not explain instead.', parameters: OrchestratorMcpRequestSecretInput, success: OrchestratorMcpRequestSecretResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 1719d3ca6bd19..c3be132f900f2 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5153, "codexAdditionalContextBytes": 6883, - "mcpToolListJsonBytes": 47538 + "mcpToolListJsonBytes": 47606 } diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index 997243d873a49..5324764875f6e 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -14,6 +14,7 @@ {"id": "wake-required-passed-rearm-or-shell-wait", "area": "pull-requests", "kind": "bad", "pattern": "on a watch wake: re-arm watch, list PRs, or wait in the shell", "experience": "On a 'required checks passed' wake with a non-required job still running, agents re-called watch_pull_request and listed PRs, or unwatched and started gh run watch.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "File PR. Babysit. Merge when you are confident it's good to go."}, {"role": "assistant", "text": "Opened [#124](https://github.com/acme/app/pull/124). T3 Code is watching it; I will merge once checks pass and review findings are clear.", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/socket-retry"}, "result": "To github.com:acme/app.git\n * [new branch] fix/socket-retry -> fix/socket-retry\nbranch 'fix/socket-retry' set up to track 'origin/fix/socket-retry'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(web): retry the socket once before showing an error\" --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "watching": true, "wasWatching": false}}]}], "mocks": {"watch_pull_request": "default", "list_thread_pull_requests": "default", "link_pull_request": "default", "unwatch_pull_request": "default"}, "world": {"branch": "fix/socket-retry", "commits": [{"sha": "abc1234", "subject": "fix(web): retry the socket once before showing an error"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -10,1 +10,2 @@\n- if (error) return showError(error);\n+ if (error && !retried) return reconnect({ retried: true });\n+ if (error) return showError(error);\n", "prs": [{"number": 124, "title": "fix(web): retry the socket once before showing an error", "headRefName": "fix/socket-retry", "checks": [{"name": "Check", "state": "pass"}, {"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test Web", "state": "pending"}], "mergeStateStatus": "UNSTABLE"}]}}, "prompt": "Update on pull request #124 (https://github.com/acme/app/pull/124), which T3 Code is watching for you:\n- All 9 required checks passed on abc1234.\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests", "link_pull_request", "link_pull_requests", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"]}, "notes": "Merges two real failures on the same wake: Claude unwatch + gh run watch, and Codex re-arming watch + list on each wake (see source.also). Wake text is copied from pullRequestWatch.ts. Merging now or ending the turn to wait for the next wake both pass."} {"id": "secret-webhook-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "secret asked in chat, or request_secret promised but not called", "experience": "Agents told the user to paste secrets in chat, or said 'give me a token through request_secret' and never called it. request_secret has 0 real calls.", "rule": "When you need a secret from the user ... call request_secret so they enter it privately ... Never ask for a secret in chat", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Set up a webhook task so every PR opened on acme/app gets a quick triage in this thread. Have GitHub sign the requests."}, {"role": "assistant", "text": "I can set that up. I need the webhook secret so T3 Code can check GitHub's signature. Create a secret for the webhook in the repo's webhook settings first."}], "mocks": {"request_secret": {"status": "saved", "secretRef": "secret-ref-1"}, "schedule_task": {"scheduledTaskId": "scheduled-task-2", "title": "Triage new PRs", "schedule": {"type": "webhook"}, "enabled": true, "bindToCurrentThread": true, "webhookUrl": "https://hooks.example.test/t3/abc"}}}, "prompt": "Ok, I made the secret in GitHub. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret"}, {"tool": "schedule_task", "args": {"schedule.type": "webhook", "schedule.signature.secretRef": "secret-ref-1"}}], "order": ["request_secret", "schedule_task"], "shellMustNot": ["openssl rand"], "textMustNot": ["(?=2026-10-23\" --limit 500 --json number --jq length"}, "result": "290"}, {"tool": "delegate_task", "input": {"title": "Haiku PR audit", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-haiku-5-5"}, "task": "Audit the 290 PRs with activity in the last 24 hours for mergeability, value, and novelty. Flag the ones that look mergeable."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "31 PRs look mergeable: #101-#107, #110, ..."}}, {"tool": "delegate_task", "input": {"title": "Opus verify", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "status": "completed", "workState": "result_available", "summary": "#101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE. #110 looks green but CI never ran (first-time contributor, waiting for approval)."}}, {"tool": "delegate_task", "input": {"title": "Sol verify", "mode": "wait", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "status": "completed", "workState": "result_available", "summary": "Agrees on #101-#107. SKIP the rest."}}]}], "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-1", "title": "Nightly PR triage", "schedule": {"type": "fixed_time", "timeOfDay": "21:00"}, "enabled": true, "bindToCurrentThread": true, "nextRunAt": "2026-10-24T21:00:00-07:00"}}}, "prompt": "This looks good, set it with a schedule", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "schedule.timeOfDay": "21:00"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "shellMustNot": ["\\bgh (api|pr|search|issue)\\b"], "textMust": ["(9(:00)? ?[Pp]\\.?[Mm]|21:00)"]}, "notes": "Correction theme H. The only schedule correction in the data."} {"id": "scheduled-run-lists-tasks", "area": "pull-requests", "kind": "bad", "pattern": "list_scheduled_tasks on every scheduled wake", "experience": "Each 4-minute scheduled run called list_scheduled_tasks and got the same single task back (9 times in 33 minutes).", "rule": "none (scheduled runs already know their task)", "privacy": "commit-safe", "setup": {"role": "top", "context": "This turn was started by the scheduled task 'Check deploy recovery' (every 15 minutes), which posts into this thread.", "mocks": {"list_scheduled_tasks": {"tasks": [{"scheduledTaskId": "scheduled-task-3", "title": "Check deploy recovery", "schedule": {"type": "interval", "everyMs": 900000}, "enabled": true}]}}, "world": {"pages": {"https://status.example.test": "

Status

Deployments: paused for all plans. Next update in 30 minutes.

"}}}, "prompt": "Check https://status.example.test for an explicit announcement that deployments resumed. Do not count green component tiles as recovery. If deployments are still paused, reply in one line and change nothing.", "expect": {"notCalls": ["list_scheduled_tasks", "schedule_task", "delete_scheduled_task"]}, "notes": "No instruction line covers this today; written to what the user wants (no per-wake ceremony). The agent may use WebFetch instead of curl."} {"id": "creator-links-own-pr", "area": "pull-requests", "kind": "guard", "pattern": "created PR ends up linked, with no watch or list", "experience": "Must keep working: an opened PR ends up linked to the thread, and the agent does not watch or list when no babysit was asked.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Can we pre-bundle the hot startup paths on shared dev servers? First load takes over a minute."}, {"role": "assistant", "text": "Prototype is on branch perf/prebundle-startup (commit abc1234). First page load on a shared dev server drops from about 64 s to 21 s. Want a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "git checkout -b perf/prebundle-startup"}, "result": "Switched to a new branch 'perf/prebundle-startup'"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts", "content": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts", "content": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts"}, {"tool": "Bash", "input": {"command": "node scripts/bench-first-load.mjs --runs 3"}, "result": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"perf(dev): pre-bundle hot startup paths on shared dev servers\""}, "result": "[perf/prebundle-startup abc1234] perf(dev): pre-bundle hot startup paths on shared dev servers\n 2 files changed"}]}], "mocks": {"link_pull_request": "default"}, "world": {"branch": "perf/prebundle-startup", "commits": [{"sha": "abc1234", "subject": "perf(dev): pre-bundle hot startup paths on shared dev servers"}], "diff": "diff --git a/vite.config.ts b/vite.config.ts\n--- a/vite.config.ts\n+++ b/vite.config.ts\n@@ -1,5 +1,6 @@\n import { defineConfig } from 'vite';\n+import { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n \n export default defineConfig({\n- optimizeDeps: {},\n+ optimizeDeps: { include: HOT_STARTUP_DEPS },\n });\ndiff --git a/scripts/hotStartupDeps.ts b/scripts/hotStartupDeps.ts\n--- a/scripts/hotStartupDeps.ts\n+++ b/scripts/hotStartupDeps.ts\n@@ -0,0 +1,2 @@\n+// Measured with scripts/bench-first-load.mjs on a shared dev server.\n+export const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "files": {"vite.config.ts": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n", "scripts/hotStartupDeps.ts": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "scripts/bench-first-load.mjs": "// Starts the dev server and times the first page load.\n// Usage: node scripts/bench-first-load.mjs --runs 3\nimport { startDevServer, timeFirstLoad } from './bench-lib.mjs';\n\nconst runs = Number(process.argv[3] ?? 3);\nfor (let i = 0; i < runs; i++) console.log(await timeFirstLoad(await startDevServer()));\n", "scripts/bench-lib.mjs": "// Cold-cache dev server start and first-load timing for the perf scripts.\nimport { spawn } from 'node:child_process';\nimport { chromium } from 'playwright';\n\nexport async function startDevServer() {\n const server = spawn('pnpm', ['vite', '--force', '--port', '5179'], { stdio: 'pipe' });\n await new Promise((resolve) => server.stdout.on('data', (b) => String(b).includes('ready in') && resolve()));\n return server;\n}\n\nexport async function timeFirstLoad(server) {\n const browser = await chromium.launch();\n const page = await browser.newPage();\n const start = performance.now();\n await page.goto('http://localhost:5179', { waitUntil: 'load' });\n const seconds = ((performance.now() - start) / 1000).toFixed(1);\n await browser.close();\n server.kill();\n return `${seconds}s`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "^node .*bench-first-load", "output": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}]}, "prompt": "Okay do it. Make a good PR.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests"], "linked": ["~/pull/124"]}, "notes": "117 of 117 PR threads linked after the fixes. A shorter text must keep this. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 14} diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 3e6bb84f697be..a7aa9f0c20924 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -1107,7 +1107,8 @@ export const t3Default = ( } case "request_secret": { // T3 names both a ref and a file with 32 hex characters, and keeps files in its secrets dir. - const id = "5f0c9e2a7b4d4183a6e1c8d2b9f07a64"; + // Each card gets its own file, so an agent asking for several secrets sees distinct paths. + const id = (sample.nextTask++).toString(16).padStart(32, "0"); const home = /^\/(?:home|Users)\/[^/]+/.exec(sample.base.environment.cwd)?.[0] ?? "/home/user"; return input.delivery === "file" From 3bad363c63c107888f27f1f827a9cb3db54cfb3e Mon Sep 17 00:00:00 2001 From: Roger Vilaseca <12006627+rgrvlsk@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:19:04 +0100 Subject: [PATCH 2/5] fix(server): keep secret guidance within context budget --- apps/server/src/mcp/toolkits/orchestrator/tools.ts | 2 +- apps/server/src/provider/agentContextBudget.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index ccfef93b0b414..232026be15161 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -197,7 +197,7 @@ const DeleteScheduledTaskTool = Tool.make("delete_scheduled_task", { const RequestSecretTool = Tool.make("request_secret", { description: - 'Ask the user to enter one secret in a private card and wait. The card has one masked, single-line field. Call once per secret. You never see the value. Returns a one-use `secretRef` for a tool that accepts it, or with `delivery: "file"` a private file path to read inside a shell command. Asked how to provide a secret: call this now, do not explain instead.', + 'Ask the user to enter one secret in a private card and wait. You never see the value. Returns a one-use `secretRef` for a tool that accepts it, or with `delivery: "file"` a private file path to read in a shell command. Asked how to provide a secret: call this now, do not explain instead.', parameters: OrchestratorMcpRequestSecretInput, success: OrchestratorMcpRequestSecretResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index c3be132f900f2..1d19191f9a0f4 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5153, "codexAdditionalContextBytes": 6883, - "mcpToolListJsonBytes": 47606 + "mcpToolListJsonBytes": 47536 } From 73ee8b91cc4e2d37a67bc84e9c3643575356f718 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Wed, 4 Nov 2026 14:34:13 +0000 Subject: [PATCH 3/5] fix(server): require a separate card for each secret --- apps/server/src/mcp/toolkits/orchestrator/tools.ts | 2 +- apps/server/src/provider/agentContextBudget.json | 2 +- evals/mcp-tool-use/cases/pull-requests.ndjson | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 232026be15161..8bd0f7fbbef28 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -197,7 +197,7 @@ const DeleteScheduledTaskTool = Tool.make("delete_scheduled_task", { const RequestSecretTool = Tool.make("request_secret", { description: - 'Ask the user to enter one secret in a private card and wait. You never see the value. Returns a one-use `secretRef` for a tool that accepts it, or with `delivery: "file"` a private file path to read in a shell command. Asked how to provide a secret: call this now, do not explain instead.', + 'Ask for one secret per call in a private single-line card and wait. Never combine values. Returns a one-use `secretRef` for a tool, or with `delivery: "file"` a private file path to read in a command. Asked how to provide a secret: call now, do not explain.', parameters: OrchestratorMcpRequestSecretInput, success: OrchestratorMcpRequestSecretResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 1d19191f9a0f4..0f7d880638fe2 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5153, "codexAdditionalContextBytes": 6883, - "mcpToolListJsonBytes": 47536 + "mcpToolListJsonBytes": 47505 } diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index 5324764875f6e..e9474d3c755dd 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -14,7 +14,7 @@ {"id": "wake-required-passed-rearm-or-shell-wait", "area": "pull-requests", "kind": "bad", "pattern": "on a watch wake: re-arm watch, list PRs, or wait in the shell", "experience": "On a 'required checks passed' wake with a non-required job still running, agents re-called watch_pull_request and listed PRs, or unwatched and started gh run watch.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "File PR. Babysit. Merge when you are confident it's good to go."}, {"role": "assistant", "text": "Opened [#124](https://github.com/acme/app/pull/124). T3 Code is watching it; I will merge once checks pass and review findings are clear.", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/socket-retry"}, "result": "To github.com:acme/app.git\n * [new branch] fix/socket-retry -> fix/socket-retry\nbranch 'fix/socket-retry' set up to track 'origin/fix/socket-retry'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(web): retry the socket once before showing an error\" --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "watching": true, "wasWatching": false}}]}], "mocks": {"watch_pull_request": "default", "list_thread_pull_requests": "default", "link_pull_request": "default", "unwatch_pull_request": "default"}, "world": {"branch": "fix/socket-retry", "commits": [{"sha": "abc1234", "subject": "fix(web): retry the socket once before showing an error"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -10,1 +10,2 @@\n- if (error) return showError(error);\n+ if (error && !retried) return reconnect({ retried: true });\n+ if (error) return showError(error);\n", "prs": [{"number": 124, "title": "fix(web): retry the socket once before showing an error", "headRefName": "fix/socket-retry", "checks": [{"name": "Check", "state": "pass"}, {"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test Web", "state": "pending"}], "mergeStateStatus": "UNSTABLE"}]}}, "prompt": "Update on pull request #124 (https://github.com/acme/app/pull/124), which T3 Code is watching for you:\n- All 9 required checks passed on abc1234.\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests", "link_pull_request", "link_pull_requests", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"]}, "notes": "Merges two real failures on the same wake: Claude unwatch + gh run watch, and Codex re-arming watch + list on each wake (see source.also). Wake text is copied from pullRequestWatch.ts. Merging now or ending the turn to wait for the next wake both pass."} {"id": "secret-webhook-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "secret asked in chat, or request_secret promised but not called", "experience": "Agents told the user to paste secrets in chat, or said 'give me a token through request_secret' and never called it. request_secret has 0 real calls.", "rule": "When you need a secret from the user ... call request_secret so they enter it privately ... Never ask for a secret in chat", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Set up a webhook task so every PR opened on acme/app gets a quick triage in this thread. Have GitHub sign the requests."}, {"role": "assistant", "text": "I can set that up. I need the webhook secret so T3 Code can check GitHub's signature. Create a secret for the webhook in the repo's webhook settings first."}], "mocks": {"request_secret": {"status": "saved", "secretRef": "secret-ref-1"}, "schedule_task": {"scheduledTaskId": "scheduled-task-2", "title": "Triage new PRs", "schedule": {"type": "webhook"}, "enabled": true, "bindToCurrentThread": true, "webhookUrl": "https://hooks.example.test/t3/abc"}}}, "prompt": "Ok, I made the secret in GitHub. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret"}, {"tool": "schedule_task", "args": {"schedule.type": "webhook", "schedule.signature.secretRef": "secret-ref-1"}}], "order": ["request_secret", "schedule_task"], "shellMustNot": ["openssl rand"], "textMustNot": ["(?=2026-10-23\" --limit 500 --json number --jq length"}, "result": "290"}, {"tool": "delegate_task", "input": {"title": "Haiku PR audit", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-haiku-5-5"}, "task": "Audit the 290 PRs with activity in the last 24 hours for mergeability, value, and novelty. Flag the ones that look mergeable."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "31 PRs look mergeable: #101-#107, #110, ..."}}, {"tool": "delegate_task", "input": {"title": "Opus verify", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "status": "completed", "workState": "result_available", "summary": "#101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE. #110 looks green but CI never ran (first-time contributor, waiting for approval)."}}, {"tool": "delegate_task", "input": {"title": "Sol verify", "mode": "wait", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "status": "completed", "workState": "result_available", "summary": "Agrees on #101-#107. SKIP the rest."}}]}], "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-1", "title": "Nightly PR triage", "schedule": {"type": "fixed_time", "timeOfDay": "21:00"}, "enabled": true, "bindToCurrentThread": true, "nextRunAt": "2026-10-24T21:00:00-07:00"}}}, "prompt": "This looks good, set it with a schedule", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "schedule.timeOfDay": "21:00"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "shellMustNot": ["\\bgh (api|pr|search|issue)\\b"], "textMust": ["(9(:00)? ?[Pp]\\.?[Mm]|21:00)"]}, "notes": "Correction theme H. The only schedule correction in the data."} {"id": "scheduled-run-lists-tasks", "area": "pull-requests", "kind": "bad", "pattern": "list_scheduled_tasks on every scheduled wake", "experience": "Each 4-minute scheduled run called list_scheduled_tasks and got the same single task back (9 times in 33 minutes).", "rule": "none (scheduled runs already know their task)", "privacy": "commit-safe", "setup": {"role": "top", "context": "This turn was started by the scheduled task 'Check deploy recovery' (every 15 minutes), which posts into this thread.", "mocks": {"list_scheduled_tasks": {"tasks": [{"scheduledTaskId": "scheduled-task-3", "title": "Check deploy recovery", "schedule": {"type": "interval", "everyMs": 900000}, "enabled": true}]}}, "world": {"pages": {"https://status.example.test": "

Status

Deployments: paused for all plans. Next update in 30 minutes.

"}}}, "prompt": "Check https://status.example.test for an explicit announcement that deployments resumed. Do not count green component tiles as recovery. If deployments are still paused, reply in one line and change nothing.", "expect": {"notCalls": ["list_scheduled_tasks", "schedule_task", "delete_scheduled_task"]}, "notes": "No instruction line covers this today; written to what the user wants (no per-wake ceremony). The agent may use WebFetch instead of curl."} {"id": "creator-links-own-pr", "area": "pull-requests", "kind": "guard", "pattern": "created PR ends up linked, with no watch or list", "experience": "Must keep working: an opened PR ends up linked to the thread, and the agent does not watch or list when no babysit was asked.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Can we pre-bundle the hot startup paths on shared dev servers? First load takes over a minute."}, {"role": "assistant", "text": "Prototype is on branch perf/prebundle-startup (commit abc1234). First page load on a shared dev server drops from about 64 s to 21 s. Want a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "git checkout -b perf/prebundle-startup"}, "result": "Switched to a new branch 'perf/prebundle-startup'"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts", "content": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts", "content": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts"}, {"tool": "Bash", "input": {"command": "node scripts/bench-first-load.mjs --runs 3"}, "result": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"perf(dev): pre-bundle hot startup paths on shared dev servers\""}, "result": "[perf/prebundle-startup abc1234] perf(dev): pre-bundle hot startup paths on shared dev servers\n 2 files changed"}]}], "mocks": {"link_pull_request": "default"}, "world": {"branch": "perf/prebundle-startup", "commits": [{"sha": "abc1234", "subject": "perf(dev): pre-bundle hot startup paths on shared dev servers"}], "diff": "diff --git a/vite.config.ts b/vite.config.ts\n--- a/vite.config.ts\n+++ b/vite.config.ts\n@@ -1,5 +1,6 @@\n import { defineConfig } from 'vite';\n+import { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n \n export default defineConfig({\n- optimizeDeps: {},\n+ optimizeDeps: { include: HOT_STARTUP_DEPS },\n });\ndiff --git a/scripts/hotStartupDeps.ts b/scripts/hotStartupDeps.ts\n--- a/scripts/hotStartupDeps.ts\n+++ b/scripts/hotStartupDeps.ts\n@@ -0,0 +1,2 @@\n+// Measured with scripts/bench-first-load.mjs on a shared dev server.\n+export const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "files": {"vite.config.ts": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n", "scripts/hotStartupDeps.ts": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "scripts/bench-first-load.mjs": "// Starts the dev server and times the first page load.\n// Usage: node scripts/bench-first-load.mjs --runs 3\nimport { startDevServer, timeFirstLoad } from './bench-lib.mjs';\n\nconst runs = Number(process.argv[3] ?? 3);\nfor (let i = 0; i < runs; i++) console.log(await timeFirstLoad(await startDevServer()));\n", "scripts/bench-lib.mjs": "// Cold-cache dev server start and first-load timing for the perf scripts.\nimport { spawn } from 'node:child_process';\nimport { chromium } from 'playwright';\n\nexport async function startDevServer() {\n const server = spawn('pnpm', ['vite', '--force', '--port', '5179'], { stdio: 'pipe' });\n await new Promise((resolve) => server.stdout.on('data', (b) => String(b).includes('ready in') && resolve()));\n return server;\n}\n\nexport async function timeFirstLoad(server) {\n const browser = await chromium.launch();\n const page = await browser.newPage();\n const start = performance.now();\n await page.goto('http://localhost:5179', { waitUntil: 'load' });\n const seconds = ((performance.now() - start) / 1000).toFixed(1);\n await browser.close();\n server.kill();\n return `${seconds}s`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "^node .*bench-first-load", "output": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}]}, "prompt": "Okay do it. Make a good PR.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests"], "linked": ["~/pull/124"]}, "notes": "117 of 117 PR threads linked after the fixes. A shorter text must keep this. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 14} From 348fba3a026b299f638752c6b6945eb97943be81 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Wed, 4 Nov 2026 17:23:21 +0000 Subject: [PATCH 4/5] test(evals): require secret requests to open private cards --- evals/mcp-tool-use/cases/pull-requests.ndjson | 2 +- evals/mcp-tool-use/run.test.ts | 9 ++++++++- evals/mcp-tool-use/run.ts | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index 974b0d6d24b5d..b9f19354f8a0a 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -14,7 +14,7 @@ {"id": "wake-required-passed-rearm-or-shell-wait", "area": "pull-requests", "kind": "bad", "pattern": "on a watch wake: re-arm watch, list PRs, or wait in the shell", "experience": "On a 'required checks passed' wake with a non-required job still running, agents re-called watch_pull_request and listed PRs, or unwatched and started gh run watch.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "File PR. Babysit. Merge when you are confident it's good to go."}, {"role": "assistant", "text": "Opened [#124](https://github.com/acme/app/pull/124). T3 Code is watching it; I will merge once checks pass and review findings are clear.", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/socket-retry"}, "result": "To github.com:acme/app.git\n * [new branch] fix/socket-retry -> fix/socket-retry\nbranch 'fix/socket-retry' set up to track 'origin/fix/socket-retry'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(web): retry the socket once before showing an error\" --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "watching": true, "wasWatching": false}}]}], "mocks": {"watch_pull_request": "default", "list_thread_pull_requests": "default", "link_pull_request": "default", "unwatch_pull_request": "default"}, "world": {"branch": "fix/socket-retry", "commits": [{"sha": "abc1234", "subject": "fix(web): retry the socket once before showing an error"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -10,1 +10,2 @@\n- if (error) return showError(error);\n+ if (error && !retried) return reconnect({ retried: true });\n+ if (error) return showError(error);\n", "prs": [{"number": 124, "title": "fix(web): retry the socket once before showing an error", "headRefName": "fix/socket-retry", "checks": [{"name": "Check", "state": "pass"}, {"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test Web", "state": "pending"}], "mergeStateStatus": "UNSTABLE"}]}}, "prompt": "Update on pull request #124 (https://github.com/acme/app/pull/124), which T3 Code is watching for you:\n- All 9 required checks passed on abc1234.\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests", "link_pull_request", "link_pull_requests", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"]}, "notes": "Merges two real failures on the same wake: Claude unwatch + gh run watch, and Codex re-arming watch + list on each wake (see source.also). Wake text is copied from pullRequestWatch.ts. Merging now or ending the turn to wait for the next wake both pass."} {"id": "secret-webhook-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "secret asked in chat, or request_secret promised but not called", "experience": "Agents told the user to paste secrets in chat, or said 'give me a token through request_secret' and never called it. request_secret has 0 real calls.", "rule": "When you need a secret from the user ... call request_secret so they enter it privately ... Never ask for a secret in chat", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Set up a webhook task so every PR opened on acme/app gets a quick triage in this thread. Have GitHub sign the requests."}, {"role": "assistant", "text": "I can set that up. I need the webhook secret so T3 Code can check GitHub's signature. Create a secret for the webhook in the repo's webhook settings first."}], "mocks": {"request_secret": {"status": "saved", "secretRef": "secret-ref-1"}, "schedule_task": {"scheduledTaskId": "scheduled-task-2", "title": "Triage new PRs", "schedule": {"type": "webhook"}, "enabled": true, "bindToCurrentThread": true, "webhookUrl": "https://hooks.example.test/t3/abc"}}}, "prompt": "Ok, I made the secret in GitHub. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret"}, {"tool": "schedule_task", "args": {"schedule.type": "webhook", "schedule.signature.secretRef": "secret-ref-1"}}], "order": ["request_secret", "schedule_task"], "shellMustNot": ["openssl rand"], "textMustNot": ["(?=2026-10-23\" --limit 500 --json number --jq length"}, "result": "290"}, {"tool": "delegate_task", "input": {"title": "Haiku PR audit", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-haiku-5-5"}, "task": "Audit the 290 PRs with activity in the last 24 hours for mergeability, value, and novelty. Flag the ones that look mergeable."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "31 PRs look mergeable: #101-#107, #110, ..."}}, {"tool": "delegate_task", "input": {"title": "Opus verify", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "status": "completed", "workState": "result_available", "summary": "#101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE. #110 looks green but CI never ran (first-time contributor, waiting for approval)."}}, {"tool": "delegate_task", "input": {"title": "Sol verify", "mode": "wait", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "status": "completed", "workState": "result_available", "summary": "Agrees on #101-#107. SKIP the rest."}}]}], "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-1", "title": "Nightly PR triage", "schedule": {"type": "fixed_time", "timeOfDay": "21:00"}, "enabled": true, "bindToCurrentThread": true, "nextRunAt": "2026-10-24T21:00:00-07:00"}}}, "prompt": "This looks good, set it with a schedule", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "schedule.timeOfDay": "21:00"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "shellMustNot": ["\\bgh (api|pr|search|issue)\\b"], "textMust": ["(9(:00)? ?[Pp]\\.?[Mm]|21:00)"]}, "notes": "Correction theme H. The only schedule correction in the data."} {"id": "scheduled-run-lists-tasks", "area": "pull-requests", "kind": "bad", "pattern": "list_scheduled_tasks on every scheduled wake", "experience": "Each 4-minute scheduled run called list_scheduled_tasks and got the same single task back (9 times in 33 minutes).", "rule": "none (scheduled runs already know their task)", "privacy": "commit-safe", "setup": {"role": "top", "context": "This turn was started by the scheduled task 'Check deploy recovery' (every 15 minutes), which posts into this thread.", "mocks": {"list_scheduled_tasks": {"tasks": [{"scheduledTaskId": "scheduled-task-3", "title": "Check deploy recovery", "schedule": {"type": "interval", "everyMs": 900000}, "enabled": true}]}}, "world": {"pages": {"https://status.example.test": "

Status

Deployments: paused for all plans. Next update in 30 minutes.

"}}}, "prompt": "Check https://status.example.test for an explicit announcement that deployments resumed. Do not count green component tiles as recovery. If deployments are still paused, reply in one line and change nothing.", "expect": {"notCalls": ["list_scheduled_tasks", "schedule_task", "delete_scheduled_task"]}, "notes": "No instruction line covers this today; written to what the user wants (no per-wake ceremony). The agent may use WebFetch instead of curl."} {"id": "creator-links-own-pr", "area": "pull-requests", "kind": "guard", "pattern": "created PR ends up linked, with no watch or list", "experience": "Must keep working: an opened PR ends up linked to the thread, and the agent does not watch or list when no babysit was asked.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Can we pre-bundle the hot startup paths on shared dev servers? First load takes over a minute."}, {"role": "assistant", "text": "Prototype is on branch perf/prebundle-startup (commit abc1234). First page load on a shared dev server drops from about 64 s to 21 s. Want a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "git checkout -b perf/prebundle-startup"}, "result": "Switched to a new branch 'perf/prebundle-startup'"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts", "content": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts", "content": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts"}, {"tool": "Bash", "input": {"command": "node scripts/bench-first-load.mjs --runs 3"}, "result": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"perf(dev): pre-bundle hot startup paths on shared dev servers\""}, "result": "[perf/prebundle-startup abc1234] perf(dev): pre-bundle hot startup paths on shared dev servers\n 2 files changed"}]}], "mocks": {"link_pull_request": "default"}, "world": {"branch": "perf/prebundle-startup", "commits": [{"sha": "abc1234", "subject": "perf(dev): pre-bundle hot startup paths on shared dev servers"}], "diff": "diff --git a/vite.config.ts b/vite.config.ts\n--- a/vite.config.ts\n+++ b/vite.config.ts\n@@ -1,5 +1,6 @@\n import { defineConfig } from 'vite';\n+import { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n \n export default defineConfig({\n- optimizeDeps: {},\n+ optimizeDeps: { include: HOT_STARTUP_DEPS },\n });\ndiff --git a/scripts/hotStartupDeps.ts b/scripts/hotStartupDeps.ts\n--- a/scripts/hotStartupDeps.ts\n+++ b/scripts/hotStartupDeps.ts\n@@ -0,0 +1,2 @@\n+// Measured with scripts/bench-first-load.mjs on a shared dev server.\n+export const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "files": {"vite.config.ts": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n", "scripts/hotStartupDeps.ts": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "scripts/bench-first-load.mjs": "// Starts the dev server and times the first page load.\n// Usage: node scripts/bench-first-load.mjs --runs 3\nimport { startDevServer, timeFirstLoad } from './bench-lib.mjs';\n\nconst runs = Number(process.argv[3] ?? 3);\nfor (let i = 0; i < runs; i++) console.log(await timeFirstLoad(await startDevServer()));\n", "scripts/bench-lib.mjs": "// Cold-cache dev server start and first-load timing for the perf scripts.\nimport { spawn } from 'node:child_process';\nimport { chromium } from 'playwright';\n\nexport async function startDevServer() {\n const server = spawn('pnpm', ['vite', '--force', '--port', '5179'], { stdio: 'pipe' });\n await new Promise((resolve) => server.stdout.on('data', (b) => String(b).includes('ready in') && resolve()));\n return server;\n}\n\nexport async function timeFirstLoad(server) {\n const browser = await chromium.launch();\n const page = await browser.newPage();\n const start = performance.now();\n await page.goto('http://localhost:5179', { waitUntil: 'load' });\n const seconds = ((performance.now() - start) / 1000).toFixed(1);\n await browser.close();\n server.kill();\n return `${seconds}s`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "^node .*bench-first-load", "output": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}]}, "prompt": "Okay do it. Make a good PR.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests"], "linked": ["~/pull/124"]}, "notes": "117 of 117 PR threads linked after the fixes. A shorter text must keep this. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 14} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 3e21040f055a1..b236cae430392 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -73,7 +73,7 @@ it("grades Codex shell and subagent calls with the shared rules", () => { ); const distinctCalls = ["application key", "application secret", "consumer key"].map((label) => ({ tool: "request_secret", - args: { label: "~" + label, delivery: "file" }, + args: { label: "~" + label, delivery: "file", value: null }, })); const cards = distinctCalls.map(({ tool, args }) => ({ step: 0, @@ -82,6 +82,13 @@ it("grades Codex shell and subagent calls with the shared rules", () => { })); const run = (calls: SampleRun["calls"]) => ({ calls, text: "", stopReason: "end_turn" }); assert.deepEqual(failed({ distinctCalls }, run(cards)), []); + assert.deepEqual( + failed( + { distinctCalls }, + run(cards.map((card) => ({ ...card, input: { ...card.input, value: "invented" } }))), + ), + ["distinctCalls"], + ); assert.deepEqual(failed({ distinctCalls }, run([cards[0]!])), ["distinctCalls"]); assert.deepEqual( failed( diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index c1dd412bb235b..258d2656a57d8 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -171,7 +171,7 @@ type McpToolUseCall = typeof McpToolUseCall.Type; const McpToolUseExpect = Schema.Struct({ /** Each entry needs one matching call. */ calls: Schema.optional(Schema.Array(McpToolUseCall)), - /** Each entry matches a different call. */ + /** Each entry matches a different call; tool names are exact. */ distinctCalls: Schema.optional(Schema.Array(McpToolUseCall)), /** No call may match any entry. */ callsMustNot: Schema.optional(Schema.Array(McpToolUseCall)), From 49e0eeb0f66ddd0a4bba107c5165c7d6b3b0a55b Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 5 Nov 2026 20:28:53 +0000 Subject: [PATCH 5/5] test(evals): reject a combined secret card in distinct-call grading --- evals/mcp-tool-use/run.test.ts | 13 +++++++++++++ evals/mcp-tool-use/run.ts | 15 +++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index b236cae430392..121814fe2da2f 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -114,6 +114,19 @@ it("grades Codex shell and subagent calls with the shared rules", () => { ), ["distinctCalls"], ); + assert.deepEqual( + failed( + { distinctCalls }, + run([ + { + ...cards[0]!, + input: { ...cards[0]!.input, label: "application key, application secret, consumer key" }, + }, + ...cards.slice(1), + ]), + ), + ["distinctCalls"], + ); }); it("matches shellMust, not shellMustNot, against a command's argv lists as a command line", () => { diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index af8d9613fbb1a..92ae6fe28cc4f 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -181,7 +181,7 @@ type McpToolUseCall = typeof McpToolUseCall.Type; const McpToolUseExpect = Schema.Struct({ /** Each entry needs one matching call. */ calls: Schema.optional(Schema.Array(McpToolUseCall)), - /** Each entry matches a different call; tool names are exact. */ + /** Each entry matches a different call, and no call matches two entries; tool names are exact. */ distinctCalls: Schema.optional(Schema.Array(McpToolUseCall)), /** No call may match any entry. */ callsMustNot: Schema.optional(Schema.Array(McpToolUseCall)), @@ -1703,14 +1703,17 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pa calls.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input)); const label = ({ tool, args }: McpToolUseCall) => (args ? `${tool} ${valueText(args)}` : tool); for (const call of expect.calls ?? []) add(`calls ${label(call)}`, found(call)); - if (expect.distinctCalls !== undefined) + if (expect.distinctCalls !== undefined) { + const expected = expect.distinctCalls.map(({ tool, args }) => ({ tool, input: args ?? {} })); add( "distinctCalls", - matchesDistinct( - expect.distinctCalls.map(({ tool, args }) => ({ tool, input: args ?? {} })), - calls.map((request) => ({ request, brief: "" })), - ), + calls.every((call) => expected.filter((entry) => matches(entry, call)).length <= 1) && + matchesDistinct( + expected, + calls.map((request) => ({ request, brief: "" })), + ), ); + } for (const call of expect.callsMustNot ?? []) add(`callsMustNot ${label(call)}`, !found(call)); for (const tool of expect.notCalls ?? []) add(`notCalls ${tool}`, count(tool) === 0); if (expect.order !== undefined) {