diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 150962e232a13..f0affbf6dec8c 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -197,7 +197,7 @@ const DeleteScheduledTaskTool = Tool.make("delete_scheduled_task", { const RequestSecretTool = Tool.make("request_secret", { description: - 'Ask for a secret in a private card and wait; open it, don\'t offer it. Returns a one-use `secretRef`, or with `delivery: "file"` a file to read only in its command, e.g. `adb pair "$(cat )"`. Asked how to give one: call now. Pasted secret for child or script: `value`. Not one from an old thread.', + 'Ask for one secret per single-line card; never combine values. Open it, don\'t offer it. Returns one-use `secretRef`, or `delivery: "file"` a file read only in its command, e.g. `adb pair "$(cat )"`. Asked how: call now. Pasted secret for child/script: `value`. Never from an old thread.', parameters: OrchestratorMcpRequestSecretInput, success: OrchestratorMcpRequestSecretResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 0a03b931fcc08..b71053bf272a8 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5133, "codexAdditionalContextBytes": 6863, - "mcpToolListJsonBytes": 47818 + "mcpToolListJsonBytes": 47809 } diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index e0c7144220030..2e14e45f7f250 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -14,6 +14,7 @@ {"id": "wake-required-passed-rearm-or-shell-wait", "area": "pull-requests", "kind": "bad", "pattern": "on a watch wake: re-arm watch, list PRs, or wait in the shell", "experience": "On a 'required checks passed' wake with a non-required job still running, agents re-called watch_pull_request and listed PRs, or unwatched and started gh run watch.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "File PR. Babysit. Merge when you are confident it's good to go."}, {"role": "assistant", "text": "Opened [#124](https://github.com/acme/app/pull/124). T3 Code is watching it; I will merge once checks pass and review findings are clear.", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/socket-retry"}, "result": "To github.com:acme/app.git\n * [new branch] fix/socket-retry -> fix/socket-retry\nbranch 'fix/socket-retry' set up to track 'origin/fix/socket-retry'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(web): retry the socket once before showing an error\" --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "watching": true, "wasWatching": false}}]}], "mocks": {"watch_pull_request": "default", "list_thread_pull_requests": "default", "link_pull_request": "default", "unwatch_pull_request": "default"}, "world": {"branch": "fix/socket-retry", "commits": [{"sha": "abc1234", "subject": "fix(web): retry the socket once before showing an error"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -10,1 +10,2 @@\n- if (error) return showError(error);\n+ if (error && !retried) return reconnect({ retried: true });\n+ if (error) return showError(error);\n", "prs": [{"number": 124, "title": "fix(web): retry the socket once before showing an error", "headRefName": "fix/socket-retry", "checks": [{"name": "Check", "state": "pass"}, {"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test Web", "state": "pending"}], "mergeStateStatus": "UNSTABLE"}]}}, "prompt": "Update on pull request #124 (https://github.com/acme/app/pull/124), which T3 Code is watching for you:\n- All 9 required checks passed on abc1234.\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests", "link_pull_request", "link_pull_requests", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"]}, "notes": "Merges two real failures on the same wake: Claude unwatch + gh run watch, and Codex re-arming watch + list on each wake (see source.also). Wake text is copied from pullRequestWatch.ts. Merging now or ending the turn to wait for the next wake both pass."} {"id": "secret-webhook-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "secret asked in chat, or request_secret promised but not called", "experience": "Agents told the user to paste secrets in chat, or said 'give me a token through request_secret' and never called it. request_secret has 0 real calls.", "rule": "When you need a secret from the user ... call request_secret so they enter it privately ... Never ask for a secret in chat", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Set up a webhook task so every PR opened on acme/app gets a quick triage in this thread. Have GitHub sign the requests."}, {"role": "assistant", "text": "I can set that up. I need the webhook secret so T3 Code can check GitHub's signature. Create a secret for the webhook in the repo's webhook settings first."}], "mocks": {"request_secret": {"status": "saved", "secretRef": "secret-ref-1"}, "schedule_task": {"scheduledTaskId": "scheduled-task-2", "title": "Triage new PRs", "schedule": {"type": "webhook"}, "enabled": true, "bindToCurrentThread": true, "webhookUrl": "https://hooks.example.test/t3/abc"}}}, "prompt": "Ok, I made the secret in GitHub. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret"}, {"tool": "schedule_task", "args": {"schedule.type": "webhook", "schedule.signature.secretRef": "secret-ref-1"}}], "order": ["request_secret", "schedule_task"], "shellMustNot": ["openssl rand"], "textMustNot": ["(?=2026-10-23\" --limit 500 --json number --jq length"}, "result": "290"}, {"tool": "delegate_task", "input": {"title": "Haiku PR audit", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-haiku-5-5"}, "task": "Audit the 290 PRs with activity in the last 24 hours for mergeability, value, and novelty. Flag the ones that look mergeable."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "31 PRs look mergeable: #101-#107, #110, ..."}}, {"tool": "delegate_task", "input": {"title": "Opus verify", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "status": "completed", "workState": "result_available", "summary": "#101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE. #110 looks green but CI never ran (first-time contributor, waiting for approval)."}}, {"tool": "delegate_task", "input": {"title": "Sol verify", "mode": "wait", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "status": "completed", "workState": "result_available", "summary": "Agrees on #101-#107. SKIP the rest."}}]}], "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-1", "title": "Nightly PR triage", "schedule": {"type": "fixed_time", "timeOfDay": "21:00"}, "enabled": true, "bindToCurrentThread": true, "nextRunAt": "2026-10-24T21:00:00-07:00"}}}, "prompt": "This looks good, set it with a schedule", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "schedule.timeOfDay": "21:00"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "shellMustNot": ["\\bgh (api|pr|search|issue)\\b"], "textMust": ["(9(:00)? ?[Pp]\\.?[Mm]|21:00)"]}, "notes": "Correction theme H. The only schedule correction in the data."} {"id": "scheduled-run-lists-tasks", "area": "pull-requests", "kind": "bad", "pattern": "list_scheduled_tasks on every scheduled wake", "experience": "Each 4-minute scheduled run called list_scheduled_tasks and got the same single task back (9 times in 33 minutes).", "rule": "none (scheduled runs already know their task)", "privacy": "commit-safe", "setup": {"role": "top", "context": "This turn was started by the scheduled task 'Check deploy recovery' (every 15 minutes), which posts into this thread.", "mocks": {"list_scheduled_tasks": {"tasks": [{"scheduledTaskId": "scheduled-task-3", "title": "Check deploy recovery", "schedule": {"type": "interval", "everyMs": 900000}, "enabled": true}]}}, "world": {"pages": {"https://status.example.test": "

Status

Deployments: paused for all plans. Next update in 30 minutes.

"}}}, "prompt": "Check https://status.example.test for an explicit announcement that deployments resumed. Do not count green component tiles as recovery. If deployments are still paused, reply in one line and change nothing.", "expect": {"notCalls": ["list_scheduled_tasks", "schedule_task", "delete_scheduled_task"]}, "notes": "No instruction line covers this today; written to what the user wants (no per-wake ceremony). The agent may use WebFetch instead of curl."} {"id": "creator-links-own-pr", "area": "pull-requests", "kind": "guard", "pattern": "created PR ends up linked, with no watch or list", "experience": "Must keep working: an opened PR ends up linked to the thread, and the agent does not watch or list when no babysit was asked.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Can we pre-bundle the hot startup paths on shared dev servers? First load takes over a minute."}, {"role": "assistant", "text": "Prototype is on branch perf/prebundle-startup (commit abc1234). First page load on a shared dev server drops from about 64 s to 21 s. Want a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "git checkout -b perf/prebundle-startup"}, "result": "Switched to a new branch 'perf/prebundle-startup'"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts", "content": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts", "content": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts"}, {"tool": "Bash", "input": {"command": "node scripts/bench-first-load.mjs --runs 3"}, "result": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"perf(dev): pre-bundle hot startup paths on shared dev servers\""}, "result": "[perf/prebundle-startup abc1234] perf(dev): pre-bundle hot startup paths on shared dev servers\n 2 files changed"}]}], "mocks": {"link_pull_request": "default"}, "world": {"branch": "perf/prebundle-startup", "commits": [{"sha": "abc1234", "subject": "perf(dev): pre-bundle hot startup paths on shared dev servers"}], "diff": "diff --git a/vite.config.ts b/vite.config.ts\n--- a/vite.config.ts\n+++ b/vite.config.ts\n@@ -1,5 +1,6 @@\n import { defineConfig } from 'vite';\n+import { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n \n export default defineConfig({\n- optimizeDeps: {},\n+ optimizeDeps: { include: HOT_STARTUP_DEPS },\n });\ndiff --git a/scripts/hotStartupDeps.ts b/scripts/hotStartupDeps.ts\n--- a/scripts/hotStartupDeps.ts\n+++ b/scripts/hotStartupDeps.ts\n@@ -0,0 +1,2 @@\n+// Measured with scripts/bench-first-load.mjs on a shared dev server.\n+export const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "files": {"vite.config.ts": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n", "scripts/hotStartupDeps.ts": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "scripts/bench-first-load.mjs": "// Starts the dev server and times the first page load.\n// Usage: node scripts/bench-first-load.mjs --runs 3\nimport { startDevServer, timeFirstLoad } from './bench-lib.mjs';\n\nconst runs = Number(process.argv[3] ?? 3);\nfor (let i = 0; i < runs; i++) console.log(await timeFirstLoad(await startDevServer()));\n", "scripts/bench-lib.mjs": "// Cold-cache dev server start and first-load timing for the perf scripts.\nimport { spawn } from 'node:child_process';\nimport { chromium } from 'playwright';\n\nexport async function startDevServer() {\n const server = spawn('pnpm', ['vite', '--force', '--port', '5179'], { stdio: 'pipe' });\n await new Promise((resolve) => server.stdout.on('data', (b) => String(b).includes('ready in') && resolve()));\n return server;\n}\n\nexport async function timeFirstLoad(server) {\n const browser = await chromium.launch();\n const page = await browser.newPage();\n const start = performance.now();\n await page.goto('http://localhost:5179', { waitUntil: 'load' });\n const seconds = ((performance.now() - start) / 1000).toFixed(1);\n await browser.close();\n server.kill();\n return `${seconds}s`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "^node .*bench-first-load", "output": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}]}, "prompt": "Okay do it. Make a good PR.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests"], "linked": ["~/pull/124"]}, "notes": "117 of 117 PR threads linked after the fixes. A shorter text must keep this. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 14} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index d158015b4fb60..121814fe2da2f 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -71,6 +71,62 @@ it("grades Codex shell and subagent calls with the shared rules", () => { ), ["notCalls Agent", "shellMustNot /sleep/"], ); + const distinctCalls = ["application key", "application secret", "consumer key"].map((label) => ({ + tool: "request_secret", + args: { label: "~" + label, delivery: "file", value: null }, + })); + const cards = distinctCalls.map(({ tool, args }) => ({ + step: 0, + tool, + input: { ...args, label: args.label.slice(1) }, + })); + const run = (calls: SampleRun["calls"]) => ({ calls, text: "", stopReason: "end_turn" }); + assert.deepEqual(failed({ distinctCalls }, run(cards)), []); + assert.deepEqual( + failed( + { distinctCalls }, + run(cards.map((card) => ({ ...card, input: { ...card.input, value: "invented" } }))), + ), + ["distinctCalls"], + ); + assert.deepEqual(failed({ distinctCalls }, run([cards[0]!])), ["distinctCalls"]); + assert.deepEqual( + failed( + { distinctCalls }, + run( + cards.map((card, i) => + i === 1 ? { ...card, input: { ...card.input, delivery: "secretRef" } } : card, + ), + ), + ), + ["distinctCalls"], + ); + assert.deepEqual( + failed( + { distinctCalls }, + run([ + { + step: 0, + tool: "request_secret", + input: { label: "application key, application secret, consumer key", delivery: "file" }, + }, + ]), + ), + ["distinctCalls"], + ); + assert.deepEqual( + failed( + { distinctCalls }, + run([ + { + ...cards[0]!, + input: { ...cards[0]!.input, label: "application key, application secret, consumer key" }, + }, + ...cards.slice(1), + ]), + ), + ["distinctCalls"], + ); }); it("matches shellMust, not shellMustNot, against a command's argv lists as a command line", () => { diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 5b0cf7d1e1ee0..92ae6fe28cc4f 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -181,6 +181,8 @@ type McpToolUseCall = typeof McpToolUseCall.Type; const McpToolUseExpect = Schema.Struct({ /** Each entry needs one matching call. */ calls: Schema.optional(Schema.Array(McpToolUseCall)), + /** Each entry matches a different call, and no call matches two entries; tool names are exact. */ + distinctCalls: Schema.optional(Schema.Array(McpToolUseCall)), /** No call may match any entry. */ callsMustNot: Schema.optional(Schema.Array(McpToolUseCall)), notCalls: Schema.optional(Schema.Array(Schema.String)), @@ -1701,6 +1703,17 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pa calls.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input)); const label = ({ tool, args }: McpToolUseCall) => (args ? `${tool} ${valueText(args)}` : tool); for (const call of expect.calls ?? []) add(`calls ${label(call)}`, found(call)); + if (expect.distinctCalls !== undefined) { + const expected = expect.distinctCalls.map(({ tool, args }) => ({ tool, input: args ?? {} })); + add( + "distinctCalls", + calls.every((call) => expected.filter((entry) => matches(entry, call)).length <= 1) && + matchesDistinct( + expected, + calls.map((request) => ({ request, brief: "" })), + ), + ); + } for (const call of expect.callsMustNot ?? []) add(`callsMustNot ${label(call)}`, !found(call)); for (const tool of expect.notCalls ?? []) add(`notCalls ${tool}`, count(tool) === 0); if (expect.order !== undefined) {