diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 65f2b6b6c451e..e7901a8656ebe 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5597, - "codexAdditionalContextBytes": 7327, + "claudeSystemAppendBytes": 5594, + "codexAdditionalContextBytes": 7324, "mcpToolListJsonBytes": 48094 } diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 28e89df5597e8..ead48236bd922 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -34,6 +34,8 @@ {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} +{"id": "vps-delegate-links-off-names-setting", "area": "orchestration", "kind": "bad", "pattern": "hidden link tools, setting never named", "experience": "With Linked environments off, the default, an agent asked to delegate to the user's VPS searched for t3_environment_link, then said the tool was not in this build and never named the setting that turns it on.", "rule": "The link tools are absent while Settings → Labs → Linked environments is off; the agent tells the user to turn it on there.", "privacy": "commit-safe", "setup": {"role": "top"}, "prompt": "Delegate this to my vps: run `uname -a` there and tell me the result. Do not edit any files here.", "expect": {"textMust": ["Labs", "Linked environments"], "notCalls": ["delegate_task", "t3_environment_links", "t3_environment_link"]}, "source": {"issue": "pingdotgg/t3code#30358"}} +{"id": "vps-delegate-links-on-calls-link", "area": "orchestration", "kind": "control", "pattern": "unlinked machine with Linked environments on", "experience": "With Linked environments on, a machine the user names that is not linked gets a t3_environment_link card, not advice to turn the setting on.", "rule": "A machine the user names that is not linked: call t3_environment_link with a hint from their words.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_environment_link": {"status": "pending", "turnItemId": "turn-item-link-card-1"}}}, "prompt": "Delegate this to my vps: run `uname -a` there and tell me the result. Do not edit any files here.", "expect": {"calls": [{"tool": "t3_environment_link"}], "textMustNot": ["Labs", "([Tt]urn on|[Ee]nable)[^.\\n]*[Ll]inked environments"]}, "source": {"issue": "pingdotgg/t3code#30358"}} {"id": "background-wait-unmatched-id", "area": "orchestration", "kind": "guard", "pattern": "waits on a task ID that matches no background work, then ends the turn as if waiting", "experience": "One agent called t3_background_wait with the literal ID 'placeholder', another with a Claude subagent ID. The tool returned a normal result listing the ID under `unmatched`, so the agent said it would continue when the work came back and ended its turn. Nothing ever woke it.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["placeholder"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"calls": [{"tool": "t3_background_wait", "args": {"taskIds": ["bq7m2x9k"]}}]}, "source": {"issue": "#29772"}, "notes": "The seeded wait gets T3's real result for an ID the session does not know. The tests are still running, so the only way to keep the thread waiting is a wait on bq7m2x9k. In replay, Sonnet 5.5 and Opus 5.5 also recover from main's success result, so this guards the recovery rather than reproducing the incident. Claude only: t3_background_wait is a Claude tool.", "maxSteps": 8} {"id": "background-wait-matched-id", "area": "orchestration", "kind": "control", "pattern": "waits on the right task ID, then ends the turn", "experience": "With the background command's own ID registered, ending the turn is correct: T3 keeps the thread working and wakes it when the tests finish.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["bq7m2x9k"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"maxCalls": {"t3_background_wait": 1, "Bash": 3}, "notCalls": ["ScheduleWakeup"]}, "source": {"issue": "#29772"}, "notes": "Same thread as background-wait-unmatched-id, with a wait T3 matched. Guards against a fix that makes agents distrust a good wait and poll; a few looks at the output are ordinary. Claude only.", "maxSteps": 8} {"id": "settle-this-thread-after-cleanup", "area": "orchestration", "kind": "bad", "pattern": "settle read as plain English for finish", "experience": "Asked to clean up scratch files and settle this thread, the agent deleted the files and ended the turn without calling t3_thread_organize (#26749).", "rule": "A request to settle this thread calls t3_thread_organize with action settle.", "privacy": "commit-safe", "notes": "From #26749. Claude defers MCP tools, so only the tool's name is visible until ToolSearch loads it; 'settle' appears only in the t3_thread_organize description.", "setup": {"role": "top", "world": {"files": {"scratch/run-1.log": "ok\n", "scratch/run-2.log": "ok\n", "src/index.ts": "export const x = 1;\n"}}, "mocks": {"t3_thread_organize": {"settlesWhenTurnEnds": true}}}, "prompt": "Clean up the scratch files and settle this thread.", "expect": {"calls": [{"tool": "t3_thread_organize", "args": {"action": "settle"}}], "notCalls": ["t3_thread_update"]}} diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index bb4d33d2c56e2..4ce08d71119fe 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -13,8 +13,8 @@ Ordinary coding tasks need no T3 tools. - As a subagent or child, do the work yourself. Start no subagents unless your task says to.`, `- \`t3_thread_launch\` and \`create_threads\` make new top-level threads, only when the user asks for a new thread. "Subagent", "delegate", and "parallel" never mean one. Worktree thread: set \`workspaceStrategy\`. Honor a requested base. Otherwise independent work uses the default branch as \`remote/branch\` with \`startFromOrigin:true\`; with no remote, use the local default. Required unpublished commits and dependent work use their local base. Shell worktrees don't bind it, and no checkout setup in its prompt. No \`workspaceStrategy\`: project root, not your worktree. - Given a thread id, \`t3_thread_read\` it before you answer about it. Asked to settle a thread, this one too: \`t3_thread_organize\`. -- Another machine: \`t3_environment_links\` lists linked ones. Pass its \`environmentId\` to a T3 tool to act there. Ids are then that machine's, so pass \`threadId\`. -- A machine the user names that isn't linked ("delegate this to my vps"): call \`t3_environment_link\` with a \`hint\` from their words. Never ask for a pairing code in chat. +- Another machine: \`t3_environment_links\` lists linked ones. Pass its \`environmentId\` and that machine's \`threadId\` to T3 tools. +- Unlinked ("delegate this to my vps"): \`t3_environment_link\` with a \`hint\` from the user's words, never a pairing code in chat. No link tools: say to enable Settings → Labs → Linked environments. - In your reply, mention a thread by its \`link\`, verbatim, never a plain title or bare id. - A login, token, PIN, pairing code or sudo password only the user can give: call \`request_secret\`, not a chat ask or a handoff. Not for access you have. Sudo: if \`sudo -n \` needs a password, use \`delivery: "file"\` and run \`sudo -S -p '' < "$file"\`; the password takes stdin, so give other input as a file argument. Asked only to schedule: \`schedule_task\` without running its prompt. - T3 tools may load late. Try the name once before you call it absent.`,