diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 4439c50940722..7c2442ed824f4 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5597, - "codexAdditionalContextBytes": 7327, + "claudeSystemAppendBytes": 5736, + "codexAdditionalContextBytes": 7466, "mcpToolListJsonBytes": 48217 } diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index e69f14f7cc547..da65829d483d1 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -34,4 +34,6 @@ {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} +{"id": "user-named-cli-route-for-review", "area": "orchestration", "kind": "bad", "pattern": "host delegation overrides the user's CLI route", "experience": "Project instructions said Grok reviews run through the grok CLI in a tmux session the user attaches to. Asked to send a change to Grok for review, the agent used delegate_task and said T3's instructions forbade the CLI.", "rule": "Use `delegate_task` for another provider's model ... Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Contents of /home/user/app/AGENTS.md (project instructions, checked into the codebase):\n\n## Reviews\n\nGrok reviews always run through the grok CLI from the shell, inside a tmux session named `grok-review`, so I can attach to it and watch: `tmux new-session -d -s grok-review \"grok -p '' | tee /tmp/grok-review.log\"`. Do not route Grok reviews any other way.", "priorTurns": [{"role": "user", "text": "Rename formatDate to formatIsoDate in src/date.ts and update its callers."}, {"role": "assistant", "text": "Done: renamed formatDate to formatIsoDate in src/date.ts and updated the two callers in src/report.ts and src/export.ts. Tests pass."}], "world": {"branch": "rename-format-date", "status": " M src/date.ts\n M src/report.ts\n M src/export.ts"}, "shellMocks": [{"match": "command -v grok|which grok", "output": "/home/user/.local/bin/grok"}, {"match": "command -v tmux|which tmux", "output": "/usr/bin/tmux"}]}, "prompt": "Send it to Grok for review.", "expect": {"calls": [{"tool": "Bash", "args": {"command": "~-s grok-review"}}], "notCalls": ["delegate_task", "Agent", "Workflow"]}, "notes": "The route is in standing project instructions, not the user's message; asked in the message, Sol already uses tmux. Passing needs the shell call that creates the grok-review tmux session, and no delegate_task. Probing grok or tmux first does not count.", "maxSteps": 12, "source": {"issue": "#30663"}} +{"id": "control-unnamed-route-review-delegates", "area": "orchestration", "kind": "control", "pattern": "no route named: another provider's review goes through delegate_task", "experience": "Guards the #30663 fix: with no instruction naming a route, a Grok review still goes through delegate_task, and the grok CLI on PATH stays unused.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Rename formatDate to formatIsoDate in src/date.ts and update its callers."}, {"role": "assistant", "text": "Done: renamed formatDate to formatIsoDate in src/date.ts and updated the two callers in src/report.ts and src/export.ts. Tests pass."}], "world": {"branch": "rename-format-date", "status": " M src/date.ts\n M src/report.ts\n M src/export.ts"}, "shellMocks": [{"match": "command -v grok|which grok", "output": "/home/user/.local/bin/grok"}, {"match": "command -v tmux|which tmux", "output": "/usr/bin/tmux"}]}, "prompt": "Send it to Grok for review.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "grok"}}], "shellMustNot": ["\\bgrok\\s+-[pf]\\b", "tmux\\s+new(-session)?\\b"]}, "maxSteps": 12, "source": {"issue": "#30663"}} {"id": "thread-send-fresh-context", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send that should start a fresh provider session", "experience": "The user wanted a confused long-running thread to start over in the same thread without its old context; a plain send would keep feeding it the history.", "rule": "`freshContext: true` on t3_thread_send starts a new provider session in the same thread, like /clear.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The thread working on the CSV importer keeps reverting to the old column mapping from early in its history."}, {"role": "assistant", "text": "That thread has a long history with three different mappings in it, so the model keeps picking up the stale one."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "messageId": "msg-1", "runId": "run-1", "status": "starting", "delivery": "started"}}}, "prompt": "Send it the final mapping below and have it start over with a clean context in that same thread, so the old history stops leaking in: thread 11111111-1111-4111-8111-111111111111.\n\nFinal mapping: id -> customer_id, amt -> amount_cents, ts -> created_at.", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "freshContext": true}}], "notCalls": ["delegate_task", "t3_thread_launch", "create_threads"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: fresh provider sessions in one T3 thread"}, "notes": "Pairs with thread-send-to-named-top-level-thread, where an ordinary send must not set freshContext."} diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 9c9720edf18b7..7ea5e9698578a 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -6,8 +6,8 @@ export const T3_CODE_ORCHESTRATION_SECTIONS: ReadonlyArray = [ Ordinary coding tasks need no T3 tools. -- Subagents: use your native tool when the child uses your provider, a model it offers, and no settings beyond yours (Claude \`Agent\`, \`run_in_background:false\`; Codex \`spawn_agent\`, then wait). Use \`delegate_task\` for another provider's model, user-asked settings, a separate checkout (\`isolation:"worktree"\`), or \`delegate_task\` by name. A native type on another provider's model counts too. Background agent: \`delegate_task\`, never a background \`Agent\`. Workflow only when asked. -- Call \`orchestrator_capabilities\` once per thread, before the first \`delegate_task\`. Again only after an error or a user-named provider change. Settings go in \`target.options\`. Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs. Never make a child's checkout in the shell or in \`task\`. +- Subagents: use your native tool when the child uses your provider, a model it offers, and no settings beyond yours (Claude \`Agent\`, \`run_in_background:false\`; Codex \`spawn_agent\`, then wait). Use \`delegate_task\` for another provider's model, user-asked settings, a separate checkout (\`isolation:"worktree"\`), or \`delegate_task\` by name. A native type on another provider's model counts too. Background agent: \`delegate_task\`, never a background \`Agent\`. Workflow only when asked. A route the user or their instructions name for another provider's model, such as its CLI in the shell or tmux, wins over \`delegate_task\`. +- Call \`orchestrator_capabilities\` once per thread, before the first \`delegate_task\`. Again only after an error or a user-named provider change. Settings go in \`target.options\`. Provider or model unavailable: say so. Unasked, never switch or fall back to native tools, Workflow, or CLIs. Never make a child's checkout in the shell or in \`task\`. - Every new delegate review or re-check is a new \`delegate_task\` (brief, findings, objections), never a \`t3_thread_send\`. Fix small review findings yourself; start no delegate for them. When told to proceed without a delegate, or to run fewer, \`task_cancel\` the extras. Start no replacement. - Follow a running delegate's \`next\`. Do not poll it (\`task_status\`, \`sleep\`, watchers, Monitor, ScheduleWakeup). Stop a delegate run you replace or no longer need with \`task_cancel\`, never \`t3_thread_interrupt\` on a \`childThreadId\`. - If you are a subagent or child, do the work yourself. Start no subagents unless your task says to.`,