diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 0a03b931fcc08..402c5f0892cf4 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5133, - "codexAdditionalContextBytes": 6863, + "claudeSystemAppendBytes": 5432, + "codexAdditionalContextBytes": 7162, "mcpToolListJsonBytes": 47818 } diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 41eb03e0939fc..7d2ab0a67059f 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -35,3 +35,5 @@ {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} {"id": "thread-send-fresh-context", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send that should start a fresh provider session", "experience": "The user wanted a confused long-running thread to start over in the same thread without its old context; a plain send would keep feeding it the history.", "rule": "`freshContext: true` on t3_thread_send starts a new provider session in the same thread, like /clear.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The thread working on the CSV importer keeps reverting to the old column mapping from early in its history."}, {"role": "assistant", "text": "That thread has a long history with three different mappings in it, so the model keeps picking up the stale one."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "messageId": "msg-1", "runId": "run-1", "status": "starting", "delivery": "started"}}}, "prompt": "Send it the final mapping below and have it start over with a clean context in that same thread, so the old history stops leaking in: thread 11111111-1111-4111-8111-111111111111.\n\nFinal mapping: id -> customer_id, amt -> amount_cents, ts -> created_at.", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "freshContext": true}}], "notCalls": ["delegate_task", "t3_thread_launch", "create_threads"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: fresh provider sessions in one T3 thread"}, "notes": "Pairs with thread-send-to-named-top-level-thread, where an ordinary send must not set freshContext."} +{"id": "explicit-cli-tmux-review", "area": "orchestration", "kind": "bad", "pattern": "internal delegation overrides an explicit CLI/tmux workflow", "experience": "Codex dispatched a Grok review through T3 delegation despite project instructions requiring a visible Bash/tmux session.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/30663"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "Project AGENTS.md: Grok reviews must run through the installed Grok CLI in a visible tmux session so I can attach and inspect. Use `tmux new-session -d -s grok-review \"grok --prompt 'Review the current diff in /home/user/acme/app and report findings.'\"`. The CLI is authenticated, tmux is installed, and this shell permits both.", "shellMocks": [{"match": "tmux new-session", "output": "(Bash completed with no output)"}, {"match": "tmux (list-sessions|has-session|capture-pane)", "output": "grok-review: 1 windows (created today)\nGrok review: no findings."}]}, "prompt": "Get a Grok review of the current diff.", "expect": {"calls": [{"tool": "Bash", "args": {"command": "~tmux new-session"}}, {"tool": "Bash", "args": {"command": "~tmux new-session.*grok"}}], "notCalls": ["delegate_task", "Agent", "Workflow", "t3_thread_launch", "create_threads"], "shellMustNot": ["^grok(?:\\s|$)"]}, "maxSteps": 8} +{"id": "explicit-cli-unavailable-no-substitution", "area": "orchestration", "kind": "bad", "pattern": "unavailable explicit CLI route silently replaced by internal delegation", "experience": "The reported workflow requires the agent to explain an unavailable CLI route before dispatching rather than substitute internal delegation.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/30663"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "Project AGENTS.md: All Grok reviews use the local Grok CLI inside tmux so I can attach. Environment: tmux is installed, but Grok CLI is not installed.", "shellMocks": [{"match": "(command -v|which) grok", "output": "Exit code 1: grok not found"}, {"match": "grok", "output": "Exit code 127: grok: command not found"}]}, "prompt": "Get a Grok review of the current diff.", "expect": {"notCalls": ["delegate_task", "Agent", "Workflow", "t3_thread_launch", "create_threads"], "textMust": ["[Gg]rok", "([Nn]ot|n['’]t) (installed|found|available)|[Mm]issing|[Uu]navailable|[Cc]annot|[Cc]an['’]t"], "callsMustNot": [{"tool": "Bash", "args": {"command": "~claude"}}, {"tool": "Bash", "args": {"command": "~codex"}}]}, "maxSteps": 8} diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index efe3fc242d9a5..ffbe8ea6d01ea 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -6,6 +6,8 @@ export const T3_CODE_ORCHESTRATION_SECTIONS: ReadonlyArray = [ Ordinary coding tasks need no T3 tools. +Explicit user or project instructions to run an external agent CLI (including Bash/tmux) take precedence over the delegation routing below. Use that route when supported; if unavailable or blocked, explain why before dispatching and stop. Never substitute another route without consent. Otherwise: + - Subagents: use your native tool when the child uses your provider, a model it offers, and no settings beyond yours (Claude \`Agent\`, \`run_in_background:false\`; Codex \`spawn_agent\`, then wait). Use \`delegate_task\` for another provider's model, user-asked settings, a separate checkout (\`isolation:"worktree"\`), or \`delegate_task\` by name. A native type on another provider's model counts too. Background agent: \`delegate_task\`, never a background \`Agent\`. Workflow only when asked. - Call \`orchestrator_capabilities\` once per thread, before the first \`delegate_task\`. Again only after an error or a user-named provider change. Settings go in \`target.options\`. Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs. Never make a child's checkout in the shell or in \`task\`. - Every new delegate review or re-check is a new \`delegate_task\` (brief, findings, objections), never a \`t3_thread_send\`. Fix small review findings yourself; start no delegate for them. When told to proceed without a delegate, or to run fewer, \`task_cancel\` the extras. Start no replacement.