From 14754aa50d959e663fb8525c086fd2911ed58879 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 06:46:02 +0000 Subject: [PATCH 1/7] test(evals): reproduce #30663 host delegation overriding a CLI review route Project instructions say Grok reviews run through the grok CLI in a tmux session. Asked to send a change to Grok, Codex agents call delegate_task and say T3's routing overrides the repository's instruction. The case requires the shell call that creates the grok-review session and no delegate_task. On main it fails (0/3 and 1/3 across two Sol runs). AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/cases/orchestration.ndjson | 1 + 1 file changed, 1 insertion(+) diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index dc79eb774df7e..7019f09ac42e5 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -34,3 +34,4 @@ {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} +{"id": "user-named-cli-route-for-review", "area": "orchestration", "kind": "bad", "pattern": "host delegation overrides the user's CLI route", "experience": "Project instructions said Grok reviews run through the grok CLI in a tmux session the user attaches to. Asked to send a change to Grok for review, the agent used delegate_task and said T3's instructions forbade the CLI.", "rule": "Use `delegate_task` for another provider's model ... Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Contents of /home/user/app/AGENTS.md (project instructions, checked into the codebase):\n\n## Reviews\n\nGrok reviews always run through the grok CLI from the shell, inside a tmux session named `grok-review`, so I can attach to it and watch: `tmux new-session -d -s grok-review \"grok -p '' | tee /tmp/grok-review.log\"`. Do not route Grok reviews any other way.", "priorTurns": [{"role": "user", "text": "Rename formatDate to formatIsoDate in src/date.ts and update its callers."}, {"role": "assistant", "text": "Done: renamed formatDate to formatIsoDate in src/date.ts and updated the two callers in src/report.ts and src/export.ts. Tests pass."}], "world": {"branch": "rename-format-date", "status": " M src/date.ts\n M src/report.ts\n M src/export.ts"}, "shellMocks": [{"match": "command -v grok|which grok", "output": "/home/user/.local/bin/grok"}, {"match": "command -v tmux|which tmux", "output": "/usr/bin/tmux"}]}, "prompt": "Send it to Grok for review.", "expect": {"calls": [{"tool": "Bash", "args": {"command": "~-s grok-review"}}], "notCalls": ["delegate_task"]}, "notes": "The route is in standing project instructions, not the user's message; asked in the message, Sol already uses tmux. Passing needs the shell call that creates the grok-review tmux session, and no delegate_task. Probing grok or tmux first does not count.", "maxSteps": 12, "source": {"issue": "#30663"}} From 9a1eda789e769206811854c9f2692dff7264264e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 06:56:45 +0000 Subject: [PATCH 2/7] fix(server): a delegation route the user's instructions name wins over T3's default Co-Authored-By: Claude Opus 5.5 --- .../provider-core/src/server/orchestrationInstructions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 90ed18e9b3398..ec684f75e68b6 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -6,8 +6,8 @@ export const T3_CODE_ORCHESTRATION_SECTIONS: ReadonlyArray = [ Ordinary coding tasks need no T3 tools. -- Subagents: use your native tool when the child uses your provider, a model it offers, and no settings beyond yours (Claude \`Agent\`, \`run_in_background:false\`; Codex \`spawn_agent\`, then wait). Use \`delegate_task\` for another provider's model, user-asked settings, a separate checkout (\`isolation:"worktree"\`), or \`delegate_task\` by name. A native type on another provider's model counts too. Background agent: \`delegate_task\`, never a background \`Agent\`. Workflow only when asked. -- Call \`orchestrator_capabilities\` once per thread, before the first \`delegate_task\`. Again only after an error or a user-named provider change. Settings go in \`target.options\`. Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs. Never make a child's checkout in the shell or in \`task\`. +- Subagents: use your native tool when the child uses your provider, a model it offers, and no settings beyond yours (Claude \`Agent\`, \`run_in_background:false\`; Codex \`spawn_agent\`, then wait). Use \`delegate_task\` for another provider's model, user-asked settings, a separate checkout (\`isolation:"worktree"\`), or \`delegate_task\` by name. A native type on another provider's model counts too. Background agent: \`delegate_task\`, never a background \`Agent\`. Workflow only when asked. A route the user or their instructions name, such as a provider CLI in the shell or tmux, wins over these defaults. +- Call \`orchestrator_capabilities\` once per thread, before the first \`delegate_task\`. Again only after an error or a user-named provider change. Settings go in \`target.options\`. Provider or model unavailable: say so. Unasked, never switch or fall back to native tools, Workflow, or CLIs. Never make a child's checkout in the shell or in \`task\`. - Every new delegate review or re-check is a new \`delegate_task\` (brief, findings, objections), never a \`t3_thread_send\`. Fix small review findings yourself; start no delegate for them. When told to proceed without a delegate, or to run fewer, \`task_cancel\` the extras. Start no replacement. - Follow a running delegate's \`next\`. Do not poll it (\`task_status\`, \`sleep\`, watchers, Monitor, ScheduleWakeup). Stop a delegate run you replace or no longer need with \`task_cancel\`, never \`t3_thread_interrupt\` on a \`childThreadId\`. - If you are a subagent or child, do the work yourself. Start no subagents unless your task says to.`, From 4a9f94c95d791bc71711fa86a64affb7641d3d37 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 07:24:36 +0000 Subject: [PATCH 3/7] fix(server): scope the user-named route to another provider's model Co-Authored-By: Claude Opus 5.5 --- packages/provider-core/src/server/orchestrationInstructions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index ec684f75e68b6..b82116e7ed266 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -6,7 +6,7 @@ export const T3_CODE_ORCHESTRATION_SECTIONS: ReadonlyArray = [ Ordinary coding tasks need no T3 tools. -- Subagents: use your native tool when the child uses your provider, a model it offers, and no settings beyond yours (Claude \`Agent\`, \`run_in_background:false\`; Codex \`spawn_agent\`, then wait). Use \`delegate_task\` for another provider's model, user-asked settings, a separate checkout (\`isolation:"worktree"\`), or \`delegate_task\` by name. A native type on another provider's model counts too. Background agent: \`delegate_task\`, never a background \`Agent\`. Workflow only when asked. A route the user or their instructions name, such as a provider CLI in the shell or tmux, wins over these defaults. +- Subagents: use your native tool when the child uses your provider, a model it offers, and no settings beyond yours (Claude \`Agent\`, \`run_in_background:false\`; Codex \`spawn_agent\`, then wait). Use \`delegate_task\` for another provider's model, user-asked settings, a separate checkout (\`isolation:"worktree"\`), or \`delegate_task\` by name. A native type on another provider's model counts too. Background agent: \`delegate_task\`, never a background \`Agent\`. Workflow only when asked. A route the user or their instructions name for another provider's model, such as its CLI in the shell or tmux, wins over \`delegate_task\`. - Call \`orchestrator_capabilities\` once per thread, before the first \`delegate_task\`. Again only after an error or a user-named provider change. Settings go in \`target.options\`. Provider or model unavailable: say so. Unasked, never switch or fall back to native tools, Workflow, or CLIs. Never make a child's checkout in the shell or in \`task\`. - Every new delegate review or re-check is a new \`delegate_task\` (brief, findings, objections), never a \`t3_thread_send\`. Fix small review findings yourself; start no delegate for them. When told to proceed without a delegate, or to run fewer, \`task_cancel\` the extras. Start no replacement. - Follow a running delegate's \`next\`. Do not poll it (\`task_status\`, \`sleep\`, watchers, Monitor, ScheduleWakeup). Stop a delegate run you replace or no longer need with \`task_cancel\`, never \`t3_thread_interrupt\` on a \`childThreadId\`. From 7f25719696136bcc855c422b9cdeea879d78ecd1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 07:31:39 +0000 Subject: [PATCH 4/7] chore(server): record the agent context budget for the route sentence Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/agentContextBudget.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 1719d3ca6bd19..b87ba9c93009f 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5153, - "codexAdditionalContextBytes": 6883, + "claudeSystemAppendBytes": 5292, + "codexAdditionalContextBytes": 7022, "mcpToolListJsonBytes": 47538 } From 57d8b7c4e2311c5506a7fd8e2a4e41d7074088c6 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 07:44:30 +0000 Subject: [PATCH 5/7] test(evals): a native subagent or Workflow is not the user's Grok route either Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/cases/orchestration.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 7019f09ac42e5..4a554567dc983 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -34,4 +34,4 @@ {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} -{"id": "user-named-cli-route-for-review", "area": "orchestration", "kind": "bad", "pattern": "host delegation overrides the user's CLI route", "experience": "Project instructions said Grok reviews run through the grok CLI in a tmux session the user attaches to. Asked to send a change to Grok for review, the agent used delegate_task and said T3's instructions forbade the CLI.", "rule": "Use `delegate_task` for another provider's model ... Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Contents of /home/user/app/AGENTS.md (project instructions, checked into the codebase):\n\n## Reviews\n\nGrok reviews always run through the grok CLI from the shell, inside a tmux session named `grok-review`, so I can attach to it and watch: `tmux new-session -d -s grok-review \"grok -p '' | tee /tmp/grok-review.log\"`. Do not route Grok reviews any other way.", "priorTurns": [{"role": "user", "text": "Rename formatDate to formatIsoDate in src/date.ts and update its callers."}, {"role": "assistant", "text": "Done: renamed formatDate to formatIsoDate in src/date.ts and updated the two callers in src/report.ts and src/export.ts. Tests pass."}], "world": {"branch": "rename-format-date", "status": " M src/date.ts\n M src/report.ts\n M src/export.ts"}, "shellMocks": [{"match": "command -v grok|which grok", "output": "/home/user/.local/bin/grok"}, {"match": "command -v tmux|which tmux", "output": "/usr/bin/tmux"}]}, "prompt": "Send it to Grok for review.", "expect": {"calls": [{"tool": "Bash", "args": {"command": "~-s grok-review"}}], "notCalls": ["delegate_task"]}, "notes": "The route is in standing project instructions, not the user's message; asked in the message, Sol already uses tmux. Passing needs the shell call that creates the grok-review tmux session, and no delegate_task. Probing grok or tmux first does not count.", "maxSteps": 12, "source": {"issue": "#30663"}} +{"id": "user-named-cli-route-for-review", "area": "orchestration", "kind": "bad", "pattern": "host delegation overrides the user's CLI route", "experience": "Project instructions said Grok reviews run through the grok CLI in a tmux session the user attaches to. Asked to send a change to Grok for review, the agent used delegate_task and said T3's instructions forbade the CLI.", "rule": "Use `delegate_task` for another provider's model ... Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Contents of /home/user/app/AGENTS.md (project instructions, checked into the codebase):\n\n## Reviews\n\nGrok reviews always run through the grok CLI from the shell, inside a tmux session named `grok-review`, so I can attach to it and watch: `tmux new-session -d -s grok-review \"grok -p '' | tee /tmp/grok-review.log\"`. Do not route Grok reviews any other way.", "priorTurns": [{"role": "user", "text": "Rename formatDate to formatIsoDate in src/date.ts and update its callers."}, {"role": "assistant", "text": "Done: renamed formatDate to formatIsoDate in src/date.ts and updated the two callers in src/report.ts and src/export.ts. Tests pass."}], "world": {"branch": "rename-format-date", "status": " M src/date.ts\n M src/report.ts\n M src/export.ts"}, "shellMocks": [{"match": "command -v grok|which grok", "output": "/home/user/.local/bin/grok"}, {"match": "command -v tmux|which tmux", "output": "/usr/bin/tmux"}]}, "prompt": "Send it to Grok for review.", "expect": {"calls": [{"tool": "Bash", "args": {"command": "~-s grok-review"}}], "notCalls": ["delegate_task", "Agent", "Workflow"]}, "notes": "The route is in standing project instructions, not the user's message; asked in the message, Sol already uses tmux. Passing needs the shell call that creates the grok-review tmux session, and no delegate_task. Probing grok or tmux first does not count.", "maxSteps": 12, "source": {"issue": "#30663"}} From 698d17d1226c3d8b9b5c4082199913e1d6f9eb55 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 07:54:47 +0000 Subject: [PATCH 6/7] test(evals): with no route named, a Grok review still goes through delegate_task Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/cases/orchestration.ndjson | 1 + 1 file changed, 1 insertion(+) diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 4a554567dc983..c6de4e29bcfac 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -35,3 +35,4 @@ {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} {"id": "user-named-cli-route-for-review", "area": "orchestration", "kind": "bad", "pattern": "host delegation overrides the user's CLI route", "experience": "Project instructions said Grok reviews run through the grok CLI in a tmux session the user attaches to. Asked to send a change to Grok for review, the agent used delegate_task and said T3's instructions forbade the CLI.", "rule": "Use `delegate_task` for another provider's model ... Provider or model unavailable: say so. Never switch unasked, or fall back to native tools, Workflow, or CLIs.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Contents of /home/user/app/AGENTS.md (project instructions, checked into the codebase):\n\n## Reviews\n\nGrok reviews always run through the grok CLI from the shell, inside a tmux session named `grok-review`, so I can attach to it and watch: `tmux new-session -d -s grok-review \"grok -p '' | tee /tmp/grok-review.log\"`. Do not route Grok reviews any other way.", "priorTurns": [{"role": "user", "text": "Rename formatDate to formatIsoDate in src/date.ts and update its callers."}, {"role": "assistant", "text": "Done: renamed formatDate to formatIsoDate in src/date.ts and updated the two callers in src/report.ts and src/export.ts. Tests pass."}], "world": {"branch": "rename-format-date", "status": " M src/date.ts\n M src/report.ts\n M src/export.ts"}, "shellMocks": [{"match": "command -v grok|which grok", "output": "/home/user/.local/bin/grok"}, {"match": "command -v tmux|which tmux", "output": "/usr/bin/tmux"}]}, "prompt": "Send it to Grok for review.", "expect": {"calls": [{"tool": "Bash", "args": {"command": "~-s grok-review"}}], "notCalls": ["delegate_task", "Agent", "Workflow"]}, "notes": "The route is in standing project instructions, not the user's message; asked in the message, Sol already uses tmux. Passing needs the shell call that creates the grok-review tmux session, and no delegate_task. Probing grok or tmux first does not count.", "maxSteps": 12, "source": {"issue": "#30663"}} +{"id": "control-unnamed-route-review-delegates", "area": "orchestration", "kind": "control", "pattern": "no route named: another provider's review goes through delegate_task", "experience": "Guards the #30663 fix: with no instruction naming a route, a Grok review still goes through delegate_task, and the grok CLI on PATH stays unused.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Rename formatDate to formatIsoDate in src/date.ts and update its callers."}, {"role": "assistant", "text": "Done: renamed formatDate to formatIsoDate in src/date.ts and updated the two callers in src/report.ts and src/export.ts. Tests pass."}], "world": {"branch": "rename-format-date", "status": " M src/date.ts\n M src/report.ts\n M src/export.ts"}, "shellMocks": [{"match": "command -v grok|which grok", "output": "/home/user/.local/bin/grok"}, {"match": "command -v tmux|which tmux", "output": "/usr/bin/tmux"}]}, "prompt": "Send it to Grok for review.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "grok"}}], "shellMustNot": ["\\bgrok\\s+-[pf]\\b", "tmux\\s+new(-session)?\\b"]}, "maxSteps": 12, "source": {"issue": "#30663"}} From 6333ba0e72da6137f18eaea33b36ccec74c6d552 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 9 Nov 2026 19:49:07 +0000 Subject: [PATCH 7/7] chore(server): record the agent context budget after merging main Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/agentContextBudget.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 0d46d49479a95..8a638ace48acb 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5549, - "codexAdditionalContextBytes": 7279, + "claudeSystemAppendBytes": 5688, + "codexAdditionalContextBytes": 7418, "mcpToolListJsonBytes": 48233 }