From 20183e0adb718dd215c82d5ee0ab065bc6f591eb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:07:26 -0800 Subject: [PATCH] fix(mcp): expose account pool members for delegation checks --- apps/server/src/mcp/OrchestratorMcpService.test.ts | 6 ++++++ apps/server/src/mcp/OrchestratorMcpService.ts | 3 +++ evals/mcp-tool-use/cases/provider-pools.ndjson | 2 ++ packages/contracts/src/orchestratorMcp.ts | 14 ++++++++++++-- 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 evals/mcp-tool-use/cases/provider-pools.ndjson diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 864affc61bf1a..308d389650464 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -1582,6 +1582,7 @@ describe("OrchestratorMcpService provider resolution", () => { ); assert.isTrue(entry!.canRunCrossProviderChildTask); assert.deepEqual(entry!.constraints, []); + assert.isUndefined(entry!.poolMemberInstanceIds); } assert.isUndefined(byId.get(disabledAntigravityInstanceId)); @@ -2419,6 +2420,9 @@ describe("OrchestratorMcpService provider resolution", () => { capabilities.providers.map((provider) => provider.providerInstanceId), [poolInstanceId], ); + assert.deepInclude(capabilities.providers[0], { + poolMemberInstanceIds: [codexInstanceId, workInstanceId], + }); const result = yield* service.delegateTask(scope, { task: "Summarize the diff.", target: { driverKind: codex }, @@ -2426,6 +2430,8 @@ describe("OrchestratorMcpService provider resolution", () => { clientRequestId: "delegate-pool-driver-1", }); assert.equal(result.status, "running"); + assert.equal(result.providerInstanceId, workInstanceId); + assert.equal(result.model, "gpt-5.4"); const request = (yield* Ref.get(dispatched))[0] as { modelSelection: { instanceId: string; model: string }; }; diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 053899ac9f8da..db453f5ee7c09 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -2382,6 +2382,9 @@ const make = Effect.gen(function* () { }); return { providerInstanceId: provider.instanceId, + ...(provider.pool?.role === "pool" + ? { poolMemberInstanceIds: provider.pool.memberInstanceIds } + : {}), driverKind: provider.driver, displayName: provider.displayName ?? null, models: listed.map((model) => { diff --git a/evals/mcp-tool-use/cases/provider-pools.ndjson b/evals/mcp-tool-use/cases/provider-pools.ndjson new file mode 100644 index 0000000000000..1d877fce89ce4 --- /dev/null +++ b/evals/mcp-tool-use/cases/provider-pools.ndjson @@ -0,0 +1,2 @@ +{"id":"pool-member-provider-mismatch","area":"orchestration","kind":"bad","pattern":"pool route compared literally with selected account","experience":"An agent reported pool_claudeAgent versus claudeAgent as an unconfirmed provider mismatch despite matching model and effort.","source":{"report":"Maintainer report, 2026-11-05"},"privacy":"commit-safe","setup":{"role":"top","priorTurns":[{"role":"user","text":"Run the requested review with pool_claudeAgent, Claude Sonnet 5.5, high effort. Report any provider, model, or option mismatch."},{"role":"assistant","toolCalls":[{"tool":"orchestrator_capabilities","input":{},"result":{"providers":[{"providerInstanceId":"pool_claudeAgent","driverKind":"claudeAgent","displayName":"Claude pool","models":[{"id":"claude-sonnet-5-5","options":[{"id":"effort","values":["high"]}]}],"poolMemberInstanceIds":["claudeAgent","claudeAgent_work"]}]}}]},{"role":"assistant","toolCalls":[{"tool":"delegate_task","input":{"task":"Review the diff. Do not edit files.","role":"review","mode":"wait","target":{"providerInstanceId":"pool_claudeAgent","model":"claude-sonnet-5-5","options":{"effort":"high"}}},"result":{"taskId":"task-pool-review","childThreadId":"44444444-4444-4444-8444-444444444444","childRunId":"run-pool-review","status":"completed","workState":"result_available","providerInstanceId":"claudeAgent","model":"claude-sonnet-5-5","options":[{"id":"effort","value":"high"}],"summary":"No findings.","hasPendingQuestion":false,"pendingApproval":null}}]}]},"prompt":"Is the provider result a real mismatch? Use the tool evidence above to give a short, definite answer. Do not launch more work.","expect":{"notCalls":["delegate_task","t3_thread_send","task_status","task_cancel"],"finalTextMust":["(?i:(expected|confirmed|valid|correct|matches|no (?:provider )?mismatch|not a (?:provider )?mismatch))","(?i:(pool|member|account))"],"textMustNot":["(?i:(unconfirmed|haven.t confirmed|cannot confirm|can.t confirm|unable to confirm|assuming|presum(?:e|ing)|appears|likely|probably))"]},"maxSteps":4} +{"id":"pool-unrelated-provider-mismatch","area":"orchestration","kind":"guard","pattern":"pool route compared literally with selected account","experience":"An agent reported pool_claudeAgent versus claudeAgent as an unconfirmed provider mismatch despite matching model and effort.","source":{"report":"Maintainer report, 2026-11-05"},"privacy":"commit-safe","setup":{"role":"top","priorTurns":[{"role":"user","text":"Run the requested review with pool_claudeAgent, Claude Sonnet 5.5, high effort. Report any provider, model, or option mismatch."},{"role":"assistant","toolCalls":[{"tool":"orchestrator_capabilities","input":{},"result":{"providers":[{"providerInstanceId":"pool_claudeAgent","driverKind":"claudeAgent","displayName":"Claude pool","models":[{"id":"claude-sonnet-5-5","options":[{"id":"effort","values":["high"]}]}],"poolMemberInstanceIds":["claudeAgent","claudeAgent_work"]}]}}]},{"role":"assistant","toolCalls":[{"tool":"delegate_task","input":{"task":"Review the diff. Do not edit files.","role":"review","mode":"wait","target":{"providerInstanceId":"pool_claudeAgent","model":"claude-sonnet-5-5","options":{"effort":"high"}}},"result":{"taskId":"task-pool-review","childThreadId":"44444444-4444-4444-8444-444444444444","childRunId":"run-pool-review","status":"completed","workState":"result_available","providerInstanceId":"claudeAgent_other","model":"claude-sonnet-5-5","options":[{"id":"effort","value":"high"}],"summary":"No findings.","hasPendingQuestion":false,"pendingApproval":null}}]}]},"prompt":"Is the provider result a real mismatch? Use the tool evidence above to give a short, definite answer. Do not launch more work.","expect":{"notCalls":["delegate_task","t3_thread_send","task_status","task_cancel"],"finalTextMust":["(?i:(mismatch|not (?:a |in (?:the )?)?(?:pool )?member|outside|not listed))","claudeAgent_other"],"textMustNot":["(?i:(no (?:provider )?mismatch|not a (?:provider )?mismatch|expected (?:pool )?routing|confirmed (?:pool )?routing))"]},"maxSteps":4} diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 70caaf172a976..e98e8b535bd96 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -90,7 +90,8 @@ export type OrchestratorMcpTargetOptions = typeof OrchestratorMcpTargetOptions.T export const OrchestratorMcpTarget = Schema.Struct({ providerInstanceId: Schema.optional( ProviderInstanceId.annotate({ - description: "Provider instance id from `orchestrator_capabilities`.", + description: + "Provider instance id from `orchestrator_capabilities`. Account pools route to a concrete member; results report that member's id. Verify it against the pool's poolMemberInstanceIds, and check model and options separately.", }), ), driverKind: Schema.optional( @@ -214,7 +215,10 @@ export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ latestTerminalStatus: Schema.NullOr(OrchestratorMcpTerminalDelegatedTaskStatus), latestTerminalSummary: Schema.NullOr(Schema.String), latestTerminalResultContextTransferId: Schema.NullOr(ContextTransferId), - providerInstanceId: ProviderInstanceId, + providerInstanceId: ProviderInstanceId.annotate({ + description: + "Concrete provider account used by the task. For a pool target, this is a member id from orchestrator_capabilities, not the pool id.", + }), model: Schema.NullOr(Schema.String), options: Schema.optional(Schema.Array(ProviderOptionSelection)), summary: Schema.NullOr(Schema.String), @@ -595,6 +599,12 @@ export type OrchestratorMcpModelOption = typeof OrchestratorMcpModelOption.Type; export const OrchestratorMcpProviderCapability = Schema.Struct({ providerInstanceId: ProviderInstanceId, + poolMemberInstanceIds: Schema.optional( + Schema.Array(ProviderInstanceId).annotate({ + description: + "Present for account pools. Configured member ids, including temporarily unavailable accounts. Delegation reports the selected concrete member as providerInstanceId; a listed member is expected pool routing. Model and options must still match.", + }), + ), driverKind: ProviderDriverKind, displayName: Schema.NullOr(Schema.String), models: Schema.Array(