From 785dce35521062be2408e7ccbf21cf1a2f13f480 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Fri, 30 Oct 2026 17:48:03 +0000 Subject: [PATCH 01/21] fix(server): delegated Claude runs report the model that actually ran When Claude substitutes a model (for example an organization that disallows Opus runs the turn on Sonnet), init and every root assistant frame report the substitute, but T3 only kept the requested selection. Parent agents read task_status and thread runs and saw the requested Opus. The Claude adapter now records the root model Claude reports on each turn's provider turn, ignoring subagent and synthetic frames and starting empty for every turn. task_status and t3_thread_read runs expose it as executedModel beside the requested model, taken from that run's own attempt; it is null when the provider reported none. The requested selection is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../OrchestratorMcpService.activity.test.ts | 20 ++++- .../src/mcp/OrchestratorMcpService.test.ts | 34 +++++++- apps/server/src/mcp/OrchestratorMcpService.ts | 21 ++++- .../Adapters/ClaudeAdapterV2.test.ts | 83 ++++++++++++++++++- .../Adapters/ClaudeAdapterV2.ts | 25 ++++++ evals/mcp-tool-use/cases/orchestration.ndjson | 1 + packages/contracts/src/orchestrationV2.ts | 2 + packages/contracts/src/orchestratorMcp.ts | 8 ++ 8 files changed, 185 insertions(+), 9 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts index c35b605dca4fb..2125383710290 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts @@ -6,6 +6,7 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, + RunAttemptId, RunId, ScheduledTaskId, SECRET_REQUEST_UNSAVED_NOTE, @@ -48,6 +49,7 @@ const childThreadId = ThreadId.make("thread-mcp-orchestrator-child"); const activeRunId = RunId.make("run-mcp-active"); const cancelledRunId = RunId.make("run-mcp-cancelled"); const childRunId = RunId.make("run-mcp-child"); +const activeAttemptId = RunAttemptId.make("attempt-mcp-active"); const taskId = NodeId.make("node-mcp-task-1"); const now = DateTime.makeUnsafe("2026-08-04T12:00:00.000Z"); const codexDriver = ProviderDriverKind.make("codex"); @@ -132,9 +134,10 @@ it("readThread preserves caller access errors and prefers activity-run status ov model: "gpt-5.4", }), runs: [ - makeRun({ id: activeRunId, ordinal: 1, status: "running" }), + { ...makeRun({ id: activeRunId, ordinal: 1, status: "running" }), activeAttemptId }, makeRun({ id: cancelledRunId, ordinal: 2, status: "cancelled" }), ], + providerTurns: [{ runAttemptId: activeAttemptId, ordinal: 1, model: "gpt-5.4-mini" }], visibleTurnItems: [], runtimeRequests: [], messages: [], @@ -167,9 +170,13 @@ it("readThread preserves caller access errors and prefers activity-run status ov ? (projection.thread as unknown as OrchestrationV2ThreadShell) : null, ), - getProjectThreadRecords: (input) => + getProjectThreadRecords: (input, fields) => input.threadId === parentThreadId - ? Effect.succeed(projection) + ? Effect.succeed( + fields?.some((field) => field === "providerTurns") + ? projection + : { ...projection, providerTurns: [] }, + ) : Effect.die(`unexpected thread ${input.threadId}`), } satisfies Partial), ServerSettings.layerTest(), @@ -196,6 +203,12 @@ it("readThread preserves caller access errors and prefers activity-run status ov expect(result.thread.status).toBe("running"); expect(result.thread.latestRunId).toBe(cancelledRunId); expect(result.thread.activeRunId).toBe(activeRunId); + expect( + result.recentRuns.map(({ runId, model, executedModel }) => ({ runId, model, executedModel })), + ).toEqual([ + { runId: cancelledRunId, model: "gpt-5.4", executedModel: null }, + { runId: activeRunId, model: "gpt-5.4", executedModel: "gpt-5.4-mini" }, + ]); callerUnavailable = true; const denied = yield* service @@ -377,6 +390,7 @@ it("taskStatus returns task.providerInstanceId rather than the driver kind", asy ], subagents: [], providerThreads: [], + providerTurns: [], updatedAt: now, } as unknown as OrchestrationV2ThreadProjection; diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 681f5ae3368ae..20b909e155f55 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -20,6 +20,7 @@ import { ProviderInstanceId, ProviderThreadId, ProviderTurnId, + RunAttemptId, RunId, ThreadId, type OrchestrationV2ServerCommand, @@ -145,6 +146,7 @@ describe("OrchestratorMcpService", () => { turnItems: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], } as unknown as OrchestrationV2ThreadProjection; let hasNestedWork = true; @@ -288,6 +290,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], turnItems: [], } as unknown as OrchestrationV2ThreadProjection; @@ -385,6 +388,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: pendingRequest ? [{ status: "pending" }] : [], nodes: [], turnItems: [], @@ -485,6 +489,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], } as unknown as OrchestrationV2ThreadProjection; const layerDependencies = Layer.mergeAll( @@ -575,6 +580,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], turnItems: [], } as unknown as OrchestrationV2ThreadProjection; @@ -678,6 +684,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], } as unknown as OrchestrationV2ThreadProjection; const layerDependencies = Layer.mergeAll( @@ -761,6 +768,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], } as unknown as OrchestrationV2ThreadProjection; const layerDependencies = Layer.mergeAll( @@ -850,6 +858,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], } as unknown as OrchestrationV2ThreadProjection; const layerDependencies = Layer.mergeAll( @@ -973,6 +982,7 @@ describe("OrchestratorMcpService", () => { appOwnedTask("node:mcp-cancel-grandchild-below", childThreadId, grandchildThreadId), ], providerThreads: [], + providerTurns: [], runtimeRequests: [], }, ], @@ -997,6 +1007,7 @@ describe("OrchestratorMcpService", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], }, ], @@ -1233,6 +1244,7 @@ describe("OrchestratorMcpService provider resolution", () => { messages: [], subagents: [], providerThreads: [], + providerTurns: [], runtimeRequests: [], turnItems: [], } as unknown as OrchestrationV2ThreadProjection; @@ -3042,11 +3054,21 @@ describe("OrchestratorMcpService provider resolution", () => { status: "running", result: null, }; + const restartedAttemptId = RunAttemptId.make("attempt:mcp-restarted-child"); + // Claude ran the restarted attempt on Sonnet; a later turn of another attempt reported Haiku. + const providerTurns = [ + { runAttemptId: restartedAttemptId, ordinal: 1, model: "claude-sonnet-4-6" }, + { + runAttemptId: RunAttemptId.make("attempt:mcp-other-run"), + ordinal: 2, + model: "claude-haiku-4-5", + }, + ]; const dependencies = Layer.mergeAll( NodeServices.layer, NodeCrypto.layer, Layer.mock(ThreadManagementService.ThreadManagementService)({ - getThreadRecords: (threadId) => + getThreadRecords: (threadId, fields) => Effect.succeed( threadId === parentThreadId ? parentProjection([task]) @@ -3060,9 +3082,13 @@ describe("OrchestratorMcpService provider resolution", () => { ordinal: 1, status: "running", modelSelection: runSelection, + activeAttemptId: restartedAttemptId, }, ] : [], + providerTurns: fields?.some((field) => field === "providerTurns") + ? providerTurns + : [], } as unknown as OrchestrationV2ThreadProjection), ), }), @@ -3076,11 +3102,15 @@ describe("OrchestratorMcpService provider resolution", () => { ); yield* Effect.gen(function* () { const service = yield* OrchestratorMcpService.OrchestratorMcpService; - for (const selection of [runSelection, threadSelection]) { + for (const [selection, executedModel] of [ + [runSelection, "claude-sonnet-4-6"], + [threadSelection, null], + ] as const) { const result = yield* service.taskStatus(scope, taskId); assert.equal(result.providerInstanceId, selection.instanceId); assert.equal(result.model, selection.model); assert.deepEqual(result.options, selection.options); + assert.equal(result.executedModel, executedModel); hasRun = false; } }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 074b19fad3fec..577d22ec48925 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -63,6 +63,7 @@ import { type ServerProviderModel, ThreadId, isOrchestrationV2WorkActive, + latestProviderTurnForAttempt, resolveProjectProviderInstanceEnabled, SECRET_REQUEST_UNSAVED_NOTE, } from "@t3tools/contracts"; @@ -875,13 +876,24 @@ function threadDetail( }; } -function threadRun(run: OrchestrationV2Run): OrchestratorMcpThreadRun { +function runExecutedModel( + projection: Pick, + run: Pick, +): string | null { + return latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId)?.model ?? null; +} + +function threadRun( + projection: Pick, + run: OrchestrationV2Run, +): OrchestratorMcpThreadRun { return { runId: run.id, ordinal: run.ordinal, status: run.status, providerInstanceId: run.modelSelection.instanceId, model: run.modelSelection.model, + executedModel: runExecutedModel(projection, run), requestedAt: DateTime.formatIso(run.requestedAt), startedAt: run.startedAt === null ? null : DateTime.formatIso(run.startedAt), completedAt: run.completedAt === null ? null : DateTime.formatIso(run.completedAt), @@ -1280,10 +1292,11 @@ const make = Effect.gen(function* () { const records = parent === undefined ? threadManagement - .getProjectThreadRecords(input, ["runs", "runtimeRequests"]) + .getProjectThreadRecords(input, ["runs", "providerTurns", "runtimeRequests"]) .pipe(Effect.map((target) => ({ ...target, contextTransfers: [] }))) : threadManagement.getProjectThreadRecords(input, [ "runs", + "providerTurns", "runtimeRequests", "contextTransfers", ]); @@ -1506,6 +1519,7 @@ const make = Effect.gen(function* () { "contextTransfers", "subagents", "providerThreads", + "providerTurns", "runtimeRequests", "nodes", ], @@ -1617,6 +1631,7 @@ const make = Effect.gen(function* () { hasPendingChildRuns: hasPendingChildRuns(childProjection, childRun), providerInstanceId: modelSelection.instanceId, model: modelSelection.model, + executedModel: childRun === undefined ? null : runExecutedModel(childControls, childRun), options: modelSelection.options ?? [], summary: derivedResult, resultContextTransferId, @@ -3124,7 +3139,7 @@ const make = Effect.gen(function* () { recentRuns: target.runs .toSorted((left, right) => right.ordinal - left.ordinal) .slice(0, input.runLimit ?? DEFAULT_THREAD_RUN_LIMIT) - .map(threadRun), + .map((run) => threadRun(target, run)), items: page.map((row) => timelineItem({ row, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index bc70c5bf7a161..f9ea4f61c928e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2979,11 +2979,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly text?: string; readonly parentToolUseId?: string | null; readonly apiErrorParams?: unknown; + readonly model?: string; }) => claudeSdkFrame({ type: "assistant", message: { - model: "claude-sonnet-4-6", + model: input.model ?? "claude-sonnet-4-6", id: `msg_${input.uuid}`, type: "message", role: "assistant", @@ -8626,6 +8627,86 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("records the root model Claude reports for each turn, not the requested one", () => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const turnInput = { + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attachments: [], + modelSelection: { + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + model: "claude-opus-4-6", + }, + }; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + ...turnInput, + attemptId: RunAttemptId.make("attempt-claude-restricted-model-1"), + text: "Review this.", + }), + ); + // An organization policy that disallows the requested model shows up only as init's model. + yield* harness.offerAndWait(claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" })); + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000661", + error: undefined, + parentToolUseId: "toolu_restricted_model_subagent", + model: "claude-haiku-4-5-20251001", + }), + turnOneResult, + ]); + yield* Queue.take(harness.terminalReceipts); + assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + ...turnInput, + attemptId: RunAttemptId.make("attempt-claude-restricted-model-2"), + providerTurnOrdinal: 2, + text: "Continue.", + }), + ); + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000662", + error: undefined, + model: "claude-haiku-4-5-20251001", + }), + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000663", result: "Done." }), + ]); + yield* Queue.take(harness.terminalReceipts); + assert.equal(lastProviderTurn(harness.events).model, "claude-haiku-4-5"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + ...turnInput, + attemptId: RunAttemptId.make("attempt-claude-restricted-model-3"), + providerTurnOrdinal: 3, + text: "Once more.", + }), + ); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000664", result: "Done." }), + ); + yield* Queue.take(harness.terminalReceipts); + assert.isUndefined(lastProviderTurn(harness.events).model); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("preserves typed Claude plans and todos through generic tool completion", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3c6295494ec83..905c515eab2d2 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4384,6 +4384,8 @@ interface ActiveClaudeTurnContext { nativeWakeStarted: boolean; gatedFramesBeforeEcho: number; readonly heldRootFrames: Array; + // Claude can run another model than requested, such as when an organization disallows it. + executedModel: string | undefined; } interface ActiveClaudeProviderRetry { @@ -4485,6 +4487,10 @@ interface PendingClaudeSubagentLaunch { readonly ownerToolUseId?: string; } +function executedModelField(context: ActiveClaudeTurnContext): { readonly model?: string } { + return context.executedModel === undefined ? {} : { model: context.executedModel }; +} + function canonicalClaudeSubagentModel(catalog: ClaudeModelCatalog, model: string): string { return resolveClaudeModelSlug(catalog, model); } @@ -5434,6 +5440,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( startedAt: context.startedAt, completedAt: null, tokenUsage, + ...executedModelField(context), }, }); }); @@ -5944,8 +5951,17 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( status: input.status, startedAt: input.context.startedAt, completedAt: input.completedAt, + ...executedModelField(input.context), }); + const recordExecutedModel = (context: ActiveClaudeTurnContext, model: unknown) => { + const reported = typeof model === "string" ? model.trim() : ""; + // "" identifies CLI notices, not a model. + if (reported.length > 0 && reported !== "") { + context.executedModel = resolveClaudeModelSlug(subagentModelCatalog, reported); + } + }; + const buildToolCallArtifacts = (input: { readonly context: ActiveClaudeTurnContext | null; readonly nativeItemId: string; @@ -8913,6 +8929,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return; } + if ( + message.type === "system" && + message.subtype === "init" && + context === (yield* Ref.get(activeTurn)) + ) { + recordExecutedModel(context, message.model); + } const focusedPlugin = claudeFocusRequestedPanePlugin(message); if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { yield* Effect.logWarning("orchestration-v2.claude-mod-pane-focus-unsupported", { @@ -9049,6 +9072,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if (message.message.model && message.message.model !== "") { context.rootModels.add(message.message.model); } + recordExecutedModel(context, message.message.model); // Subagent uuids exist only in the sidechain transcript, so resume and fork cannot target them. context.nativeMessageCursor = message.uuid; context.latestAssistantRateLimit = @@ -11159,6 +11183,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeWakeStarted: false, gatedFramesBeforeEcho: 0, heldRootFrames: [], + executedModel: undefined, }; const attachTurn = Effect.gen(function* () { querySession.lastTurn = context; diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 41eb03e0939fc..c364dd44e711f 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -35,3 +35,4 @@ {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} {"id": "thread-send-fresh-context", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send that should start a fresh provider session", "experience": "The user wanted a confused long-running thread to start over in the same thread without its old context; a plain send would keep feeding it the history.", "rule": "`freshContext: true` on t3_thread_send starts a new provider session in the same thread, like /clear.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The thread working on the CSV importer keeps reverting to the old column mapping from early in its history."}, {"role": "assistant", "text": "That thread has a long history with three different mappings in it, so the model keeps picking up the stale one."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "messageId": "msg-1", "runId": "run-1", "status": "starting", "delivery": "started"}}}, "prompt": "Send it the final mapping below and have it start over with a clean context in that same thread, so the old history stops leaking in: thread 11111111-1111-4111-8111-111111111111.\n\nFinal mapping: id -> customer_id, amt -> amount_cents, ts -> created_at.", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "freshContext": true}}], "notCalls": ["delegate_task", "t3_thread_launch", "create_threads"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: fresh provider sessions in one T3 thread"}, "notes": "Pairs with thread-send-to-named-top-level-thread, where an ordinary send must not set freshContext."} +{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "No blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["[Ss]onnet"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it. Main returned no executedModel, so the parent only saw the requested Opus."} diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 6e70cc66a7203..7378c10955104 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1186,6 +1186,8 @@ export const OrchestrationV2ProviderTurn = Schema.Struct({ completedAt: Schema.NullOr(Schema.DateTimeUtc), tokenUsage: Schema.optional(OrchestrationV2ProviderTurnTokenUsage), turnTokenUsage: Schema.optional(TurnTokenUsage), + /** The model the provider reported running this turn's root agent, which can differ from the requested one. */ + model: Schema.optional(TrimmedNonEmptyString), }); export type OrchestrationV2ProviderTurn = typeof OrchestrationV2ProviderTurn.Type; diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index c73bbd48e26f5..21382fdbd1691 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -201,6 +201,12 @@ export const ORCHESTRATOR_MCP_RUNNING_TASK_NEXT = "Completion wakes this thread, so end your turn. Do not wait on it or poll it."; const OrchestratorMcpRunningTaskNext = Schema.Literal(ORCHESTRATOR_MCP_RUNNING_TASK_NEXT); +// Optional: linked environments on older versions omit it. +const OrchestratorMcpExecutedModel = Schema.NullOr(Schema.String).annotate({ + description: + "Model the provider reported running, or null if unreported. Can differ from the requested `model`.", +}); + export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ taskId: NodeId, /** Null for a task that runs in a linked environment; see `remoteChild`. */ @@ -221,6 +227,7 @@ export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ latestTerminalResultContextTransferId: Schema.NullOr(ContextTransferId), providerInstanceId: ProviderInstanceId, model: Schema.NullOr(Schema.String), + executedModel: Schema.optional(OrchestratorMcpExecutedModel), options: Schema.optional(Schema.Array(ProviderOptionSelection)), summary: Schema.NullOr(Schema.String), resultContextTransferId: Schema.NullOr(ContextTransferId), @@ -462,6 +469,7 @@ export const OrchestratorMcpThreadRun = Schema.Struct({ status: OrchestrationV2RunStatus, providerInstanceId: ProviderInstanceId, model: Schema.String, + executedModel: Schema.optional(OrchestratorMcpExecutedModel), requestedAt: IsoDateTime, startedAt: Schema.NullOr(IsoDateTime), completedAt: Schema.NullOr(IsoDateTime), From 7e95fb296616e871c1c396373d8ceb703ed11f3f Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Fri, 30 Oct 2026 17:58:09 +0000 Subject: [PATCH 02/21] fix(server): delegated completions say when Claude ran another model A parent that read a delegate's result through the normal completion message or task_status summary saw only the child's answer, so a Sonnet run that Opus was requested for still read as an Opus result unless the parent fetched activity. subagentResultForRun now puts "Requested model X, but the provider ran Y." ahead of the result when the run's own provider turn reported another model, followed by that run's root provider notices verbatim, such as Claude's organization fallback warning. Notices from native subagents, other runs, and T3's own weak-ref notices stay out. The orchestrator and task_status load system_notice items with the result so the stored result, inline wake, and derived summary match. The requested thread and run selection is unchanged, and results stored before this change keep their text. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/OrchestratorMcpService.ts | 4 +- .../DelegatedCompletionDelivery.test.ts | 91 +++++++++++++++++- .../src/orchestration-v2/Orchestrator.ts | 2 +- .../SubagentProjection.test.ts | 94 +++++++++++++++++++ evals/mcp-tool-use/cases/orchestration.ndjson | 3 +- .../src/server/subagentProjection.ts | 36 ++++++- 6 files changed, 220 insertions(+), 10 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 577d22ec48925..13e6b76a11e6c 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -615,7 +615,7 @@ function pageIncludesTerminalTaskResult(input: { readonly task: OrchestrationV2Subagent; readonly target: Pick< OrchestrationV2ThreadProjection, - "thread" | "runs" | "contextTransfers" | "messages" | "turnItems" + "thread" | "runs" | "contextTransfers" | "messages" | "turnItems" | "providerTurns" >; readonly maxChars: number; }): boolean { @@ -1552,7 +1552,7 @@ const make = Effect.gen(function* () { messageRoles: ["assistant"], messageRunIds: resultRunIds, turnItemRunIds: resultRunIds, - turnItemTypes: ["assistant_message", "error"], + turnItemTypes: ["assistant_message", "error", "system_notice"], }) .pipe(Effect.mapError(threadManagementFailure)); const childProjection = { diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index db3a2759e3ac5..2c9da0d0082e6 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -15,6 +15,8 @@ import { ProviderDriverKind, ProviderInstanceId, ProviderThreadId, + ProviderTurnId, + RunAttemptId, RunId, RuntimeRequestId, ThreadId, @@ -2703,6 +2705,9 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { const parent = yield* orchestrator.getThreadProjection(threadId); const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId); const providerThreadId = ProviderThreadId.make("provider-thread:result-message-child"); + const childRootNodeId = NodeId.make("node:result-message-child-root"); + const childAttemptId = RunAttemptId.make("attempt:result-message-child"); + const fallbackNotice = "Model gpt-5.4 is restricted. Using gpt-5.4-mini instead."; yield* eventSink.write({ commandId: CommandId.make("command:result-message-child-provider"), events: [ @@ -2723,7 +2728,83 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { threadId: child.childThreadId, runId: child.childRunId, occurredAt: now, - payload: { ...childProjection.runs[0]!, providerThreadId }, + payload: { + ...childProjection.runs[0]!, + providerThreadId, + rootNodeId: childRootNodeId, + activeAttemptId: childAttemptId, + }, + }, + { + id: EventId.make("event:result-message-child-provider-turn"), + type: "provider-turn.updated", + threadId: child.childThreadId, + runId: child.childRunId, + nodeId: childRootNodeId, + occurredAt: now, + payload: { + id: ProviderTurnId.make("provider-turn:result-message-child"), + providerThreadId, + nodeId: childRootNodeId, + runAttemptId: childAttemptId, + nativeTurnRef: null, + ordinal: 1, + status: "completed", + startedAt: now, + completedAt: now, + model: "gpt-5.4-mini", + }, + }, + { + id: EventId.make("event:result-message-child-notice"), + type: "turn-item.updated", + threadId: child.childThreadId, + runId: child.childRunId, + nodeId: childRootNodeId, + occurredAt: now, + payload: { + id: TurnItemId.make("turn-item:result-message-child-notice"), + type: "system_notice", + threadId: child.childThreadId, + runId: child.childRunId, + nodeId: childRootNodeId, + providerThreadId, + providerTurnId: ProviderTurnId.make("provider-turn:result-message-child"), + nativeItemRef: { + driver, + nativeId: "result-message-child-notice", + strength: "strong", + }, + parentItemId: null, + ordinal: 0, + status: "completed", + title: fallbackNotice, + message: fallbackNotice, + startedAt: now, + completedAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("event:result-message-child-answer"), + type: "message.updated", + threadId: child.childThreadId, + runId: child.childRunId, + occurredAt: now, + payload: { + id: MessageId.make("message:result-message-child-answer"), + threadId: child.childThreadId, + runId: child.childRunId, + nodeId: childRootNodeId, + role: "assistant", + text: "Review done.", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, }, ], }); @@ -2732,7 +2813,13 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { const recovered = yield* orchestrator.getThreadProjection(threadId); const task = recovered.subagents.find((candidate) => candidate.id === child.taskId)!; assert.equal(task.status, "completed"); - assert.isNotNull(task.result); + assert.equal( + task.result, + `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\nProvider notice: ${fallbackNotice}\n\nReview done.`, + ); + const recoveredChild = yield* orchestrator.getThreadProjection(child.childThreadId); + assert.deepEqual(recoveredChild.thread.modelSelection, modelSelection); + assert.deepEqual(recoveredChild.runs[0]?.modelSelection, modelSelection); assert.isEmpty(recovered.contextHandoffs); assert.equal(recovered.contextTransfers.length, 1); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index d40ca624a11d2..589626ac9110b 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -12780,7 +12780,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio messageRoles: ["assistant"], messageRunIds: [childRun.id], turnItemRunId: childRun.id, - turnItemTypes: ["assistant_message", "error"], + turnItemTypes: ["assistant_message", "error", "system_notice"], }, ); const childProjection = { diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index ae8f7e09d2068..ad4532b0e64af 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -9,8 +9,10 @@ import { EventId, TurnItemId, type OrchestrationV2Run, + type OrchestrationV2TurnItem, type OrchestrationV2AppThread, ProjectId, + ProviderDriverKind, ProviderInstanceId, ProviderThreadId, ThreadId, @@ -415,6 +417,98 @@ it("orders a task's assistant messages by turn item, not by timestamps", () => { assert.equal(result.turnItemId, report.turnItem.id); }); +type SystemNotice = Extract; + +it("notes a substituted model with the root provider notice from the same run", () => { + const { projection, run } = taskFixture(); + const answer = makeSubagentConversationArtifacts({ + messageId: MessageId.make("answer"), + turnItemId: TurnItemId.make("answer"), + threadId: parentThreadId, + rootNodeId: NodeId.make("root"), + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + role: "assistant", + text: "Review done.", + ordinal: 1, + now: childCreatedAt, + }); + const fallback = "Model gpt-5.4 is restricted by your organization. Using gpt-5.4-mini instead."; + const notice = ( + id: string, + message: string, + overrides: Partial> = {}, + ): SystemNotice => ({ + id: TurnItemId.make(id), + threadId: parentThreadId, + runId: run.id, + nodeId: NodeId.make("root"), + providerThreadId: null, + providerTurnId: null, + nativeItemRef: { + driver: ProviderDriverKind.make("claudeAgent"), + nativeId: id, + strength: "strong", + }, + parentItemId: null, + ordinal: 0, + status: "completed", + title: message, + startedAt: childCreatedAt, + completedAt: childCreatedAt, + updatedAt: childCreatedAt, + type: "system_notice", + message, + ...overrides, + }); + const result = subagentResultForRun( + { + ...projection, + messages: [{ ...answer.message, runId: run.id }], + turnItems: [ + { ...answer.turnItem, runId: run.id }, + notice("fallback", fallback), + notice("native-subagent", "Subagent fell back to Haiku.", { + nodeId: NodeId.make("native-subagent"), + }), + notice("other-run", "Another run's notice.", { runId: RunId.make("other") }), + notice("usage-limit", "T3 usage limit notice.", { + nativeItemRef: { + driver: ProviderDriverKind.make("claudeAgent"), + nativeId: "usage-limit", + strength: "weak", + }, + }), + ], + providerTurns: [ + { runAttemptId: run.activeAttemptId, ordinal: 1, model: "gpt-5.4-mini" }, + { runAttemptId: RunAttemptId.make("other"), ordinal: 2, model: "gpt-5.4-nano" }, + ] as never, + }, + run, + ); + assert.equal( + result.text, + `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\nProvider notice: ${fallback}\n\nReview done.`, + ); + assert.equal(result.messageId, answer.message.id); + assert.equal( + subagentResultForRun( + { + ...projection, + messages: [{ ...answer.message, runId: run.id }], + turnItems: [notice("fallback", fallback)], + providerTurns: [ + { runAttemptId: run.activeAttemptId, ordinal: 1, model: "gpt-5.4" }, + ] as never, + }, + run, + ).text, + "Review done.", + ); +}); + it.effect("ends an orphaned native subagent's node when only its record already ended", () => Effect.gen(function* () { const id = NodeId.make("node:orphaned-native-subagent"); diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index c364dd44e711f..74c4331793d23 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -35,4 +35,5 @@ {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} {"id": "thread-send-fresh-context", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send that should start a fresh provider session", "experience": "The user wanted a confused long-running thread to start over in the same thread without its old context; a plain send would keep feeding it the history.", "rule": "`freshContext: true` on t3_thread_send starts a new provider session in the same thread, like /clear.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The thread working on the CSV importer keeps reverting to the old column mapping from early in its history."}, {"role": "assistant", "text": "That thread has a long history with three different mappings in it, so the model keeps picking up the stale one."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "messageId": "msg-1", "runId": "run-1", "status": "starting", "delivery": "started"}}}, "prompt": "Send it the final mapping below and have it start over with a clean context in that same thread, so the old history stops leaking in: thread 11111111-1111-4111-8111-111111111111.\n\nFinal mapping: id -> customer_id, amt -> amount_cents, ts -> created_at.", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "freshContext": true}}], "notCalls": ["delegate_task", "t3_thread_launch", "create_threads"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: fresh provider sessions in one T3 thread"}, "notes": "Pairs with thread-send-to-named-top-level-thread, where an ordinary send must not set freshContext."} -{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "No blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["[Ss]onnet"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it. Main returned no executedModel, so the parent only saw the requested Opus."} +{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["[Ss]onnet"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it, summary included. Main returned no executedModel and only the child's answer as the summary, so the parent only saw the requested Opus."} +{"id": "delegate-substituted-model-inline-result", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran, from the inline result", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The completion message inlined only the child's answer, so the parent called it an Opus review.", "rule": "Delegated completion messages put \"Requested model X, but the provider ran Y.\" and the provider's notice before a child's result when the provider ran another model.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 (Opus review PR 123) [completed]:\nRequested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "expect": {"textMust": ["[Ss]onnet"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The prompt is the inline completion message as the fix writes it for a short result. On main the same message carried only the child's answer. To check the old text, replace the prompt with 'Delegated task task-1 (Opus review PR 123) [completed]:\\n' plus the answer alone."} diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index dbd929192e04d..38b71a5b322ef 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -22,7 +22,7 @@ import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2Pe import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; -import { isOrchestrationV2WorkActive } from "@t3tools/contracts"; +import { isOrchestrationV2WorkActive, latestProviderTurnForAttempt } from "@t3tools/contracts"; function trimmed(value: string | null | undefined): string | undefined { const result = value?.trim(); @@ -176,8 +176,35 @@ export function makeSubagentConversationArtifacts(input: { return { message, turnItem }; } +function substitutedModelNote( + projection: Pick, + run: OrchestrationV2Run, +): string | undefined { + const executedModel = latestProviderTurnForAttempt( + projection.providerTurns, + run.activeAttemptId, + )?.model; + if (executedModel === undefined || executedModel === run.modelSelection.model) return undefined; + const notices = projection.turnItems + .filter( + (item): item is Extract => + item.type === "system_notice" && + item.runId === run.id && + item.nodeId === run.rootNodeId && + item.parentItemId === null && + // T3's own notices, such as usage limits, carry weak refs. + item.nativeItemRef?.strength === "strong", + ) + .toSorted((left, right) => left.ordinal - right.ordinal) + .map((item) => `Provider notice: ${item.message}`); + return [ + `Requested model ${run.modelSelection.model}, but the provider ran ${executedModel}.`, + ...notices, + ].join("\n"); +} + export function subagentResultForRun( - projection: Pick, + projection: Pick, run: OrchestrationV2Run, ): { readonly text: string; @@ -216,15 +243,16 @@ export function subagentResultForRun( .filter((item) => item.runId === run.id && item.type === "error") .toSorted((left, right) => right.ordinal - left.ordinal)[0] : undefined; - const text = + const result = (failure?.type === "error" ? failure.failure.message : undefined) ?? message?.text ?? turnItem?.text ?? (run.status === "completed" ? "Child task completed without an assistant result." : `Child task ended with status ${run.status}.`); + const note = substitutedModelNote(projection, run); return { - text, + text: note === undefined ? result : `${note}\n\n${result}`, messageId: failure === undefined ? (message?.id ?? turnItem?.messageId ?? null) : null, turnItemId: failure?.id ?? turnItem?.id ?? null, }; From 73f61ba6bf79f529ad7184a858a536c30422a111 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Fri, 30 Oct 2026 18:02:39 +0000 Subject: [PATCH 03/21] fix(server): report the model of the delegated continuation --- .../src/mcp/OrchestratorMcpService.test.ts | 20 ++++++++++++++++++- apps/server/src/mcp/OrchestratorMcpService.ts | 6 ++++-- evals/mcp-tool-use/cases/orchestration.ndjson | 4 ++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 20b909e155f55..a78c5fcb5df69 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -3054,10 +3054,13 @@ describe("OrchestratorMcpService provider resolution", () => { status: "running", result: null, }; + let hasContinuation = false; const restartedAttemptId = RunAttemptId.make("attempt:mcp-restarted-child"); + const continuationAttemptId = RunAttemptId.make("attempt:mcp-child-continuation"); // Claude ran the restarted attempt on Sonnet; a later turn of another attempt reported Haiku. const providerTurns = [ { runAttemptId: restartedAttemptId, ordinal: 1, model: "claude-sonnet-4-6" }, + { runAttemptId: continuationAttemptId, ordinal: 2, model: "claude-haiku-4-5" }, { runAttemptId: RunAttemptId.make("attempt:mcp-other-run"), ordinal: 2, @@ -3083,7 +3086,20 @@ describe("OrchestratorMcpService provider resolution", () => { status: "running", modelSelection: runSelection, activeAttemptId: restartedAttemptId, + delegatedTaskId: taskId, }, + ...(hasContinuation + ? [ + { + id: RunId.make("run:mcp-child-continuation"), + ordinal: 2, + status: "running", + modelSelection: runSelection, + activeAttemptId: continuationAttemptId, + delegatedTaskId: taskId, + }, + ] + : []), ] : [], providerTurns: fields?.some((field) => field === "providerTurns") @@ -3104,6 +3120,7 @@ describe("OrchestratorMcpService provider resolution", () => { const service = yield* OrchestratorMcpService.OrchestratorMcpService; for (const [selection, executedModel] of [ [runSelection, "claude-sonnet-4-6"], + [runSelection, "claude-haiku-4-5"], [threadSelection, null], ] as const) { const result = yield* service.taskStatus(scope, taskId); @@ -3111,7 +3128,8 @@ describe("OrchestratorMcpService provider resolution", () => { assert.equal(result.model, selection.model); assert.deepEqual(result.options, selection.options); assert.equal(result.executedModel, executedModel); - hasRun = false; + if (hasContinuation) hasRun = false; + hasContinuation = true; } }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); }), diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 13e6b76a11e6c..9527e9478692b 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1612,7 +1612,8 @@ const make = Effect.gen(function* () { : null; const resultContextTransferId = resultTransfer?.id ?? null; const latestTerminalResultContextTransferId = resultTransferForRun(terminalRun)?.id ?? null; - const modelSelection = childRun?.modelSelection ?? + const executionRun = taskResultRun ?? childRun; + const modelSelection = executionRun?.modelSelection ?? childControls.thread.modelSelection ?? { instanceId: task.providerInstanceId, model: task.model, @@ -1631,7 +1632,8 @@ const make = Effect.gen(function* () { hasPendingChildRuns: hasPendingChildRuns(childProjection, childRun), providerInstanceId: modelSelection.instanceId, model: modelSelection.model, - executedModel: childRun === undefined ? null : runExecutedModel(childControls, childRun), + executedModel: + executionRun === undefined ? null : runExecutedModel(childControls, executionRun), options: modelSelection.options ?? [], summary: derivedResult, resultContextTransferId, diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 74c4331793d23..c556c91cbaa8e 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -35,5 +35,5 @@ {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} {"id": "thread-send-fresh-context", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send that should start a fresh provider session", "experience": "The user wanted a confused long-running thread to start over in the same thread without its old context; a plain send would keep feeding it the history.", "rule": "`freshContext: true` on t3_thread_send starts a new provider session in the same thread, like /clear.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The thread working on the CSV importer keeps reverting to the old column mapping from early in its history."}, {"role": "assistant", "text": "That thread has a long history with three different mappings in it, so the model keeps picking up the stale one."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "messageId": "msg-1", "runId": "run-1", "status": "starting", "delivery": "started"}}}, "prompt": "Send it the final mapping below and have it start over with a clean context in that same thread, so the old history stops leaking in: thread 11111111-1111-4111-8111-111111111111.\n\nFinal mapping: id -> customer_id, amt -> amount_cents, ts -> created_at.", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "freshContext": true}}], "notCalls": ["delegate_task", "t3_thread_launch", "create_threads"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: fresh provider sessions in one T3 thread"}, "notes": "Pairs with thread-send-to-named-top-level-thread, where an ordinary send must not set freshContext."} -{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["[Ss]onnet"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it, summary included. Main returned no executedModel and only the child's answer as the summary, so the parent only saw the requested Opus."} -{"id": "delegate-substituted-model-inline-result", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran, from the inline result", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The completion message inlined only the child's answer, so the parent called it an Opus review.", "rule": "Delegated completion messages put \"Requested model X, but the provider ran Y.\" and the provider's notice before a child's result when the provider ran another model.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 (Opus review PR 123) [completed]:\nRequested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "expect": {"textMust": ["[Ss]onnet"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The prompt is the inline completion message as the fix writes it for a short result. On main the same message carried only the child's answer. To check the old text, replace the prompt with 'Delegated task task-1 (Opus review PR 123) [completed]:\\n' plus the answer alone."} +{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["(?:[Ss]onnet[^.!?\\n]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review)[^.!?\\n]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it, summary included. Main returned no executedModel and only the child's answer as the summary, so the parent only saw the requested Opus."} +{"id": "delegate-substituted-model-inline-result", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran, from the inline result", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The completion message inlined only the child's answer, so the parent called it an Opus review.", "rule": "Delegated completion messages put \"Requested model X, but the provider ran Y.\" and the provider's notice before a child's result when the provider ran another model.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 (Opus review PR 123) [completed]:\nRequested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "expect": {"textMust": ["(?:[Ss]onnet[^.!?\\n]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review)[^.!?\\n]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The prompt is the inline completion message as the fix writes it for a short result. On main the same message carried only the child's answer. To check the old text, replace the prompt with 'Delegated task task-1 (Opus review PR 123) [completed]:\\n' plus the answer alone."} From 629dac9af3d105a452d17ce4f4fe979f673ae776 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Fri, 30 Oct 2026 18:13:15 +0000 Subject: [PATCH 04/21] fix(server): remote delegated tasks report the model that ran there A task delegated to a linked environment completed here with only the reply text, and task_status reported the model the task was launched with, so a run the provider there substituted another model for still read as the requested one. The follower now reads the result run's requested and executed model from the same t3_thread_read it takes the reply from, matched to that exact run, and completes the task with them. The result carries the same "Requested model X, but the provider ran Y." lead as a local task, so the stored result, inline wake, and task_status summary agree, and task_status reports that run's model and executedModel. The models are kept on remoteChild; the task's configured selection is unchanged. Older environments omit executedModel, which reads as unreported and leaves the result as before. Provider notices from the run there are not copied: thread reads carry no node or native-ref strength, so the root-only boundary a local task uses cannot be applied. They stay readable in that thread's activity view. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/OrchestratorMcpService.ts | 3 +- .../src/orchestration-v2/Orchestrator.ts | 5 +- apps/server/src/peer/RemoteDelegation.test.ts | 63 ++++++++++++++++--- apps/server/src/peer/RemoteDelegation.ts | 34 ++++++++-- packages/contracts/src/orchestrationV2.ts | 10 +++ .../src/server/subagentProjection.ts | 53 +++++++++++----- 6 files changed, 136 insertions(+), 32 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 9527e9478692b..70fa2f500649d 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1801,7 +1801,8 @@ const make = Effect.gen(function* () { workState: terminal ? "result_available" : "working", hasPendingChildRuns: false, providerInstanceId: task.providerInstanceId, - model: task.model, + model: task.remoteChild?.resultModel?.model ?? task.model, + executedModel: task.remoteChild?.resultModel?.executedModel ?? null, summary: task.result, resultContextTransferId: null, latestTerminalRunId: null, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 589626ac9110b..1e96781b5f6e0 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -8565,7 +8565,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const now = yield* DateTime.now; const updatedTask: OrchestrationV2Subagent = { ...task, - remoteChild, + remoteChild: + command.resultModel === undefined + ? remoteChild + : { ...remoteChild!, resultModel: command.resultModel }, status: command.status, result: command.result, completedAt: now, diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 00b7b5d6f6963..dfe07d1841573 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -7,13 +7,16 @@ import { EventId, MessageId, NodeId, + type OrchestrationV2ProviderTurn, type OrchestrationV2Run, ProjectId, ProviderDriverKind, ProviderInstanceId, ProviderThreadId, + ProviderTurnId, type Project, type RepositoryIdentity, + RunAttemptId, RunId, ThreadId, TurnItemId, @@ -100,6 +103,9 @@ interface BoxThread { readonly laterError: Ref.Ref; /** A later run's reply, after the task's own, as a follow-up message would leave. */ readonly laterReply: Ref.Ref; + /** The model the provider there reported running the task's own run, and the later one. */ + readonly executedModel: Ref.Ref; + readonly laterExecutedModel: Ref.Ref; /** The run each wait asked for, in order. */ readonly waitedFor: Ref.Ref>; /** The thread's run ends with a task of its own still running, whose result wakes it again. */ @@ -145,6 +151,8 @@ const makeBoxThread = Effect.gen(function* () { error: yield* Ref.make(null), laterError: yield* Ref.make(null), laterReply: yield* Ref.make(null), + executedModel: yield* Ref.make(null), + laterExecutedModel: yield* Ref.make(null), waitedFor: yield* Ref.make>([]), nested: yield* Ref.make(false), nestedWaiting: yield* Deferred.make(), @@ -219,25 +227,43 @@ const boxToolkitLayer = (thread: BoxThread) => { ...(linkOrigin === null ? {} : { linkOrigin }), }; }); - const runOf = (status: OrchestrationV2Run["status"]) => + const runOf = (status: OrchestrationV2Run["status"], id = runId) => ({ - id: runId, + id, threadId: ThreadId.make("thread:box"), - ordinal: 1, + ordinal: id === runId ? 1 : 2, status, modelSelection, + activeAttemptId: RunAttemptId.make(`attempt:${id}`), requestedAt: DateTime.makeUnsafe(0), startedAt: DateTime.makeUnsafe(0), completedAt: null, }) as unknown as OrchestrationV2Run; + const providerTurnOf = (id: RunId, model: string | null) => + ({ + id: ProviderTurnId.make(`provider-turn:${id}`), + runAttemptId: RunAttemptId.make(`attempt:${id}`), + ordinal: 1, + ...(model === null ? {} : { model }), + }) as unknown as OrchestrationV2ProviderTurn; const projectionOf = (threadId: ThreadId) => Effect.gen(function* () { const shell = yield* shellOf(threadId); const status = yield* Ref.get(thread.status); const runless = yield* Ref.get(thread.runless); + const later = (yield* Ref.get(thread.laterReply)) !== null; return shell === null ? (null as never) - : { ...idleThreadProjection(shell), runs: runless ? [] : [runOf(status)] }; + : { + ...idleThreadProjection(shell), + runs: runless + ? [] + : [runOf(status), ...(later ? [runOf("completed", boxLaterRunId)] : [])], + providerTurns: [ + providerTurnOf(runId, yield* Ref.get(thread.executedModel)), + providerTurnOf(boxLaterRunId, yield* Ref.get(thread.laterExecutedModel)), + ], + }; }); // One record per command id, as its receipt replays a repeat. const recordStop = (commandId: string, kind: "interrupt" | "thread.stop" | "delegated-tasks") => @@ -906,6 +932,9 @@ it.effect("a task delegated to the box wakes the laptop's parent when it ends th expect(status.structuredContent).toMatchObject({ status: "completed", summary: "All 412 tests pass on the box.", + model: "gpt-5.4", + // The provider there reported no model, as an older box never does. + executedModel: null, }); // Delivery validation still reads messages; choosing the observer does not need another read. expect(messageReads).toHaveLength(2); @@ -1264,11 +1293,19 @@ it.effect("a task's result is its own run's whole last reply, not a later run's" const long = "x".repeat(45_000); yield* Ref.set(thread.reply, long); yield* Ref.set(thread.laterReply, "Someone else asked a follow-up."); + // The provider there ran the task's run on another model, the later run on the one asked for. + yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); + yield* Ref.set(thread.laterExecutedModel, "gpt-5.4"); yield* Ref.set(thread.status, "completed"); yield* Deferred.succeed(thread.finished, undefined); const task = yield* taskResult(a, taskId); expect(task?.status).toBe("completed"); - expect(task?.result).toBe(long); + expect(task?.result).toBe( + `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\n\n${long}`, + ); + expect((yield* a.call("task_status", { taskId })).structuredContent).toMatchObject({ + executedModel: "gpt-5.4-mini", + }); // Every wait was for the run the launch started. const waits = yield* Ref.get(thread.waitedFor); expect(waits.length).toBeGreaterThan(0); @@ -1302,12 +1339,20 @@ it.effect("a task whose thread there waits on its own task ends with the answer ); expect(first).toBe("waiting there"); - // The review reports and wakes the thread there, whose next run answers. + // The review reports and wakes the thread there, whose next run answers, + // on another model than the provider ran the first run on. + yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); + yield* Ref.set(thread.laterExecutedModel, "gpt-5.4-nano"); yield* Ref.set(thread.laterReply, "The review found two issues; both are fixed."); yield* Deferred.succeed(thread.nestedSettled, undefined); - expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ - status: "completed", - result: "The review found two issues; both are fixed.", + const result = + "Requested model gpt-5.4, but the provider ran gpt-5.4-nano.\n\nThe review found two issues; both are fixed."; + expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "completed", result }); + const status = yield* a.call("task_status", { taskId: delegated.taskId }); + expect(status.structuredContent).toMatchObject({ + summary: result, + model: "gpt-5.4", + executedModel: "gpt-5.4-nano", }); }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 89917026ff8f7..7140db34543ca 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -24,6 +24,7 @@ import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type * as Tool from "effect/ai/Tool"; +import { withSubstitutedModelNote } from "@t3tools/provider-core/server/subagentProjection"; import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import { ProjectToolkit } from "../mcp/toolkits/project/tools.ts"; @@ -368,6 +369,7 @@ const make = Effect.gen(function* () { result: string, stoppedThere: "ran-out" | "stopped" | undefined, stopCount?: number, + resultModel?: OrchestrationV2RemoteTaskChild["resultModel"], ) { const attempt = yield* random.uuidV4(); yield* threads @@ -380,6 +382,7 @@ const make = Effect.gen(function* () { result, ...(stoppedThere === undefined ? {} : { stoppedThere }), ...(stopCount === undefined ? {} : { stopCount }), + ...(resultModel === undefined ? {} : { resultModel }), }) .pipe( Effect.catch((error) => @@ -435,6 +438,7 @@ const make = Effect.gen(function* () { * The last reply `runId` gave on the task's thread there, which is its * result, read whole however long it is. A failed run prefers its error * from the activity view over an earlier reply, without taking another run's. + * Also the model that run asked for and the one its provider reported running. */ const lastReply = Effect.fn("RemoteDelegation.lastReply")(function* ( scope: ThreadScope, @@ -445,13 +449,21 @@ const make = Effect.gen(function* () { let reply: OrchestratorMcpThreadTimelineItem | undefined; let error: OrchestratorMcpThreadTimelineItem | undefined; let afterPosition: number | undefined; + let resultModel: OrchestrationV2RemoteTaskChild["resultModel"]; while (true) { const read = yield* forwarding.readThread(scope, remote.environmentId, { threadId: remote.threadId, view: failed ? "activity" : "messages", limit: 100, + runLimit: 50, ...(afterPosition === undefined ? {} : { afterPosition }), }); + const run = read.recentRuns.find((candidate) => candidate.runId === runId); + // Older environments omit executedModel. + resultModel ??= + run === undefined + ? undefined + : { model: run.model, executedModel: run.executedModel ?? null }; reply = read.items.findLast( (item) => item.type === "assistant_message" && (runId === null || item.runId === runId), @@ -466,7 +478,7 @@ const make = Effect.gen(function* () { afterPosition = read.nextPosition; } reply = error ?? reply; - if (reply === undefined) return undefined; + if (reply === undefined) return { text: undefined, resultModel }; let text = reply.text ?? ""; let textOffset = reply.textTruncated ? (reply.nextTextOffset ?? null) : null; while (textOffset !== null) { @@ -481,7 +493,7 @@ const make = Effect.gen(function* () { text += part.text ?? ""; textOffset = part.textTruncated ? (part.nextTextOffset ?? null) : null; } - return text; + return { text, resultModel }; }); /** @@ -613,7 +625,7 @@ const make = Effect.gen(function* () { // A task recorded without its run waits on the thread's latest; a // thread there with none has nothing left to wait for. if (waited.runId === null) { - const reply = yield* lastReply(scope, remote, null); + const reply = (yield* lastReply(scope, remote, null)).text; yield* complete( parentThreadId, taskId, @@ -628,13 +640,25 @@ const make = Effect.gen(function* () { runId: remote.runId ?? waited.runId, status: waited.status, }; - const result = yield* lastReply(scope, remote, resultRun.runId, resultRun.status === "failed"); + const { text, resultModel } = yield* lastReply( + scope, + remote, + resultRun.runId, + resultRun.status === "failed", + ); yield* complete( parentThreadId, taskId, terminalStatus(resultRun.status), - result ?? noReply, + resultModel === undefined + ? (text ?? noReply) + : withSubstitutedModelNote(text ?? noReply, { + requestedModel: resultModel.model, + executedModel: resultModel.executedModel, + }), "ran-out", + undefined, + resultModel, ); return yield* settled(parentThreadId, taskId); }); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 7378c10955104..a90bfd88627c8 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -862,6 +862,12 @@ const DedupedBodyMarker = Schema.optionalKey(Schema.Literal(true)); * of in a child thread here: that environment and the ordinary thread it runs * as there. Such a task has no `childThreadId`. */ +// executedModel is null when the environment there reported none, as older versions never do. +const OrchestrationV2RemoteTaskResultModel = Schema.Struct({ + model: Schema.String, + executedModel: Schema.NullOr(Schema.String), +}); + export const OrchestrationV2RemoteTaskChild = Schema.Struct({ environmentId: EnvironmentId, threadId: ThreadId, @@ -880,6 +886,8 @@ export const OrchestrationV2RemoteTaskChild = Schema.Struct({ * one owed, so each reaches the thread there once however often it is sent. */ stopCount: Schema.optional(NonNegativeInt), + /** The result's run there: the model it asked for and the one its provider reported running. */ + resultModel: Schema.optional(OrchestrationV2RemoteTaskResultModel), }); export type OrchestrationV2RemoteTaskChild = typeof OrchestrationV2RemoteTaskChild.Type; @@ -3472,6 +3480,8 @@ const OrchestrationV2InternalCommand = Schema.Union([ stoppedThere: Schema.optional(Schema.Literals(["ran-out", "stopped"])), /** The stop generation this report acknowledges; absent for the original unnumbered stop. */ stopCount: Schema.optional(NonNegativeInt), + /** Kept on `remoteChild` with the result this report completes the task with. */ + resultModel: Schema.optional(OrchestrationV2RemoteTaskResultModel), }), /** * Owes a stop to the thread there of a remote task that already has its result, as Stop on diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index 38b71a5b322ef..ea1f539d7c413 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -176,16 +176,31 @@ export function makeSubagentConversationArtifacts(input: { return { message, turnItem }; } -function substitutedModelNote( - projection: Pick, +/** + * A delegated result, led by a note when the provider ran its run on another + * model than the run requested, with that run's own provider notices. + */ +export function withSubstitutedModelNote( + result: string, + input: { + readonly requestedModel: string; + readonly executedModel: string | null | undefined; + readonly notices?: ReadonlyArray; + }, +): string { + if (input.executedModel == null || input.executedModel === input.requestedModel) return result; + const note = [ + `Requested model ${input.requestedModel}, but the provider ran ${input.executedModel}.`, + ...(input.notices ?? []).map((notice) => `Provider notice: ${notice}`), + ].join("\n"); + return `${note}\n\n${result}`; +} + +function rootProviderNotices( + projection: Pick, run: OrchestrationV2Run, -): string | undefined { - const executedModel = latestProviderTurnForAttempt( - projection.providerTurns, - run.activeAttemptId, - )?.model; - if (executedModel === undefined || executedModel === run.modelSelection.model) return undefined; - const notices = projection.turnItems +): ReadonlyArray { + return projection.turnItems .filter( (item): item is Extract => item.type === "system_notice" && @@ -196,11 +211,7 @@ function substitutedModelNote( item.nativeItemRef?.strength === "strong", ) .toSorted((left, right) => left.ordinal - right.ordinal) - .map((item) => `Provider notice: ${item.message}`); - return [ - `Requested model ${run.modelSelection.model}, but the provider ran ${executedModel}.`, - ...notices, - ].join("\n"); + .map((item) => item.message); } export function subagentResultForRun( @@ -250,9 +261,19 @@ export function subagentResultForRun( (run.status === "completed" ? "Child task completed without an assistant result." : `Child task ended with status ${run.status}.`); - const note = substitutedModelNote(projection, run); + const executedModel = latestProviderTurnForAttempt( + projection.providerTurns, + run.activeAttemptId, + )?.model; return { - text: note === undefined ? result : `${note}\n\n${result}`, + text: + executedModel === undefined + ? result + : withSubstitutedModelNote(result, { + requestedModel: run.modelSelection.model, + executedModel, + notices: rootProviderNotices(projection, run), + }), messageId: failure === undefined ? (message?.id ?? turnItem?.messageId ?? null) : null, turnItemId: failure?.id ?? turnItem?.id ?? null, }; From 3e94ca092019208672097be07743c6a4e086ea41 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Fri, 30 Oct 2026 18:16:08 +0000 Subject: [PATCH 05/21] fix(server): keep local Claude fallbacks out of root model reports --- .../Adapters/ClaudeAdapterV2.test.ts | 28 +++++++++++++++++-- .../Adapters/ClaudeAdapterV2.ts | 5 +++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index f9ea4f61c928e..3addc002203d8 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -8640,6 +8640,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { model: "claude-opus-4-6", }, }; + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" }), + ); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ ...turnInput, @@ -8647,8 +8651,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { text: "Review this.", }), ); - // An organization policy that disallows the requested model shows up only as init's model. - yield* harness.offerAndWait(claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" })); yield* Queue.offerAll(harness.sdkMessages, [ makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000661", @@ -8656,10 +8658,32 @@ describe("ClaudeAdapterV2 background wake turns", () => { parentToolUseId: "toolu_restricted_model_subagent", model: "claude-haiku-4-5-20251001", }), + claudeSdkFrame({ + type: "system", + subtype: "model_refusal_fallback", + scope: "local", + trigger: "refusal", + direction: "retry", + original_model: "claude-opus-4-6", + fallback_model: "claude-haiku-4-5", + request_id: null, + content: "A side question used a fallback model.", + uuid: "00000000-0000-4000-8000-000000000665", + session_id: WAKE_NATIVE_SESSION, + }), turnOneResult, ]); yield* Queue.take(harness.terminalReceipts); assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); + const localNotices = harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "system_notice" && + event.turnItem.message === "A side question used a fallback model." + ? [event.turnItem] + : [], + ); + assert.lengthOf(localNotices, 1); + assert.isNull(localNotices[0]?.nodeId); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 905c515eab2d2..8c73aca995695 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -9253,7 +9253,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), threadId: context.input.threadId, runId: context.input.runId, - nodeId: context.input.rootNodeId, + nodeId: + message.subtype === "model_refusal_fallback" && message.scope === "local" + ? null + : context.input.rootNodeId, providerThreadId: context.input.providerThread.id, providerTurnId: context.providerTurnId, nativeItemRef: { From 3ceb5aa5468a8b74d5a7321ff87c7e54f50ea66c Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Fri, 30 Oct 2026 18:21:41 +0000 Subject: [PATCH 06/21] test(evals): accept explicit model substitution reports --- evals/mcp-tool-use/cases/orchestration.ndjson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index c556c91cbaa8e..04f5b30eb7633 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -35,5 +35,5 @@ {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} {"id": "thread-send-fresh-context", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send that should start a fresh provider session", "experience": "The user wanted a confused long-running thread to start over in the same thread without its old context; a plain send would keep feeding it the history.", "rule": "`freshContext: true` on t3_thread_send starts a new provider session in the same thread, like /clear.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The thread working on the CSV importer keeps reverting to the old column mapping from early in its history."}, {"role": "assistant", "text": "That thread has a long history with three different mappings in it, so the model keeps picking up the stale one."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "messageId": "msg-1", "runId": "run-1", "status": "starting", "delivery": "started"}}}, "prompt": "Send it the final mapping below and have it start over with a clean context in that same thread, so the old history stops leaking in: thread 11111111-1111-4111-8111-111111111111.\n\nFinal mapping: id -> customer_id, amt -> amount_cents, ts -> created_at.", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "freshContext": true}}], "notCalls": ["delegate_task", "t3_thread_launch", "create_threads"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: fresh provider sessions in one T3 thread"}, "notes": "Pairs with thread-send-to-named-top-level-thread, where an ordinary send must not set freshContext."} -{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["(?:[Ss]onnet[^.!?\\n]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review)[^.!?\\n]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it, summary included. Main returned no executedModel and only the child's answer as the summary, so the parent only saw the requested Opus."} -{"id": "delegate-substituted-model-inline-result", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran, from the inline result", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The completion message inlined only the child's answer, so the parent called it an Opus review.", "rule": "Delegated completion messages put \"Requested model X, but the provider ran Y.\" and the provider's notice before a child's result when the provider ran another model.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 (Opus review PR 123) [completed]:\nRequested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "expect": {"textMust": ["(?:[Ss]onnet[^.!?\\n]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review)[^.!?\\n]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The prompt is the inline completion message as the fix writes it for a short result. On main the same message carried only the child's answer. To check the old text, replace the prompt with 'Delegated task task-1 (Opus review PR 123) [completed]:\\n' plus the answer alone."} +{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["(?:[Ss]onnet[^.!?]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review|substituted)[^.!?]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it, summary included. Main returned no executedModel and only the child's answer as the summary, so the parent only saw the requested Opus."} +{"id": "delegate-substituted-model-inline-result", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran, from the inline result", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The completion message inlined only the child's answer, so the parent called it an Opus review.", "rule": "Delegated completion messages put \"Requested model X, but the provider ran Y.\" and the provider's notice before a child's result when the provider ran another model.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 (Opus review PR 123) [completed]:\nRequested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "expect": {"textMust": ["(?:[Ss]onnet[^.!?]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review|substituted)[^.!?]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The prompt is the inline completion message as the fix writes it for a short result. On main the same message carried only the child's answer. To check the old text, replace the prompt with 'Delegated task task-1 (Opus review PR 123) [completed]:\\n' plus the answer alone."} From 901607bd7f07a4370ef6603b31c303d583be0776 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Fri, 30 Oct 2026 18:55:10 +0000 Subject: [PATCH 07/21] fix(server): keep delegated model reports tied to their result attempt --- .../src/mcp/OrchestratorMcpService.test.ts | 49 ++++++++++++++----- apps/server/src/mcp/OrchestratorMcpService.ts | 6 ++- .../SubagentProjection.test.ts | 26 ++++++++-- .../src/server/subagentProjection.ts | 9 +++- 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index a78c5fcb5df69..b8df76b7cae69 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -3055,8 +3055,10 @@ describe("OrchestratorMcpService provider resolution", () => { result: null, }; let hasContinuation = false; + let hasStoredResult = false; const restartedAttemptId = RunAttemptId.make("attempt:mcp-restarted-child"); const continuationAttemptId = RunAttemptId.make("attempt:mcp-child-continuation"); + const continuationSelection = { ...runSelection, model: "claude-sonnet-4-6" }; // Claude ran the restarted attempt on Sonnet; a later turn of another attempt reported Haiku. const providerTurns = [ { runAttemptId: restartedAttemptId, ordinal: 1, model: "claude-sonnet-4-6" }, @@ -3074,7 +3076,27 @@ describe("OrchestratorMcpService provider resolution", () => { getThreadRecords: (threadId, fields) => Effect.succeed( threadId === parentThreadId - ? parentProjection([task]) + ? ({ + ...parentProjection([ + { + ...task, + status: hasStoredResult ? "completed" : task.status, + result: hasStoredResult ? "Stored review result." : null, + completionDelivery: { state: "acknowledged" }, + }, + ]), + contextTransfers: hasStoredResult + ? [ + { + id: "transfer:mcp-stored-result", + type: "subagent_result", + sourceThreadId: childThreadId, + targetThreadId: parentThreadId, + sourcePoint: { runId: RunId.make("run:mcp-restarted-child") }, + }, + ] + : [], + } as unknown as OrchestrationV2ThreadProjection) : ({ ...childProjection, thread: { ...childProjection.thread, modelSelection: threadSelection }, @@ -3083,20 +3105,20 @@ describe("OrchestratorMcpService provider resolution", () => { { id: RunId.make("run:mcp-restarted-child"), ordinal: 1, - status: "running", + status: hasStoredResult ? "completed" : "running", modelSelection: runSelection, activeAttemptId: restartedAttemptId, - delegatedTaskId: taskId, + delegatedTaskId: hasStoredResult ? undefined : taskId, }, ...(hasContinuation ? [ { id: RunId.make("run:mcp-child-continuation"), ordinal: 2, - status: "running", - modelSelection: runSelection, + status: hasStoredResult ? "completed" : "running", + modelSelection: continuationSelection, activeAttemptId: continuationAttemptId, - delegatedTaskId: taskId, + delegatedTaskId: hasStoredResult ? undefined : taskId, }, ] : []), @@ -3118,18 +3140,21 @@ describe("OrchestratorMcpService provider resolution", () => { ); yield* Effect.gen(function* () { const service = yield* OrchestratorMcpService.OrchestratorMcpService; - for (const [selection, executedModel] of [ - [runSelection, "claude-sonnet-4-6"], - [runSelection, "claude-haiku-4-5"], - [threadSelection, null], + for (const [runExists, continuationExists, storedResult, selection, executedModel] of [ + [true, false, false, runSelection, "claude-sonnet-4-6"], + [true, true, false, continuationSelection, "claude-haiku-4-5"], + [true, true, true, runSelection, "claude-sonnet-4-6"], + [false, false, false, threadSelection, null], ] as const) { + hasRun = runExists; + hasContinuation = continuationExists; + hasStoredResult = storedResult; const result = yield* service.taskStatus(scope, taskId); assert.equal(result.providerInstanceId, selection.instanceId); assert.equal(result.model, selection.model); assert.deepEqual(result.options, selection.options); assert.equal(result.executedModel, executedModel); - if (hasContinuation) hasRun = false; - hasContinuation = true; + if (hasStoredResult) assert.equal(result.summary, "Stored review result."); } }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); }), diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 70fa2f500649d..c4138a04036b6 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1612,7 +1612,11 @@ const make = Effect.gen(function* () { : null; const resultContextTransferId = resultTransfer?.id ?? null; const latestTerminalResultContextTransferId = resultTransferForRun(terminalRun)?.id ?? null; - const executionRun = taskResultRun ?? childRun; + const executionRun = + task.result === null + ? (taskResultRun ?? childRun) + : (childControls.runs.find((run) => run.id === resultTransfer?.sourcePoint.runId) ?? + childRun); const modelSelection = executionRun?.modelSelection ?? childControls.thread.modelSelection ?? { instanceId: task.providerInstanceId, diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index ad4532b0e64af..6c23ebf7759c1 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -15,6 +15,7 @@ import { ProviderDriverKind, ProviderInstanceId, ProviderThreadId, + ProviderTurnId, ThreadId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; @@ -438,14 +439,16 @@ it("notes a substituted model with the root provider notice from the same run", const notice = ( id: string, message: string, - overrides: Partial> = {}, + overrides: Partial< + Pick + > = {}, ): SystemNotice => ({ id: TurnItemId.make(id), threadId: parentThreadId, runId: run.id, nodeId: NodeId.make("root"), providerThreadId: null, - providerTurnId: null, + providerTurnId: ProviderTurnId.make("active-notice"), nativeItemRef: { driver: ProviderDriverKind.make("claudeAgent"), nativeId: id, @@ -473,6 +476,10 @@ it("notes a substituted model with the root provider notice from the same run", nodeId: NodeId.make("native-subagent"), }), notice("other-run", "Another run's notice.", { runId: RunId.make("other") }), + notice("old-attempt", "An earlier attempt's notice.", { + providerTurnId: ProviderTurnId.make("old-attempt"), + }), + notice("unattributed", "A notice without attempt attribution.", { providerTurnId: null }), notice("usage-limit", "T3 usage limit notice.", { nativeItemRef: { driver: ProviderDriverKind.make("claudeAgent"), @@ -482,8 +489,19 @@ it("notes a substituted model with the root provider notice from the same run", }), ], providerTurns: [ - { runAttemptId: run.activeAttemptId, ordinal: 1, model: "gpt-5.4-mini" }, - { runAttemptId: RunAttemptId.make("other"), ordinal: 2, model: "gpt-5.4-nano" }, + { id: ProviderTurnId.make("active-notice"), runAttemptId: run.activeAttemptId, ordinal: 1 }, + { + id: ProviderTurnId.make("active-result"), + runAttemptId: run.activeAttemptId, + ordinal: 2, + model: "gpt-5.4-mini", + }, + { + id: ProviderTurnId.make("old-attempt"), + runAttemptId: RunAttemptId.make("other"), + ordinal: 3, + model: "gpt-5.4-nano", + }, ] as never, }, run, diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index ea1f539d7c413..6379d338b2849 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -197,9 +197,14 @@ export function withSubstitutedModelNote( } function rootProviderNotices( - projection: Pick, + projection: Pick, run: OrchestrationV2Run, ): ReadonlyArray { + const attemptTurnIds = new Set( + projection.providerTurns + .filter((turn) => run.activeAttemptId !== null && turn.runAttemptId === run.activeAttemptId) + .map((turn) => turn.id), + ); return projection.turnItems .filter( (item): item is Extract => @@ -207,6 +212,8 @@ function rootProviderNotices( item.runId === run.id && item.nodeId === run.rootNodeId && item.parentItemId === null && + item.providerTurnId !== null && + attemptTurnIds.has(item.providerTurnId) && // T3's own notices, such as usage limits, carry weak refs. item.nativeItemRef?.strength === "strong", ) From bfb74e0a3a281f33b68c02c95998febba5fafe9a Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Sat, 31 Oct 2026 02:01:12 +0000 Subject: [PATCH 08/21] fix(claude): retain the model reported by root stream starts --- .../Adapters/ClaudeAdapterV2.test.ts | 25 +++++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 1 + 2 files changed, 26 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 3addc002203d8..11d0c21a6552d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -8718,6 +8718,31 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); yield* Queue.take(harness.terminalReceipts); assert.isUndefined(lastProviderTurn(harness.events).model); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + ...turnInput, + attemptId: RunAttemptId.make("attempt-claude-restricted-model-4"), + providerTurnOrdinal: 4, + text: "Stream only.", + }), + ); + yield* Queue.offerAll(harness.sdkMessages, [ + usageStreamFrame({ + type: "message_start", + message: { id: "stream-model", model: "claude-sonnet-4-6", usage: zeroUsage }, + }), + usageStreamFrame( + { + type: "message_start", + message: { id: "child-model", model: "claude-haiku-4-5", usage: zeroUsage }, + }, + "native-child", + ), + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000666", result: "Done." }), + ]); + yield* Queue.take(harness.terminalReceipts); + assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); }).pipe( Effect.scoped, Effect.provide( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 8c73aca995695..fb985a9e43990 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -8964,6 +8964,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const event = message.event; const reasoning = context.reasoning; if (event.type === "message_start") { + recordExecutedModel(context, event.message.model); reasoning.messageId = event.message.id; reasoning.streamBlocks.clear(); const usage: ClaudeMessageUsage | undefined = event.message.usage; From 77a68055d4a83bf9e54f3afbbd1fcbf627d886b3 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Sat, 31 Oct 2026 02:50:54 +0000 Subject: [PATCH 09/21] fix(claude): requested aliases are not model substitutions A run that requested a catalog alias such as `opus` recorded Claude's canonical slug, so delegated results claimed a substitution when the alias named the model that ran. The adapter now keeps the run's own label when the catalog resolves it to the reported model, which covers local and remote result paths without new contract fields. An idle root `message_start` stream frame now also opens a wake turn, so a wake that streams before its init or snapshot keeps its reported model. Child stream starts still do not wake the parent. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 172 ++++++++++++------ .../Adapters/ClaudeAdapterV2.ts | 16 +- .../SubagentProjection.test.ts | 16 ++ 3 files changed, 146 insertions(+), 58 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 11d0c21a6552d..9f51dd86673c8 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -93,6 +93,10 @@ import { WorktreeToolkit } from "../../mcp/toolkits/worktree/tools.ts"; import { ThreadToolkit } from "../../mcp/toolkits/thread/tools.ts"; import { OrchestratorToolkit } from "../../mcp/toolkits/orchestrator/tools.ts"; import { ClaudeExecutableFileCheck } from "../../provider/Drivers/ClaudeExecutable.ts"; +import { + BUNDLED_CLAUDE_MODEL_CATALOG, + resolveClaudeModelSlug, +} from "../../provider/ClaudeModelCatalog.ts"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; @@ -8743,6 +8747,37 @@ describe("ClaudeAdapterV2 background wake turns", () => { ]); yield* Queue.take(harness.terminalReceipts); assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); + + // A requested alias that the catalog resolves to the reported model is no substitution. + const aliasTurn = (ordinal: number, reported: string) => + Effect.gen(function* () { + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + ...turnInput, + modelSelection: { ...turnInput.modelSelection, model: "opus" }, + attemptId: RunAttemptId.make(`attempt-claude-restricted-model-${ordinal}`), + providerTurnOrdinal: ordinal, + text: "Use the alias.", + }), + ); + yield* Queue.offerAll(harness.sdkMessages, [ + usageStreamFrame({ + type: "message_start", + message: { id: `alias-model-${ordinal}`, model: reported, usage: zeroUsage }, + }), + makeResultFrame({ + uuid: `00000000-0000-4000-8000-00000000067${ordinal}`, + result: "Done.", + }), + ]); + yield* Queue.take(harness.terminalReceipts); + return lastProviderTurn(harness.events).model; + }); + assert.equal( + yield* aliasTurn(5, resolveClaudeModelSlug(BUNDLED_CLAUDE_MODEL_CATALOG, "opus")), + "opus", + ); + assert.equal(yield* aliasTurn(6, "claude-sonnet-4-6"), "claude-sonnet-4-6"); }).pipe( Effect.scoped, Effect.provide( @@ -12006,68 +12041,95 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect("starts the wake run when Claude opens the wake turn, before its output", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeWakeHarness; - const now = yield* DateTime.now; + it.effect.each(["init", "stream start"] as const)( + "starts the wake run when Claude opens the wake turn with %s, before its output", + (opening) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now, - attemptId: RunAttemptId.make("attempt-claude-wake-init-1"), - text: "Run the build in the background.", - attachments: [], - }), - ); - yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); - yield* Queue.offer(harness.sdkMessages, turnOneResult); - yield* harness.awaitTerminal(1); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-init-1"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* harness.awaitTerminal(1); - yield* harness.offerAndWait(wakeNotification); - assert.lengthOf(harness.continuationRequests, 0); - yield* harness.offerAndWait(wakeTurnInit); - assert.lengthOf(harness.continuationRequests, 1); - assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); + yield* harness.offerAndWait(wakeNotification); + assert.lengthOf(harness.continuationRequests, 0); + if (opening === "init") { + yield* harness.offerAndWait(wakeTurnInit); + } else { + yield* harness.offerAndWait( + usageStreamFrame( + { + type: "message_start", + message: { id: "wake-child", model: "claude-haiku-4-5", usage: zeroUsage }, + }, + "native-child", + ), + ); + assert.lengthOf(harness.continuationRequests, 0); + yield* harness.offerAndWait( + usageStreamFrame({ + type: "message_start", + message: { id: "wake-root", model: "claude-sonnet-4-6", usage: zeroUsage }, + }), + ); + } + assert.lengthOf(harness.continuationRequests, 1); + assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now, - attemptId: RunAttemptId.make("attempt-claude-wake-init-2"), - text: "Background task completed.", - attachments: [], - providerTurnOrdinal: 2, - messageCreatedBy: "agent", - messageCreationSource: "provider", - }), - ); - assert.lengthOf(harness.terminalEvents(), 1); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-init-2"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + assert.lengthOf(harness.terminalEvents(), 1); - yield* harness.offerAndWait(wakeAssistant); - yield* harness.offerAndWait(wakeResult); - yield* harness.awaitTerminal(2); - assert.equal(harness.terminalEvents()[1]?.status, "completed"); - assert.lengthOf(harness.continuationRequests, 1); - assert.isTrue( - harness.events.some( - (event) => event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, - ), - ); - }).pipe( - Effect.provide( - Layer.mergeAll( - IdAllocator.layer, - McpProviderSessions.layer, - NodeServices.layer, - NodeCrypto.layer, + if (opening === "init") { + yield* harness.offerAndWait(wakeAssistant); + } + yield* harness.offerAndWait(wakeResult); + yield* harness.awaitTerminal(2); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.lengthOf(harness.continuationRequests, 1); + assert.isTrue( + harness.events.some( + (event) => + event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, + ), + ); + if (opening === "stream start") { + assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); + } + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), ), ), ), - ), ); it.effect.each([true, false])( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index fb985a9e43990..3f8d44f16bc9e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -5958,7 +5958,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const reported = typeof model === "string" ? model.trim() : ""; // "" identifies CLI notices, not a model. if (reported.length > 0 && reported !== "") { - context.executedModel = resolveClaudeModelSlug(subagentModelCatalog, reported); + const executed = resolveClaudeModelSlug(subagentModelCatalog, reported); + const requested = context.input.modelSelection.model; + // Keep the run's own label, such as an alias, when it names the model that ran. + context.executedModel = + resolveClaudeModelSlug(subagentModelCatalog, requested) === executed + ? requested + : executed; } }; @@ -8481,7 +8487,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); }); } - const isWakeTurnStart = isClaudeTurnStartMessage(message); + const isWakeTurnStart = + isClaudeTurnStartMessage(message) || + (message.type === "stream_event" && + message.event.type === "message_start" && + message.parent_tool_use_id == null); const isWakeEvidence = isPendingTaskNotification || isPendingSubagentNotification || @@ -8546,7 +8556,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( message.task_id, ); } - // Open continuations only after root init, user, assistant, or result frames prove a wake began. + // Open continuations only after root init, message start, user, assistant, or result frames prove a wake began. // Background child frames can stream while the root is idle. const buffered = (yield* Ref.get(wakeBuffers)).get(wakeInput.nativeThreadId); const hasBufferedNotification = diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index 6c23ebf7759c1..a2f1dc073bee7 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -525,6 +525,22 @@ it("notes a substituted model with the root provider notice from the same run", ).text, "Review done.", ); + // The newest turn reported no model, so an earlier turn's substitution is not attributed to it. + assert.equal( + subagentResultForRun( + { + ...projection, + messages: [{ ...answer.message, runId: run.id }], + turnItems: [notice("fallback", fallback)], + providerTurns: [ + { runAttemptId: run.activeAttemptId, ordinal: 1, model: "gpt-5.4-mini" }, + { runAttemptId: run.activeAttemptId, ordinal: 2 }, + ] as never, + }, + run, + ).text, + "Review done.", + ); }); it.effect("ends an orphaned native subagent's node when only its record already ended", () => From 98ffaa0c96e094f06d8af03a65b50f30ad11b575 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Sat, 31 Oct 2026 03:08:21 +0000 Subject: [PATCH 10/21] fix(claude): report substitutions without rewriting the model that ran The previous commit stored a requested alias such as `opus` as the provider turn's model, which lost the concrete model Claude reported. The adapter now keeps the canonical reported slug and records an optional `modelSubstituted` flag, comparing the catalog-resolved requested and reported models. Delegated results, MCP task and thread reads, and remote results carry the flag and trust it when present. Results from environments that omit it still compare the labels. A root `message_start` now also marks a newer turn after an empty buffered result, the same frames that open a wake, so that result no longer settles the continuation that turn owns. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/OrchestratorMcpService.ts | 25 +++++++---- .../Adapters/ClaudeAdapterV2.test.ts | 39 ++++++++++------- .../Adapters/ClaudeAdapterV2.ts | 35 +++++++++------- .../SubagentProjection.test.ts | 35 ++++++++-------- apps/server/src/peer/RemoteDelegation.test.ts | 42 +++++++++++-------- apps/server/src/peer/RemoteDelegation.ts | 11 ++++- packages/contracts/src/orchestrationV2.ts | 5 ++- packages/contracts/src/orchestratorMcp.ts | 6 +++ .../src/server/subagentProjection.ts | 19 +++++---- 9 files changed, 135 insertions(+), 82 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index c4138a04036b6..7ac266caff021 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -878,9 +878,18 @@ function threadDetail( function runExecutedModel( projection: Pick, - run: Pick, -): string | null { - return latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId)?.model ?? null; + run: Pick | undefined, +): { readonly executedModel: string | null; readonly modelSubstituted?: boolean } { + const turn = + run === undefined + ? undefined + : latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId); + return turn?.model === undefined + ? { executedModel: null } + : { + executedModel: turn.model, + ...(turn.modelSubstituted === undefined ? {} : { modelSubstituted: turn.modelSubstituted }), + }; } function threadRun( @@ -893,7 +902,7 @@ function threadRun( status: run.status, providerInstanceId: run.modelSelection.instanceId, model: run.modelSelection.model, - executedModel: runExecutedModel(projection, run), + ...runExecutedModel(projection, run), requestedAt: DateTime.formatIso(run.requestedAt), startedAt: run.startedAt === null ? null : DateTime.formatIso(run.startedAt), completedAt: run.completedAt === null ? null : DateTime.formatIso(run.completedAt), @@ -1636,8 +1645,7 @@ const make = Effect.gen(function* () { hasPendingChildRuns: hasPendingChildRuns(childProjection, childRun), providerInstanceId: modelSelection.instanceId, model: modelSelection.model, - executedModel: - executionRun === undefined ? null : runExecutedModel(childControls, executionRun), + ...runExecutedModel(childControls, executionRun), options: modelSelection.options ?? [], summary: derivedResult, resultContextTransferId, @@ -1805,8 +1813,9 @@ const make = Effect.gen(function* () { workState: terminal ? "result_available" : "working", hasPendingChildRuns: false, providerInstanceId: task.providerInstanceId, - model: task.remoteChild?.resultModel?.model ?? task.model, - executedModel: task.remoteChild?.resultModel?.executedModel ?? null, + model: task.model, + executedModel: null, + ...task.remoteChild?.resultModel, summary: task.result, resultContextTransferId: null, latestTerminalRunId: null, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 9f51dd86673c8..e8b23c53c1849 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -8634,6 +8634,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { it.effect("records the root model Claude reports for each turn, not the requested one", () => Effect.gen(function* () { const harness = yield* makeWakeHarness; + const executed = () => { + const { model, modelSubstituted } = lastProviderTurn(harness.events); + return { model, modelSubstituted }; + }; const turnInput = { threadId: harness.threadId, providerThread: harness.providerThread, @@ -8678,7 +8682,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { turnOneResult, ]); yield* Queue.take(harness.terminalReceipts); - assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); + assert.deepEqual(executed(), { model: "claude-sonnet-4-6", modelSubstituted: true }); const localNotices = harness.events.flatMap((event) => event.type === "turn_item.updated" && event.turnItem.type === "system_notice" && @@ -8706,7 +8710,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000663", result: "Done." }), ]); yield* Queue.take(harness.terminalReceipts); - assert.equal(lastProviderTurn(harness.events).model, "claude-haiku-4-5"); + assert.deepEqual(executed(), { model: "claude-haiku-4-5", modelSubstituted: true }); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ @@ -8721,7 +8725,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000664", result: "Done." }), ); yield* Queue.take(harness.terminalReceipts); - assert.isUndefined(lastProviderTurn(harness.events).model); + assert.deepEqual(executed(), { model: undefined, modelSubstituted: undefined }); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ @@ -8746,7 +8750,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000666", result: "Done." }), ]); yield* Queue.take(harness.terminalReceipts); - assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); + assert.deepEqual(executed(), { model: "claude-sonnet-4-6", modelSubstituted: true }); // A requested alias that the catalog resolves to the reported model is no substitution. const aliasTurn = (ordinal: number, reported: string) => @@ -8771,13 +8775,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ]); yield* Queue.take(harness.terminalReceipts); - return lastProviderTurn(harness.events).model; + return executed(); }); - assert.equal( - yield* aliasTurn(5, resolveClaudeModelSlug(BUNDLED_CLAUDE_MODEL_CATALOG, "opus")), - "opus", - ); - assert.equal(yield* aliasTurn(6, "claude-sonnet-4-6"), "claude-sonnet-4-6"); + const opus = resolveClaudeModelSlug(BUNDLED_CLAUDE_MODEL_CATALOG, "opus"); + assert.deepEqual(yield* aliasTurn(5, opus), { model: opus, modelSubstituted: false }); + assert.deepEqual(yield* aliasTurn(6, "claude-sonnet-4-6"), { + model: "claude-sonnet-4-6", + modelSubstituted: true, + }); }).pipe( Effect.scoped, Effect.provide( @@ -12805,11 +12810,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { }; it.effect.each([ - { label: "an empty one", quotaFailureFirst: false }, - { label: "a quota failure and an empty one", quotaFailureFirst: true }, + { label: "an empty one", quotaFailureFirst: false, nextOpensWith: "init" }, + { label: "a quota failure and an empty one", quotaFailureFirst: true, nextOpensWith: "init" }, + { label: "an empty one by streaming", quotaFailureFirst: false, nextOpensWith: "stream" }, ])( "keeps the wake run open for the turn Claude opens right after $label", - ({ quotaFailureFirst }) => + ({ quotaFailureFirst, nextOpensWith }) => Effect.scoped( Effect.gen(function* () { const harness = yield* makeWakeHarness; @@ -12890,7 +12896,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); yield* harness.offerAndWait( - claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000963" }), + nextOpensWith === "init" + ? claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000963" }) + : usageStreamFrame({ + type: "message_start", + message: { id: "wake-pair-next", model: "claude-sonnet-4-6", usage: zeroUsage }, + }), ); assert.lengthOf(harness.continuationRequests, 1); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3f8d44f16bc9e..1b55139999cb9 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -3222,8 +3222,14 @@ function isClaudeBackgroundTasksChangedMessage(message: SDKMessage): boolean { return message.type === "system" && (message.subtype as string) === "background_tasks_changed"; } +// A wake can stream its root message before its init frame. function isClaudeTurnStartMessage(message: SDKMessage): boolean { - return message.type === "system" && message.subtype === "init"; + return ( + (message.type === "system" && message.subtype === "init") || + (message.type === "stream_event" && + message.event.type === "message_start" && + message.parent_tool_use_id == null) + ); } function isClaudeRootTurnFrame(message: SDKMessage): boolean { @@ -4385,7 +4391,7 @@ interface ActiveClaudeTurnContext { gatedFramesBeforeEcho: number; readonly heldRootFrames: Array; // Claude can run another model than requested, such as when an organization disallows it. - executedModel: string | undefined; + executedModel: { readonly model: string; readonly modelSubstituted: boolean } | undefined; } interface ActiveClaudeProviderRetry { @@ -4487,8 +4493,11 @@ interface PendingClaudeSubagentLaunch { readonly ownerToolUseId?: string; } -function executedModelField(context: ActiveClaudeTurnContext): { readonly model?: string } { - return context.executedModel === undefined ? {} : { model: context.executedModel }; +function executedModelField(context: ActiveClaudeTurnContext): { + readonly model?: string; + readonly modelSubstituted?: boolean; +} { + return context.executedModel ?? {}; } function canonicalClaudeSubagentModel(catalog: ClaudeModelCatalog, model: string): string { @@ -5959,12 +5968,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // "" identifies CLI notices, not a model. if (reported.length > 0 && reported !== "") { const executed = resolveClaudeModelSlug(subagentModelCatalog, reported); - const requested = context.input.modelSelection.model; - // Keep the run's own label, such as an alias, when it names the model that ran. - context.executedModel = - resolveClaudeModelSlug(subagentModelCatalog, requested) === executed - ? requested - : executed; + // An alias such as `opus` requests the model its canonical slug names. + const requested = resolveClaudeModelSlug( + subagentModelCatalog, + context.input.modelSelection.model, + ); + context.executedModel = { model: executed, modelSubstituted: requested !== executed }; } }; @@ -8487,11 +8496,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); }); } - const isWakeTurnStart = - isClaudeTurnStartMessage(message) || - (message.type === "stream_event" && - message.event.type === "message_start" && - message.parent_tool_use_id == null); + const isWakeTurnStart = isClaudeTurnStartMessage(message); const isWakeEvidence = isPendingTaskNotification || isPendingSubagentNotification || diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index a2f1dc073bee7..0b1630ffcbb56 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -511,34 +511,35 @@ it("notes a substituted model with the root provider notice from the same run", `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\nProvider notice: ${fallback}\n\nReview done.`, ); assert.equal(result.messageId, answer.message.id); - assert.equal( + const resultWithTurns = (providerTurns: ReadonlyArray) => subagentResultForRun( { ...projection, messages: [{ ...answer.message, runId: run.id }], turnItems: [notice("fallback", fallback)], - providerTurns: [ - { runAttemptId: run.activeAttemptId, ordinal: 1, model: "gpt-5.4" }, - ] as never, + providerTurns: providerTurns as never, }, run, - ).text, + ).text; + const attempt = { id: ProviderTurnId.make("active-notice"), runAttemptId: run.activeAttemptId }; + // The provider's own verdict wins over comparing labels, such as for an alias it resolved. + assert.equal( + resultWithTurns([ + { ...attempt, ordinal: 1, model: "gpt-5.4-2026-03-05", modelSubstituted: false }, + ]), "Review done.", ); + assert.equal( + resultWithTurns([{ ...attempt, ordinal: 1, model: "gpt-5.4-mini", modelSubstituted: true }]), + `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\nProvider notice: ${fallback}\n\nReview done.`, + ); + assert.equal(resultWithTurns([{ ...attempt, ordinal: 1, model: "gpt-5.4" }]), "Review done."); // The newest turn reported no model, so an earlier turn's substitution is not attributed to it. assert.equal( - subagentResultForRun( - { - ...projection, - messages: [{ ...answer.message, runId: run.id }], - turnItems: [notice("fallback", fallback)], - providerTurns: [ - { runAttemptId: run.activeAttemptId, ordinal: 1, model: "gpt-5.4-mini" }, - { runAttemptId: run.activeAttemptId, ordinal: 2 }, - ] as never, - }, - run, - ).text, + resultWithTurns([ + { ...attempt, ordinal: 1, model: "gpt-5.4-mini", modelSubstituted: true }, + { ...attempt, ordinal: 2 }, + ]), "Review done.", ); }); diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index dfe07d1841573..ba53ce0d57d36 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -87,6 +87,8 @@ type LaunchedWith = Pick< >; /** The thread the box launches for the task, which the test finishes. */ +type ReportedModel = Pick; + interface BoxThread { readonly launched: Ref.Ref; /** The link session that launched it, as the box stamps it. */ @@ -104,8 +106,8 @@ interface BoxThread { /** A later run's reply, after the task's own, as a follow-up message would leave. */ readonly laterReply: Ref.Ref; /** The model the provider there reported running the task's own run, and the later one. */ - readonly executedModel: Ref.Ref; - readonly laterExecutedModel: Ref.Ref; + readonly executedModel: Ref.Ref; + readonly laterExecutedModel: Ref.Ref; /** The run each wait asked for, in order. */ readonly waitedFor: Ref.Ref>; /** The thread's run ends with a task of its own still running, whose result wakes it again. */ @@ -151,8 +153,8 @@ const makeBoxThread = Effect.gen(function* () { error: yield* Ref.make(null), laterError: yield* Ref.make(null), laterReply: yield* Ref.make(null), - executedModel: yield* Ref.make(null), - laterExecutedModel: yield* Ref.make(null), + executedModel: yield* Ref.make({}), + laterExecutedModel: yield* Ref.make({}), waitedFor: yield* Ref.make>([]), nested: yield* Ref.make(false), nestedWaiting: yield* Deferred.make(), @@ -239,12 +241,12 @@ const boxToolkitLayer = (thread: BoxThread) => { startedAt: DateTime.makeUnsafe(0), completedAt: null, }) as unknown as OrchestrationV2Run; - const providerTurnOf = (id: RunId, model: string | null) => + const providerTurnOf = (id: RunId, reported: ReportedModel) => ({ id: ProviderTurnId.make(`provider-turn:${id}`), runAttemptId: RunAttemptId.make(`attempt:${id}`), ordinal: 1, - ...(model === null ? {} : { model }), + ...reported, }) as unknown as OrchestrationV2ProviderTurn; const projectionOf = (threadId: ThreadId) => Effect.gen(function* () { @@ -1294,8 +1296,9 @@ it.effect("a task's result is its own run's whole last reply, not a later run's" yield* Ref.set(thread.reply, long); yield* Ref.set(thread.laterReply, "Someone else asked a follow-up."); // The provider there ran the task's run on another model, the later run on the one asked for. - yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); - yield* Ref.set(thread.laterExecutedModel, "gpt-5.4"); + // Neither turn says whether that was a substitution, as an older box's never do, so the labels decide. + yield* Ref.set(thread.executedModel, { model: "gpt-5.4-mini" }); + yield* Ref.set(thread.laterExecutedModel, { model: "gpt-5.4" }); yield* Ref.set(thread.status, "completed"); yield* Deferred.succeed(thread.finished, undefined); const task = yield* taskResult(a, taskId); @@ -1303,9 +1306,9 @@ it.effect("a task's result is its own run's whole last reply, not a later run's" expect(task?.result).toBe( `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\n\n${long}`, ); - expect((yield* a.call("task_status", { taskId })).structuredContent).toMatchObject({ - executedModel: "gpt-5.4-mini", - }); + const status = (yield* a.call("task_status", { taskId })).structuredContent; + expect(status).toMatchObject({ executedModel: "gpt-5.4-mini" }); + expect(status).not.toHaveProperty("modelSubstituted"); // Every wait was for the run the launch started. const waits = yield* Ref.get(thread.waitedFor); expect(waits.length).toBeGreaterThan(0); @@ -1339,20 +1342,23 @@ it.effect("a task whose thread there waits on its own task ends with the answer ); expect(first).toBe("waiting there"); - // The review reports and wakes the thread there, whose next run answers, - // on another model than the provider ran the first run on. - yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); - yield* Ref.set(thread.laterExecutedModel, "gpt-5.4-nano"); + // The review reports and wakes the thread there, whose next run answers on + // the dated slug of the model it asked for, unlike the first run's substitute. + yield* Ref.set(thread.executedModel, { model: "gpt-5.4-mini", modelSubstituted: true }); + yield* Ref.set(thread.laterExecutedModel, { + model: "gpt-5.4-2026-03-05", + modelSubstituted: false, + }); yield* Ref.set(thread.laterReply, "The review found two issues; both are fixed."); yield* Deferred.succeed(thread.nestedSettled, undefined); - const result = - "Requested model gpt-5.4, but the provider ran gpt-5.4-nano.\n\nThe review found two issues; both are fixed."; + const result = "The review found two issues; both are fixed."; expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "completed", result }); const status = yield* a.call("task_status", { taskId: delegated.taskId }); expect(status.structuredContent).toMatchObject({ summary: result, model: "gpt-5.4", - executedModel: "gpt-5.4-nano", + executedModel: "gpt-5.4-2026-03-05", + modelSubstituted: false, }); }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 7140db34543ca..ca7b5df1c183e 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -459,11 +459,17 @@ const make = Effect.gen(function* () { ...(afterPosition === undefined ? {} : { afterPosition }), }); const run = read.recentRuns.find((candidate) => candidate.runId === runId); - // Older environments omit executedModel. + // Older environments omit executedModel and modelSubstituted. resultModel ??= run === undefined ? undefined - : { model: run.model, executedModel: run.executedModel ?? null }; + : { + model: run.model, + executedModel: run.executedModel ?? null, + ...(run.modelSubstituted === undefined + ? {} + : { modelSubstituted: run.modelSubstituted }), + }; reply = read.items.findLast( (item) => item.type === "assistant_message" && (runId === null || item.runId === runId), @@ -655,6 +661,7 @@ const make = Effect.gen(function* () { : withSubstitutedModelNote(text ?? noReply, { requestedModel: resultModel.model, executedModel: resultModel.executedModel, + modelSubstituted: resultModel.modelSubstituted, }), "ran-out", undefined, diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index a90bfd88627c8..b6476bdda8972 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -862,10 +862,11 @@ const DedupedBodyMarker = Schema.optionalKey(Schema.Literal(true)); * of in a child thread here: that environment and the ordinary thread it runs * as there. Such a task has no `childThreadId`. */ -// executedModel is null when the environment there reported none, as older versions never do. +// executedModel is null and modelSubstituted absent when the environment there reported none, as older versions never do. const OrchestrationV2RemoteTaskResultModel = Schema.Struct({ model: Schema.String, executedModel: Schema.NullOr(Schema.String), + modelSubstituted: Schema.optional(Schema.Boolean), }); export const OrchestrationV2RemoteTaskChild = Schema.Struct({ @@ -1196,6 +1197,8 @@ export const OrchestrationV2ProviderTurn = Schema.Struct({ turnTokenUsage: Schema.optional(TurnTokenUsage), /** The model the provider reported running this turn's root agent, which can differ from the requested one. */ model: Schema.optional(TrimmedNonEmptyString), + /** Whether `model` is another model than the run requested, which an alias for it is not. */ + modelSubstituted: Schema.optional(Schema.Boolean), }); export type OrchestrationV2ProviderTurn = typeof OrchestrationV2ProviderTurn.Type; diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 21382fdbd1691..ea722836cc144 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -206,6 +206,10 @@ const OrchestratorMcpExecutedModel = Schema.NullOr(Schema.String).annotate({ description: "Model the provider reported running, or null if unreported. Can differ from the requested `model`.", }); +const OrchestratorMcpModelSubstituted = Schema.Boolean.annotate({ + description: + "Whether `executedModel` is another model than the requested `model`, not just its alias. Absent when unknown.", +}); export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ taskId: NodeId, @@ -228,6 +232,7 @@ export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ providerInstanceId: ProviderInstanceId, model: Schema.NullOr(Schema.String), executedModel: Schema.optional(OrchestratorMcpExecutedModel), + modelSubstituted: Schema.optional(OrchestratorMcpModelSubstituted), options: Schema.optional(Schema.Array(ProviderOptionSelection)), summary: Schema.NullOr(Schema.String), resultContextTransferId: Schema.NullOr(ContextTransferId), @@ -470,6 +475,7 @@ export const OrchestratorMcpThreadRun = Schema.Struct({ providerInstanceId: ProviderInstanceId, model: Schema.String, executedModel: Schema.optional(OrchestratorMcpExecutedModel), + modelSubstituted: Schema.optional(OrchestratorMcpModelSubstituted), requestedAt: IsoDateTime, startedAt: Schema.NullOr(IsoDateTime), completedAt: Schema.NullOr(IsoDateTime), diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index 6379d338b2849..5220fd1ce7064 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -185,10 +185,17 @@ export function withSubstitutedModelNote( input: { readonly requestedModel: string; readonly executedModel: string | null | undefined; + // Set where the provider resolved aliases; older environments omit it. + readonly modelSubstituted?: boolean | undefined; readonly notices?: ReadonlyArray; }, ): string { - if (input.executedModel == null || input.executedModel === input.requestedModel) return result; + if ( + input.executedModel == null || + !(input.modelSubstituted ?? input.executedModel !== input.requestedModel) + ) { + return result; + } const note = [ `Requested model ${input.requestedModel}, but the provider ran ${input.executedModel}.`, ...(input.notices ?? []).map((notice) => `Provider notice: ${notice}`), @@ -268,17 +275,15 @@ export function subagentResultForRun( (run.status === "completed" ? "Child task completed without an assistant result." : `Child task ended with status ${run.status}.`); - const executedModel = latestProviderTurnForAttempt( - projection.providerTurns, - run.activeAttemptId, - )?.model; + const latestTurn = latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId); return { text: - executedModel === undefined + latestTurn?.model === undefined ? result : withSubstitutedModelNote(result, { requestedModel: run.modelSelection.model, - executedModel, + executedModel: latestTurn.model, + modelSubstituted: latestTurn.modelSubstituted, notices: rootProviderNotices(projection, run), }), messageId: failure === undefined ? (message?.id ?? turnItem?.messageId ?? null) : null, From 1254025d6611e44e8f1d59694c34ea8b29557096 Mon Sep 17 00:00:00 2001 From: PR Batch Tester Date: Mon, 2 Nov 2026 03:06:52 +0000 Subject: [PATCH 11/21] test(server): include provider turns in pagination fixture --- apps/server/src/mcp/OrchestratorMcpService.activity.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts index 2125383710290..683d389821e86 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts @@ -234,6 +234,7 @@ it("readThread pages cut-off text without splitting an emoji across pages", asyn model: "gpt-5.4", }), runs: [makeRun({ id: activeRunId, ordinal: 1, status: "running" })], + providerTurns: [], visibleTurnItems: [], runtimeRequests: [], messages: [], From a785062b688e0ab683ff7c062ffa7406267d08cf Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:24:08 +0000 Subject: [PATCH 12/21] test(claude): retain execution metadata during compaction --- .../Adapters/ClaudeAdapterV2.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index e8b23c53c1849..74c531f023b9c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -8679,9 +8679,27 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000000665", session_id: WAKE_NATIVE_SESSION, }), + claudeSdkFrame({ + type: "system", + subtype: "compact_boundary", + compact_metadata: { trigger: "auto", pre_tokens: 10_000, post_tokens: 4_321 }, + uuid: "00000000-0000-4000-8000-000000000666", + session_id: WAKE_NATIVE_SESSION, + }), turnOneResult, ]); yield* Queue.take(harness.terminalReceipts); + const compactionTurn = harness.events.find( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.status === "running" && + event.providerTurn.tokenUsage?.usedTokens === 4_321, + ); + assert.equal(compactionTurn?.type, "provider_turn.updated"); + if (compactionTurn?.type === "provider_turn.updated") { + assert.equal(compactionTurn.providerTurn.model, "claude-sonnet-4-6"); + assert.isTrue(compactionTurn.providerTurn.modelSubstituted); + } assert.deepEqual(executed(), { model: "claude-sonnet-4-6", modelSubstituted: true }); const localNotices = harness.events.flatMap((event) => event.type === "turn_item.updated" && From 0cd5d8053963c25db161360d1b9b2daa25da0e8b Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:03:04 +0000 Subject: [PATCH 13/21] fix(claude): discovered model aliases are not substitutions Claude Code now advertises models it reports, and a discovered alias such as `gateway/best` resolves to its own canonical model. The adapter only knew the bundled and custom catalog, so a run of that alias marked its reported canonical model as a substitution and delegated results warned. The driver now shares each instance's reported models with its adapter, and the substitution check resolves both the requested and reported model through the catalog, then through Claude Code's reported alias mappings. The recorded executed model and the requested selection are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 34 ++++++++++++++++--- .../Adapters/ClaudeAdapterV2.ts | 28 +++++++++++---- .../server/src/provider/ClaudeModelCatalog.ts | 14 ++++++-- .../src/provider/Drivers/ClaudeDriver.ts | 14 ++++++-- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 74c531f023b9c..a039a951c114f 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -68,6 +68,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -3139,6 +3140,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly freshQueueOnReopen?: boolean; readonly forkSession?: ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerShape["forkSession"]; readonly allocateSessionId?: Effect.Effect; + readonly reportedModels?: ReadonlyArray; }) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -3201,6 +3203,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { path: yield* Path.Path, random: yield* SecureRandom.SecureRandom, idAllocator, + ...(options?.reportedModels === undefined + ? {} + : { reportedModels: yield* Ref.make(options.reportedModels) }), continuationRequests: { offer: (request) => Effect.sync(() => { @@ -8633,7 +8638,17 @@ describe("ClaudeAdapterV2 background wake turns", () => { it.effect("records the root model Claude reports for each turn, not the requested one", () => Effect.gen(function* () { - const harness = yield* makeWakeHarness; + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + reportedModels: [ + { + value: "gateway/best", + resolvedModel: "gateway/claude-future", + displayName: "Best", + description: "", + }, + ], + }); const executed = () => { const { model, modelSubstituted } = lastProviderTurn(harness.events); return { model, modelSubstituted }; @@ -8770,19 +8785,19 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* Queue.take(harness.terminalReceipts); assert.deepEqual(executed(), { model: "claude-sonnet-4-6", modelSubstituted: true }); - // A requested alias that the catalog resolves to the reported model is no substitution. - const aliasTurn = (ordinal: number, reported: string) => + // An alias the catalog or Claude Code resolves to the reported model is no substitution. + const aliasTurn = (ordinal: number, reported: string, requested = "opus") => Effect.gen(function* () { yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ ...turnInput, - modelSelection: { ...turnInput.modelSelection, model: "opus" }, + modelSelection: { ...turnInput.modelSelection, model: requested }, attemptId: RunAttemptId.make(`attempt-claude-restricted-model-${ordinal}`), providerTurnOrdinal: ordinal, text: "Use the alias.", }), ); - yield* Queue.offerAll(harness.sdkMessages, [ + yield* Queue.offerAll(harness.processQueues.at(-1) ?? harness.sdkMessages, [ usageStreamFrame({ type: "message_start", message: { id: `alias-model-${ordinal}`, model: reported, usage: zeroUsage }, @@ -8801,6 +8816,15 @@ describe("ClaudeAdapterV2 background wake turns", () => { model: "claude-sonnet-4-6", modelSubstituted: true, }); + assert.deepEqual(yield* aliasTurn(7, "gateway/claude-future", "gateway/best"), { + model: "gateway/claude-future", + modelSubstituted: false, + }); + assert.equal(harness.getOpenedOptions()?.model, "gateway/best"); + assert.deepEqual(yield* aliasTurn(8, "claude-sonnet-4-6", "gateway/best"), { + model: "claude-sonnet-4-6", + modelSubstituted: true, + }); }).pipe( Effect.scoped, Effect.provide( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 1b55139999cb9..44569f1d390ee 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -48,6 +48,7 @@ import { getSubagentMessages, type HookCallback, type McpServerStatus as ClaudeMcpServerStatus, + type ModelInfo, query, resolveSettings as resolveClaudeSettings, type Options as ClaudeQueryOptions, @@ -4852,6 +4853,8 @@ export interface ClaudeAdapterV2Options { readonly idAllocator: IdAllocator.IdAllocatorV2["Service"]; readonly queryRunner: ClaudeAgentSdkQueryRunnerShape; readonly scopedLimitNames?: Ref.Ref; + /** Models this instance's Claude Code reported, with the model each alias resolves to. */ + readonly reportedModels?: Ref.Ref>; readonly onUsageLimits?: ManagedServerProvider["applyUsageLimits"]; readonly continuationRequests?: { readonly offer: ( @@ -5963,19 +5966,27 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ...executedModelField(input.context), }); - const recordExecutedModel = (context: ActiveClaudeTurnContext, model: unknown) => { + // Untraced: it runs for every root assistant frame. + const recordExecutedModel = Effect.fnUntraced(function* ( + context: ActiveClaudeTurnContext, + model: unknown, + ) { const reported = typeof model === "string" ? model.trim() : ""; // "" identifies CLI notices, not a model. if (reported.length > 0 && reported !== "") { - const executed = resolveClaudeModelSlug(subagentModelCatalog, reported); + const discovered = adapterOptions.reportedModels + ? yield* Ref.get(adapterOptions.reportedModels) + : []; + const executed = resolveClaudeModelSlug(subagentModelCatalog, reported, discovered); // An alias such as `opus` requests the model its canonical slug names. const requested = resolveClaudeModelSlug( subagentModelCatalog, context.input.modelSelection.model, + discovered, ); context.executedModel = { model: executed, modelSubstituted: requested !== executed }; } - }; + }); const buildToolCallArtifacts = (input: { readonly context: ActiveClaudeTurnContext | null; @@ -8949,7 +8960,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( message.subtype === "init" && context === (yield* Ref.get(activeTurn)) ) { - recordExecutedModel(context, message.model); + yield* recordExecutedModel(context, message.model); } const focusedPlugin = claudeFocusRequestedPanePlugin(message); if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { @@ -8979,7 +8990,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const event = message.event; const reasoning = context.reasoning; if (event.type === "message_start") { - recordExecutedModel(context, event.message.model); + yield* recordExecutedModel(context, event.message.model); reasoning.messageId = event.message.id; reasoning.streamBlocks.clear(); const usage: ClaudeMessageUsage | undefined = event.message.usage; @@ -9088,7 +9099,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if (message.message.model && message.message.model !== "") { context.rootModels.add(message.message.model); } - recordExecutedModel(context, message.message.model); + yield* recordExecutedModel(context, message.message.model); // Subagent uuids exist only in the sidechain transcript, so resume and fork cannot target them. context.nativeMessageCursor = message.uuid; context.latestAssistantRateLimit = @@ -12321,7 +12332,10 @@ export type ClaudeAdapterV2DriverEnv = export const createClaudeAdapterV2 = Effect.fn("ClaudeAdapterV2Driver.create")( function* ( input: ProviderAdapterDriverCreateInput, - hooks: Pick = {}, + hooks: Pick< + ClaudeAdapterV2Options, + "scopedLimitNames" | "reportedModels" | "onUsageLimits" + > = {}, ) { const { instanceId, environment, enabled, config } = input; const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/provider/ClaudeModelCatalog.ts b/apps/server/src/provider/ClaudeModelCatalog.ts index d4fba4253866b..31a61cb281e77 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.ts @@ -189,8 +189,18 @@ function resolveClaudeCatalogModel( ); } -export function resolveClaudeModelSlug(catalog: ClaudeModelCatalog, slugOrAlias: string): string { - return resolveClaudeCatalogModel(catalog, slugOrAlias)?.model.slug ?? slugOrAlias; +/** `discovered` rows from Claude Code name the model each of their values resolves to. */ +export function resolveClaudeModelSlug( + catalog: ClaudeModelCatalog, + slugOrAlias: string, + discovered: ReadonlyArray = [], +): string { + const known = resolveClaudeCatalogModel(catalog, slugOrAlias); + if (known) return known.model.slug; + const resolved = discovered + .find((info) => info.value.trim() === slugOrAlias) + ?.resolvedModel?.trim(); + return resolved ? resolveClaudeModelSlug(catalog, resolved) : slugOrAlias; } export function getClaudeCatalogModelCapabilities( diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index c0a9c5698d92c..ed5e1cd95929b 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -1,3 +1,4 @@ +import type { ModelInfo as ClaudeModelInfo } from "@anthropic-ai/claude-agent-sdk"; import { ClaudeSettings, ProviderDriverKind } from "@t3tools/contracts"; import * as Cache from "effect/Cache"; import * as Duration from "effect/Duration"; @@ -146,6 +147,7 @@ export const ClaudeDriver: ProviderDriver>([]); const orchestrationAdapter = yield* createClaudeAdapterV2( { instanceId, @@ -155,7 +157,11 @@ export const ClaudeDriver: ProviderDriver snapshot.applyUsageLimits(update) }, + { + scopedLimitNames, + reportedModels, + onUsageLimits: (update) => snapshot.applyUsageLimits(update), + }, ).pipe( Effect.mapError( (cause) => @@ -189,7 +195,11 @@ export const ClaudeDriver: ProviderDriver Ref.set(lastGoodCapabilities, capabilities)), + Effect.tap(({ capabilities }) => + Ref.set(lastGoodCapabilities, capabilities).pipe( + Effect.andThen(Ref.set(reportedModels, capabilities.models ?? [])), + ), + ), Effect.catch((error) => Ref.getAndSet(lastGoodCapabilities, undefined).pipe( Effect.flatMap((capabilities) => From 4605780f41d17e322ec5556cea06e70a83812744 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:18:37 +0000 Subject: [PATCH 14/21] fix(claude): keep discovered model identity stable during a turn --- .../Adapters/ClaudeAdapterV2.test.ts | 35 +++++++++++++------ .../Adapters/ClaudeAdapterV2.ts | 26 +++++++------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index a039a951c114f..6f765ddd84885 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3140,7 +3140,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly freshQueueOnReopen?: boolean; readonly forkSession?: ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerShape["forkSession"]; readonly allocateSessionId?: Effect.Effect; - readonly reportedModels?: ReadonlyArray; + readonly reportedModels?: Ref.Ref>; }) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -3205,7 +3205,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { idAllocator, ...(options?.reportedModels === undefined ? {} - : { reportedModels: yield* Ref.make(options.reportedModels) }), + : { reportedModels: options.reportedModels }), continuationRequests: { offer: (request) => Effect.sync(() => { @@ -8638,16 +8638,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { it.effect("records the root model Claude reports for each turn, not the requested one", () => Effect.gen(function* () { + const discoveredAlias = { + value: "gateway/best", + resolvedModel: "gateway/claude-future", + displayName: "Best", + description: "", + }; + const reportedModels = yield* Ref.make>([discoveredAlias]); const harness = yield* makeWakeHarnessWithOptions({ freshQueueOnReopen: true, - reportedModels: [ - { - value: "gateway/best", - resolvedModel: "gateway/claude-future", - displayName: "Best", - description: "", - }, - ], + reportedModels, }); const executed = () => { const { model, modelSubstituted } = lastProviderTurn(harness.events); @@ -8797,11 +8797,19 @@ describe("ClaudeAdapterV2 background wake turns", () => { text: "Use the alias.", }), ); - yield* Queue.offerAll(harness.processQueues.at(-1) ?? harness.sdkMessages, [ + yield* harness.offerAndWait( usageStreamFrame({ type: "message_start", message: { id: `alias-model-${ordinal}`, model: reported, usage: zeroUsage }, }), + ); + if (ordinal === 7) yield* Ref.set(reportedModels, []); + yield* Queue.offerAll(harness.processQueues.at(-1) ?? harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: `00000000-0000-4000-8000-00000000068${ordinal}`, + error: undefined, + model: reported, + }), makeResultFrame({ uuid: `00000000-0000-4000-8000-00000000067${ordinal}`, result: "Done.", @@ -8825,6 +8833,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { model: "claude-sonnet-4-6", modelSubstituted: true, }); + yield* Ref.set(reportedModels, [{ ...discoveredAlias, resolvedModel: "claude-sonnet-4-6" }]); + assert.deepEqual(yield* aliasTurn(9, "claude-sonnet-4-6", "gateway/best"), { + model: "claude-sonnet-4-6", + modelSubstituted: false, + }); }).pipe( Effect.scoped, Effect.provide( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 44569f1d390ee..425dd83be2998 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4321,6 +4321,7 @@ function formatClaudeUsageLimitWait(waitMs: number): string { } interface ActiveClaudeTurnContext { + readonly reportedModels: ReadonlyArray; readonly queryPolicy: ClaudeRuntimeQueryPolicy; readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly nativeThreadId: string; @@ -5966,17 +5967,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ...executedModelField(input.context), }); - // Untraced: it runs for every root assistant frame. - const recordExecutedModel = Effect.fnUntraced(function* ( - context: ActiveClaudeTurnContext, - model: unknown, - ) { + const recordExecutedModel = (context: ActiveClaudeTurnContext, model: unknown) => { const reported = typeof model === "string" ? model.trim() : ""; // "" identifies CLI notices, not a model. if (reported.length > 0 && reported !== "") { - const discovered = adapterOptions.reportedModels - ? yield* Ref.get(adapterOptions.reportedModels) - : []; + const discovered = context.reportedModels; const executed = resolveClaudeModelSlug(subagentModelCatalog, reported, discovered); // An alias such as `opus` requests the model its canonical slug names. const requested = resolveClaudeModelSlug( @@ -5986,7 +5981,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); context.executedModel = { model: executed, modelSubstituted: requested !== executed }; } - }); + }; const buildToolCallArtifacts = (input: { readonly context: ActiveClaudeTurnContext | null; @@ -8960,7 +8955,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( message.subtype === "init" && context === (yield* Ref.get(activeTurn)) ) { - yield* recordExecutedModel(context, message.model); + recordExecutedModel(context, message.model); } const focusedPlugin = claudeFocusRequestedPanePlugin(message); if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { @@ -8990,7 +8985,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const event = message.event; const reasoning = context.reasoning; if (event.type === "message_start") { - yield* recordExecutedModel(context, event.message.model); + recordExecutedModel(context, event.message.model); reasoning.messageId = event.message.id; reasoning.streamBlocks.clear(); const usage: ClaudeMessageUsage | undefined = event.message.usage; @@ -9099,7 +9094,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if (message.message.model && message.message.model !== "") { context.rootModels.add(message.message.model); } - yield* recordExecutedModel(context, message.message.model); + recordExecutedModel(context, message.message.model); // Subagent uuids exist only in the sidechain transcript, so resume and fork cannot target them. context.nativeMessageCursor = message.uuid; context.latestAssistantRateLimit = @@ -11165,8 +11160,15 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(Path.Path, path), ); + const reportedModels = adapterOptions.reportedModels + ? yield* Ref.get(adapterOptions.reportedModels) + : []; const querySession = yield* openQuery(turnInput, nativeThreadId); const context: ActiveClaudeTurnContext = { + // Continuations consume the existing query's output, including its model identity. + reportedModels: isClaudeProviderContinuationTurn(turnInput) + ? (querySession.lastTurn?.reportedModels ?? reportedModels) + : reportedModels, input: turnInput, queryPolicy: querySession.queryPolicy, nativeThreadId, From 7bac2a69363736b04e00a2af0e1c5a407fabf258 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:34:48 +0000 Subject: [PATCH 15/21] test(claude): keep execution identity across catalog refreshes --- .../Adapters/ClaudeAdapterV2.test.ts | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index eae01033b625c..722e0bad07012 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3138,6 +3138,19 @@ describe("ClaudeAdapterV2 background wake turns", () => { for (let tick = 0; tick < ticks; tick++) yield* Effect.yieldNow; }); + const modelCatalogWithAlias = (alias: string, slug: string) => ({ + models: BUNDLED_CLAUDE_MODEL_CATALOG.models.map((entry) => { + const aliases = (entry.model.aliases ?? []).filter((value) => value !== alias); + return { + ...entry, + model: { + ...entry.model, + aliases: entry.model.slug === slug ? [...aliases, alias] : aliases, + }, + }; + }), + }); + const makeWakeHarnessWithOptions = (options?: { readonly close?: (sdkMessages: Queue.Queue) => Effect.Effect; readonly interrupt?: Effect.Effect; @@ -8681,9 +8694,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { description: "", }; const reportedModels = yield* Ref.make>([discoveredAlias]); + const initialCatalog = modelCatalogWithAlias("opus", "claude-opus-4-6"); + const modelCatalog = yield* Ref.make(initialCatalog); const harness = yield* makeWakeHarnessWithOptions({ freshQueueOnReopen: true, reportedModels, + modelCatalog: Ref.get(modelCatalog), }); const executed = () => { const { model, modelSubstituted } = lastProviderTurn(harness.events); @@ -8840,6 +8856,22 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); if (ordinal === 7) yield* Ref.set(reportedModels, []); + if (ordinal === 5) { + yield* Ref.set(modelCatalog, BUNDLED_CLAUDE_MODEL_CATALOG); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: RunId.make(`run-attempt-claude-restricted-model-${ordinal}`), + providerThread: harness.providerThread, + providerTurnId: lastProviderTurn(harness.events).id, + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("catalog-refresh-steer"), + text: "Keep reviewing.", + attachments: [], + }, + }); + } yield* Queue.offerAll(harness.processQueues.at(-1) ?? harness.sdkMessages, [ makeAssistantErrorFrame({ uuid: `00000000-0000-4000-8000-00000000068${ordinal}`, @@ -8854,7 +8886,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* Queue.take(harness.terminalReceipts); return executed(); }); - const opus = resolveClaudeModelSlug(BUNDLED_CLAUDE_MODEL_CATALOG, "opus"); + const opus = resolveClaudeModelSlug(initialCatalog, "opus"); assert.deepEqual(yield* aliasTurn(5, opus), { model: opus, modelSubstituted: false }); assert.deepEqual(yield* aliasTurn(6, "claude-sonnet-4-6"), { model: "claude-sonnet-4-6", @@ -8874,6 +8906,17 @@ describe("ClaudeAdapterV2 background wake turns", () => { model: "claude-sonnet-4-6", modelSubstituted: false, }); + yield* Ref.set(modelCatalog, modelCatalogWithAlias("downloaded-opus", opus)); + assert.deepEqual(yield* aliasTurn(10, opus, "downloaded-opus"), { + model: opus, + modelSubstituted: false, + }); + yield* Ref.set(modelCatalog, BUNDLED_CLAUDE_MODEL_CATALOG); + const refreshedOpus = resolveClaudeModelSlug(BUNDLED_CLAUDE_MODEL_CATALOG, "opus"); + assert.deepEqual(yield* aliasTurn(11, refreshedOpus), { + model: refreshedOpus, + modelSubstituted: false, + }); }).pipe( Effect.scoped, Effect.provide( @@ -12142,7 +12185,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { (opening) => Effect.scoped( Effect.gen(function* () { - const harness = yield* makeWakeHarness; + const modelCatalog = yield* Ref.make( + modelCatalogWithAlias("sonnet", "claude-sonnet-4-6"), + ); + const harness = yield* makeWakeHarnessWithOptions({ modelCatalog: Ref.get(modelCatalog) }); const now = yield* DateTime.now; yield* harness.runtime.startTurn( @@ -12151,6 +12197,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { providerThread: harness.providerThread, now, attemptId: RunAttemptId.make("attempt-claude-wake-init-1"), + modelSelection: { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "sonnet", + }, text: "Run the build in the background.", attachments: [], }), @@ -12162,7 +12212,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* harness.offerAndWait(wakeNotification); assert.lengthOf(harness.continuationRequests, 0); if (opening === "init") { - yield* harness.offerAndWait(wakeTurnInit); + yield* harness.offerAndWait(claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" })); } else { yield* harness.offerAndWait( usageStreamFrame( @@ -12184,12 +12234,17 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.lengthOf(harness.continuationRequests, 1); assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); + yield* Ref.set(modelCatalog, BUNDLED_CLAUDE_MODEL_CATALOG); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ threadId: harness.threadId, providerThread: harness.providerThread, now, attemptId: RunAttemptId.make("attempt-claude-wake-init-2"), + modelSelection: { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "sonnet", + }, text: "Background task completed.", attachments: [], providerTurnOrdinal: 2, @@ -12212,9 +12267,8 @@ describe("ClaudeAdapterV2 background wake turns", () => { event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, ), ); - if (opening === "stream start") { - assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); - } + assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); + assert.isFalse(lastProviderTurn(harness.events).modelSubstituted); }).pipe( Effect.provide( Layer.mergeAll( From dae6340e91bafc1746cfd3318eb55e15586ddde5 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:40:42 +0000 Subject: [PATCH 16/21] fix(claude): keep catalog identity tied to the producing turn --- .../Adapters/ClaudeAdapterV2.test.ts | 8 +++++--- .../Adapters/ClaudeAdapterV2.ts | 20 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 722e0bad07012..e5204f9d81021 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -8695,7 +8695,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }; const reportedModels = yield* Ref.make>([discoveredAlias]); const initialCatalog = modelCatalogWithAlias("opus", "claude-opus-4-6"); - const modelCatalog = yield* Ref.make(initialCatalog); + const modelCatalog = yield* Ref.make(initialCatalog); const harness = yield* makeWakeHarnessWithOptions({ freshQueueOnReopen: true, reportedModels, @@ -12185,7 +12185,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { (opening) => Effect.scoped( Effect.gen(function* () { - const modelCatalog = yield* Ref.make( + const modelCatalog = yield* Ref.make( modelCatalogWithAlias("sonnet", "claude-sonnet-4-6"), ); const harness = yield* makeWakeHarnessWithOptions({ modelCatalog: Ref.get(modelCatalog) }); @@ -12212,7 +12212,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* harness.offerAndWait(wakeNotification); assert.lengthOf(harness.continuationRequests, 0); if (opening === "init") { - yield* harness.offerAndWait(claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" })); + yield* harness.offerAndWait( + claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" }), + ); } else { yield* harness.offerAndWait( usageStreamFrame( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 006f9a44c8d90..9c543eef6eb4e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4322,6 +4322,7 @@ function formatClaudeUsageLimitWait(waitMs: number): string { } interface ActiveClaudeTurnContext { + readonly modelCatalog: ClaudeModelCatalog; readonly reportedModels: ReadonlyArray; readonly queryPolicy: ClaudeRuntimeQueryPolicy; readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; @@ -5979,10 +5980,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // "" identifies CLI notices, not a model. if (reported.length > 0 && reported !== "") { const discovered = context.reportedModels; - const executed = resolveClaudeModelSlug(modelCatalog, reported, discovered); + const executed = resolveClaudeModelSlug(context.modelCatalog, reported, discovered); // An alias such as `opus` requests the model its canonical slug names. const requested = resolveClaudeModelSlug( - modelCatalog, + context.modelCatalog, context.input.modelSelection.model, discovered, ); @@ -10782,6 +10783,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const openQuery = Effect.fnUntraced(function* ( turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, nativeThreadId: string, + turnModelCatalog: ClaudeModelCatalog, ) { const queryPolicy = yield* queryPolicyForTurn(turnInput); const mcpSession = yield* mcpSessions.read(turnInput.threadId); @@ -10796,7 +10798,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const queryPolicyKey = claudeEffectiveQueryPolicyKey(queryPolicy, mcpOverrides); const compiledSelection = compileClaudeModelSelection( turnInput.modelSelection, - modelCatalog, + turnModelCatalog, ); const resumeSessionAt = yield* getNativeConversationHeadId(turnInput.providerThread); const existing = yield* Ref.get(queryContext); @@ -10912,7 +10914,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); const queryOptions = makeClaudeQueryOptions({ modelSelection: turnInput.modelSelection, - modelCatalog, + modelCatalog: turnModelCatalog, nativeThreadId, title: turnInput.appThread.title, resume: shouldResume, @@ -11102,7 +11104,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const startTurn = Effect.fn("ClaudeAdapterV2.startTurn")( function* (turnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { - modelCatalog = yield* loadModelCatalog; + const turnModelCatalog = yield* loadModelCatalog; + modelCatalog = turnModelCatalog; const startedAt = yield* DateTime.now; const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); const nativeTurnId = `turn:${turnInput.attemptId}`; @@ -11161,7 +11164,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( turnInput.message.text, - compileClaudeModelSelection(turnInput.modelSelection, modelCatalog) + compileClaudeModelSelection(turnInput.modelSelection, turnModelCatalog) .promptEffort, ), attachments: turnInput.message.attachments, @@ -11177,9 +11180,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const reportedModels = adapterOptions.reportedModels ? yield* Ref.get(adapterOptions.reportedModels) : []; - const querySession = yield* openQuery(turnInput, nativeThreadId); + const querySession = yield* openQuery(turnInput, nativeThreadId, turnModelCatalog); const context: ActiveClaudeTurnContext = { // Continuations consume the existing query's output, including its model identity. + modelCatalog: isClaudeProviderContinuationTurn(turnInput) + ? (querySession.lastTurn?.modelCatalog ?? turnModelCatalog) + : turnModelCatalog, reportedModels: isClaudeProviderContinuationTurn(turnInput) ? (querySession.lastTurn?.reportedModels ?? reportedModels) : reportedModels, From be302fb82b6d2916b495579675b37b28251ebd4b Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:57:39 +0000 Subject: [PATCH 17/21] style(server): format Claude catalog regression setup --- .../src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index e5204f9d81021..76a3e5eb85429 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -12188,7 +12188,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { const modelCatalog = yield* Ref.make( modelCatalogWithAlias("sonnet", "claude-sonnet-4-6"), ); - const harness = yield* makeWakeHarnessWithOptions({ modelCatalog: Ref.get(modelCatalog) }); + const harness = yield* makeWakeHarnessWithOptions({ + modelCatalog: Ref.get(modelCatalog), + }); const now = yield* DateTime.now; yield* harness.runtime.startTurn( From 202e2539f9228e989a352de5a086adb8e186a400 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 6 Nov 2026 05:09:11 +0000 Subject: [PATCH 18/21] fix(claude): report the init model that lands before the first turn attaches Resolving the settings compaction window now awaits the filesystem after the query's SDK consumer starts, so a fresh process's system/init could be routed before its opening turn attached and its root model was dropped. The live query now keeps that early init model and its first attached turn records it; later turns on the process still start without one. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/Adapters/ClaudeAdapterV2.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 1855c56db390a..e2256dd644340 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4448,6 +4448,8 @@ interface ClaudeLiveQueryContext { // The CLI may fall back from the requested mode on its first init frame. openedPermissionMode: PermissionMode; receivedInit: boolean; + // Init can arrive before the turn that opened this process attaches; that turn reports it. + initModelBeforeFirstTurn: string | undefined; // From system/init; feature-detects control requests such as MCP Apps. capabilities: ReadonlySet; permissionMode: PermissionMode; @@ -11053,6 +11055,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( promptEchoMode: "unknown", openedPermissionMode: queryOptions.permissionMode, receivedInit: false, + initModelBeforeFirstTurn: undefined, capabilities: new Set(), permissionMode: queryOptions.permissionMode, stopping: false, @@ -11093,6 +11096,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( context.openedPermissionMode = message.permissionMode ?? context.openedPermissionMode; context.receivedInit = true; + if (context.lastTurn === null) context.initModelBeforeFirstTurn = message.model; context.capabilities = new Set(message.capabilities ?? []); } if ( @@ -11299,6 +11303,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( executedModel: undefined, }; const attachTurn = Effect.gen(function* () { + if (querySession.lastTurn === null && querySession.initModelBeforeFirstTurn) { + recordExecutedModel(context, querySession.initModelBeforeFirstTurn); + } querySession.lastTurn = context; yield* Ref.set(activeTurn, context); if ( From a6d4fb4024ee359f02ea6896b41818bfbbba00e7 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:47:47 +0000 Subject: [PATCH 19/21] refactor(server): report the executed model with the smallest change Keep the root model Claude reports on each provider turn and surface it as executedModel on task status and thread runs, with a short note on delegated results. Drop substitution classification, alias discovery, catalog snapshots, init attribution, wake-start and notice-forwarding changes, the stored linked-task model, and the non-discriminating eval cases. Co-Authored-By: Claude Opus 5.5 --- .../OrchestratorMcpService.activity.test.ts | 29 +- .../src/mcp/OrchestratorMcpService.test.ts | 1 - apps/server/src/mcp/OrchestratorMcpService.ts | 22 +- .../Adapters/ClaudeAdapterV2.test.ts | 448 ++++-------------- .../Adapters/ClaudeAdapterV2.ts | 95 +--- .../DelegatedCompletionDelivery.test.ts | 91 +--- .../src/orchestration-v2/Orchestrator.ts | 7 +- .../SubagentProjection.test.ts | 115 +---- apps/server/src/peer/RemoteDelegation.test.ts | 32 +- apps/server/src/peer/RemoteDelegation.ts | 45 +- .../server/src/provider/ClaudeModelCatalog.ts | 14 +- .../src/provider/Drivers/ClaudeDriver.ts | 9 +- evals/mcp-tool-use/cases/orchestration.ndjson | 2 - packages/contracts/src/orchestrationV2.ts | 13 - packages/contracts/src/orchestratorMcp.ts | 8 +- .../src/server/subagentProjection.ts | 73 +-- 16 files changed, 190 insertions(+), 814 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts index 683d389821e86..dfaebe9228d52 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts @@ -6,7 +6,6 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, - RunAttemptId, RunId, ScheduledTaskId, SECRET_REQUEST_UNSAVED_NOTE, @@ -49,7 +48,6 @@ const childThreadId = ThreadId.make("thread-mcp-orchestrator-child"); const activeRunId = RunId.make("run-mcp-active"); const cancelledRunId = RunId.make("run-mcp-cancelled"); const childRunId = RunId.make("run-mcp-child"); -const activeAttemptId = RunAttemptId.make("attempt-mcp-active"); const taskId = NodeId.make("node-mcp-task-1"); const now = DateTime.makeUnsafe("2026-08-04T12:00:00.000Z"); const codexDriver = ProviderDriverKind.make("codex"); @@ -134,11 +132,11 @@ it("readThread preserves caller access errors and prefers activity-run status ov model: "gpt-5.4", }), runs: [ - { ...makeRun({ id: activeRunId, ordinal: 1, status: "running" }), activeAttemptId }, + makeRun({ id: activeRunId, ordinal: 1, status: "running" }), makeRun({ id: cancelledRunId, ordinal: 2, status: "cancelled" }), ], - providerTurns: [{ runAttemptId: activeAttemptId, ordinal: 1, model: "gpt-5.4-mini" }], visibleTurnItems: [], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [], @@ -170,13 +168,9 @@ it("readThread preserves caller access errors and prefers activity-run status ov ? (projection.thread as unknown as OrchestrationV2ThreadShell) : null, ), - getProjectThreadRecords: (input, fields) => + getProjectThreadRecords: (input) => input.threadId === parentThreadId - ? Effect.succeed( - fields?.some((field) => field === "providerTurns") - ? projection - : { ...projection, providerTurns: [] }, - ) + ? Effect.succeed(projection) : Effect.die(`unexpected thread ${input.threadId}`), } satisfies Partial), ServerSettings.layerTest(), @@ -203,12 +197,6 @@ it("readThread preserves caller access errors and prefers activity-run status ov expect(result.thread.status).toBe("running"); expect(result.thread.latestRunId).toBe(cancelledRunId); expect(result.thread.activeRunId).toBe(activeRunId); - expect( - result.recentRuns.map(({ runId, model, executedModel }) => ({ runId, model, executedModel })), - ).toEqual([ - { runId: cancelledRunId, model: "gpt-5.4", executedModel: null }, - { runId: activeRunId, model: "gpt-5.4", executedModel: "gpt-5.4-mini" }, - ]); callerUnavailable = true; const denied = yield* service @@ -234,8 +222,8 @@ it("readThread pages cut-off text without splitting an emoji across pages", asyn model: "gpt-5.4", }), runs: [makeRun({ id: activeRunId, ordinal: 1, status: "running" })], - providerTurns: [], visibleTurnItems: [], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [], @@ -326,6 +314,7 @@ it("taskStatus returns task.providerInstanceId rather than the driver kind", asy }), runs: [makeRun({ id: activeRunId, ordinal: 1, status: "running" })], visibleTurnItems: [], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [], @@ -379,6 +368,7 @@ it("taskStatus returns task.providerInstanceId rather than the driver kind", asy }), ], visibleTurnItems: [], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [ @@ -391,7 +381,6 @@ it("taskStatus returns task.providerInstanceId rather than the driver kind", asy ], subagents: [], providerThreads: [], - providerTurns: [], updatedAt: now, } as unknown as OrchestrationV2ThreadProjection; @@ -453,6 +442,7 @@ it("readThread and sendToThread reach threads in other projects", async () => { return parentRuns; }, visibleTurnItems: [], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [], @@ -489,6 +479,7 @@ it("readThread and sendToThread reach threads in other projects", async () => { }, }, ], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [], @@ -685,6 +676,7 @@ it("requestSecret saves a pasted value to a file without opening a card", async }), runs: [{ ...makeRun({ id: activeRunId, ordinal: 1, status: "running" }), rootNodeId: taskId }], visibleTurnItems: [], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [], @@ -806,6 +798,7 @@ it("requestSecret returns the unsaved note for a declined, cancelled or timed-ou card === undefined ? [] : [{ type: "secret_request", id: card.turnItemId, secretStatus: outcome }], + providerTurns: [], runtimeRequests: [], messages: [], contextTransfers: [], diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index bc0e11f28ab7b..d6b7410876f68 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -3154,7 +3154,6 @@ describe("OrchestratorMcpService provider resolution", () => { assert.equal(result.model, selection.model); assert.deepEqual(result.options, selection.options); assert.equal(result.executedModel, executedModel); - if (hasStoredResult) assert.equal(result.summary, "Stored review result."); } }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); }), diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 267a245073d63..c11dc67cf5bb9 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -879,17 +879,10 @@ function threadDetail( function runExecutedModel( projection: Pick, run: Pick | undefined, -): { readonly executedModel: string | null; readonly modelSubstituted?: boolean } { - const turn = - run === undefined - ? undefined - : latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId); - return turn?.model === undefined - ? { executedModel: null } - : { - executedModel: turn.model, - ...(turn.modelSubstituted === undefined ? {} : { modelSubstituted: turn.modelSubstituted }), - }; +): string | null { + return run === undefined + ? null + : (latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId)?.model ?? null); } function threadRun( @@ -902,7 +895,7 @@ function threadRun( status: run.status, providerInstanceId: run.modelSelection.instanceId, model: run.modelSelection.model, - ...runExecutedModel(projection, run), + executedModel: runExecutedModel(projection, run), requestedAt: DateTime.formatIso(run.requestedAt), startedAt: run.startedAt === null ? null : DateTime.formatIso(run.startedAt), completedAt: run.completedAt === null ? null : DateTime.formatIso(run.completedAt), @@ -1564,7 +1557,7 @@ const make = Effect.gen(function* () { messageRoles: ["assistant"], messageRunIds: resultRunIds, turnItemRunIds: resultRunIds, - turnItemTypes: ["assistant_message", "error", "system_notice"], + turnItemTypes: ["assistant_message", "error"], }) .pipe(Effect.mapError(threadManagementFailure)); const childProjection = { @@ -1648,7 +1641,7 @@ const make = Effect.gen(function* () { hasPendingChildRuns: hasPendingChildRuns(childProjection, childRun), providerInstanceId: modelSelection.instanceId, model: modelSelection.model, - ...runExecutedModel(childControls, executionRun), + executedModel: runExecutedModel(childControls, executionRun), options: modelSelection.options ?? [], summary: derivedResult, resultContextTransferId, @@ -1818,7 +1811,6 @@ const make = Effect.gen(function* () { providerInstanceId: task.providerInstanceId, model: task.model, executedModel: null, - ...task.remoteChild?.resultModel, summary: task.result, resultContextTransferId: null, latestTerminalRunId: null, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index c3d320ab21cdb..d7229e12fb20d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -68,7 +68,6 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; -import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -96,7 +95,6 @@ import { OrchestratorToolkit } from "../../mcp/toolkits/orchestrator/tools.ts"; import { ClaudeExecutableFileCheck } from "../../provider/Drivers/ClaudeExecutable.ts"; import { BUNDLED_CLAUDE_MODEL_CATALOG, - resolveClaudeModelSlug, type ClaudeModelCatalog, } from "../../provider/ClaudeModelCatalog.ts"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; @@ -3138,19 +3136,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { for (let tick = 0; tick < ticks; tick++) yield* Effect.yieldNow; }); - const modelCatalogWithAlias = (alias: string, slug: string) => ({ - models: BUNDLED_CLAUDE_MODEL_CATALOG.models.map((entry) => { - const aliases = (entry.model.aliases ?? []).filter((value) => value !== alias); - return { - ...entry, - model: { - ...entry.model, - aliases: entry.model.slug === slug ? [...aliases, alias] : aliases, - }, - }; - }), - }); - const makeWakeHarnessWithOptions = (options?: { readonly close?: (sdkMessages: Queue.Queue) => Effect.Effect; readonly interrupt?: Effect.Effect; @@ -3175,7 +3160,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly freshQueueOnReopen?: boolean; readonly forkSession?: ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerShape["forkSession"]; readonly allocateSessionId?: Effect.Effect; - readonly reportedModels?: Ref.Ref>; }) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -3239,9 +3223,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { path: yield* Path.Path, random: yield* SecureRandom.SecureRandom, idAllocator, - ...(options?.reportedModels === undefined - ? {} - : { reportedModels: options.reportedModels }), continuationRequests: { offer: (request) => Effect.sync(() => { @@ -8878,236 +8859,65 @@ describe("ClaudeAdapterV2 background wake turns", () => { it.effect("records the root model Claude reports for each turn, not the requested one", () => Effect.gen(function* () { - const discoveredAlias = { - value: "gateway/best", - resolvedModel: "gateway/claude-future", - displayName: "Best", - description: "", - }; - const reportedModels = yield* Ref.make>([discoveredAlias]); - const initialCatalog = modelCatalogWithAlias("opus", "claude-opus-4-6"); - const modelCatalog = yield* Ref.make(initialCatalog); - const harness = yield* makeWakeHarnessWithOptions({ - freshQueueOnReopen: true, - reportedModels, - modelCatalog: Ref.get(modelCatalog), - }); - const executed = () => { - const { model, modelSubstituted } = lastProviderTurn(harness.events); - return { model, modelSubstituted }; - }; - const turnInput = { - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attachments: [], - modelSelection: { - instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, - model: "claude-opus-4-6", - }, - }; - yield* Queue.offer( - harness.sdkMessages, - claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" }), - ); - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - ...turnInput, - attemptId: RunAttemptId.make("attempt-claude-restricted-model-1"), - text: "Review this.", - }), - ); - yield* Queue.offerAll(harness.sdkMessages, [ - makeAssistantErrorFrame({ - uuid: "00000000-0000-4000-8000-000000000661", - error: undefined, - parentToolUseId: "toolu_restricted_model_subagent", - model: "claude-haiku-4-5-20251001", - }), - claudeSdkFrame({ - type: "system", - subtype: "model_refusal_fallback", - scope: "local", - trigger: "refusal", - direction: "retry", - original_model: "claude-opus-4-6", - fallback_model: "claude-haiku-4-5", - request_id: null, - content: "A side question used a fallback model.", - uuid: "00000000-0000-4000-8000-000000000665", - session_id: WAKE_NATIVE_SESSION, - }), - claudeSdkFrame({ - type: "system", - subtype: "compact_boundary", - compact_metadata: { trigger: "auto", pre_tokens: 10_000, post_tokens: 4_321 }, - uuid: "00000000-0000-4000-8000-000000000666", - session_id: WAKE_NATIVE_SESSION, - }), - turnOneResult, - ]); - yield* Queue.take(harness.terminalReceipts); - const compactionTurn = harness.events.find( - (event) => - event.type === "provider_turn.updated" && - event.providerTurn.status === "running" && - event.providerTurn.tokenUsage?.usedTokens === 4_321, - ); - assert.equal(compactionTurn?.type, "provider_turn.updated"); - if (compactionTurn?.type === "provider_turn.updated") { - assert.equal(compactionTurn.providerTurn.model, "claude-sonnet-4-6"); - assert.isTrue(compactionTurn.providerTurn.modelSubstituted); - } - assert.deepEqual(executed(), { model: "claude-sonnet-4-6", modelSubstituted: true }); - const localNotices = harness.events.flatMap((event) => - event.type === "turn_item.updated" && - event.turnItem.type === "system_notice" && - event.turnItem.message === "A side question used a fallback model." - ? [event.turnItem] - : [], - ); - assert.lengthOf(localNotices, 1); - assert.isNull(localNotices[0]?.nodeId); - - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - ...turnInput, - attemptId: RunAttemptId.make("attempt-claude-restricted-model-2"), - providerTurnOrdinal: 2, - text: "Continue.", - }), - ); - yield* Queue.offerAll(harness.sdkMessages, [ - makeAssistantErrorFrame({ - uuid: "00000000-0000-4000-8000-000000000662", - error: undefined, - model: "claude-haiku-4-5-20251001", - }), - makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000663", result: "Done." }), - ]); - yield* Queue.take(harness.terminalReceipts); - assert.deepEqual(executed(), { model: "claude-haiku-4-5", modelSubstituted: true }); - - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - ...turnInput, - attemptId: RunAttemptId.make("attempt-claude-restricted-model-3"), - providerTurnOrdinal: 3, - text: "Once more.", - }), - ); - yield* Queue.offer( - harness.sdkMessages, - makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000664", result: "Done." }), - ); - yield* Queue.take(harness.terminalReceipts); - assert.deepEqual(executed(), { model: undefined, modelSubstituted: undefined }); - - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - ...turnInput, - attemptId: RunAttemptId.make("attempt-claude-restricted-model-4"), - providerTurnOrdinal: 4, - text: "Stream only.", - }), - ); - yield* Queue.offerAll(harness.sdkMessages, [ - usageStreamFrame({ - type: "message_start", - message: { id: "stream-model", model: "claude-sonnet-4-6", usage: zeroUsage }, - }), - usageStreamFrame( - { - type: "message_start", - message: { id: "child-model", model: "claude-haiku-4-5", usage: zeroUsage }, - }, - "native-child", - ), - makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000666", result: "Done." }), - ]); - yield* Queue.take(harness.terminalReceipts); - assert.deepEqual(executed(), { model: "claude-sonnet-4-6", modelSubstituted: true }); - - // An alias the catalog or Claude Code resolves to the reported model is no substitution. - const aliasTurn = (ordinal: number, reported: string, requested = "opus") => + const harness = yield* makeWakeHarnessWithOptions({ freshQueueOnReopen: true }); + const runTurn = (ordinal: number, frames: ReadonlyArray) => Effect.gen(function* () { yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ - ...turnInput, - modelSelection: { ...turnInput.modelSelection, model: requested }, - attemptId: RunAttemptId.make(`attempt-claude-restricted-model-${ordinal}`), - providerTurnOrdinal: ordinal, - text: "Use the alias.", - }), - ); - yield* harness.offerAndWait( - usageStreamFrame({ - type: "message_start", - message: { id: `alias-model-${ordinal}`, model: reported, usage: zeroUsage }, - }), - ); - if (ordinal === 7) yield* Ref.set(reportedModels, []); - if (ordinal === 5) { - yield* Ref.set(modelCatalog, BUNDLED_CLAUDE_MODEL_CATALOG); - yield* harness.runtime.steerTurn({ threadId: harness.threadId, - runId: RunId.make(`run-attempt-claude-restricted-model-${ordinal}`), providerThread: harness.providerThread, - providerTurnId: lastProviderTurn(harness.events).id, - message: { - createdBy: "user", - creationSource: "web", - messageId: MessageId.make("catalog-refresh-steer"), - text: "Keep reviewing.", - attachments: [], + now: yield* DateTime.now, + attachments: [], + modelSelection: { + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + model: "claude-opus-4-6", }, - }); - } - yield* Queue.offerAll(harness.processQueues.at(-1) ?? harness.sdkMessages, [ - makeAssistantErrorFrame({ - uuid: `00000000-0000-4000-8000-00000000068${ordinal}`, - error: undefined, - model: reported, + attemptId: RunAttemptId.make(`attempt-claude-executed-model-${ordinal}`), + providerTurnOrdinal: ordinal, + text: "Review this.", }), + ); + yield* Queue.offerAll(harness.sdkMessages, [ + ...frames, makeResultFrame({ - uuid: `00000000-0000-4000-8000-00000000067${ordinal}`, + uuid: `00000000-0000-4000-8000-00000000066${ordinal}`, result: "Done.", }), ]); yield* Queue.take(harness.terminalReceipts); - return executed(); + return lastProviderTurn(harness.events).model; }); - const opus = resolveClaudeModelSlug(initialCatalog, "opus"); - assert.deepEqual(yield* aliasTurn(5, opus), { model: opus, modelSubstituted: false }); - assert.deepEqual(yield* aliasTurn(6, "claude-sonnet-4-6"), { - model: "claude-sonnet-4-6", - modelSubstituted: true, - }); - assert.deepEqual(yield* aliasTurn(7, "gateway/claude-future", "gateway/best"), { - model: "gateway/claude-future", - modelSubstituted: false, - }); - assert.equal(harness.getOpenedOptions()?.model, "gateway/best"); - assert.deepEqual(yield* aliasTurn(8, "claude-sonnet-4-6", "gateway/best"), { - model: "claude-sonnet-4-6", - modelSubstituted: true, - }); - yield* Ref.set(reportedModels, [{ ...discoveredAlias, resolvedModel: "claude-sonnet-4-6" }]); - assert.deepEqual(yield* aliasTurn(9, "claude-sonnet-4-6", "gateway/best"), { - model: "claude-sonnet-4-6", - modelSubstituted: false, - }); - yield* Ref.set(modelCatalog, modelCatalogWithAlias("downloaded-opus", opus)); - assert.deepEqual(yield* aliasTurn(10, opus, "downloaded-opus"), { - model: opus, - modelSubstituted: false, - }); - yield* Ref.set(modelCatalog, BUNDLED_CLAUDE_MODEL_CATALOG); - const refreshedOpus = resolveClaudeModelSlug(BUNDLED_CLAUDE_MODEL_CATALOG, "opus"); - assert.deepEqual(yield* aliasTurn(11, refreshedOpus), { - model: refreshedOpus, - modelSubstituted: false, - }); + yield* Queue.offer(harness.sdkMessages, wakeTurnInit); + // A native subagent's model is not the run's. + assert.equal( + yield* runTurn(1, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000671", + error: undefined, + model: "claude-sonnet-4-6", + }), + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000672", + error: undefined, + parentToolUseId: "toolu_executed_model_subagent", + model: "claude-haiku-4-5-20251001", + }), + ]), + "claude-sonnet-4-6", + ); + // A dated slug reports its catalog model, so it is not mistaken for another one. + assert.equal( + yield* runTurn(2, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000673", + error: undefined, + model: "claude-haiku-4-5-20251001", + }), + ]), + "claude-haiku-4-5", + ); + // A turn that reports no model does not inherit the previous one's. + assert.isUndefined(yield* runTurn(3, [])); }).pipe( Effect.scoped, Effect.provide( @@ -12371,110 +12181,68 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect.each(["init", "stream start"] as const)( - "starts the wake run when Claude opens the wake turn with %s, before its output", - (opening) => - Effect.scoped( - Effect.gen(function* () { - const modelCatalog = yield* Ref.make( - modelCatalogWithAlias("sonnet", "claude-sonnet-4-6"), - ); - const harness = yield* makeWakeHarnessWithOptions({ - modelCatalog: Ref.get(modelCatalog), - }); - const now = yield* DateTime.now; + it.effect("starts the wake run when Claude opens the wake turn, before its output", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now, - attemptId: RunAttemptId.make("attempt-claude-wake-init-1"), - modelSelection: { - ...CLAUDE_TEST_MODEL_SELECTION, - model: "sonnet", - }, - text: "Run the build in the background.", - attachments: [], - }), - ); - yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); - yield* Queue.offer(harness.sdkMessages, turnOneResult); - yield* harness.awaitTerminal(1); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-init-1"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* harness.awaitTerminal(1); - yield* harness.offerAndWait(wakeNotification); - assert.lengthOf(harness.continuationRequests, 0); - if (opening === "init") { - yield* harness.offerAndWait( - claudeSdkFrame({ ...wakeTurnInit, model: "claude-sonnet-4-6" }), - ); - } else { - yield* harness.offerAndWait( - usageStreamFrame( - { - type: "message_start", - message: { id: "wake-child", model: "claude-haiku-4-5", usage: zeroUsage }, - }, - "native-child", - ), - ); - assert.lengthOf(harness.continuationRequests, 0); - yield* harness.offerAndWait( - usageStreamFrame({ - type: "message_start", - message: { id: "wake-root", model: "claude-sonnet-4-6", usage: zeroUsage }, - }), - ); - } - assert.lengthOf(harness.continuationRequests, 1); - assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); + yield* harness.offerAndWait(wakeNotification); + assert.lengthOf(harness.continuationRequests, 0); + yield* harness.offerAndWait(wakeTurnInit); + assert.lengthOf(harness.continuationRequests, 1); + assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); - yield* Ref.set(modelCatalog, BUNDLED_CLAUDE_MODEL_CATALOG); - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now, - attemptId: RunAttemptId.make("attempt-claude-wake-init-2"), - modelSelection: { - ...CLAUDE_TEST_MODEL_SELECTION, - model: "sonnet", - }, - text: "Background task completed.", - attachments: [], - providerTurnOrdinal: 2, - messageCreatedBy: "agent", - messageCreationSource: "provider", - }), - ); - assert.lengthOf(harness.terminalEvents(), 1); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-init-2"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + assert.lengthOf(harness.terminalEvents(), 1); - if (opening === "init") { - yield* harness.offerAndWait(wakeAssistant); - } - yield* harness.offerAndWait(wakeResult); - yield* harness.awaitTerminal(2); - assert.equal(harness.terminalEvents()[1]?.status, "completed"); - assert.lengthOf(harness.continuationRequests, 1); - assert.isTrue( - harness.events.some( - (event) => - event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, - ), - ); - assert.equal(lastProviderTurn(harness.events).model, "claude-sonnet-4-6"); - assert.isFalse(lastProviderTurn(harness.events).modelSubstituted); - }).pipe( - Effect.provide( - Layer.mergeAll( - IdAllocator.layer, - McpProviderSessions.layer, - NodeServices.layer, - NodeCrypto.layer, - ), + yield* harness.offerAndWait(wakeAssistant); + yield* harness.offerAndWait(wakeResult); + yield* harness.awaitTerminal(2); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.lengthOf(harness.continuationRequests, 1); + assert.isTrue( + harness.events.some( + (event) => event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, + ), + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, ), ), ), + ), ); it.effect.each([true, false])( @@ -13150,12 +12918,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { }; it.effect.each([ - { label: "an empty one", quotaFailureFirst: false, nextOpensWith: "init" }, - { label: "a quota failure and an empty one", quotaFailureFirst: true, nextOpensWith: "init" }, - { label: "an empty one by streaming", quotaFailureFirst: false, nextOpensWith: "stream" }, + { label: "an empty one", quotaFailureFirst: false }, + { label: "a quota failure and an empty one", quotaFailureFirst: true }, ])( "keeps the wake run open for the turn Claude opens right after $label", - ({ quotaFailureFirst, nextOpensWith }) => + ({ quotaFailureFirst }) => Effect.scoped( Effect.gen(function* () { const harness = yield* makeWakeHarness; @@ -13236,12 +13003,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); yield* harness.offerAndWait( - nextOpensWith === "init" - ? claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000963" }) - : usageStreamFrame({ - type: "message_start", - message: { id: "wake-pair-next", model: "claude-sonnet-4-6", usage: zeroUsage }, - }), + claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000963" }), ); assert.lengthOf(harness.continuationRequests, 1); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 90b09185b026d..6df4b70baf80a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -48,7 +48,6 @@ import { getSubagentMessages, type HookCallback, type McpServerStatus as ClaudeMcpServerStatus, - type ModelInfo, query, resolveSettings as resolveClaudeSettings, type Options as ClaudeQueryOptions, @@ -3233,14 +3232,8 @@ function isClaudeBackgroundTasksChangedMessage(message: SDKMessage): boolean { return message.type === "system" && (message.subtype as string) === "background_tasks_changed"; } -// A wake can stream its root message before its init frame. function isClaudeTurnStartMessage(message: SDKMessage): boolean { - return ( - (message.type === "system" && message.subtype === "init") || - (message.type === "stream_event" && - message.event.type === "message_start" && - message.parent_tool_use_id == null) - ); + return message.type === "system" && message.subtype === "init"; } function isClaudeRootTurnFrame(message: SDKMessage): boolean { @@ -4331,8 +4324,6 @@ function formatClaudeUsageLimitWait(waitMs: number): string { } interface ActiveClaudeTurnContext { - readonly modelCatalog: ClaudeModelCatalog; - readonly reportedModels: ReadonlyArray; readonly queryPolicy: ClaudeRuntimeQueryPolicy; readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly autoCompactWindow: string; @@ -4405,7 +4396,7 @@ interface ActiveClaudeTurnContext { gatedFramesBeforeEcho: number; readonly heldRootFrames: Array; // Claude can run another model than requested, such as when an organization disallows it. - executedModel: { readonly model: string; readonly modelSubstituted: boolean } | undefined; + executedModel: string | undefined; } interface ActiveClaudeProviderRetry { @@ -4448,8 +4439,6 @@ interface ClaudeLiveQueryContext { // The CLI may fall back from the requested mode on its first init frame. openedPermissionMode: PermissionMode; receivedInit: boolean; - // Init can arrive before the turn that opened this process attaches; that turn reports it. - initModelBeforeFirstTurn: string | undefined; // From system/init; feature-detects control requests such as MCP Apps. capabilities: ReadonlySet; permissionMode: PermissionMode; @@ -4511,13 +4500,6 @@ interface PendingClaudeSubagentLaunch { readonly ownerToolUseId?: string; } -function executedModelField(context: ActiveClaudeTurnContext): { - readonly model?: string; - readonly modelSubstituted?: boolean; -} { - return context.executedModel ?? {}; -} - function canonicalClaudeSubagentModel(catalog: ClaudeModelCatalog, model: string): string { return resolveClaudeModelSlug(catalog, model); } @@ -4871,8 +4853,6 @@ export interface ClaudeAdapterV2Options { readonly idAllocator: IdAllocator.IdAllocatorV2["Service"]; readonly queryRunner: ClaudeAgentSdkQueryRunnerShape; readonly scopedLimitNames?: Ref.Ref; - /** Models this instance's Claude Code reported, with the model each alias resolves to. */ - readonly reportedModels?: Ref.Ref>; readonly onUsageLimits?: ManagedServerProvider["applyUsageLimits"]; readonly continuationRequests?: { readonly offer: ( @@ -5516,7 +5496,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( startedAt: context.startedAt, completedAt: null, tokenUsage, - ...executedModelField(context), + ...(context.executedModel === undefined ? {} : { model: context.executedModel }), }, }); }); @@ -6027,25 +6007,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( status: input.status, startedAt: input.context.startedAt, completedAt: input.completedAt, - ...executedModelField(input.context), + ...(input.context.executedModel === undefined + ? {} + : { model: input.context.executedModel }), }); - const recordExecutedModel = (context: ActiveClaudeTurnContext, model: unknown) => { - const reported = typeof model === "string" ? model.trim() : ""; - // "" identifies CLI notices, not a model. - if (reported.length > 0 && reported !== "") { - const discovered = context.reportedModels; - const executed = resolveClaudeModelSlug(context.modelCatalog, reported, discovered); - // An alias such as `opus` requests the model its canonical slug names. - const requested = resolveClaudeModelSlug( - context.modelCatalog, - context.input.modelSelection.model, - discovered, - ); - context.executedModel = { model: executed, modelSubstituted: requested !== executed }; - } - }; - const buildToolCallArtifacts = (input: { readonly context: ActiveClaudeTurnContext | null; readonly nativeItemId: string; @@ -8631,7 +8597,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( message.task_id, ); } - // Open continuations only after root init, message start, user, assistant, or result frames prove a wake began. + // Open continuations only after root init, user, assistant, or result frames prove a wake began. // Background child frames can stream while the root is idle. const buffered = (yield* Ref.get(wakeBuffers)).get(wakeInput.nativeThreadId); const hasBufferedNotification = @@ -9014,13 +8980,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return; } - if ( - message.type === "system" && - message.subtype === "init" && - context === (yield* Ref.get(activeTurn)) - ) { - recordExecutedModel(context, message.model); - } const focusedPlugin = claudeFocusRequestedPanePlugin(message); if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { yield* Effect.logWarning("orchestration-v2.claude-mod-pane-focus-unsupported", { @@ -9049,7 +9008,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const event = message.event; const reasoning = context.reasoning; if (event.type === "message_start") { - recordExecutedModel(context, event.message.model); reasoning.messageId = event.message.id; reasoning.streamBlocks.clear(); const usage: ClaudeMessageUsage | undefined = event.message.usage; @@ -9157,8 +9115,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // CLI notices and API errors arrive as assistant messages from "". if (message.message.model && message.message.model !== "") { context.rootModels.add(message.message.model); + context.executedModel = resolveClaudeModelSlug(modelCatalog, message.message.model); } - recordExecutedModel(context, message.message.model); // Subagent uuids exist only in the sidechain transcript, so resume and fork cannot target them. context.nativeMessageCursor = message.uuid; context.latestAssistantRateLimit = @@ -9339,10 +9297,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), threadId: context.input.threadId, runId: context.input.runId, - nodeId: - message.subtype === "model_refusal_fallback" && message.scope === "local" - ? null - : context.input.rootNodeId, + nodeId: context.input.rootNodeId, providerThreadId: context.input.providerThread.id, providerTurnId: context.providerTurnId, nativeItemRef: { @@ -10838,7 +10793,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const openQuery = Effect.fnUntraced(function* ( turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, nativeThreadId: string, - turnModelCatalog: ClaudeModelCatalog, ) { const queryPolicy = yield* queryPolicyForTurn(turnInput); const mcpSession = yield* mcpSessions.read(turnInput.threadId); @@ -10853,7 +10807,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const queryPolicyKey = claudeEffectiveQueryPolicyKey(queryPolicy, mcpOverrides); const compiledSelection = compileClaudeModelSelection( turnInput.modelSelection, - turnModelCatalog, + modelCatalog, ); const resumeSessionAt = yield* getNativeConversationHeadId(turnInput.providerThread); const existing = yield* Ref.get(queryContext); @@ -10969,7 +10923,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); const queryOptions = makeClaudeQueryOptions({ modelSelection: turnInput.modelSelection, - modelCatalog: turnModelCatalog, + modelCatalog, nativeThreadId, title: turnInput.appThread.title, resume: shouldResume, @@ -11055,7 +11009,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( promptEchoMode: "unknown", openedPermissionMode: queryOptions.permissionMode, receivedInit: false, - initModelBeforeFirstTurn: undefined, capabilities: new Set(), permissionMode: queryOptions.permissionMode, stopping: false, @@ -11096,7 +11049,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( context.openedPermissionMode = message.permissionMode ?? context.openedPermissionMode; context.receivedInit = true; - if (context.lastTurn === null) context.initModelBeforeFirstTurn = message.model; context.capabilities = new Set(message.capabilities ?? []); } if ( @@ -11163,8 +11115,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const startTurn = Effect.fn("ClaudeAdapterV2.startTurn")( function* (turnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { - const turnModelCatalog = yield* loadModelCatalog; - modelCatalog = turnModelCatalog; + modelCatalog = yield* loadModelCatalog; const startedAt = yield* DateTime.now; const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); const nativeTurnId = `turn:${turnInput.attemptId}`; @@ -11223,7 +11174,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( turnInput.message.text, - compileClaudeModelSelection(turnInput.modelSelection, turnModelCatalog) + compileClaudeModelSelection(turnInput.modelSelection, modelCatalog) .promptEffort, ), attachments: turnInput.message.attachments, @@ -11236,23 +11187,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(Path.Path, path), ); - const reportedModels = adapterOptions.reportedModels - ? yield* Ref.get(adapterOptions.reportedModels) - : []; - const querySession = yield* openQuery(turnInput, nativeThreadId, turnModelCatalog); + const querySession = yield* openQuery(turnInput, nativeThreadId); const autoCompactWindow = yield* resolveAutoCompactWindow( querySession.settingsCwd, turnInput.modelSelection, querySession.launchArgSettings, ); const context: ActiveClaudeTurnContext = { - // Continuations consume the existing query's output, including its model identity. - modelCatalog: isClaudeProviderContinuationTurn(turnInput) - ? (querySession.lastTurn?.modelCatalog ?? turnModelCatalog) - : turnModelCatalog, - reportedModels: isClaudeProviderContinuationTurn(turnInput) - ? (querySession.lastTurn?.reportedModels ?? reportedModels) - : reportedModels, input: turnInput, autoCompactWindow, queryPolicy: querySession.queryPolicy, @@ -11303,9 +11244,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( executedModel: undefined, }; const attachTurn = Effect.gen(function* () { - if (querySession.lastTurn === null && querySession.initModelBeforeFirstTurn) { - recordExecutedModel(context, querySession.initModelBeforeFirstTurn); - } querySession.lastTurn = context; yield* Ref.set(activeTurn, context); if ( @@ -12423,10 +12361,7 @@ export type ClaudeAdapterV2DriverEnv = export const createClaudeAdapterV2 = Effect.fn("ClaudeAdapterV2Driver.create")( function* ( input: ProviderAdapterDriverCreateInput, - hooks: Pick< - ClaudeAdapterV2Options, - "scopedLimitNames" | "reportedModels" | "onUsageLimits" | "modelCatalog" - > = {}, + hooks: Pick = {}, ) { const { instanceId, environment, enabled, config } = input; const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 2c9da0d0082e6..db3a2759e3ac5 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -15,8 +15,6 @@ import { ProviderDriverKind, ProviderInstanceId, ProviderThreadId, - ProviderTurnId, - RunAttemptId, RunId, RuntimeRequestId, ThreadId, @@ -2705,9 +2703,6 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { const parent = yield* orchestrator.getThreadProjection(threadId); const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId); const providerThreadId = ProviderThreadId.make("provider-thread:result-message-child"); - const childRootNodeId = NodeId.make("node:result-message-child-root"); - const childAttemptId = RunAttemptId.make("attempt:result-message-child"); - const fallbackNotice = "Model gpt-5.4 is restricted. Using gpt-5.4-mini instead."; yield* eventSink.write({ commandId: CommandId.make("command:result-message-child-provider"), events: [ @@ -2728,83 +2723,7 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { threadId: child.childThreadId, runId: child.childRunId, occurredAt: now, - payload: { - ...childProjection.runs[0]!, - providerThreadId, - rootNodeId: childRootNodeId, - activeAttemptId: childAttemptId, - }, - }, - { - id: EventId.make("event:result-message-child-provider-turn"), - type: "provider-turn.updated", - threadId: child.childThreadId, - runId: child.childRunId, - nodeId: childRootNodeId, - occurredAt: now, - payload: { - id: ProviderTurnId.make("provider-turn:result-message-child"), - providerThreadId, - nodeId: childRootNodeId, - runAttemptId: childAttemptId, - nativeTurnRef: null, - ordinal: 1, - status: "completed", - startedAt: now, - completedAt: now, - model: "gpt-5.4-mini", - }, - }, - { - id: EventId.make("event:result-message-child-notice"), - type: "turn-item.updated", - threadId: child.childThreadId, - runId: child.childRunId, - nodeId: childRootNodeId, - occurredAt: now, - payload: { - id: TurnItemId.make("turn-item:result-message-child-notice"), - type: "system_notice", - threadId: child.childThreadId, - runId: child.childRunId, - nodeId: childRootNodeId, - providerThreadId, - providerTurnId: ProviderTurnId.make("provider-turn:result-message-child"), - nativeItemRef: { - driver, - nativeId: "result-message-child-notice", - strength: "strong", - }, - parentItemId: null, - ordinal: 0, - status: "completed", - title: fallbackNotice, - message: fallbackNotice, - startedAt: now, - completedAt: now, - updatedAt: now, - }, - }, - { - id: EventId.make("event:result-message-child-answer"), - type: "message.updated", - threadId: child.childThreadId, - runId: child.childRunId, - occurredAt: now, - payload: { - id: MessageId.make("message:result-message-child-answer"), - threadId: child.childThreadId, - runId: child.childRunId, - nodeId: childRootNodeId, - role: "assistant", - text: "Review done.", - attachments: [], - streaming: false, - createdBy: "agent", - creationSource: "server", - createdAt: now, - updatedAt: now, - }, + payload: { ...childProjection.runs[0]!, providerThreadId }, }, ], }); @@ -2813,13 +2732,7 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { const recovered = yield* orchestrator.getThreadProjection(threadId); const task = recovered.subagents.find((candidate) => candidate.id === child.taskId)!; assert.equal(task.status, "completed"); - assert.equal( - task.result, - `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\nProvider notice: ${fallbackNotice}\n\nReview done.`, - ); - const recoveredChild = yield* orchestrator.getThreadProjection(child.childThreadId); - assert.deepEqual(recoveredChild.thread.modelSelection, modelSelection); - assert.deepEqual(recoveredChild.runs[0]?.modelSelection, modelSelection); + assert.isNotNull(task.result); assert.isEmpty(recovered.contextHandoffs); assert.equal(recovered.contextTransfers.length, 1); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 985a321f212fa..5d3c4391b1fe8 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -8588,10 +8588,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const now = yield* DateTime.now; const updatedTask: OrchestrationV2Subagent = { ...task, - remoteChild: - command.resultModel === undefined - ? remoteChild - : { ...remoteChild!, resultModel: command.resultModel }, + remoteChild, status: command.status, result: command.result, completedAt: now, @@ -12806,7 +12803,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio messageRoles: ["assistant"], messageRunIds: [childRun.id], turnItemRunId: childRun.id, - turnItemTypes: ["assistant_message", "error", "system_notice"], + turnItemTypes: ["assistant_message", "error"], }, ); const childProjection = { diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index 0b1630ffcbb56..e689b20bc43df 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -9,13 +9,10 @@ import { EventId, TurnItemId, type OrchestrationV2Run, - type OrchestrationV2TurnItem, type OrchestrationV2AppThread, ProjectId, - ProviderDriverKind, ProviderInstanceId, ProviderThreadId, - ProviderTurnId, ThreadId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; @@ -418,9 +415,7 @@ it("orders a task's assistant messages by turn item, not by timestamps", () => { assert.equal(result.turnItemId, report.turnItem.id); }); -type SystemNotice = Extract; - -it("notes a substituted model with the root provider notice from the same run", () => { +it("leads a task's result with the model its own attempt reported running", () => { const { projection, run } = taskFixture(); const answer = makeSubagentConversationArtifacts({ messageId: MessageId.make("answer"), @@ -435,113 +430,27 @@ it("notes a substituted model with the root provider notice from the same run", ordinal: 1, now: childCreatedAt, }); - const fallback = "Model gpt-5.4 is restricted by your organization. Using gpt-5.4-mini instead."; - const notice = ( - id: string, - message: string, - overrides: Partial< - Pick - > = {}, - ): SystemNotice => ({ - id: TurnItemId.make(id), - threadId: parentThreadId, - runId: run.id, - nodeId: NodeId.make("root"), - providerThreadId: null, - providerTurnId: ProviderTurnId.make("active-notice"), - nativeItemRef: { - driver: ProviderDriverKind.make("claudeAgent"), - nativeId: id, - strength: "strong", - }, - parentItemId: null, - ordinal: 0, - status: "completed", - title: message, - startedAt: childCreatedAt, - completedAt: childCreatedAt, - updatedAt: childCreatedAt, - type: "system_notice", - message, - ...overrides, - }); - const result = subagentResultForRun( - { - ...projection, - messages: [{ ...answer.message, runId: run.id }], - turnItems: [ - { ...answer.turnItem, runId: run.id }, - notice("fallback", fallback), - notice("native-subagent", "Subagent fell back to Haiku.", { - nodeId: NodeId.make("native-subagent"), - }), - notice("other-run", "Another run's notice.", { runId: RunId.make("other") }), - notice("old-attempt", "An earlier attempt's notice.", { - providerTurnId: ProviderTurnId.make("old-attempt"), - }), - notice("unattributed", "A notice without attempt attribution.", { providerTurnId: null }), - notice("usage-limit", "T3 usage limit notice.", { - nativeItemRef: { - driver: ProviderDriverKind.make("claudeAgent"), - nativeId: "usage-limit", - strength: "weak", - }, - }), - ], - providerTurns: [ - { id: ProviderTurnId.make("active-notice"), runAttemptId: run.activeAttemptId, ordinal: 1 }, - { - id: ProviderTurnId.make("active-result"), - runAttemptId: run.activeAttemptId, - ordinal: 2, - model: "gpt-5.4-mini", - }, - { - id: ProviderTurnId.make("old-attempt"), - runAttemptId: RunAttemptId.make("other"), - ordinal: 3, - model: "gpt-5.4-nano", - }, - ] as never, - }, - run, - ); - assert.equal( - result.text, - `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\nProvider notice: ${fallback}\n\nReview done.`, - ); - assert.equal(result.messageId, answer.message.id); - const resultWithTurns = (providerTurns: ReadonlyArray) => + const resultWith = (providerTurns: ReadonlyArray) => subagentResultForRun( { ...projection, messages: [{ ...answer.message, runId: run.id }], - turnItems: [notice("fallback", fallback)], providerTurns: providerTurns as never, }, run, ).text; - const attempt = { id: ProviderTurnId.make("active-notice"), runAttemptId: run.activeAttemptId }; - // The provider's own verdict wins over comparing labels, such as for an alias it resolved. - assert.equal( - resultWithTurns([ - { ...attempt, ordinal: 1, model: "gpt-5.4-2026-03-05", modelSubstituted: false }, - ]), - "Review done.", - ); - assert.equal( - resultWithTurns([{ ...attempt, ordinal: 1, model: "gpt-5.4-mini", modelSubstituted: true }]), - `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\nProvider notice: ${fallback}\n\nReview done.`, - ); - assert.equal(resultWithTurns([{ ...attempt, ordinal: 1, model: "gpt-5.4" }]), "Review done."); - // The newest turn reported no model, so an earlier turn's substitution is not attributed to it. + const turn = (ordinal: number, model?: string, runAttemptId = run.activeAttemptId) => ({ + runAttemptId, + ordinal, + ...(model === undefined ? {} : { model }), + }); assert.equal( - resultWithTurns([ - { ...attempt, ordinal: 1, model: "gpt-5.4-mini", modelSubstituted: true }, - { ...attempt, ordinal: 2 }, - ]), - "Review done.", + resultWith([turn(1, "gpt-5.4-mini"), turn(2, "gpt-5.4-nano", RunAttemptId.make("other"))]), + "Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\nReview done.", ); + assert.equal(resultWith([turn(1, "gpt-5.4")]), "Review done."); + // The newest turn reported no model, so an earlier turn's is not attributed to it. + assert.equal(resultWith([turn(1, "gpt-5.4-mini"), turn(2)]), "Review done."); }); it.effect("ends an orphaned native subagent's node when only its record already ended", () => diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index ba53ce0d57d36..29fbb28e84200 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -86,9 +86,9 @@ type LaunchedWith = Pick< "modelSelection" | "runtimeMode" | "interactionMode" >; -/** The thread the box launches for the task, which the test finishes. */ -type ReportedModel = Pick; +type ReportedModel = Pick; +/** The thread the box launches for the task, which the test finishes. */ interface BoxThread { readonly launched: Ref.Ref; /** The link session that launched it, as the box stamps it. */ @@ -934,9 +934,6 @@ it.effect("a task delegated to the box wakes the laptop's parent when it ends th expect(status.structuredContent).toMatchObject({ status: "completed", summary: "All 412 tests pass on the box.", - model: "gpt-5.4", - // The provider there reported no model, as an older box never does. - executedModel: null, }); // Delivery validation still reads messages; choosing the observer does not need another read. expect(messageReads).toHaveLength(2); @@ -1296,7 +1293,6 @@ it.effect("a task's result is its own run's whole last reply, not a later run's" yield* Ref.set(thread.reply, long); yield* Ref.set(thread.laterReply, "Someone else asked a follow-up."); // The provider there ran the task's run on another model, the later run on the one asked for. - // Neither turn says whether that was a substitution, as an older box's never do, so the labels decide. yield* Ref.set(thread.executedModel, { model: "gpt-5.4-mini" }); yield* Ref.set(thread.laterExecutedModel, { model: "gpt-5.4" }); yield* Ref.set(thread.status, "completed"); @@ -1304,11 +1300,8 @@ it.effect("a task's result is its own run's whole last reply, not a later run's" const task = yield* taskResult(a, taskId); expect(task?.status).toBe("completed"); expect(task?.result).toBe( - `Requested model gpt-5.4, but the provider ran gpt-5.4-mini.\n\n${long}`, + `Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\n${long}`, ); - const status = (yield* a.call("task_status", { taskId })).structuredContent; - expect(status).toMatchObject({ executedModel: "gpt-5.4-mini" }); - expect(status).not.toHaveProperty("modelSubstituted"); // Every wait was for the run the launch started. const waits = yield* Ref.get(thread.waitedFor); expect(waits.length).toBeGreaterThan(0); @@ -1342,23 +1335,12 @@ it.effect("a task whose thread there waits on its own task ends with the answer ); expect(first).toBe("waiting there"); - // The review reports and wakes the thread there, whose next run answers on - // the dated slug of the model it asked for, unlike the first run's substitute. - yield* Ref.set(thread.executedModel, { model: "gpt-5.4-mini", modelSubstituted: true }); - yield* Ref.set(thread.laterExecutedModel, { - model: "gpt-5.4-2026-03-05", - modelSubstituted: false, - }); + // The review reports and wakes the thread there, whose next run answers. yield* Ref.set(thread.laterReply, "The review found two issues; both are fixed."); yield* Deferred.succeed(thread.nestedSettled, undefined); - const result = "The review found two issues; both are fixed."; - expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ status: "completed", result }); - const status = yield* a.call("task_status", { taskId: delegated.taskId }); - expect(status.structuredContent).toMatchObject({ - summary: result, - model: "gpt-5.4", - executedModel: "gpt-5.4-2026-03-05", - modelSubstituted: false, + expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ + status: "completed", + result: "The review found two issues; both are fixed.", }); }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index ca7b5df1c183e..221043f351030 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -5,6 +5,7 @@ import { OrchestratorMcpFailure, type OrchestratorMcpDelegateTaskInput, type OrchestratorMcpTaskCancelResult, + type OrchestratorMcpThreadRun, type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, @@ -24,7 +25,7 @@ import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type * as Tool from "effect/ai/Tool"; -import { withSubstitutedModelNote } from "@t3tools/provider-core/server/subagentProjection"; +import { withExecutedModelNote } from "@t3tools/provider-core/server/subagentProjection"; import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import { ProjectToolkit } from "../mcp/toolkits/project/tools.ts"; @@ -369,7 +370,6 @@ const make = Effect.gen(function* () { result: string, stoppedThere: "ran-out" | "stopped" | undefined, stopCount?: number, - resultModel?: OrchestrationV2RemoteTaskChild["resultModel"], ) { const attempt = yield* random.uuidV4(); yield* threads @@ -382,7 +382,6 @@ const make = Effect.gen(function* () { result, ...(stoppedThere === undefined ? {} : { stoppedThere }), ...(stopCount === undefined ? {} : { stopCount }), - ...(resultModel === undefined ? {} : { resultModel }), }) .pipe( Effect.catch((error) => @@ -438,7 +437,7 @@ const make = Effect.gen(function* () { * The last reply `runId` gave on the task's thread there, which is its * result, read whole however long it is. A failed run prefers its error * from the activity view over an earlier reply, without taking another run's. - * Also the model that run asked for and the one its provider reported running. + * Led by a note when its provider reported running another model than it asked for. */ const lastReply = Effect.fn("RemoteDelegation.lastReply")(function* ( scope: ThreadScope, @@ -449,7 +448,7 @@ const make = Effect.gen(function* () { let reply: OrchestratorMcpThreadTimelineItem | undefined; let error: OrchestratorMcpThreadTimelineItem | undefined; let afterPosition: number | undefined; - let resultModel: OrchestrationV2RemoteTaskChild["resultModel"]; + let run: OrchestratorMcpThreadRun | undefined; while (true) { const read = yield* forwarding.readThread(scope, remote.environmentId, { threadId: remote.threadId, @@ -458,18 +457,7 @@ const make = Effect.gen(function* () { runLimit: 50, ...(afterPosition === undefined ? {} : { afterPosition }), }); - const run = read.recentRuns.find((candidate) => candidate.runId === runId); - // Older environments omit executedModel and modelSubstituted. - resultModel ??= - run === undefined - ? undefined - : { - model: run.model, - executedModel: run.executedModel ?? null, - ...(run.modelSubstituted === undefined - ? {} - : { modelSubstituted: run.modelSubstituted }), - }; + run ??= read.recentRuns.find((candidate) => candidate.runId === runId); reply = read.items.findLast( (item) => item.type === "assistant_message" && (runId === null || item.runId === runId), @@ -484,7 +472,7 @@ const make = Effect.gen(function* () { afterPosition = read.nextPosition; } reply = error ?? reply; - if (reply === undefined) return { text: undefined, resultModel }; + if (reply === undefined) return undefined; let text = reply.text ?? ""; let textOffset = reply.textTruncated ? (reply.nextTextOffset ?? null) : null; while (textOffset !== null) { @@ -499,7 +487,7 @@ const make = Effect.gen(function* () { text += part.text ?? ""; textOffset = part.textTruncated ? (part.nextTextOffset ?? null) : null; } - return { text, resultModel }; + return run === undefined ? text : withExecutedModelNote(text, run.model, run.executedModel); }); /** @@ -631,7 +619,7 @@ const make = Effect.gen(function* () { // A task recorded without its run waits on the thread's latest; a // thread there with none has nothing left to wait for. if (waited.runId === null) { - const reply = (yield* lastReply(scope, remote, null)).text; + const reply = yield* lastReply(scope, remote, null); yield* complete( parentThreadId, taskId, @@ -646,26 +634,13 @@ const make = Effect.gen(function* () { runId: remote.runId ?? waited.runId, status: waited.status, }; - const { text, resultModel } = yield* lastReply( - scope, - remote, - resultRun.runId, - resultRun.status === "failed", - ); + const result = yield* lastReply(scope, remote, resultRun.runId, resultRun.status === "failed"); yield* complete( parentThreadId, taskId, terminalStatus(resultRun.status), - resultModel === undefined - ? (text ?? noReply) - : withSubstitutedModelNote(text ?? noReply, { - requestedModel: resultModel.model, - executedModel: resultModel.executedModel, - modelSubstituted: resultModel.modelSubstituted, - }), + result ?? noReply, "ran-out", - undefined, - resultModel, ); return yield* settled(parentThreadId, taskId); }); diff --git a/apps/server/src/provider/ClaudeModelCatalog.ts b/apps/server/src/provider/ClaudeModelCatalog.ts index c35fa0b40390b..57132e652ae08 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.ts @@ -235,18 +235,8 @@ function resolveClaudeCatalogModel( ); } -/** `discovered` rows from Claude Code name the model each of their values resolves to. */ -export function resolveClaudeModelSlug( - catalog: ClaudeModelCatalog, - slugOrAlias: string, - discovered: ReadonlyArray = [], -): string { - const known = resolveClaudeCatalogModel(catalog, slugOrAlias); - if (known) return known.model.slug; - const resolved = discovered - .find((info) => info.value.trim() === slugOrAlias) - ?.resolvedModel?.trim(); - return resolved ? resolveClaudeModelSlug(catalog, resolved) : slugOrAlias; +export function resolveClaudeModelSlug(catalog: ClaudeModelCatalog, slugOrAlias: string): string { + return resolveClaudeCatalogModel(catalog, slugOrAlias)?.model.slug ?? slugOrAlias; } export function getClaudeCatalogModelCapabilities( diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index d924d60d84dc4..c2d4b63861074 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -1,4 +1,3 @@ -import type { ModelInfo as ClaudeModelInfo } from "@anthropic-ai/claude-agent-sdk"; import { ClaudeSettings, ProviderDriverKind } from "@t3tools/contracts"; import * as Cache from "effect/Cache"; import * as Duration from "effect/Duration"; @@ -147,7 +146,6 @@ export const ClaudeDriver: ProviderDriver>([]); const orchestrationAdapter = yield* createClaudeAdapterV2( { instanceId, @@ -159,7 +157,6 @@ export const ClaudeDriver: ProviderDriver snapshot.applyUsageLimits(update), modelCatalog, }, @@ -196,11 +193,7 @@ export const ClaudeDriver: ProviderDriver - Ref.set(lastGoodCapabilities, capabilities).pipe( - Effect.andThen(Ref.set(reportedModels, capabilities.models ?? [])), - ), - ), + Effect.tap(({ capabilities }) => Ref.set(lastGoodCapabilities, capabilities)), Effect.catch((error) => Ref.getAndSet(lastGoodCapabilities, undefined).pipe( Effect.flatMap((capabilities) => diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 04f5b30eb7633..41eb03e0939fc 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -35,5 +35,3 @@ {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} {"id": "thread-send-fresh-context", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send that should start a fresh provider session", "experience": "The user wanted a confused long-running thread to start over in the same thread without its old context; a plain send would keep feeding it the history.", "rule": "`freshContext: true` on t3_thread_send starts a new provider session in the same thread, like /clear.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The thread working on the CSV importer keeps reverting to the old column mapping from early in its history."}, {"role": "assistant", "text": "That thread has a long history with three different mappings in it, so the model keeps picking up the stale one."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "", "messageId": "msg-1", "runId": "run-1", "status": "starting", "delivery": "started"}}}, "prompt": "Send it the final mapping below and have it start over with a clean context in that same thread, so the old history stops leaking in: thread 11111111-1111-4111-8111-111111111111.\n\nFinal mapping: id -> customer_id, amt -> amount_cents, ts -> created_at.", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111", "freshContext": true}}], "notCalls": ["delegate_task", "t3_thread_launch", "create_threads"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: fresh provider sessions in one T3 thread"}, "notes": "Pairs with thread-send-to-named-top-level-thread, where an ordinary send must not set freshContext."} -{"id": "delegate-substituted-model-reported", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The parent called the result an Opus review.", "rule": "executedModel: Model the provider reported running, or null if unreported. Can differ from the requested `model`.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 reached a terminal state. Use task_status with taskId task-1 to read the result.", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-1"}}], "textMust": ["(?:[Ss]onnet[^.!?]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review|substituted)[^.!?]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The mock is task_status as the fix returns it, summary included. Main returned no executedModel and only the child's answer as the summary, so the parent only saw the requested Opus."} -{"id": "delegate-substituted-model-inline-result", "area": "orchestration", "kind": "bad", "pattern": "requested model reported as the one that ran, from the inline result", "experience": "An organization that disallows Opus made Claude run an Opus delegate on Sonnet. The completion message inlined only the child's answer, so the parent called it an Opus review.", "rule": "Delegated completion messages put \"Requested model X, but the provider ran Y.\" and the provider's notice before a child's result when the provider ran another model.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have an Opus delegate review https://github.com/acme/app/pull/123 and tell me what it found."}, {"role": "assistant", "text": "An Opus review of PR 123 is running. I'll report back when it finishes.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "options": {"effort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report blocking findings first."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": null}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "hasPendingChildRuns": false, "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "executedModel": "claude-sonnet-5-5", "options": [{"id": "effort", "value": "high"}], "summary": "Requested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "latestTerminalRunId": "run-child-1", "latestTerminalStatus": "completed", "latestTerminalSummary": null, "latestTerminalResultContextTransferId": null, "resultContextTransferId": null, "hasPendingQuestion": false, "pendingApproval": null, "waitTimedOut": false}}, "world": {"prs": [{"number": 123, "title": "fix: install script paths", "headRefName": "fix/install-paths"}]}}, "prompt": "Delegated task task-1 (Opus review PR 123) [completed]:\nRequested model claude-opus-5-5, but the provider ran claude-sonnet-5-5.\nProvider notice: Model \"claude-opus-5-5\" is restricted by your organization's settings. Using claude-sonnet-5-5 instead.\n\nNo blocking findings. One nit: `formatInstallPath` in scripts/install.ts ignores a trailing slash.", "expect": {"textMust": ["(?:[Ss]onnet[^.!?]*(?:ran|performed|executed|completed|review)|(?:ran|performed|executed|completed|review|substituted)[^.!?]*[Ss]onnet)"], "textMustNot": ["[Oo]pus review (?:found|reported|completed|returned)", "[Oo]pus (?:ran|performed|executed|completed) (?:the|this) review"]}, "source": {"issue": "https://github.com/pingdotgg/t3code/issues/27705"}, "notes": "The prompt is the inline completion message as the fix writes it for a short result. On main the same message carried only the child's answer. To check the old text, replace the prompt with 'Delegated task task-1 (Opus review PR 123) [completed]:\\n' plus the answer alone."} diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 5b59068ae9164..a0c75c315bd2e 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -863,13 +863,6 @@ const DedupedBodyMarker = Schema.optionalKey(Schema.Literal(true)); * of in a child thread here: that environment and the ordinary thread it runs * as there. Such a task has no `childThreadId`. */ -// executedModel is null and modelSubstituted absent when the environment there reported none, as older versions never do. -const OrchestrationV2RemoteTaskResultModel = Schema.Struct({ - model: Schema.String, - executedModel: Schema.NullOr(Schema.String), - modelSubstituted: Schema.optional(Schema.Boolean), -}); - export const OrchestrationV2RemoteTaskChild = Schema.Struct({ environmentId: EnvironmentId, threadId: ThreadId, @@ -888,8 +881,6 @@ export const OrchestrationV2RemoteTaskChild = Schema.Struct({ * one owed, so each reaches the thread there once however often it is sent. */ stopCount: Schema.optional(NonNegativeInt), - /** The result's run there: the model it asked for and the one its provider reported running. */ - resultModel: Schema.optional(OrchestrationV2RemoteTaskResultModel), }); export type OrchestrationV2RemoteTaskChild = typeof OrchestrationV2RemoteTaskChild.Type; @@ -1206,8 +1197,6 @@ export const OrchestrationV2ProviderTurn = Schema.Struct({ turnTokenUsage: Schema.optional(TurnTokenUsage), /** The model the provider reported running this turn's root agent, which can differ from the requested one. */ model: Schema.optional(TrimmedNonEmptyString), - /** Whether `model` is another model than the run requested, which an alias for it is not. */ - modelSubstituted: Schema.optional(Schema.Boolean), }); export type OrchestrationV2ProviderTurn = typeof OrchestrationV2ProviderTurn.Type; @@ -3493,8 +3482,6 @@ const OrchestrationV2InternalCommand = Schema.Union([ stoppedThere: Schema.optional(Schema.Literals(["ran-out", "stopped"])), /** The stop generation this report acknowledges; absent for the original unnumbered stop. */ stopCount: Schema.optional(NonNegativeInt), - /** Kept on `remoteChild` with the result this report completes the task with. */ - resultModel: Schema.optional(OrchestrationV2RemoteTaskResultModel), }), /** * Owes a stop to the thread there of a remote task that already has its result, as Stop on diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index ea722836cc144..e82c0fb73effc 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -204,11 +204,7 @@ const OrchestratorMcpRunningTaskNext = Schema.Literal(ORCHESTRATOR_MCP_RUNNING_T // Optional: linked environments on older versions omit it. const OrchestratorMcpExecutedModel = Schema.NullOr(Schema.String).annotate({ description: - "Model the provider reported running, or null if unreported. Can differ from the requested `model`.", -}); -const OrchestratorMcpModelSubstituted = Schema.Boolean.annotate({ - description: - "Whether `executedModel` is another model than the requested `model`, not just its alias. Absent when unknown.", + "Model the provider reported running, or null when unavailable. Can differ from the requested `model`.", }); export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ @@ -232,7 +228,6 @@ export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ providerInstanceId: ProviderInstanceId, model: Schema.NullOr(Schema.String), executedModel: Schema.optional(OrchestratorMcpExecutedModel), - modelSubstituted: Schema.optional(OrchestratorMcpModelSubstituted), options: Schema.optional(Schema.Array(ProviderOptionSelection)), summary: Schema.NullOr(Schema.String), resultContextTransferId: Schema.NullOr(ContextTransferId), @@ -475,7 +470,6 @@ export const OrchestratorMcpThreadRun = Schema.Struct({ providerInstanceId: ProviderInstanceId, model: Schema.String, executedModel: Schema.optional(OrchestratorMcpExecutedModel), - modelSubstituted: Schema.optional(OrchestratorMcpModelSubstituted), requestedAt: IsoDateTime, startedAt: Schema.NullOr(IsoDateTime), completedAt: Schema.NullOr(IsoDateTime), diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index 5220fd1ce7064..62145567f6651 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -176,56 +176,15 @@ export function makeSubagentConversationArtifacts(input: { return { message, turnItem }; } -/** - * A delegated result, led by a note when the provider ran its run on another - * model than the run requested, with that run's own provider notices. - */ -export function withSubstitutedModelNote( +/** A delegated result, led by a note when the provider reported running another model than requested. */ +export function withExecutedModelNote( result: string, - input: { - readonly requestedModel: string; - readonly executedModel: string | null | undefined; - // Set where the provider resolved aliases; older environments omit it. - readonly modelSubstituted?: boolean | undefined; - readonly notices?: ReadonlyArray; - }, + requestedModel: string, + executedModel: string | null | undefined, ): string { - if ( - input.executedModel == null || - !(input.modelSubstituted ?? input.executedModel !== input.requestedModel) - ) { - return result; - } - const note = [ - `Requested model ${input.requestedModel}, but the provider ran ${input.executedModel}.`, - ...(input.notices ?? []).map((notice) => `Provider notice: ${notice}`), - ].join("\n"); - return `${note}\n\n${result}`; -} - -function rootProviderNotices( - projection: Pick, - run: OrchestrationV2Run, -): ReadonlyArray { - const attemptTurnIds = new Set( - projection.providerTurns - .filter((turn) => run.activeAttemptId !== null && turn.runAttemptId === run.activeAttemptId) - .map((turn) => turn.id), - ); - return projection.turnItems - .filter( - (item): item is Extract => - item.type === "system_notice" && - item.runId === run.id && - item.nodeId === run.rootNodeId && - item.parentItemId === null && - item.providerTurnId !== null && - attemptTurnIds.has(item.providerTurnId) && - // T3's own notices, such as usage limits, carry weak refs. - item.nativeItemRef?.strength === "strong", - ) - .toSorted((left, right) => left.ordinal - right.ordinal) - .map((item) => item.message); + return executedModel == null || executedModel === requestedModel + ? result + : `Requested model ${requestedModel}; the provider reported running ${executedModel}.\n\n${result}`; } export function subagentResultForRun( @@ -268,24 +227,22 @@ export function subagentResultForRun( .filter((item) => item.runId === run.id && item.type === "error") .toSorted((left, right) => right.ordinal - left.ordinal)[0] : undefined; - const result = + const text = (failure?.type === "error" ? failure.failure.message : undefined) ?? message?.text ?? turnItem?.text ?? (run.status === "completed" ? "Child task completed without an assistant result." : `Child task ended with status ${run.status}.`); - const latestTurn = latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId); + const executedModel = latestProviderTurnForAttempt( + projection.providerTurns, + run.activeAttemptId, + )?.model; return { text: - latestTurn?.model === undefined - ? result - : withSubstitutedModelNote(result, { - requestedModel: run.modelSelection.model, - executedModel: latestTurn.model, - modelSubstituted: latestTurn.modelSubstituted, - notices: rootProviderNotices(projection, run), - }), + executedModel === undefined + ? text + : withExecutedModelNote(text, run.modelSelection.model, executedModel), messageId: failure === undefined ? (message?.id ?? turnItem?.messageId ?? null) : null, turnItemId: failure?.id ?? turnItem?.id ?? null, }; From f7c2ed273c92d2a0ffa6c8c44f6eea14b1a8c548 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:00:14 +0000 Subject: [PATCH 20/21] test(server): cut executed-model tests to assertions that fail on main Co-Authored-By: Claude Opus 5.5 --- .../src/mcp/OrchestratorMcpService.test.ts | 80 ++++------------ .../Adapters/ClaudeAdapterV2.test.ts | 93 +++---------------- .../SubagentProjection.test.ts | 40 +------- apps/server/src/peer/RemoteDelegation.test.ts | 43 +++------ 4 files changed, 47 insertions(+), 209 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index d6b7410876f68..df13d037005a4 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -20,7 +20,6 @@ import { ProviderInstanceId, ProviderThreadId, ProviderTurnId, - RunAttemptId, RunId, ThreadId, type OrchestrationV2ServerCommand, @@ -3054,49 +3053,14 @@ describe("OrchestratorMcpService provider resolution", () => { status: "running", result: null, }; - let hasContinuation = false; - let hasStoredResult = false; - const restartedAttemptId = RunAttemptId.make("attempt:mcp-restarted-child"); - const continuationAttemptId = RunAttemptId.make("attempt:mcp-child-continuation"); - const continuationSelection = { ...runSelection, model: "claude-sonnet-4-6" }; - // Claude ran the restarted attempt on Sonnet; a later turn of another attempt reported Haiku. - const providerTurns = [ - { runAttemptId: restartedAttemptId, ordinal: 1, model: "claude-sonnet-4-6" }, - { runAttemptId: continuationAttemptId, ordinal: 2, model: "claude-haiku-4-5" }, - { - runAttemptId: RunAttemptId.make("attempt:mcp-other-run"), - ordinal: 2, - model: "claude-haiku-4-5", - }, - ]; const dependencies = Layer.mergeAll( NodeServices.layer, NodeCrypto.layer, Layer.mock(ThreadManagementService.ThreadManagementService)({ - getThreadRecords: (threadId, fields) => + getThreadRecords: (threadId) => Effect.succeed( threadId === parentThreadId - ? ({ - ...parentProjection([ - { - ...task, - status: hasStoredResult ? "completed" : task.status, - result: hasStoredResult ? "Stored review result." : null, - completionDelivery: { state: "acknowledged" }, - }, - ]), - contextTransfers: hasStoredResult - ? [ - { - id: "transfer:mcp-stored-result", - type: "subagent_result", - sourceThreadId: childThreadId, - targetThreadId: parentThreadId, - sourcePoint: { runId: RunId.make("run:mcp-restarted-child") }, - }, - ] - : [], - } as unknown as OrchestrationV2ThreadProjection) + ? parentProjection([task]) : ({ ...childProjection, thread: { ...childProjection.thread, modelSelection: threadSelection }, @@ -3105,28 +3069,20 @@ describe("OrchestratorMcpService provider resolution", () => { { id: RunId.make("run:mcp-restarted-child"), ordinal: 1, - status: hasStoredResult ? "completed" : "running", + status: "running", modelSelection: runSelection, - activeAttemptId: restartedAttemptId, - delegatedTaskId: hasStoredResult ? undefined : taskId, + activeAttemptId: "attempt:mcp-restarted-child", }, - ...(hasContinuation - ? [ - { - id: RunId.make("run:mcp-child-continuation"), - ordinal: 2, - status: hasStoredResult ? "completed" : "running", - modelSelection: continuationSelection, - activeAttemptId: continuationAttemptId, - delegatedTaskId: hasStoredResult ? undefined : taskId, - }, - ] - : []), ] : [], - providerTurns: fields?.some((field) => field === "providerTurns") - ? providerTurns - : [], + // Claude ran the requested Opus on Sonnet. + providerTurns: [ + { + runAttemptId: "attempt:mcp-restarted-child", + ordinal: 1, + model: "claude-sonnet-4-6", + }, + ], } as unknown as OrchestrationV2ThreadProjection), ), }), @@ -3140,20 +3096,16 @@ describe("OrchestratorMcpService provider resolution", () => { ); yield* Effect.gen(function* () { const service = yield* OrchestratorMcpService.OrchestratorMcpService; - for (const [runExists, continuationExists, storedResult, selection, executedModel] of [ - [true, false, false, runSelection, "claude-sonnet-4-6"], - [true, true, false, continuationSelection, "claude-haiku-4-5"], - [true, true, true, runSelection, "claude-sonnet-4-6"], - [false, false, false, threadSelection, null], + for (const [selection, executedModel] of [ + [runSelection, "claude-sonnet-4-6"], + [threadSelection, null], ] as const) { - hasRun = runExists; - hasContinuation = continuationExists; - hasStoredResult = storedResult; const result = yield* service.taskStatus(scope, taskId); assert.equal(result.providerInstanceId, selection.instanceId); assert.equal(result.model, selection.model); assert.deepEqual(result.options, selection.options); assert.equal(result.executedModel, executedModel); + hasRun = false; } }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); }), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index d7229e12fb20d..8f29312c0ab96 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2970,11 +2970,15 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000000101", session_id: WAKE_NATIVE_SESSION, }); - const makeAssistantTextFrame = (input: { readonly uuid: string; readonly text: string }) => + const makeAssistantTextFrame = (input: { + readonly uuid: string; + readonly text: string; + readonly model?: string; + }) => claudeSdkFrame({ type: "assistant", message: { - model: "claude-sonnet-4-6", + model: input.model ?? "claude-sonnet-4-6", id: `msg_${input.uuid}`, type: "message", role: "assistant", @@ -3003,12 +3007,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly text?: string; readonly parentToolUseId?: string | null; readonly apiErrorParams?: unknown; - readonly model?: string; }) => claudeSdkFrame({ type: "assistant", message: { - model: input.model ?? "claude-sonnet-4-6", + model: "claude-sonnet-4-6", id: `msg_${input.uuid}`, type: "message", role: "assistant", @@ -6837,7 +6840,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); yield* Queue.offerAll(harness.sdkMessages, [ - makeAssistantTextFrame({ uuid: mainUuid, text: "Delegating." }), + // Claude reports the model it ran, which can differ from the one requested. + makeAssistantTextFrame({ + uuid: mainUuid, + text: "Delegating.", + model: "claude-haiku-4-5-20251001", + }), makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000642", error: undefined, @@ -6849,6 +6857,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { const providerTurn = lastProviderTurn(harness.events); assert.equal(providerTurn.nativeTurnRef?.nativeId, mainUuid); + assert.equal(providerTurn.model, "claude-haiku-4-5"); const rolledBack = yield* harness.runtime.rollbackThread({ providerThread: harness.providerThread, target: { @@ -8857,80 +8866,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect("records the root model Claude reports for each turn, not the requested one", () => - Effect.gen(function* () { - const harness = yield* makeWakeHarnessWithOptions({ freshQueueOnReopen: true }); - const runTurn = (ordinal: number, frames: ReadonlyArray) => - Effect.gen(function* () { - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attachments: [], - modelSelection: { - instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, - model: "claude-opus-4-6", - }, - attemptId: RunAttemptId.make(`attempt-claude-executed-model-${ordinal}`), - providerTurnOrdinal: ordinal, - text: "Review this.", - }), - ); - yield* Queue.offerAll(harness.sdkMessages, [ - ...frames, - makeResultFrame({ - uuid: `00000000-0000-4000-8000-00000000066${ordinal}`, - result: "Done.", - }), - ]); - yield* Queue.take(harness.terminalReceipts); - return lastProviderTurn(harness.events).model; - }); - yield* Queue.offer(harness.sdkMessages, wakeTurnInit); - // A native subagent's model is not the run's. - assert.equal( - yield* runTurn(1, [ - makeAssistantErrorFrame({ - uuid: "00000000-0000-4000-8000-000000000671", - error: undefined, - model: "claude-sonnet-4-6", - }), - makeAssistantErrorFrame({ - uuid: "00000000-0000-4000-8000-000000000672", - error: undefined, - parentToolUseId: "toolu_executed_model_subagent", - model: "claude-haiku-4-5-20251001", - }), - ]), - "claude-sonnet-4-6", - ); - // A dated slug reports its catalog model, so it is not mistaken for another one. - assert.equal( - yield* runTurn(2, [ - makeAssistantErrorFrame({ - uuid: "00000000-0000-4000-8000-000000000673", - error: undefined, - model: "claude-haiku-4-5-20251001", - }), - ]), - "claude-haiku-4-5", - ); - // A turn that reports no model does not inherit the previous one's. - assert.isUndefined(yield* runTurn(3, [])); - }).pipe( - Effect.scoped, - Effect.provide( - Layer.mergeAll( - IdAllocator.layer, - McpProviderSessions.layer, - NodeServices.layer, - NodeCrypto.layer, - ), - ), - ), - ); - it.effect("preserves typed Claude plans and todos through generic tool completion", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index e689b20bc43df..87ac8a2ad1758 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -413,44 +413,14 @@ it("orders a task's assistant messages by turn item, not by timestamps", () => { assert.equal(result.text, "All steps passed."); assert.equal(result.messageId, report.message.id); assert.equal(result.turnItemId, report.turnItem.id); -}); - -it("leads a task's result with the model its own attempt reported running", () => { - const { projection, run } = taskFixture(); - const answer = makeSubagentConversationArtifacts({ - messageId: MessageId.make("answer"), - turnItemId: TurnItemId.make("answer"), - threadId: parentThreadId, - rootNodeId: NodeId.make("root"), - providerThreadId: null, - providerTurnId: null, - nativeItemRef: null, - role: "assistant", - text: "Review done.", - ordinal: 1, - now: childCreatedAt, - }); - const resultWith = (providerTurns: ReadonlyArray) => + const reported = { runAttemptId: run.activeAttemptId, ordinal: 1, model: "gpt-5.4-mini" }; + assert.equal( subagentResultForRun( - { - ...projection, - messages: [{ ...answer.message, runId: run.id }], - providerTurns: providerTurns as never, - }, + { ...projection, messages: [report.message], providerTurns: [reported as never] }, run, - ).text; - const turn = (ordinal: number, model?: string, runAttemptId = run.activeAttemptId) => ({ - runAttemptId, - ordinal, - ...(model === undefined ? {} : { model }), - }); - assert.equal( - resultWith([turn(1, "gpt-5.4-mini"), turn(2, "gpt-5.4-nano", RunAttemptId.make("other"))]), - "Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\nReview done.", + ).text, + "Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\nAll steps passed.", ); - assert.equal(resultWith([turn(1, "gpt-5.4")]), "Review done."); - // The newest turn reported no model, so an earlier turn's is not attributed to it. - assert.equal(resultWith([turn(1, "gpt-5.4-mini"), turn(2)]), "Review done."); }); it.effect("ends an orphaned native subagent's node when only its record already ended", () => diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 29fbb28e84200..6f39437778bc5 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -7,16 +7,13 @@ import { EventId, MessageId, NodeId, - type OrchestrationV2ProviderTurn, type OrchestrationV2Run, ProjectId, ProviderDriverKind, ProviderInstanceId, ProviderThreadId, - ProviderTurnId, type Project, type RepositoryIdentity, - RunAttemptId, RunId, ThreadId, TurnItemId, @@ -86,8 +83,6 @@ type LaunchedWith = Pick< "modelSelection" | "runtimeMode" | "interactionMode" >; -type ReportedModel = Pick; - /** The thread the box launches for the task, which the test finishes. */ interface BoxThread { readonly launched: Ref.Ref; @@ -105,9 +100,8 @@ interface BoxThread { readonly laterError: Ref.Ref; /** A later run's reply, after the task's own, as a follow-up message would leave. */ readonly laterReply: Ref.Ref; - /** The model the provider there reported running the task's own run, and the later one. */ - readonly executedModel: Ref.Ref; - readonly laterExecutedModel: Ref.Ref; + /** The model the provider there reported running the task's run. */ + readonly executedModel: Ref.Ref; /** The run each wait asked for, in order. */ readonly waitedFor: Ref.Ref>; /** The thread's run ends with a task of its own still running, whose result wakes it again. */ @@ -153,8 +147,7 @@ const makeBoxThread = Effect.gen(function* () { error: yield* Ref.make(null), laterError: yield* Ref.make(null), laterReply: yield* Ref.make(null), - executedModel: yield* Ref.make({}), - laterExecutedModel: yield* Ref.make({}), + executedModel: yield* Ref.make(null), waitedFor: yield* Ref.make>([]), nested: yield* Ref.make(false), nestedWaiting: yield* Deferred.make(), @@ -229,42 +222,32 @@ const boxToolkitLayer = (thread: BoxThread) => { ...(linkOrigin === null ? {} : { linkOrigin }), }; }); - const runOf = (status: OrchestrationV2Run["status"], id = runId) => + const runOf = (status: OrchestrationV2Run["status"]) => ({ - id, + id: runId, threadId: ThreadId.make("thread:box"), - ordinal: id === runId ? 1 : 2, + ordinal: 1, status, modelSelection, - activeAttemptId: RunAttemptId.make(`attempt:${id}`), + activeAttemptId: "attempt:box", requestedAt: DateTime.makeUnsafe(0), startedAt: DateTime.makeUnsafe(0), completedAt: null, }) as unknown as OrchestrationV2Run; - const providerTurnOf = (id: RunId, reported: ReportedModel) => - ({ - id: ProviderTurnId.make(`provider-turn:${id}`), - runAttemptId: RunAttemptId.make(`attempt:${id}`), - ordinal: 1, - ...reported, - }) as unknown as OrchestrationV2ProviderTurn; const projectionOf = (threadId: ThreadId) => Effect.gen(function* () { const shell = yield* shellOf(threadId); const status = yield* Ref.get(thread.status); const runless = yield* Ref.get(thread.runless); - const later = (yield* Ref.get(thread.laterReply)) !== null; + const model = yield* Ref.get(thread.executedModel); return shell === null ? (null as never) : { ...idleThreadProjection(shell), - runs: runless + runs: runless ? [] : [runOf(status)], + providerTurns: (model === null ? [] - : [runOf(status), ...(later ? [runOf("completed", boxLaterRunId)] : [])], - providerTurns: [ - providerTurnOf(runId, yield* Ref.get(thread.executedModel)), - providerTurnOf(boxLaterRunId, yield* Ref.get(thread.laterExecutedModel)), - ], + : [{ runAttemptId: "attempt:box", ordinal: 1, model }]) as never, }; }); // One record per command id, as its receipt replays a repeat. @@ -1292,9 +1275,7 @@ it.effect("a task's result is its own run's whole last reply, not a later run's" const long = "x".repeat(45_000); yield* Ref.set(thread.reply, long); yield* Ref.set(thread.laterReply, "Someone else asked a follow-up."); - // The provider there ran the task's run on another model, the later run on the one asked for. - yield* Ref.set(thread.executedModel, { model: "gpt-5.4-mini" }); - yield* Ref.set(thread.laterExecutedModel, { model: "gpt-5.4" }); + yield* Ref.set(thread.executedModel, "gpt-5.4-mini"); yield* Ref.set(thread.status, "completed"); yield* Deferred.succeed(thread.finished, undefined); const task = yield* taskResult(a, taskId); From 54536a651b12f1a3cc1f1449ccfd5cf91a28b666 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:30:11 +0000 Subject: [PATCH 21/21] fix(server): a dated Claude model id no longer reads as another model Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 4 ++-- .../orchestration-v2/Adapters/ClaudeAdapterV2.ts | 6 +++++- apps/server/src/provider/ClaudeModelCatalog.ts | 13 +++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 8f29312c0ab96..d9b349b809781 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -6844,7 +6844,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeAssistantTextFrame({ uuid: mainUuid, text: "Delegating.", - model: "claude-haiku-4-5-20251001", + model: "claude-opus-4-5-20251101", }), makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000642", @@ -6857,7 +6857,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { const providerTurn = lastProviderTurn(harness.events); assert.equal(providerTurn.nativeTurnRef?.nativeId, mainUuid); - assert.equal(providerTurn.model, "claude-haiku-4-5"); + assert.equal(providerTurn.model, "claude-opus-4-5"); const rolledBack = yield* harness.runtime.rollbackThread({ providerThread: harness.providerThread, target: { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6df4b70baf80a..8ccb7b3ba0377 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -188,6 +188,7 @@ import { type ClaudeModelCatalog, getClaudeCatalogModelCapabilities, resolveClaudeModelSlug, + resolveClaudeReportedModelSlug, scopeClaudeModelCatalog, } from "../../provider/ClaudeModelCatalog.ts"; import { @@ -9115,7 +9116,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // CLI notices and API errors arrive as assistant messages from "". if (message.message.model && message.message.model !== "") { context.rootModels.add(message.message.model); - context.executedModel = resolveClaudeModelSlug(modelCatalog, message.message.model); + context.executedModel = resolveClaudeReportedModelSlug( + modelCatalog, + message.message.model, + ); } // Subagent uuids exist only in the sidechain transcript, so resume and fork cannot target them. context.nativeMessageCursor = message.uuid; diff --git a/apps/server/src/provider/ClaudeModelCatalog.ts b/apps/server/src/provider/ClaudeModelCatalog.ts index 57132e652ae08..4efca48f253bf 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.ts @@ -239,6 +239,19 @@ export function resolveClaudeModelSlug(catalog: ClaudeModelCatalog, slugOrAlias: return resolveClaudeCatalogModel(catalog, slugOrAlias)?.model.slug ?? slugOrAlias; } +/** Claude reports dated IDs, such as `claude-opus-4-5-20251101`, that the catalog may not list. */ +export function resolveClaudeReportedModelSlug( + catalog: ClaudeModelCatalog, + reported: string, +): string { + return ( + ( + resolveClaudeCatalogModel(catalog, reported) ?? + resolveClaudeCatalogModel(catalog, reported.replace(/-\d{8}$/, "")) + )?.model.slug ?? reported + ); +} + export function getClaudeCatalogModelCapabilities( catalog: ClaudeModelCatalog, slugOrAlias: string | null | undefined,