diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 6450f9acac2cb..84437454fffe1 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -9,6 +9,8 @@ import { type OrchestratorMcpCapabilitiesResult, OrchestratorMcpTarget, type OrchestrationV2ThreadShell, + type OrchestrationV2Run, + type OrchestrationV2Subagent, type ScheduledTask, ScheduledTaskId, ProjectId, @@ -2413,78 +2415,101 @@ describe("OrchestratorMcpService provider resolution", () => { }), ); }); - it.effect( - "reports the child run's selection after a restart and the thread selection before launch", - () => - Effect.gen(function* () { - const runSelection = { - instanceId: ProviderInstanceId.make("claudeAgent-personal"), - model: "claude-opus-4-6", - options: [ - { id: "effort", value: "max" }, - { id: "fastMode", value: true }, - ], - } as const; - const threadSelection = { - instanceId: codexInstanceId, - model: "gpt-5.4", - options: [{ id: "reasoningEffort", value: "high" }], - } as const; - let hasRun = true; - const task = { - id: taskId, - threadId: parentThreadId, - origin: "app_owned", - childThreadId, - providerInstanceId: ProviderInstanceId.make("claudeAgent-work"), - model: "claude-sonnet-4-6", - status: "running", - result: null, - }; - const dependencies = Layer.mergeAll( - NodeServices.layer, - Layer.mock(ThreadManagementService.ThreadManagementService)({ - getThreadRecords: (threadId) => - Effect.succeed( - threadId === parentThreadId - ? parentProjection([task]) - : ({ - ...childProjection, - thread: { ...childProjection.thread, modelSelection: threadSelection }, - runs: hasRun - ? [ - { - id: RunId.make("run:mcp-restarted-child"), - ordinal: 1, - status: "running", - modelSelection: runSelection, - }, - ] - : [], - } as unknown as OrchestrationV2ThreadProjection), - ), - }), - ServerSettings.layerTest(), - providerRegistryLayer([]), - adapterRegistryLayer([]), - Layer.mock(ProjectService.ProjectService)({}), - Layer.mock(SecretRequests.SecretRequests)({}), - Layer.mock(RemoteDelegation.RemoteDelegation)({}), - Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), - ); - yield* Effect.gen(function* () { - const service = yield* OrchestratorMcpService.OrchestratorMcpService; - for (const selection of [runSelection, threadSelection]) { - const result = yield* service.taskStatus(scope, taskId); - assert.equal(result.providerInstanceId, selection.instanceId); - assert.equal(result.model, selection.model); - assert.deepEqual(result.options, selection.options); - hasRun = false; - } - }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); - }), - ); + it.effect.each([ + { phase: "after restart", hasRun: true }, + { phase: "before launch", hasRun: false }, + ])("reports the child selection $phase with execution provenance when available", ({ hasRun }) => + Effect.gen(function* () { + const runSelection = { + instanceId: ProviderInstanceId.make("claudeAgent-personal"), + model: "claude-opus-5-5", + options: [ + { id: "effort", value: "max" }, + { id: "fastMode", value: true }, + ], + } as const; + const threadSelection = { + instanceId: codexInstanceId, + model: "gpt-5.4", + options: [{ id: "reasoningEffort", value: "high" }], + } as const; + const child = idleThreadProjection(liveThreadShell(childThreadId)); + const run = { + id: RunId.make("run:mcp-restarted-child"), + threadId: childThreadId, + ordinal: 1, + providerInstanceId: runSelection.instanceId, + modelSelection: runSelection, + observedModel: "claude-sonnet-5-5", + providerThreadId: null, + userMessageId: MessageId.make("message:mcp-restarted-child"), + rootNodeId: null, + activeAttemptId: null, + status: "running", + requestedAt: child.updatedAt, + startedAt: child.updatedAt, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + } satisfies OrchestrationV2Run; + const task = { + id: taskId, + threadId: parentThreadId, + parentNodeId, + runId: null, + origin: "app_owned", + createdBy: "agent", + driver: ProviderDriverKind.make("claudeAgent"), + childThreadId, + providerThreadId: null, + nativeTaskRef: null, + prompt: "Review the implementation.", + title: null, + providerInstanceId: ProviderInstanceId.make("claudeAgent-work"), + model: "claude-sonnet-4-6", + status: "running", + result: null, + startedAt: child.updatedAt, + completedAt: null, + updatedAt: child.updatedAt, + } satisfies OrchestrationV2Subagent; + const parent = { + ...idleThreadProjection(liveThreadShell(parentThreadId)), + subagents: [task], + }; + const childWithRun = { + ...child, + thread: { ...child.thread, modelSelection: threadSelection }, + runs: hasRun ? [run] : [], + }; + const dependencies = Layer.mergeAll( + NodeServices.layer, + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: (id) => Effect.succeed(id === parentThreadId ? parent : childWithRun), + }), + ServerSettings.layerTest(), + providerRegistryLayer([]), + adapterRegistryLayer([]), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({}), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ); + yield* Effect.gen(function* () { + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const result = yield* service.taskStatus(scope, taskId); + + const selection = hasRun ? runSelection : threadSelection; + const observedModel = hasRun ? run.observedModel : undefined; + assert.equal(result.providerInstanceId, selection.instanceId); + assert.equal(result.model, observedModel ?? selection.model); + assert.equal(result.requestedModel, selection.model); + assert.equal(result.observedModel, observedModel); + assert.deepEqual(result.options, selection.options); + }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); + }), + ); it.effect( "inherits delegated options, merges overrides, and resets on explicit clearing or a new target", () => diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index d8dacb35a0617..991c1247b9365 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -790,6 +790,9 @@ function threadDetail( ): OrchestratorMcpThreadDetail { const latest = ThreadManagementService.latestRun(projection); const active = ThreadManagementService.latestActiveRun(projection); + const modelSelection = + latest?.observedModel == null ? projection.thread.modelSelection : latest.modelSelection; + return { threadId: projection.thread.id, projectId: projection.thread.projectId, @@ -800,8 +803,10 @@ function threadDetail( status: active?.status ?? latest?.status ?? "idle", latestRunId: latest?.id ?? null, activeRunId: active?.id ?? null, - providerInstanceId: projection.thread.modelSelection.instanceId, - model: projection.thread.modelSelection.model, + providerInstanceId: modelSelection.instanceId, + model: latest?.observedModel ?? modelSelection.model, + requestedModel: modelSelection.model, + ...(latest?.observedModel == null ? {} : { observedModel: latest.observedModel }), runtimeMode: projection.thread.runtimeMode, interactionMode: projection.thread.interactionMode, linkedPullRequest: projection.thread.linkedPullRequest ?? null, @@ -837,7 +842,9 @@ function threadRun(run: OrchestrationV2Run): OrchestratorMcpThreadRun { ordinal: run.ordinal, status: run.status, providerInstanceId: run.modelSelection.instanceId, - model: run.modelSelection.model, + model: run.observedModel ?? run.modelSelection.model, + requestedModel: run.modelSelection.model, + ...(run.observedModel == null ? {} : { observedModel: run.observedModel }), requestedAt: DateTime.formatIso(run.requestedAt), startedAt: run.startedAt === null ? null : DateTime.formatIso(run.startedAt), completedAt: run.completedAt === null ? null : DateTime.formatIso(run.completedAt), @@ -1535,7 +1542,9 @@ const make = Effect.gen(function* () { workState, hasPendingChildRuns: hasPendingChildRuns(childProjection, childRun), providerInstanceId: modelSelection.instanceId, - model: modelSelection.model, + model: childRun?.observedModel ?? modelSelection.model, + requestedModel: modelSelection.model, + ...(childRun?.observedModel == null ? {} : { observedModel: childRun.observedModel }), options: modelSelection.options ?? [], summary: derivedResult, resultContextTransferId, diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index ec8c9434bec43..e5f871c1e6546 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { CommandId, EnvironmentId, + EventId, isProviderNativeSubagentThread, MessageId, type ModelSelection, @@ -16,10 +17,12 @@ import { type ServerProvider, ThreadId, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { McpSchema, McpServer } from "effect/ai"; @@ -27,12 +30,14 @@ import { McpSchema, McpServer } from "effect/ai"; import { CodexOrchestratorReplayHarness } from "../orchestration-v2/Adapters/CodexAdapterV2.testkit.ts"; import * as CommandReceiptStore from "../orchestration-v2/CommandReceiptStore.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; +import { makeProviderReplayGate } from "@t3tools/provider-testing/replayGate"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderReplayHarness from "../orchestration-v2/testkit/ProviderReplayHarness.ts"; import { @@ -193,12 +198,22 @@ const layerUnusedScheduledTaskStub = Layer.succeed( describe("orchestrator MCP toolkit", () => { it.live.each([ - { label: "distinct", result: queuedFollowupResult, latestSummary: queuedFollowupResult }, - { label: "identical", result: delegatedResult, latestSummary: null }, - ])("reports $label child results", ({ result, latestSummary }) => + { + label: "distinct", + result: queuedFollowupResult, + latestSummary: queuedFollowupResult, + observedModel: "provider-reported-model", + }, + { label: "identical", result: delegatedResult, latestSummary: null, observedModel: undefined }, + ])("reports $label child results", ({ result, latestSummary, observedModel }) => Effect.scoped( Effect.gen(function* () { const cwd = yield* checkpointWorkspace("delegated-task-status"); + const replayGate = makeProviderReplayGate(["child.original.item/started"]); + yield* Effect.addFinalizer(() => Effect.sync(() => replayGate.releaseAll())); + const delegatedSummary = observedModel + ? `Requested model: ${codexModel}. Provider-reported model: ${observedModel}.\n\n${delegatedResult}` + : delegatedResult; const rawTranscript = yield* readDelegatedTaskStatusTranscript(result); const transcript = yield* CodexOrchestratorReplayHarness.decodeTranscript( materializeReplayTranscriptWorkspace(rawTranscript, cwd), @@ -211,6 +226,7 @@ describe("orchestrator MCP toolkit", () => { runtimePolicyOverride: { cwd }, }, CodexOrchestratorReplayHarness, + { replayGate }, ); const layerOrchestration = Layer.merge( layerOrchestrator, @@ -233,9 +249,9 @@ describe("orchestrator MCP toolkit", () => { Layer.provideMerge(McpServer.McpServer.layer), Layer.provideMerge(layerOrchestration), Layer.provide( - CodexOrchestratorReplayHarness.makeProviderAdapterRegistryLayer(transcript).pipe( - Layer.provide(McpProviderSessions.layer), - ), + CodexOrchestratorReplayHarness.makeProviderAdapterRegistryLayer(transcript, { + replayGate, + }).pipe(Layer.provide(McpProviderSessions.layer)), ), Layer.provide(layerProviderRegistry), Layer.provide(ServerSettings.layerTest()), @@ -337,7 +353,55 @@ describe("orchestrator MCP toolkit", () => { const delegatedStart = yield* decodeDelegateTaskResult( delegatedCall.structuredContent, ).pipe(Effect.orDie); - expect(delegatedStart.childRunId).not.toBeNull(); + const childRunId = delegatedStart.childRunId; + if (childRunId === null) { + return yield* Effect.die("Delegation did not create a child run."); + } + + expect( + yield* Effect.promise(() => replayGate.waitForReached("child.original.item/started")), + ).toBe(true); + if (observedModel) { + const childThreadId = localChild(delegatedStart.childThreadId); + const startedTurn = yield* orchestrator + .streamStoredEventsFrom({ + threadId: childThreadId, + afterSequence: 0, + }) + .pipe( + Stream.filterMap((stored) => + stored.event.type === "provider-turn.updated" && + stored.event.runId === childRunId && + stored.event.payload.status === "running" + ? Result.succeed(stored.event.payload) + : Result.failVoid, + ), + Stream.runHead, + ); + if (Option.isNone(startedTurn)) { + return yield* Effect.die("Delegated child provider turn did not start."); + } + + const providerTurn = startedTurn.value; + // This Codex fixture tests completion from normalized telemetry; Claude SDK capture has its own replay. + const sink = yield* EventSink.EventSinkV2; + yield* sink.write({ + events: [ + { + id: EventId.make("event:mcp-replay-observed-model"), + type: "provider-turn.updated", + threadId: childThreadId, + runId: childRunId, + nodeId: providerTurn.nodeId, + driver: ProviderDriverKind.make("codex"), + providerInstanceId: codexInstanceId, + occurredAt: yield* DateTime.now, + payload: { ...providerTurn, observedModel }, + }, + ], + }); + } + replayGate.release("child.original.item/started"); yield* orchestrator .streamStoredEventsFrom({ threadId: parentThreadId, @@ -367,7 +431,7 @@ describe("orchestrator MCP toolkit", () => { ).pipe(Effect.orDie); expect(delegated).toMatchObject({ status: "completed", - summary: delegatedResult, + summary: delegatedSummary, providerInstanceId: codexInstanceId, hasPendingChildRuns: false, latestTerminalRunId: delegated.childRunId, @@ -376,6 +440,14 @@ describe("orchestrator MCP toolkit", () => { latestTerminalResultContextTransferId: null, }); expect(delegated.resultContextTransferId).not.toBeNull(); + const completedParent = yield* orchestrator.getThreadProjection(parentThreadId); + expect( + completedParent.subagents.find((task) => task.id === delegated.taskId), + ).toMatchObject({ + status: "completed", + model: observedModel ?? codexModel, + result: delegatedSummary, + }); const delegatedChild = yield* orchestrator.getThreadProjection( localChild(delegated.childThreadId), @@ -466,7 +538,7 @@ describe("orchestrator MCP toolkit", () => { expect(pendingStatus).toMatchObject({ childRunId: delegated.childRunId, status: "completed", - summary: delegatedResult, + summary: delegatedSummary, resultContextTransferId: delegated.resultContextTransferId, hasPendingChildRuns: true, latestTerminalRunId: delegated.childRunId, @@ -579,7 +651,7 @@ describe("orchestrator MCP toolkit", () => { expect(finalStatus).toMatchObject({ childRunId: delegated.childRunId, status: "completed", - summary: delegatedResult, + summary: delegatedSummary, resultContextTransferId: delegated.resultContextTransferId, hasPendingChildRuns: false, latestTerminalRunId: queuedFollowup.runId, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 40f33b3a1a309..1b0cab1067c83 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -31,6 +31,7 @@ import { ChatImageAttachment, CheckpointId, ClaudeSettings, + type OrchestrationV2Subagent, EnvironmentId, MessageId, type ModelSelection, @@ -74,6 +75,18 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; +import * as OrchestratorMcpService from "../../mcp/OrchestratorMcpService.ts"; +import * as ThreadManagementService from "../ThreadManagementService.ts"; +import * as ThreadLaunchService from "../ThreadLaunchService.ts"; +import * as GitVcsDriver from "../../vcs/GitVcsDriver.ts"; +import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; +import * as ProviderRegistry from "../../provider/ProviderRegistry.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as SecretRequests from "../../secrets/SecretRequests.ts"; +import * as RemoteDelegation from "../../peer/RemoteDelegation.ts"; +import * as ScheduledTaskService from "../../scheduledTasks/ScheduledTaskService.ts"; +import * as ServerSettings from "../../serverSettings.ts"; + import * as PortScanner from "../../preview/PortScanner.ts"; import { attachmentRelativePath } from "../../attachmentStore.ts"; import { WorkflowAgentTranscriptReadObserver } from "../workflowAgentTranscript.ts"; @@ -7061,6 +7074,221 @@ describe("ClaudeAdapterV2 background wake turns", () => { output: { success, taskId: input.taskId, updatedFields: ["status"] }, }); + it.effect.each([ + { sdkModel: "claude-sonnet-5-5", expectedObservedModel: "claude-sonnet-5-5" }, + { sdkModel: "claude-opus-5-5", expectedObservedModel: "claude-opus-5-5" }, + { sdkModel: "custom-provider-model", expectedObservedModel: "custom-provider-model" }, + { sdkModel: " claude-sonnet-5-5 ", expectedObservedModel: "claude-sonnet-5-5" }, + { sdkModel: "", expectedObservedModel: undefined }, + { sdkModel: "", expectedObservedModel: undefined }, + { sdkModel: undefined, expectedObservedModel: undefined }, + ])( + "reports root model $sdkModel without changing requested selection", + ({ sdkModel, expectedObservedModel }) => + Effect.gen(function* () { + let sequence = 700; + const nextUuid = () => `00000000-0000-4000-8000-${String(++sequence).padStart(12, "0")}`; + const prompt = "Track a synthetic task."; + const transcript = yield* ClaudeOrchestratorReplayHarness.decodeTranscript({ + provider: "claudeAgent", + protocol: "claude-agent-sdk.query", + version: "1", + scenario: "claude_org_fallback", + metadata: { nativeSessionId: WAKE_NATIVE_SESSION }, + entries: [ + { + type: "expect_outbound", + frame: { + type: "query.open", + options: { + model: "claude-opus-5-5", + tools: { type: "preset", preset: "claude_code" }, + permissionMode: "bypassPermissions", + allowDangerouslySkipPermissions: true, + settings: { fastMode: false, showThinkingSummaries: true }, + sessionId: WAKE_NATIVE_SESSION, + }, + }, + }, + { + type: "expect_outbound", + frame: { + type: "prompt.offer", + message: { + type: "user", + message: { role: "user", content: prompt }, + parent_tool_use_id: null, + }, + }, + }, + ...[ + claudeSdkFrame({ + type: "system", + subtype: "init", + model: sdkModel, + uuid: nextUuid(), + session_id: WAKE_NATIVE_SESSION, + }), + ...(sdkModel === "claude-sonnet-5-5" + ? [ + claudeSdkFrame({ + type: "system", + subtype: "informational", + level: "warning", + content: + 'Model "claude-opus-5-5" is restricted by your organization\'s settings. Using claude-sonnet-5-5 instead.', + uuid: nextUuid(), + session_id: WAKE_NATIVE_SESSION, + }), + ] + : []), + claudeSdkFrame({ + ...makeResultFrame({ uuid: nextUuid(), result: "Task recorded." }), + modelUsage: { + "claude-haiku-5-5": { + inputTokens: 1994, + outputTokens: 22, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + costUSD: 0, + contextWindow: 200000, + maxOutputTokens: 8192, + }, + "claude-sonnet-5-5": { + inputTokens: 22, + outputTokens: 7962, + cacheReadInputTokens: 559622, + cacheCreationInputTokens: 59129, + costUSD: 0, + contextWindow: 200000, + maxOutputTokens: 64000, + }, + }, + }), + ].map((frame) => ({ type: "emit_inbound" as const, frame })), + ], + } satisfies ProviderReplayTranscript); + const materialized = yield* materializeFixtureInput({ + scenario: transcript.scenario, + fixtureInput: { steps: [{ type: "message", text: prompt }] }, + driver: ProviderDriverKind.make("claudeAgent"), + modelSelection: { + ...CLAUDE_MODEL_SELECTION, + model: "claude-opus-5-5", + options: [{ id: "effort", value: "high" }], + }, + }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime); + const scenario = { name: transcript.scenario, transcript, ...materialized }; + yield* Effect.gen(function* () { + const result = yield* runOrchestratorV2Scenario(scenario); + assertBaseProjection({ result, transcript, runCount: 1, runStatuses: ["completed"] }); + + const projection = projectionFor(result, transcript.scenario); + assertSemanticProjectionIntegrity(projection); + const run = projection.runs[0]; + assert(run !== undefined, "Expected a persisted completed run."); + + assert.equal(projection.thread.modelSelection.model, "claude-opus-5-5"); + assert.equal(run.modelSelection.model, "claude-opus-5-5"); + assert.equal(projection.providerTurns[0]?.observedModel, expectedObservedModel); + assert.equal(run.observedModel, expectedObservedModel); + + if (sdkModel !== "claude-sonnet-5-5") { + return; + } + + const parentId = ThreadId.make("fallback-reporting-parent"); + const task: OrchestrationV2Subagent = { + id: NodeId.make("fallback-reporting-task"), + threadId: parentId, + runId: null, + parentNodeId: NodeId.make("fallback-reporting-parent-node"), + origin: "app_owned", + createdBy: "agent", + driver: ProviderDriverKind.make("claudeAgent"), + providerInstanceId: run.providerInstanceId, + providerThreadId: run.providerThreadId, + childThreadId: projection.thread.id, + nativeTaskRef: null, + prompt, + title: null, + model: run.modelSelection.model, + status: "completed", + result: null, + completionDelivery: { state: "disposed", observedByRunId: null }, + startedAt: run.startedAt, + completedAt: run.completedAt, + updatedAt: projection.updatedAt, + }; + const parent = { + ...projection, + thread: { ...projection.thread, id: parentId }, + runs: [], + subagents: [task], + }; + const reportingLayer = OrchestratorMcpService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadManagementService.ThreadManagementService)({ + delegatedTaskResultPending: () => Effect.succeed(false), + getThreadRecords: (id) => Effect.succeed(id === parentId ? parent : projection), + }), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(GitVcsDriver.GitVcsDriver)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ + list: () => Effect.succeed([]), + }), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({}), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ServerSettings.layerTest(), + NodeServices.layer, + ), + ), + ); + const report = yield* Effect.gen(function* () { + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + return yield* service.taskStatus( + { + environmentId: EnvironmentId.make("fallback-replay"), + requestNamespace: "fallback-replay", + thread: { + threadId: parentId, + providerSessionId: "fallback-replay", + providerInstanceId: run.providerInstanceId, + }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }, + task.id, + ); + }).pipe(Effect.provide(reportingLayer)); + + assert.equal(report.model, expectedObservedModel ?? "claude-opus-5-5"); + assert.equal(report.requestedModel, "claude-opus-5-5"); + assert.equal(report.observedModel, expectedObservedModel); + assert.deepEqual(report.options, [{ id: "effort", value: "high" }]); + + assert.isTrue(projection.turnItems.some((item) => item.type === "system_notice")); + assert.lengthOf( + result.domainEvents.filter( + (event) => + event.type === "provider-session.updated" && + event.payload.model === expectedObservedModel, + ), + 1, + ); + }).pipe( + Effect.provide(layerProviderReplay(scenario, ClaudeOrchestratorReplayHarness)), + provideDeterministicTestRuntime, + Effect.scoped, + ); + }), + ); + it.effect("persists a todo_list from inline Claude task-tool frames", () => Effect.gen(function* () { let sequence = 700; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 428e44e19020c..c8c4bd62637f1 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -2881,6 +2881,39 @@ function isClaudeRootTurnFrame(message: SDKMessage): boolean { ); } +function readClaudeRootModel(message: SDKMessage): string | null { + let model: string; + + // Auxiliary modelUsage entries and native child frames do not identify the root model. + switch (message.type) { + case "system": + if (message.subtype !== "init") return null; + + model = message.model; + break; + case "assistant": + if (message.parent_tool_use_id != null) return null; + + model = message.message.model; + break; + case "stream_event": + if (message.parent_tool_use_id != null || message.event.type !== "message_start") { + return null; + } + + model = message.event.message.model; + break; + default: + return null; + } + + // Some SDK frames omit model despite its required SDK type or use a synthetic placeholder. + if (typeof model !== "string") return null; + + const observedModel = model.trim(); + return observedModel.length > 0 && !observedModel.startsWith("<") ? observedModel : null; +} + function claudePendingBackgroundTasksFromRoster( roster: ReadonlyMap, awaitedTaskIds: ReadonlySet, @@ -3869,6 +3902,7 @@ interface ActiveClaudeTurnContext { readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly nativeThreadId: string; readonly nativeTurnId: string; + observedModel: string | null; nativeMessageCursor: string | null; readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; readonly providerTurnOrdinal: number; @@ -3963,6 +3997,7 @@ interface ClaudeLiveQueryContext { readonly nativeThreadId: string; readonly query: ClaudeAgentSdkQuerySession; readonly queryPolicyKey: string; + observedModel: string | null; selection: CompiledClaudeModelSelection; readonly closed: Deferred.Deferred; lastTurn: ActiveClaudeTurnContext | null; @@ -4939,6 +4974,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( startedAt: context.startedAt, completedAt: null, tokenUsage, + ...(context.observedModel === null ? {} : { observedModel: context.observedModel }), }, }); }); @@ -5430,6 +5466,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( status: input.status, startedAt: input.context.startedAt, completedAt: input.completedAt, + ...(input.context.observedModel === null + ? {} + : { observedModel: input.context.observedModel }), }); const buildToolCallArtifacts = (input: { @@ -8045,6 +8084,19 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const message = input.message; + const observedModel = readClaudeRootModel(message); + if (message.type === "system" && message.subtype === "init" && observedModel !== null) { + yield* emitProviderEvent({ + type: "provider_session.updated", + driver: CLAUDE_PROVIDER, + providerSession: { + ...session, + model: observedModel, + updatedAt: yield* DateTime.now, + }, + }); + } + // Track before routing: idle-wake Monitors report tasks before replaying their tool calls. yield* trackClaudeMonitorCalls(message); // Update before routing because the idle wake gate reads this. @@ -8198,6 +8250,33 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return; } + if (observedModel !== null) { + liveQuery.observedModel = observedModel; + if (context.observedModel !== observedModel) { + context.observedModel = observedModel; + yield* emitProviderEvent({ + type: "provider_turn.updated", + driver: CLAUDE_PROVIDER, + providerTurn: providerTurnPayload({ + context, + status: "running", + completedAt: null, + }), + }); + if (!isClaudeTurnStartMessage(message)) { + yield* emitProviderEvent({ + type: "provider_session.updated", + driver: CLAUDE_PROVIDER, + providerSession: { + ...session, + model: observedModel, + updatedAt: yield* DateTime.now, + }, + }); + } + } + } + const focusedPlugin = claudeFocusRequestedPanePlugin(message); if (focusedPlugin !== null && context === (yield* Ref.get(activeTurn))) { yield* Effect.logWarning("orchestration-v2.claude-mod-pane-focus-unsupported", { @@ -9882,6 +9961,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), ); live.selection = compiledSelection; + live.observedModel = null; } if ( existing !== null && @@ -10027,6 +10107,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeThreadId, query: querySession, queryPolicyKey, + observedModel: null, selection: compiledSelection, closed, lastTurn: null, @@ -10054,6 +10135,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( context.openedPermissionMode = message.permissionMode ?? context.openedPermissionMode; context.receivedInit = true; + context.observedModel = readClaudeRootModel(message); } if ( message.type === "system" && @@ -10176,6 +10258,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( queryPolicy: querySession.queryPolicy, nativeThreadId, nativeTurnId, + observedModel: querySession.observedModel, nativeMessageCursor: null, providerTurnId, providerTurnOrdinal, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 50f9a2acc0f39..6acb5681eb2b0 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -4967,6 +4967,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); const restartedRun: OrchestrationV2Run = { ...targetRun, + observedModel: undefined, providerInstanceId: input.modelSelection.instanceId, modelSelection: input.modelSelection, providerThreadId: restartProviderThread.id, @@ -12144,6 +12145,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const updatedTask: OrchestrationV2Subagent = { ...task, providerThreadId: childRun.providerThreadId, + model: childRun.observedModel ?? task.model, status: terminalStatus, result: result.text, completedAt: now, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 1dc7c912e7765..1b00aaa340f9c 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -1560,6 +1560,101 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { assert.deepEqual(yield* store.getTurnStartHistory(threadId, [RunId.make("run:other")]), []); }), ); + it.effect( + "records root model observations without overwriting lifecycle state or another attempt", + () => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const threadId = yield* addRolledBackRecoveryCandidate("observed-model-attempt"); + const now = yield* DateTime.now; + const original = (yield* store.getThreadProjection(threadId)).runs[0]; + assert(original !== undefined && original.rootNodeId !== null); + const run = { + ...original, + activeAttemptId: RunAttemptId.make("attempt-observed-first"), + status: "completed" as const, + completedAt: now, + }; + yield* store.apply({ + id: EventId.make("event-observed-completed"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: run, + }); + const observation = { + id: EventId.make("event-observed-first"), + type: "provider-turn.updated" as const, + threadId, + runId: run.id, + occurredAt: now, + payload: { + id: ProviderTurnId.make("turn-observed-first"), + providerThreadId: ProviderThreadId.make("thread-observed-provider"), + nodeId: original.rootNodeId, + runAttemptId: run.activeAttemptId, + nativeTurnRef: null, + ordinal: 1, + status: "running" as const, + observedModel: "claude-sonnet-5-5", + startedAt: now, + completedAt: null, + }, + }; + yield* store.apply(observation); + const recorded = (yield* store.getThreadProjection(threadId)).runs[0]; + assert.deepEqual(recorded, { ...run, observedModel: "claude-sonnet-5-5" }); + + yield* store.apply({ + id: EventId.make("event-observed-stale-snapshot"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, observedModel: "stale-snapshot-model" }, + }); + assert.deepEqual((yield* store.getThreadProjection(threadId)).runs[0], { + ...run, + observedModel: "claude-sonnet-5-5", + }); + + const restarted = { + ...run, + status: "starting" as const, + completedAt: null, + rootNodeId: NodeId.make("node-observed-second"), + activeAttemptId: RunAttemptId.make("attempt-observed-second"), + }; + yield* store.apply({ + id: EventId.make("event-observed-restart"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: restarted, + }); + yield* store.apply({ ...observation, id: EventId.make("event-observed-late-first") }); + assert.deepEqual((yield* store.getThreadProjection(threadId)).runs[0], restarted); + + yield* store.apply({ + ...observation, + id: EventId.make("event-observed-second"), + payload: { + ...observation.payload, + id: ProviderTurnId.make("turn-observed-second"), + ordinal: 2, + nodeId: restarted.rootNodeId, + runAttemptId: restarted.activeAttemptId, + observedModel: "claude-opus-5-5", + }, + }); + assert.deepEqual((yield* store.getThreadProjection(threadId)).runs[0], { + ...restarted, + observedModel: "claude-opus-5-5", + }); + }), + ); it.effect("preserves stored provider usage when a terminal update omits it", () => Effect.gen(function* () { const projectionStore = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 88a9073117dd6..afce88600ba88 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -680,11 +680,13 @@ function upsertProviderTurn( next: OrchestrationV2ProviderTurn, ): Array { const current = turns.find((turn) => turn.id === next.id); + const observedModel = next.observedModel ?? current?.observedModel; + const tokenUsage = next.tokenUsage ?? current?.tokenUsage; + return upsertById(turns, { ...next, - ...((next.tokenUsage ?? current?.tokenUsage) === undefined - ? {} - : { tokenUsage: next.tokenUsage ?? current?.tokenUsage }), + ...(observedModel === undefined ? {} : { observedModel }), + ...(tokenUsage === undefined ? {} : { tokenUsage }), }); } @@ -695,6 +697,9 @@ function preserveRunRecordedFields( if (current === undefined) return next; return { ...next, + ...(current.observedModel !== undefined && next.activeAttemptId === current.activeAttemptId + ? { observedModel: current.observedModel } + : {}), ...(next.delegatedCompletion === undefined && current.delegatedCompletion !== undefined ? { delegatedCompletion: current.delegatedCompletion } : {}), @@ -876,11 +881,22 @@ export function applyToProjection( : base.thread, providerThreads: upsertById(base.providerThreads, event.payload), }; - case "provider-turn.updated": - return { - ...base, - providerTurns: upsertProviderTurn(base.providerTurns, event.payload), - }; + case "provider-turn.updated": { + const turn = event.payload; + const observedModel = turn.observedModel; + const runs = + observedModel === undefined || turn.tokenUsage !== undefined + ? base.runs + : base.runs.map((run) => + run.rootNodeId === turn.nodeId && + run.activeAttemptId === turn.runAttemptId && + run.observedModel !== observedModel + ? { ...run, observedModel } + : run, + ); + + return { ...base, runs, providerTurns: upsertProviderTurn(base.providerTurns, turn) }; + } case "runtime-request.updated": return { ...base, @@ -1834,9 +1850,13 @@ export const layer: Layer.Layer = OR item.payload_json LIKE '%"viewedImagePath"%' OR item.payload_json LIKE '%"toolName":"%delegate_task"%')`; - const keepRecordedRunField = (payload: Statement.Fragment, path: string) => sql` + const keepRecordedRunField = ( + payload: Statement.Fragment, + path: string, + condition = sql`json_type(excluded.payload_json, ${path}) IS NULL`, + ) => sql` CASE - WHEN json_type(excluded.payload_json, ${path}) IS NULL + WHEN ${condition} AND json_type(orchestration_v2_projection_runs.payload_json, ${path}) IS NOT NULL THEN json_set( ${payload}, @@ -1973,7 +1993,15 @@ export const layer: Layer.Layer = requested_at = excluded.requested_at, completed_at = excluded.completed_at, payload_json = ${keepRecordedRunField( - keepRecordedRunField(sql`excluded.payload_json`, "$.delegatedCompletion"), + keepRecordedRunField( + keepRecordedRunField( + sql`excluded.payload_json`, + "$.observedModel", + sql`json_extract(excluded.payload_json, '$.activeAttemptId') IS + json_extract(orchestration_v2_projection_runs.payload_json, '$.activeAttemptId')`, + ), + "$.delegatedCompletion", + ), "$.restartCancelledBackgroundWork", )} `; @@ -2345,6 +2373,19 @@ export const layer: Layer.Layer = completed_at = excluded.completed_at, payload_json = excluded.payload_json `; + if ( + event.payload.observedModel !== undefined && + event.payload.tokenUsage === undefined + ) { + yield* sql` + UPDATE orchestration_v2_projection_runs + SET payload_json = json_set(payload_json, '$.observedModel', ${event.payload.observedModel}) + WHERE thread_id = ${event.threadId} + AND json_extract(payload_json, '$.rootNodeId') = ${event.payload.nodeId} + AND json_extract(payload_json, '$.activeAttemptId') IS ${event.payload.runAttemptId} + AND json_extract(payload_json, '$.observedModel') IS NOT ${event.payload.observedModel} + `; + } break; } case "runtime-request.updated": { diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index d059c4c20e9d7..701f723eed464 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -242,6 +242,7 @@ function makeRestartAdapter( strength: "strong", }, ordinal: input.providerTurnOrdinal, + observedModel: initialSelection.model, status: "running", startedAt: occurredAt, completedAt: null, @@ -343,6 +344,7 @@ it.live("retries a failed selection restart session open before completing", () yield* worker.drain(); const activeProjection = yield* orchestrator.getThreadProjection(threadId); assert.isTrue(activeProjection.providerTurns.some((turn) => turn.status === "running")); + assert.equal(activeProjection.runs[0]?.observedModel, initialSelection.model); const activeRunId = activeProjection.runs[0]?.id; if (activeRunId === undefined) { return yield* Effect.die("active restart test run is missing"); @@ -408,6 +410,7 @@ it.live("retries a failed selection restart session open before completing", () ); assert.equal(projection.runs.at(-1)?.modelSelection.model, replacementSelection.model); assert.equal(projection.runs.at(-1)?.status, "completed"); + assert.equal(projection.runs.at(-1)?.observedModel, undefined); assert.isTrue(captured.failedReplacementOpen); assert.equal(captured.closedSessionCount, 0); assert.deepEqual( diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index 95abcbaf5c1a8..d6d4c189abdda 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -8,6 +8,7 @@ import { EventId, TurnItemId, type OrchestrationV2Run, + type OrchestrationV2TurnItem, type OrchestrationV2AppThread, ProjectId, ProviderInstanceId, @@ -356,42 +357,119 @@ it("exposes the provider failure rather than a progress message from the failed assert.isNull(result.messageId); }); -it("orders a task's assistant messages by turn item, not by timestamps", () => { - const { projection, run } = taskFixture(); - const at = DateTime.makeUnsafe("2026-07-24T10:05:00.000Z"); - const message = (id: string, text: string, ordinal: number) => { - const artifacts = makeSubagentConversationArtifacts({ - messageId: MessageId.make(id), - turnItemId: TurnItemId.make(id), - threadId: parentThreadId, - rootNodeId: NodeId.make("root"), +it.each([ + { + requestedModel: "claude-opus-4-1", + observedModel: undefined, + fallbackNotice: null, + provenance: "", + }, + { + requestedModel: "claude-opus-4-1", + observedModel: "claude-opus-4-1", + fallbackNotice: null, + provenance: "", + }, + { + requestedModel: "claude-opus-5-5", + observedModel: "claude-sonnet-5-5", + fallbackNotice: + 'Model "claude-opus-5-5" is restricted by your organization\'s settings. Using claude-sonnet-5-5 instead.', + provenance: + 'Requested model: claude-opus-5-5. Provider-reported model: claude-sonnet-5-5.\nProvider notice: Model "claude-opus-5-5" is restricted by your organization\'s settings. Using claude-sonnet-5-5 instead.\n\n', + }, + { + requestedModel: "opus", + observedModel: "claude-opus-4-1", + fallbackNotice: null, + provenance: "Requested model: opus. Provider-reported model: claude-opus-4-1.\n\n", + }, + { + requestedModel: "claude-opus-4-1", + observedModel: "claude-opus-4-1[1m]", + fallbackNotice: null, + provenance: + "Requested model: claude-opus-4-1. Provider-reported model: claude-opus-4-1[1m].\n\n", + }, +])( + "orders assistant messages and reports $requestedModel / $observedModel provenance", + ({ requestedModel, observedModel, fallbackNotice, provenance }) => { + const { projection, run } = taskFixture(); + const at = DateTime.makeUnsafe("2026-07-24T10:05:00.000Z"); + const message = (id: string, text: string, ordinal: number) => { + const artifacts = makeSubagentConversationArtifacts({ + messageId: MessageId.make(id), + turnItemId: TurnItemId.make(id), + threadId: parentThreadId, + rootNodeId: NodeId.make("root"), + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + role: "assistant", + text, + ordinal, + now: at, + }); + return { + message: { ...artifacts.message, runId: run.id }, + turnItem: { ...artifacts.turnItem, runId: run.id }, + }; + }; + const progress = message("progress", "Starting step 0.", 1); + const report = message("report", "All steps passed.", 2); + + const notice = { + id: TurnItemId.make("fallback-notice"), + threadId: run.threadId, + runId: run.id, + nodeId: run.rootNodeId, providerThreadId: null, providerTurnId: null, nativeItemRef: null, - role: "assistant", - text, - ordinal, - now: at, - }); - return { - message: { ...artifacts.message, runId: run.id }, - turnItem: { ...artifacts.turnItem, runId: run.id }, - }; - }; - const progress = message("progress", "Starting step 0.", 1); - const report = message("report", "All steps passed.", 2); - const result = subagentResultForRun( - { - ...projection, - messages: [progress.message, report.message], - turnItems: [report.turnItem, progress.turnItem], - }, - run, - ); - assert.equal(result.text, "All steps passed."); - assert.equal(result.messageId, report.message.id); - assert.equal(result.turnItemId, report.turnItem.id); -}); + parentItemId: null, + ordinal: 0, + type: "system_notice", + status: "completed", + title: null, + message: fallbackNotice ?? "", + startedAt: at, + completedAt: at, + updatedAt: at, + } satisfies OrchestrationV2TurnItem; + const notices = [ + { + ...notice, + id: TurnItemId.make("other-run"), + runId: RunId.make("other"), + message: "Other run notice", + }, + { + ...notice, + id: TurnItemId.make("other-root"), + nodeId: NodeId.make("previous-root"), + message: "Previous attempt notice", + }, + { + ...notice, + id: TurnItemId.make("native-child"), + parentItemId: TurnItemId.make("native-task"), + message: "Native child notice", + }, + notice, + ]; + const result = subagentResultForRun( + { + ...projection, + messages: [progress.message, report.message], + turnItems: [report.turnItem, ...notices, progress.turnItem], + }, + { ...run, modelSelection: { ...run.modelSelection, model: requestedModel }, observedModel }, + ); + assert.equal(result.text, `${provenance}All steps passed.`); + assert.equal(result.messageId, report.message.id); + assert.equal(result.turnItemId, report.turnItem.id); + }, +); it.effect("ends an orphaned native subagent's node when only its record already ended", () => Effect.gen(function* () { diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index f6ba3e9f0c1a7..b0c3d52650f6f 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -5,12 +5,14 @@ import { type OrchestrationV2Run, type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, + EventId, MessageId, ProjectId, ProviderInstanceId, NodeId, ProviderThreadId, ProviderTurnId, + RunAttemptId, RunId, ThreadId, TurnItemId, @@ -134,6 +136,34 @@ describe("applyOrchestrationV2ProjectionEvent", () => { expect(next?.thread.pullRequests?.[0]).toBe(links[0]); }); + it("keeps observed execution provenance when a completion snapshot omits it", () => { + const projection = { + ...emptyProjection, + runs: [{ ...run, observedModel: "claude-sonnet-5-5" }], + }; + const next = applyOrchestrationV2ProjectionEvent(projection, { + id: EventId.make("event-run-provenance"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: run, + }); + + expect(next?.runs[0]?.observedModel).toBe("claude-sonnet-5-5"); + expect(next?.runs[0]?.modelSelection).toEqual(run.modelSelection); + + const restarted = applyOrchestrationV2ProjectionEvent(next, { + id: EventId.make("event-run-provenance-restart"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { ...run, activeAttemptId: RunAttemptId.make("attempt-reducer-new") }, + }); + expect(restarted?.runs[0]?.observedModel).toBeUndefined(); + }); + it("keeps live token usage when the terminal provider turn omits it", () => { const providerTurnId = ProviderTurnId.make("provider-turn-reducer"); const running = { @@ -146,6 +176,7 @@ describe("applyOrchestrationV2ProjectionEvent", () => { status: "running" as const, startedAt: now, completedAt: null, + observedModel: "claude-sonnet-5-5", tokenUsage: { usedTokens: 50_000, maxTokens: 200_000, @@ -164,12 +195,14 @@ describe("applyOrchestrationV2ProjectionEvent", () => { status: "completed", completedAt: now, tokenUsage: undefined, + observedModel: undefined, }, } as OrchestrationV2DomainEvent; const next = applyOrchestrationV2ProjectionEvent(projection, event); expect(next?.providerTurns[0]?.status).toBe("completed"); + expect(next?.providerTurns[0]?.observedModel).toBe("claude-sonnet-5-5"); expect(next?.providerTurns[0]?.tokenUsage).toEqual(running.tokenUsage); }); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index fbba7d1bd9dd4..4aa1e9268d4af 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -299,7 +299,13 @@ export function applyOrchestrationV2ProjectionEvent( return { ...projection, thread: event.payload }; case "run.created": case "run.updated": { - const next = { ...base, runs: upsertEntity(base.runs, event.payload) }; + const current = base.runs.find((run) => run.id === event.payload.id); + const payload = + current?.observedModel !== undefined && + event.payload.activeAttemptId === current.activeAttemptId + ? { ...event.payload, observedModel: current.observedModel } + : event.payload; + const next = { ...base, runs: upsertEntity(base.runs, payload) }; return { ...next, visibleTurnItems: activeVisibleTurnItems(next) }; } case "run.background-work-cancelled": @@ -347,22 +353,32 @@ export function applyOrchestrationV2ProjectionEvent( }; case "provider-thread.updated": return { ...base, providerThreads: upsertEntity(base.providerThreads, event.payload) }; - case "provider-turn.updated": + case "provider-turn.updated": { + const current = base.providerTurns.find((turn) => turn.id === event.payload.id); + const observedModel = event.payload.observedModel ?? current?.observedModel; + const tokenUsage = event.payload.tokenUsage ?? current?.tokenUsage; + + const runs = + event.payload.observedModel === undefined || event.payload.tokenUsage !== undefined + ? base.runs + : base.runs.map((run) => + run.rootNodeId === event.payload.nodeId && + run.activeAttemptId === event.payload.runAttemptId && + run.observedModel !== event.payload.observedModel + ? { ...run, observedModel: event.payload.observedModel } + : run, + ); + return { ...base, + runs, providerTurns: upsertEntity(base.providerTurns, { ...event.payload, - ...((event.payload.tokenUsage ?? - base.providerTurns.find((turn) => turn.id === event.payload.id)?.tokenUsage) === - undefined - ? {} - : { - tokenUsage: - event.payload.tokenUsage ?? - base.providerTurns.find((turn) => turn.id === event.payload.id)?.tokenUsage, - }), + ...(observedModel === undefined ? {} : { observedModel }), + ...(tokenUsage === undefined ? {} : { tokenUsage }), }), }; + } case "runtime-request.updated": return { ...base, runtimeRequests: upsertEntity(base.runtimeRequests, event.payload) }; case "message.updated": diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index d20da5780255b..88027efc5326c 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -669,6 +669,7 @@ export const OrchestrationV2Run = Schema.Struct({ ordinal: PositiveInt, providerInstanceId: ProviderInstanceId, modelSelection: ModelSelection, + observedModel: Schema.optional(TrimmedNonEmptyString), providerThreadId: Schema.NullOr(ProviderThreadId), userMessageId: MessageId, rootNodeId: Schema.NullOr(NodeId), @@ -1143,6 +1144,7 @@ export const OrchestrationV2ProviderTurn = Schema.Struct({ nodeId: NodeId, runAttemptId: Schema.NullOr(RunAttemptId), nativeTurnRef: Schema.NullOr(OrchestrationV2ProviderRef), + observedModel: Schema.optional(TrimmedNonEmptyString), ordinal: PositiveInt, status: Schema.Literals([ "pending", diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 1474a5d6e87b6..cb0e0418601e2 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -221,6 +221,8 @@ export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ latestTerminalResultContextTransferId: Schema.NullOr(ContextTransferId), providerInstanceId: ProviderInstanceId, model: Schema.NullOr(Schema.String), + requestedModel: Schema.optional(Schema.String), + observedModel: Schema.optional(Schema.String), options: Schema.optional(Schema.Array(ProviderOptionSelection)), summary: Schema.NullOr(Schema.String), resultContextTransferId: Schema.NullOr(ContextTransferId), @@ -434,6 +436,8 @@ export const OrchestratorMcpThreadDetail = Schema.Struct({ activeRunId: Schema.NullOr(RunId), providerInstanceId: ProviderInstanceId, model: Schema.String, + requestedModel: Schema.optional(Schema.String), + observedModel: Schema.optional(Schema.String), runtimeMode: RuntimeMode, interactionMode: ProviderInteractionMode, linkedPullRequest: Schema.NullOr(ThreadLinkedPullRequest), @@ -462,6 +466,8 @@ export const OrchestratorMcpThreadRun = Schema.Struct({ status: OrchestrationV2RunStatus, providerInstanceId: ProviderInstanceId, model: Schema.String, + requestedModel: Schema.optional(Schema.String), + observedModel: Schema.optional(Schema.String), requestedAt: IsoDateTime, startedAt: Schema.NullOr(IsoDateTime), completedAt: Schema.NullOr(IsoDateTime), diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index be8c4bf98aa54..9642179a4e110 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -214,8 +214,32 @@ export function subagentResultForRun( (run.status === "completed" ? "Child task completed without an assistant result." : `Child task ended with status ${run.status}.`); + + const hasModelMismatch = + run.observedModel != null && run.observedModel !== run.modelSelection.model; + const providerNotices = hasModelMismatch + ? projection.turnItems + .filter( + (item): item is Extract => + item.type === "system_notice" && + item.runId === run.id && + run.rootNodeId !== null && + item.nodeId === run.rootNodeId && + item.parentItemId === null && + item.message.trim().length > 0, + ) + .toSorted((left, right) => left.ordinal - right.ordinal) + .map((item) => `Provider notice: ${item.message.trim()}`) + : []; + const provenance = hasModelMismatch + ? [ + `Requested model: ${run.modelSelection.model}. Provider-reported model: ${run.observedModel}.`, + ...providerNotices, + ].join("\n") + "\n\n" + : ""; + return { - text, + text: `${provenance}${text}`, messageId: failure === undefined ? (message?.id ?? turnItem?.messageId ?? null) : null, turnItemId: failure?.id ?? turnItem?.id ?? null, };