diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 95432c40e6887..f6270395ed17c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -6585,6 +6585,137 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("moves the thread to the model a session refusal fallback switched to", () => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const original = { + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + model: "claude-opus-5-5", + } satisfies ModelSelection; + const fallback = { ...original, model: "claude-opus-4-8" }; + const turn = (ordinal: number, modelSelection: ModelSelection) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make(`attempt-refusal-fallback:${ordinal}`), + text: `Request ${ordinal}`, + attachments: [], + providerTurnOrdinal: ordinal, + modelSelection, + }); + const settle = Effect.gen(function* () { + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + }); + const refusalFallback = (scope: "session" | "local", uuid: string) => + claudeSdkFrame({ + type: "system", + subtype: "model_refusal_fallback", + trigger: "refusal", + direction: "retry", + scope, + original_model: original.model, + fallback_model: fallback.model, + request_id: null, + content: "Switched to Opus 4.8", + uuid, + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn(turn(1, original)); + yield* Queue.offer( + harness.sdkMessages, + refusalFallback("local", "00000000-0000-4000-8000-000000000401"), + ); + yield* Queue.offer( + harness.sdkMessages, + refusalFallback("session", "00000000-0000-4000-8000-000000000402"), + ); + yield* settle; + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "app_thread.model_selection.updated" ? [event] : [], + ), + [ + { + type: "app_thread.model_selection.updated", + runId: turn(1, original).runId, + attemptId: turn(1, original).attemptId, + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + threadId: harness.threadId, + previousModelSelection: original, + modelSelection: fallback, + }, + ], + ); + + yield* harness.runtime.startTurn(turn(2, fallback)); + yield* settle; + assert.deepEqual(harness.liveSelectionChanges, []); + yield* harness.runtime.startTurn(turn(3, original)); + yield* settle; + assert.deepEqual(harness.liveSelectionChanges[0], { model: original.model }); + const explicitDefault = { ...original, options: [{ id: "fastMode", value: false }] }; + yield* harness.runtime.startTurn(turn(4, explicitDefault)); + yield* Queue.offer( + harness.sdkMessages, + refusalFallback("session", "00000000-0000-4000-8000-000000000404"), + ); + yield* settle; + const defaultUpdate = harness.events + .filter((event) => event.type === "app_thread.model_selection.updated") + .at(-1); + assert.deepEqual(defaultUpdate?.previousModelSelection, explicitDefault); + assert.deepEqual(defaultUpdate?.modelSelection, { + ...fallback, + options: explicitDefault.options, + }); + const steered = { ...original, model: "claude-sonnet-4-6" }; + const input = turn(5, original); + yield* harness.runtime.startTurn(input); + const idAllocator = yield* IdAllocator.IdAllocatorV2; + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: "turn:" + input.attemptId, + }), + modelSelection: steered, + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("steer-fallback"), + text: "Try this model", + attachments: [], + }, + }); + yield* Queue.offer( + harness.sdkMessages, + refusalFallback("session", "00000000-0000-4000-8000-000000000403"), + ); + yield* settle; + const updates = harness.events.filter( + (event) => event.type === "app_thread.model_selection.updated", + ); + assert.deepEqual(updates.at(-1)?.previousModelSelection, steered); + assert.deepEqual(updates.at(-1)?.modelSelection, fallback); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + // regression: #34549 a model change sent with a steer was shown but never applied to the turn. it.effect("applies a model change sent with a steer to the running turn", () => Effect.scoped( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3e10e09906796..edd9de5d36e50 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4788,6 +4788,7 @@ interface ClaudeLiveQueryContext { readonly query: ClaudeAgentSdkQuerySession; readonly queryPolicyKey: string; selection: CompiledClaudeModelSelection; + modelSelection: ModelSelection; readonly closed: Deferred.Deferred; lastTurn: ActiveClaudeTurnContext | null; // "early" echoes on the first frame; "acknowledged" confirms uuid acceptance, but resume turns can still precede the prompt. @@ -10322,6 +10323,25 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const now = yield* DateTime.now; const nativeItemId = message.uuid; + // "local" covers a subagent or side question; absent (older CLIs) means "session". + if (message.subtype === "model_refusal_fallback" && message.scope !== "local") { + const previousModelSelection = liveQuery.modelSelection; + const modelSelection = { + ...previousModelSelection, + model: resolveClaudeModelSlug(modelCatalog, message.fallback_model), + }; + liveQuery.selection = compileClaudeModelSelection(modelSelection, modelCatalog); + liveQuery.modelSelection = modelSelection; + yield* emitProviderEvent({ + type: "app_thread.model_selection.updated", + runId: context.input.runId, + attemptId: context.input.attemptId, + driver: CLAUDE_PROVIDER, + threadId: context.input.threadId, + previousModelSelection, + modelSelection, + }); + } yield* emitProviderEvent({ type: "turn_item.updated", driver: CLAUDE_PROVIDER, @@ -11904,6 +11924,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const applySelectionLive = Effect.fnUntraced(function* ( live: ClaudeLiveQueryContext, selection: CompiledClaudeModelSelection, + modelSelection: ModelSelection, ) { // Preserve launcher-selected models unless the model changed or a failed update left it unknown. yield* ( @@ -11922,6 +11943,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), ); live.selection = selection; + live.modelSelection = modelSelection; }); const openQuery = Effect.fnUntraced(function* ( @@ -11964,7 +11986,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( (existing.selection.apiModelId === compiledSelection.apiModelId || existing.openedPermissionMode === queryPolicy.permissionMode) ) { - yield* applySelectionLive(existing, compiledSelection); + yield* applySelectionLive(existing, compiledSelection, turnInput.modelSelection); } if ( existing !== null && @@ -11990,6 +12012,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), ); } + existing.modelSelection = turnInput.modelSelection; return existing; } @@ -12130,6 +12153,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( query: querySession, queryPolicyKey, selection: compiledSelection, + modelSelection: turnInput.modelSelection, closed, lastTurn: null, promptEchoMode: "unknown", @@ -12844,8 +12868,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.provideService(Path.Path, path), ); yield* backgroundForegroundSubagents(existing, backgroundDeadline); - if (liveSteerSelection !== undefined) { - yield* applySelectionLive(existing, liveSteerSelection); + if (liveSteerSelection !== undefined && turnInput.modelSelection !== undefined) { + yield* applySelectionLive(existing, liveSteerSelection, turnInput.modelSelection); } // The turn can end while the moves or the selection change are answered. // A steer offered then would start a new turn the orchestrator never @@ -13245,7 +13269,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return; } // A failed update clears the live identity, so the next turn applies the selection again. - yield* applySelectionLive(live, selection).pipe( + yield* applySelectionLive(live, selection, modelSelection).pipe( Effect.catch((cause) => Effect.logWarning("orchestration-v2.claude-live-selection-failed", { cause }), ), diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 520fea7c0575b..775e6ad235971 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -2343,6 +2343,130 @@ layer("ProviderEventIngestorV2", (it) => { ), ), ); + + it.effect("moves a thread to the model its provider switched the session to", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSink.EventSinkV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const rootEvent = yield* threadCreatedEvent(now); + const threadId = rootEvent.threadId; + if (threadId === undefined) throw new Error("Expected a thread fixture event"); + yield* eventSink.write({ events: [rootEvent] }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const runId = RunId.make("run:fallback"); + const attemptId = RunAttemptId.make("attempt:fallback"); + const run = { + id: runId, + threadId, + ordinal: 1, + providerInstanceId: modelSelection.instanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make("message:fallback"), + rootNodeId: NodeId.make("node:fallback"), + activeAttemptId: attemptId, + status: "running", + requestedAt: now, + startedAt: now, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + } satisfies OrchestrationV2Run; + yield* eventSink.write({ + events: [ + { + ...rootEvent, + id: yield* idAllocator.allocate.event({ threadId }), + type: "run.created", + payload: run, + }, + ], + }); + const reportModel = (reported: ModelSelection) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId, + event: { + type: "app_thread.model_selection.updated", + runId, + attemptId, + driver: CODEX_DRIVER, + threadId, + previousModelSelection: modelSelection, + modelSelection: reported, + }, + }); + const switched = { ...modelSelection, model: "gpt-fallback" }; + + const otherInstance = yield* reportModel({ + ...switched, + instanceId: ProviderInstanceId.make("codex-work"), + }); + + assert.lengthOf(otherInstance, 0); + const first = yield* reportModel(switched); + const repeated = yield* reportModel(switched); + + assert.deepEqual( + first.map((stored) => stored.event.type), + ["thread.model-selection-updated"], + ); + assert.lengthOf(repeated, 0); + assert.deepEqual((yield* projectionStore.getThread(threadId)).modelSelection, switched); + for (const newer of [ + { ...modelSelection, model: "user-selected-model" }, + { ...modelSelection, options: [{ id: "reasoningEffort", value: "low" }] }, + ]) { + yield* eventSink.write({ + events: [ + { + ...rootEvent, + id: yield* idAllocator.allocate.event({ threadId }), + type: "thread.model-selection-updated", + payload: { + ...(yield* projectionStore.getThread(threadId)), + modelSelection: newer, + updatedAt: now, + }, + }, + ], + }); + assert.lengthOf(yield* reportModel(switched), 0); + assert.deepEqual((yield* projectionStore.getThread(threadId)).modelSelection, newer); + } + yield* eventSink.write({ + events: [ + { + ...rootEvent, + id: yield* idAllocator.allocate.event({ threadId }), + type: "thread.model-selection-updated", + payload: { ...(yield* projectionStore.getThread(threadId)), modelSelection }, + }, + { + ...rootEvent, + id: yield* idAllocator.allocate.event({ threadId }), + type: "run.updated", + payload: { ...run, status: "interrupted", completedAt: now }, + }, + { + ...rootEvent, + id: yield* idAllocator.allocate.event({ threadId }), + type: "run.created", + payload: { ...run, id: RunId.make("run:newer"), ordinal: 2 }, + }, + ], + }); + assert.lengthOf(yield* reportModel(switched), 0); + assert.deepEqual((yield* projectionStore.getThread(threadId)).modelSelection, modelSelection); + }), + ); }); const ingestCompletedProviderTurn = ( diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 623fddb491108..d86ba4831731b 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -338,23 +338,14 @@ export const layer: Layer.Layer< }, ); - /** Providers may reveal native subagent models late; lock full-row updates against concurrent thread commands. */ - const syncSubagentThreadModel = Effect.fn("ProviderEventIngestor.syncSubagentThreadModel")( - function* (input: ProviderEventIngestInput, subagent: OrchestrationV2Subagent) { - const { childThreadId, model } = subagent; - if ( - subagent.origin !== "provider_native" || - childThreadId === null || - (model === null && subagent.modelSelection === undefined) - ) { - return []; - } - const reportedSelection = ({ modelSelection }: { modelSelection: ModelSelection }) => - subagent.modelSelection ?? - (model === null || model === modelSelection.model - ? modelSelection - : { instanceId: modelSelection.instanceId, model }); - const staleThread = projections.getThread(childThreadId).pipe( + /** Thread commands rewrite the full row; lock this read and write against them. */ + const syncThreadModelSelection = Effect.fn("ProviderEventIngestor.syncThreadModelSelection")( + function* ( + input: ProviderEventIngestInput, + threadId: ThreadId, + reportedSelection: (thread: { modelSelection: ModelSelection }) => ModelSelection, + ) { + const staleThread = projections.getThread(threadId).pipe( Effect.map((thread) => modelSelectionsEqual(thread.modelSelection, reportedSelection(thread)) ? null : thread, ), @@ -362,7 +353,7 @@ export const layer: Layer.Layer< ); if ((yield* staleThread) === null) return []; return yield* threadCommands.withLock( - childThreadId, + threadId, Effect.gen(function* () { const thread = yield* staleThread; if (thread === null) return []; @@ -377,12 +368,55 @@ export const layer: Layer.Layer< }, occurredAt: now, }); + if (input.event.type === "app_thread.model_selection.updated") { + const source = input.event; + const run = yield* projections.getRun(threadId, source.runId); + if ( + run === undefined || + run.activeAttemptId !== source.attemptId || + (run.status !== "running" && run.status !== "waiting") + ) + return []; + const result = yield* eventSink + .writeIfRunCurrent({ + threadId, + runId: source.runId, + activeAttemptId: source.attemptId, + expectedStatus: run.status, + events: [event], + }) + .pipe(retryLockTimeout); + return result.storedEvents; + } return yield* eventSink.write({ events: [event] }).pipe(retryLockTimeout); }), ); }, ); + /** Providers may reveal native subagent models late. */ + const syncSubagentThreadModel = Effect.fn("ProviderEventIngestor.syncSubagentThreadModel")( + function* (input: ProviderEventIngestInput, subagent: OrchestrationV2Subagent) { + const { childThreadId, model } = subagent; + if ( + subagent.origin !== "provider_native" || + childThreadId === null || + (model === null && subagent.modelSelection === undefined) + ) { + return []; + } + return yield* syncThreadModelSelection( + input, + childThreadId, + ({ modelSelection }) => + subagent.modelSelection ?? + (model === null || model === modelSelection.model + ? modelSelection + : { instanceId: modelSelection.instanceId, model }), + ); + }, + ); + /** Retire rewound runs like CheckpointRollbackService; retain the reporting run and its lastRunOrdinal. */ const rollBackRewoundRuns = Effect.fn("ProviderEventIngestor.rollBackRewoundRuns")(function* ( input: ProviderEventIngestInput, @@ -493,6 +527,9 @@ export const layer: Layer.Layer< }), ]; } + case "app_thread.model_selection.updated": + // Written under the thread's lock after ingestion. + return []; case "provider_session.updated": return [ yield* makeDomainEvent(input, { @@ -802,11 +839,23 @@ export const layer: Layer.Layer< ), ), ), - // @effect-diagnostics-next-line flatMapConditionalToFilterOrFail:off - The predicate narrows the captured event for the subagent update. - Effect.flatMap((storedEvents) => - storedEvents.length === 0 || input.event.type !== "subagent.updated" + Effect.flatMap((storedEvents) => { + const event = input.event; + const sync = + event.type === "app_thread.model_selection.updated" + ? // Preserve selections the user changed while the run was active. + syncThreadModelSelection(input, event.threadId, ({ modelSelection }) => + modelSelectionsEqual(modelSelection, event.previousModelSelection) && + modelSelection.instanceId === event.modelSelection.instanceId + ? event.modelSelection + : modelSelection, + ) + : storedEvents.length > 0 && event.type === "subagent.updated" + ? syncSubagentThreadModel(input, event.subagent) + : null; + return sync === null ? Effect.succeed(storedEvents) - : syncSubagentThreadModel(input, input.event.subagent).pipe( + : sync.pipe( Effect.map((synced) => [...storedEvents, ...synced]), Effect.mapError( (cause) => @@ -816,8 +865,8 @@ export const layer: Layer.Layer< cause, }), ), - ), - ), + ); + }), Effect.tap((storedEvents) => Effect.gen(function* () { if (storedEvents.length === 0 || input.event.type !== "provider_turn.updated") diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 612a130e019aa..7bfd04a0000f9 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -409,6 +409,8 @@ export function routeProviderEvent( }, ]; } + case "app_thread.model_selection.updated": + return [ownsThread(event.threadId), state]; case "provider_thread.updated": { const belongs = state.ownedProviderThreadIds.has(event.providerThread.id) || diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 2a2537c8bf866..39d4eb4c640ca 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -87,6 +87,16 @@ export const ProviderAdapterV2Event = Schema.Union([ appThread: OrchestrationV2AppThread, createOnly: Schema.optional(Schema.Boolean), }), + /** The provider moved the thread's session to another model on its own, such as Claude's refusal fallback. */ + Schema.Struct({ + type: Schema.Literal("app_thread.model_selection.updated"), + runId: RunId, + attemptId: RunAttemptId, + driver: ProviderDriverKind, + threadId: ThreadId, + previousModelSelection: ModelSelection, + modelSelection: ModelSelection, + }), Schema.Struct({ type: Schema.Literal("provider_session.updated"), driver: ProviderDriverKind,