From 781ee0e45713eb743e9594300b4185f781356255 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:29:45 +0100 Subject: [PATCH 01/29] fix(claude): preserve the session selected by clear across process replacement --- .../Adapters/ClaudeAdapterV2.test.ts | 86 ++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 127 ++++++++++++++++-- 2 files changed, 205 insertions(+), 8 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 0c8b2ad588acc..f94b168c32d27 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2383,6 +2383,92 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + it.effect.each(["active", "idle"] as const)( + "persists a session cleared while %s and resumes it after process replacement", + (phase) => + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }); + const turn = (ordinal: number, providerThread: OrchestrationV2ProviderThread) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`clear-session-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: [], + providerTurnOrdinal: ordinal, + }); + yield* harness.runtime.startTurn(turn(1, harness.providerThread)); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "subagent-native-session", + parent_tool_use_id: "subagent-launch", + uuid: "00000000-0000-4000-8000-000000000500", + }), + ); + if (phase === "idle") { + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + } + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000501", + }), + ); + if (phase === "active") { + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + } else { + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === "cleared-native-session", + ), + ); + } + assert.isFalse( + harness.events.some( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === "subagent-native-session", + ), + ); + const completed = harness.events.findLast( + (event) => event.type === "provider_thread.updated", + ); + assert.equal(completed?.type, "provider_thread.updated"); + if (completed?.type !== "provider_thread.updated") return; + assert.equal(completed.providerThread.nativeThreadRef?.nativeId, "cleared-native-session"); + assert.equal(completed.providerThread.id, harness.providerThread.id); + yield* harness.runtime.startTurn(turn(2, completed.providerThread)); + assert.equal(harness.processQueues.length, 1); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + yield* harness.runtime.interruptTurn({ + providerThread: completed.providerThread, + providerTurnId: lastProviderTurn(harness.events).id, + requestRuntimeRestart: true, + }); + yield* harness.runtime.startTurn(turn(3, completed.providerThread)); + assert.equal(harness.processQueues.length, 2); + assert.equal(harness.getOpenedOptions()?.resume, "cleared-native-session"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + describe("Claude settings restrictions", () => { it.effect("resolves cached managed policy from the configured home through the real SDK", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 0e0868918eaa4..3efb3c200f50c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -3401,8 +3401,8 @@ function formatClaudeUsageLimitWait(waitMs: number): string { interface ActiveClaudeTurnContext { readonly queryPolicy: ClaudeRuntimeQueryPolicy; - readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; - readonly nativeThreadId: string; + input: ProviderAdapter.ProviderAdapterV2TurnInput; + nativeThreadId: string; readonly nativeTurnId: string; nativeMessageCursor: string | null; readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; @@ -3499,7 +3499,7 @@ interface ActiveClaudeSubagent { interface ClaudeLiveQueryContext { readonly queryPolicy: ClaudeRuntimeQueryPolicy; - readonly nativeThreadId: string; + nativeThreadId: string; readonly query: ClaudeAgentSdkQuerySession; readonly queryPolicyKey: string; selection: CompiledClaudeModelSelection; @@ -4426,6 +4426,99 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); }); + const adoptNativeSessionId = Effect.fnUntraced(function* ( + query: ClaudeLiveQueryContext, + nativeThreadId: string, + ) { + if ((yield* Ref.get(queryContext)) !== query || query.stopping) return; + const previousId = query.nativeThreadId; + if (nativeThreadId.length === 0 || nativeThreadId === previousId) return; + const previous = (yield* Ref.get(lastProviderThreadByNativeThread)).get(previousId); + if (previous === undefined) return; + const providerThread: OrchestrationV2ProviderThread = { + ...previous, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: nativeThreadId, + strength: "strong", + }, + nativeConversationHeadRef: null, + updatedAt: yield* DateTime.now, + }; + const rekey = (entries: ReadonlyMap): Map => { + const next = new Map(entries); + if (next.has(previousId)) { + const value = next.get(previousId)!; + next.delete(previousId); + next.set(nativeThreadId, value); + } + return next; + }; + // /clear replaces the transcript, not the live process and its background routing. + yield* Ref.update(lastTurnRouteByNativeThread, rekey); + yield* Ref.update(pendingBackgroundTasksByNativeThread, rekey); + yield* Ref.update(wakeEligibleBackgroundTasksByNativeThread, rekey); + yield* Ref.update(opaqueBackgroundTaskReplayTombstonesByNativeThread, rekey); + yield* Ref.update(taskListsByNativeThread, rekey); + yield* Ref.update(wakeBuffers, rekey); + yield* Ref.update(wakeReportsByNativeThread, (state) => ({ + userTurns: rekey(state.userTurns), + reports: rekey(state.reports), + })); + yield* Ref.update(lastProviderThreadByNativeThread, (entries) => + rekey(entries).set(nativeThreadId, providerThread), + ); + yield* Ref.update(openedNativeThreads, (entries) => { + const next = new Set(entries); + next.delete(previousId); + next.add(nativeThreadId); + return next; + }); + yield* Ref.update(requestedContinuations, (entries) => { + if (!entries.has(previousId)) return entries; + const next = new Set(entries); + next.delete(previousId); + next.add(nativeThreadId); + return next; + }); + yield* Ref.update(claudeMonitors, (state) => ({ + ...state, + tasks: new Map( + [...state.tasks].map(([id, task]) => [ + id, + task.nativeThreadId === previousId ? { ...task, nativeThreadId } : task, + ]), + ), + })); + if (goalsByNativeThread.has(previousId)) { + goalsByNativeThread.set(nativeThreadId, goalsByNativeThread.get(previousId) ?? null); + goalsByNativeThread.delete(previousId); + } + const settled = settledTurnByNativeThread.get(previousId); + if (settled !== undefined) { + settledTurnByNativeThread.delete(previousId); + settledTurnByNativeThread.set(nativeThreadId, settled); + } + for (const turn of new Set([yield* Ref.get(activeTurn), query.lastTurn, settled])) { + if (turn?.nativeThreadId !== previousId) continue; + turn.nativeThreadId = nativeThreadId; + turn.input = { + ...turn.input, + providerThread: { + ...turn.input.providerThread, + nativeThreadRef: providerThread.nativeThreadRef, + nativeConversationHeadRef: null, + }, + }; + } + query.nativeThreadId = nativeThreadId; + yield* emitProviderEvent({ + type: "provider_thread.updated", + driver: CLAUDE_PROVIDER, + providerThread, + }); + }); + const rosterForNativeThread = ( all: ReadonlyMap>, nativeThreadId: string, @@ -9386,7 +9479,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( yield* querySession.messages.pipe( Stream.runForEach((message) => sdkMessageLock.withPermits(1)( - Effect.suspend(() => { + Effect.gen(function* () { + if ( + message.type === "system" && + message.subtype === "init" && + Reflect.get(message, "parent_tool_use_id") == null + ) { + yield* adoptNativeSessionId(context, message.session_id); + } if ( message.type === "system" && message.subtype === "init" && @@ -9403,7 +9503,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ) { context.permissionMode = message.permissionMode; } - return handleSdkMessage({ query: querySession, message }); + yield* handleSdkMessage({ query: querySession, message }); }), ), ), @@ -9442,7 +9542,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const startTurn = Effect.fn("ClaudeAdapterV2.startTurn")( function* (turnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { const startedAt = yield* DateTime.now; - const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); + let nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); const nativeTurnId = `turn:${turnInput.attemptId}`; // Fresh for every offer: Claude acks a prompt whose uuid its // transcript already holds without running a turn, and a session @@ -9548,6 +9648,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( heldRootFrames: [], }; const attachTurn = Effect.gen(function* () { + if (querySession.nativeThreadId !== context.nativeThreadId) { + context.nativeThreadId = querySession.nativeThreadId; + const providerThread = (yield* Ref.get(lastProviderThreadByNativeThread)).get( + querySession.nativeThreadId, + ); + if (providerThread !== undefined) context.input = { ...turnInput, providerThread }; + } + nativeThreadId = context.nativeThreadId; querySession.lastTurn = context; yield* Ref.set(activeTurn, context); if ( @@ -9606,8 +9714,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); }); if (userMessage !== null) { - yield* attachTurn; - yield* startUserTurnForWakeReports(nativeThreadId); + yield* sdkMessageLock.withPermits(1)( + attachTurn.pipe( + Effect.andThen(() => startUserTurnForWakeReports(context.nativeThreadId)), + ), + ); yield* querySession.query.offer(userMessage); return; } From 63401a505d3021e9bada5692fdaf68978301f88b Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:59:20 +0100 Subject: [PATCH 02/29] fix(claude): persist a session adopted while idle With no run subscribed, ProviderSessionManager published the adopted provider thread to an empty subscriber set, so the projection kept the pre-clear native ID and a later resume reopened the old conversation. An unsubscribed provider_thread.updated now goes straight to the ingestor, like session-scoped runtime requests. Co-Authored-By: Claude Opus 5.5 --- .../ProviderSessionManager.test.ts | 70 +++++++++++++++++++ .../ProviderSessionManager.ts | 12 +++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index b60758e80c728..96a968fba43c6 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -2945,6 +2945,76 @@ it.effect("ProviderSessionManagerV2 persists session-scoped runtime requests wit }), ); +it.effect("ProviderSessionManagerV2 persists a provider thread update without a run", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-idle-thread-update", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-idle-thread-update", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const afterSequence = yield* eventSink.latestSequence({ threadId }); + const persistedFiber = yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter((stored) => stored.event.type === "provider-thread.updated"), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterEvents); + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "cleared-session", + strength: "strong", + }, + }, + }); + yield* Fiber.join(persistedFiber); + + const projection = yield* projectionStore.getThreadProjection(threadId); + assert.deepEqual( + projection.providerThreads.map((candidate) => candidate.nativeThreadRef?.nativeId), + ["cleared-session"], + ); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); + }), +); + it.effect( "ProviderSessionManagerV2 opens one shared runtime, broadcasts events, and detaches threads independently", () => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index e82995d1db77f..1d6dc6f8f19fe 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1992,7 +1992,17 @@ export const layerWithOptions = ( // (project trust, login, or session-switch hooks). Persist // their runless request artifacts directly so the normal T3 // request UI can answer them and unblock session setup. - const threadId = sessionScopedRuntimeRequestThreadId(event); + // A provider thread can also change while idle (Claude adopts + // a new session after `/clear`). With no run subscribed, that + // update would be dropped and a later resume would reopen the + // old native session. + const threadId = + sessionScopedRuntimeRequestThreadId(event) ?? + (event.type === "provider_thread.updated" && + event.providerThread.appThreadId !== null && + (yield* Ref.get(entry.eventSubscribers)).size === 0 + ? event.providerThread.appThreadId + : undefined); if (threadId !== undefined) { yield* Effect.gen(function* () { const current = (yield* Ref.get(sessions)).get( From aa5ccb60d28d24756b485507b414d70163fa19d9 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:06:15 +0100 Subject: [PATCH 03/29] fix(claude): start a fresh task list after /clear Claude keys its task store by native session unless CLAUDE_CODE_TASK_LIST_ID is set. Moving the cached list to the adopted session merged pre-clear tasks into the new session's list. Drop the cached entry instead, so the next task tool reloads the right store. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 38 ++++++++++++++++++- .../Adapters/ClaudeAdapterV2.ts | 8 +++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index f94b168c32d27..af67196b96a28 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -5754,7 +5754,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { session_id: WAKE_NATIVE_SESSION, }), ); - return { configDir, startTurn, call, launchSubagent, finishTurn, plans }; + return { + configDir, + startTurn, + call, + launchSubagent, + finishTurn, + plans, + offer: harness.offerAndWait, + nextUuid, + }; }); const makeTaskToolHarness = makeTaskToolHarnessWithOptions(); @@ -5887,6 +5896,33 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("starts a fresh task list when /clear selects a new native session", () => + Effect.gen(function* () { + const harness = yield* makeTaskToolHarness; + yield* harness.startTurn(); + yield* harness.call(createTask("1", "Before clear")); + yield* harness.call(createTask("2", "Also before clear")); + yield* harness.offer( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: harness.nextUuid(), + }), + ); + yield* harness.call(createTask("1", "After clear")); + yield* harness.finishTurn(); + assert.deepEqual(harness.plans().at(-1)?.steps, [ + { id: "task-1", text: "After clear", status: "pending" }, + ]); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("removes deleted tasks without resurrecting them on later updates or turns", () => Effect.gen(function* () { const harness = yield* makeTaskToolHarness; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3efb3c200f50c..112f44af9336d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4459,7 +4459,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( yield* Ref.update(pendingBackgroundTasksByNativeThread, rekey); yield* Ref.update(wakeEligibleBackgroundTasksByNativeThread, rekey); yield* Ref.update(opaqueBackgroundTaskReplayTombstonesByNativeThread, rekey); - yield* Ref.update(taskListsByNativeThread, rekey); + // Claude keys its task store by session unless CLAUDE_CODE_TASK_LIST_ID is set, so + // drop the cached list and let the next task tool reload the new session's store. + yield* Ref.update(taskListsByNativeThread, (entries) => { + const next = new Map(entries); + next.delete(previousId); + return next; + }); yield* Ref.update(wakeBuffers, rekey); yield* Ref.update(wakeReportsByNativeThread, (state) => ({ userTurns: rekey(state.userTurns), From 3461eecd9086be32357804f5271613d8ee134745 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:37:14 +0100 Subject: [PATCH 04/29] fix(claude): follow an idle /clear when the next turn is stale A turn built before the idle adoption reached the projection carried the pre-clear native ID, so openQuery replaced the live process and resumed the old session. The adapter now records each adoption and redirects a turn that still names the replaced session. The pre-clear todo list also stayed the thread's active list when no task tool ran afterward. Adoption now supersedes it, unless CLAUDE_CODE_TASK_LIST_ID shares the list across sessions. An idle adoption publishes the supersede at the next turn, because nothing persists adapter events while no run is subscribed. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 79 +++++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 61 +++++++++++++- 2 files changed, 139 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index af67196b96a28..ff665152ed307 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2469,6 +2469,53 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("keeps the cleared session when the next turn carries the pre-clear thread", () => + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }); + const turn = (ordinal: number) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`stale-clear-session-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: [], + providerTurnOrdinal: ordinal, + }); + yield* harness.runtime.startTurn(turn(1)); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000511", + }), + ); + yield* harness.runtime.startTurn(turn(2)); + assert.equal(harness.processQueues.length, 1); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + assert.equal(lastProviderTurn(harness.events).providerThreadId, harness.providerThread.id); + const thread = harness.events.findLast((event) => event.type === "provider_thread.updated"); + assert.equal( + thread?.type === "provider_thread.updated" + ? thread.providerThread.nativeThreadRef?.nativeId + : undefined, + "cleared-native-session", + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + describe("Claude settings restrictions", () => { it.effect("resolves cached managed policy from the configured home through the real SDK", () => Effect.gen(function* () { @@ -5923,6 +5970,38 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect.each([ + { listId: undefined, status: "superseded" }, + { listId: "shared-list", status: "active" }, + ] as const)( + "retires a session task list after an idle /clear (shared list: $listId)", + ({ listId, status }) => + Effect.gen(function* () { + const harness = yield* makeTaskToolHarnessWithOptions({ CLAUDE_CODE_TASK_LIST_ID: listId }); + yield* harness.startTurn(); + yield* harness.call(createTask("1", "Before clear")); + yield* harness.finishTurn(); + yield* harness.offer( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: harness.nextUuid(), + }), + ); + yield* harness.startTurn(); + yield* harness.finishTurn(); + const plans = harness.plans(); + assert.equal(plans.at(-1)?.id, plans[0]?.id); + assert.equal(plans.at(-1)?.status, status); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("removes deleted tasks without resurrecting them on later updates or turns", () => Effect.gen(function* () { const harness = yield* makeTaskToolHarness; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 112f44af9336d..7d12af6692ab2 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4426,6 +4426,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); }); + // Old native ID -> the session `/clear` replaced it with, for turns built from a + // projection that has not caught up yet. Adoption removes the new ID as a key + // first, so following the chain cannot loop. + const adoptedNativeSessionIds = new Map(); + // Todo lists retired by an idle `/clear`. Nothing persists adapter events while no + // run is subscribed, so the next turn publishes them. + const supersededTodoPlans: Array = []; const adoptNativeSessionId = Effect.fnUntraced(function* ( query: ClaudeLiveQueryContext, nativeThreadId: string, @@ -4517,6 +4524,31 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, }; } + // Without a shared CLAUDE_CODE_TASK_LIST_ID the new session starts with no tasks, + // so the pre-clear todo list must stop being the thread's current one. + const todoKey = `${previous.appThreadId}:todo_list`; + const todoPlan = (yield* Ref.get(latestPlanByKind)).get(todoKey); + if ( + !adapterOptions.environment.CLAUDE_CODE_TASK_LIST_ID?.trim() && + previous.appThreadId !== null && + todoPlan !== undefined + ) { + yield* Ref.update(latestPlanByKind, (current) => { + const updated = new Map(current); + updated.delete(todoKey); + return updated; + }); + const superseded = { ...todoPlan, status: "superseded" as const }; + if ((yield* Ref.get(activeTurn)) === null) supersededTodoPlans.push(superseded); + else + yield* emitProviderEvent({ + type: "plan.updated", + driver: CLAUDE_PROVIDER, + plan: superseded, + }); + } + adoptedNativeSessionIds.delete(nativeThreadId); + adoptedNativeSessionIds.set(previousId, nativeThreadId); query.nativeThreadId = nativeThreadId; yield* emitProviderEvent({ type: "provider_thread.updated", @@ -9546,7 +9578,31 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); const startTurn = Effect.fn("ClaudeAdapterV2.startTurn")( - function* (turnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { + function* (requestedTurnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { + // A turn can be built from a projection read before an idle `/clear` adoption + // landed. Opening its native ID would replace the live process with the + // pre-clear session, so follow the adoption this adapter already made. + const requestedNativeId = + requestedTurnInput.providerThread.nativeThreadRef?.nativeId ?? undefined; + let adoptedNativeId = requestedNativeId; + while (adoptedNativeId !== undefined && adoptedNativeSessionIds.has(adoptedNativeId)) { + adoptedNativeId = adoptedNativeSessionIds.get(adoptedNativeId); + } + const turnInput = + adoptedNativeId === undefined || adoptedNativeId === requestedNativeId + ? requestedTurnInput + : { + ...requestedTurnInput, + providerThread: { + ...requestedTurnInput.providerThread, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: adoptedNativeId, + strength: "strong" as const, + }, + nativeConversationHeadRef: null, + }, + }; const startedAt = yield* DateTime.now; let nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); const nativeTurnId = `turn:${turnInput.attemptId}`; @@ -9664,6 +9720,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeThreadId = context.nativeThreadId; querySession.lastTurn = context; yield* Ref.set(activeTurn, context); + for (const plan of supersededTodoPlans.splice(0)) { + yield* emitProviderEvent({ type: "plan.updated", driver: CLAUDE_PROVIDER, plan }); + } if ( !isClaudeProviderContinuationTurn(turnInput) && permissionModeForClaudeRuntimePolicy(turnInput.runtimePolicy) === From b8dc55892a5674004d10a0d969d19463c28f0bf9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 10:58:18 +0000 Subject: [PATCH 05/29] test(claude): reproduce /clear races that resume the pre-clear session (#24153) Two deterministic failing tests for races in the existing fix: - ProviderSessionManager routes a provider_thread.updated to a run subscriber that is finalizing its terminal; closing the subscription drops it, so the cleared session is never persisted. - ClaudeAdapterV2 resolves the /clear alias before attachment prep; a root init arriving during prep makes openQuery replace the live process and resume the pre-clear session. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 68 +++++++++++ .../ProviderSessionManager.test.ts | 108 ++++++++++++++++++ 2 files changed, 176 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index ff665152ed307..6558b3a0d1708 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2516,6 +2516,74 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("keeps the session a /clear selects while the next turn reads its attachments", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const reading = yield* Deferred.make(); + const release = yield* Deferred.make(); + const attachment = ChatImageAttachment.make({ + type: "image", + id: ChatAttachmentId.make("thread-claude-wake-12345678-1234-1234-1234-123456789abc"), + name: "diagram.png", + mimeType: "image/png", + sizeBytes: 4, + }); + const attachmentPath = attachmentRelativePath(attachment)!; + const gatedFileSystem: FileSystem.FileSystem = { + ...fileSystem, + readFile: (filePath) => + filePath.endsWith(attachmentPath) + ? Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(fileSystem.readFile(filePath)), + ) + : fileSystem.readFile(filePath), + }; + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }).pipe(Effect.provideService(FileSystem.FileSystem, gatedFileSystem)); + yield* fileSystem.writeFile( + NodePath.join(harness.attachmentsDir, attachmentPath), + Uint8Array.from([1, 2, 3, 4]), + ); + const turn = (ordinal: number) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`clear-during-prep-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: ordinal === 1 ? [] : [attachment], + providerTurnOrdinal: ordinal, + }); + yield* harness.runtime.startTurn(turn(1)); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + + const started = yield* harness.runtime.startTurn(turn(2)).pipe(Effect.forkScoped); + yield* Deferred.await(reading); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000521", + }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(started); + + assert.equal(harness.processQueues.length, 1); + assert.notEqual(harness.getOpenedOptions()?.resume, WAKE_NATIVE_SESSION); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + describe("Claude settings restrictions", () => { it.effect("resolves cached managed policy from the configured home through the real SDK", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 96a968fba43c6..56f6ab47cdab8 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3015,6 +3015,114 @@ it.effect("ProviderSessionManagerV2 persists a provider thread update without a }), ); +it.effect( + "ProviderSessionManagerV2 persists a provider thread update queued behind a closing run", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-thread-update-behind-terminal", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-thread-update-behind-terminal", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + // The run consumer is still subscribed while it finalizes the terminal. + const runSubscription = yield* runtime.subscribeEvents!; + const afterSequence = yield* eventSink.latestSequence({ threadId }); + const sessionUpdates = yield* Queue.unbounded(); + yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter((stored) => stored.event.type === "provider-session.updated"), + Stream.runForEach(() => Queue.offer(sessionUpdates, undefined)), + Effect.forkScoped, + ); + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterEvents); + yield* Queue.offer(adapterEvents!, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "turn-before-clear", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + // Claude's root init for the cleared session lands before the run closes. + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "cleared-session", + strength: "strong", + }, + }, + }); + // The event pump handles events in order, so once this sentinel is + // persisted the thread update has already been routed. + yield* Queue.offer(adapterEvents!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + yield* Queue.take(sessionUpdates); + + const consumed = yield* runSubscription.events.pipe(Stream.take(1), Stream.runCollect); + assert.deepEqual( + Array.from(consumed).map((event) => event.type), + ["turn.terminal"], + ); + yield* runSubscription.close; + yield* Queue.offer(adapterEvents!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + yield* Queue.take(sessionUpdates); + + const projection = yield* projectionStore.getThreadProjection(threadId); + assert.deepEqual( + projection.providerThreads.map((candidate) => candidate.nativeThreadRef?.nativeId), + ["cleared-session"], + ); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 }))); + }), +); + it.effect( "ProviderSessionManagerV2 opens one shared runtime, broadcasts events, and detaches threads independently", () => From dcd53799358a73d7a38c159a5a1c940182e45bd4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:00:38 +0000 Subject: [PATCH 06/29] test(claude): reproduce a restart after /clear starting the cleared session as new (#24153) The /clear attempt records the pre-clear native ID, so after a provider session restart the next turn reports nativeThreadHasTurns: false for the cleared session. A fresh Claude adapter then opens it with sessionId, which Claude rejects because /clear already wrote its transcript. Co-Authored-By: Claude Opus 5.5 --- .../ProviderSwitch.integration.test.ts | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 72aae60c0d075..50a85165a6f0a 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -104,6 +104,7 @@ function makeTestAdapter(input: { readonly nativeThreadGeneration?: Ref.Ref; readonly failResume?: boolean; readonly failResumeOnce?: Ref.Ref; + readonly clearNativeThreadOnRunOrdinal?: number; readonly initialContextUsage?: OrchestrationV2ProviderThread["contextUsage"]; readonly getModelContextWindow?: (selection: ModelSelection) => number | undefined; readonly canReuseContextUsage?: ProviderAdapter.ProviderAdapterV2SessionRuntime["canReuseContextUsage"]; @@ -356,6 +357,23 @@ function makeTestAdapter(input: { }, }); } + // Claude's `/clear` starts a new native session inside the turn. + if (turnInput.runOrdinal === input.clearNativeThreadOnRunOrdinal) { + yield* PubSub.publish(events, { + type: "provider_thread.updated", + driver: input.driver, + providerThread: { + ...turnInput.providerThread, + nativeThreadRef: { + driver: input.driver, + nativeId: `${turnInput.providerThread.nativeThreadRef?.nativeId}:cleared`, + strength: "strong", + }, + nativeConversationHeadRef: null, + updatedAt: eventTime, + }, + }); + } for (const event of providerEvents) { yield* PubSub.publish(events, event); } @@ -1182,6 +1200,107 @@ describe("orchestration v2 provider switching", () => { ), ); + it.live("resumes the native session /clear selected after the provider session restarts", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("clear-native-session-restart"); + const capturedTurns = yield* Ref.make>([]); + const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([ + makeTestAdapter({ + instanceId: CLAUDE_MODEL_SELECTION.instanceId, + driver: CLAUDE_DRIVER, + capabilities: ClaudeProviderCapabilitiesV2, + modelSelection: CLAUDE_MODEL_SELECTION, + responseByRunOrdinal: {}, + capturedTurns, + clearNativeThreadOnRunOrdinal: 1, + }), + ]); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const runToCompletion = Effect.fn("runToCompletion")(function* ( + ordinal: number, + text: string, + ) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`clear-restart:${ordinal}`), + threadId, + messageId: MessageId.make(`clear-restart:${ordinal}`), + createdBy: "user", + creationSource: "web", + text, + attachments: [], + modelSelection: CLAUDE_MODEL_SELECTION, + dispatchMode: { type: "start_immediately" }, + }); + yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && + event.payload.ordinal === ordinal && + (event.payload.status === "completed" || event.payload.status === "failed"), + ), + Stream.runHead, + ); + yield* worker.drain(); + }); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("clear-restart:create"), + threadId, + projectId, + createdBy: "user", + creationSource: "web", + title: "Clear then restart", + modelSelection: CLAUDE_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + yield* runToCompletion(1, "/clear"); + const providerThread = (yield* orchestrator.getThreadProjection( + threadId, + )).providerThreads.at(0); + if (providerThread === undefined || providerThread.providerSessionId === null) { + return yield* Effect.die("Expected a provider session after /clear."); + } + yield* orchestrator.dispatch({ + type: "provider-session.detach", + commandId: CommandId.make("clear-restart:detach"), + threadId, + providerSessionId: providerThread.providerSessionId, + }); + yield* worker.drain(); + yield* runToCompletion(2, "Continue after the restart"); + + const turns = yield* Ref.get(capturedTurns); + assert.equal(turns.length, 2); + assert.equal(turns[1]?.nativeThreadId, `${turns[0]?.nativeThreadId}:cleared`); + // Claude writes the cleared session's transcript during /clear, so + // opening it with a fresh sessionId fails with "already in use". + assert.isTrue(turns[1]?.nativeThreadHasTurns); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { + name: "clear-native-session-restart", + runtimePolicyOverride: { + cwd, + approvalPolicy: "never", + sandboxPolicy: { type: "readOnly" }, + }, + }, + layerRegistry, + ), + ), + ); + }), + ), + ); + it.live.each([ ["background work", new ClaudeBackgroundWorkBlocksQueryReplacementError()], ["an unreadable append file", new ClaudeAppendSystemPromptFileError({ path: "missing.md" })], From cdacbe071c626765bd45b7efdffa99afc003a0d5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:12:27 +0000 Subject: [PATCH 07/29] fix(claude): keep the session /clear selects across run close and restart (#24153) - Persist provider thread updates a settled run never read when its subscription closes, so a /clear adoption queued behind the terminal is not dropped. Subscriptions pull one signal at a time so unread events are still queued at close. - Resolve /clear aliases together with the live query under the SDK lock, after async query prep, so a turn whose attachments are still loading keeps the adopted process. - Treat a native id no attempt started on as provider-adopted, so a fresh process resumes the cleared session instead of creating it again. - attachTurn overrides only the native identity and head of the turn's provider thread. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.ts | 108 +++++++++++------- .../ProviderSessionManager.ts | 102 +++++++++++++---- .../ProviderTurnStartService.ts | 14 +++ 3 files changed, 159 insertions(+), 65 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 7d12af6692ab2..0f3f5cdaa651a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -9315,35 +9315,78 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return false; }); - const openQuery = Effect.fnUntraced(function* ( + // A turn can be built from a projection read before an idle `/clear` adoption + // landed. Opening its native ID would replace the live process with the + // pre-clear session, so follow the adoption this adapter already made. + const followAdoptedNativeSession = ( turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, - nativeThreadId: string, + ): ProviderAdapter.ProviderAdapterV2TurnInput => { + const requestedNativeId = turnInput.providerThread.nativeThreadRef?.nativeId ?? undefined; + let adoptedNativeId = requestedNativeId; + while (adoptedNativeId !== undefined && adoptedNativeSessionIds.has(adoptedNativeId)) { + adoptedNativeId = adoptedNativeSessionIds.get(adoptedNativeId); + } + return adoptedNativeId === undefined || adoptedNativeId === requestedNativeId + ? turnInput + : { + ...turnInput, + providerThread: { + ...turnInput.providerThread, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: adoptedNativeId, + strength: "strong", + }, + nativeConversationHeadRef: null, + }, + }; + }; + + const openQuery = Effect.fnUntraced(function* ( + requestedTurnInput: ProviderAdapter.ProviderAdapterV2TurnInput, ) { - const queryPolicy = yield* queryPolicyForTurn(turnInput); + const queryPolicy = yield* queryPolicyForTurn(requestedTurnInput); const mcpOverrides = claudeMcpQueryOverrides({ - mcpSession: yield* mcpSessions.read(turnInput.threadId), + mcpSession: yield* mcpSessions.read(requestedTurnInput.threadId), readOnlySandbox: - sandboxPolicyKindForClaudeRuntimePolicy(turnInput.runtimePolicy) === "readOnly", + sandboxPolicyKindForClaudeRuntimePolicy(requestedTurnInput.runtimePolicy) === + "readOnly", ...(queryPolicy.allowedTools === undefined ? {} : { allowedTools: queryPolicy.allowedTools }), }); const queryPolicyKey = claudeEffectiveQueryPolicyKey(queryPolicy, mcpOverrides); - const compiledSelection = compileClaudeModelSelection(turnInput.modelSelection); + const compiledSelection = compileClaudeModelSelection(requestedTurnInput.modelSelection); + // Adoption renames the live query under the SDK lock, so read it together with the + // alias it implies. A later adoption only renames this same process, which the turn + // picks up when it attaches. + const [existing, existingNativeThreadId, turnInput] = yield* sdkMessageLock.withPermits( + 1, + )( + Effect.map( + Ref.get(queryContext), + (live) => + [ + live, + live?.nativeThreadId, + followAdoptedNativeSession(requestedTurnInput), + ] as const, + ), + ); + const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); const resumeSessionAt = yield* getNativeConversationHeadId(turnInput.providerThread); - const existing = yield* Ref.get(queryContext); // A continuation prompts nothing: it drains output the live process // already produced, so it keeps that process whatever its selection. if ( existing !== null && - existing.nativeThreadId === nativeThreadId && + existingNativeThreadId === nativeThreadId && isClaudeProviderContinuationTurn(turnInput) ) { return existing; } if ( existing !== null && - existing.nativeThreadId === nativeThreadId && + existingNativeThreadId === nativeThreadId && existing.queryPolicyKey === queryPolicyKey && !existing.stopping && existing.selection.queryIdentity !== compiledSelection.queryIdentity && @@ -9366,7 +9409,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } if ( existing !== null && - existing.nativeThreadId === nativeThreadId && + existingNativeThreadId === nativeThreadId && existing.queryPolicyKey === queryPolicyKey && existing.selection.queryIdentity === compiledSelection.queryIdentity ) { @@ -9395,7 +9438,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // which closes the process. if ( existing !== null && - existing.nativeThreadId === nativeThreadId && + existingNativeThreadId === nativeThreadId && !existing.stopping && (yield* liveProcessRunsBackgroundWork(existing)) ) { @@ -9579,30 +9622,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const startTurn = Effect.fn("ClaudeAdapterV2.startTurn")( function* (requestedTurnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { - // A turn can be built from a projection read before an idle `/clear` adoption - // landed. Opening its native ID would replace the live process with the - // pre-clear session, so follow the adoption this adapter already made. - const requestedNativeId = - requestedTurnInput.providerThread.nativeThreadRef?.nativeId ?? undefined; - let adoptedNativeId = requestedNativeId; - while (adoptedNativeId !== undefined && adoptedNativeSessionIds.has(adoptedNativeId)) { - adoptedNativeId = adoptedNativeSessionIds.get(adoptedNativeId); - } - const turnInput = - adoptedNativeId === undefined || adoptedNativeId === requestedNativeId - ? requestedTurnInput - : { - ...requestedTurnInput, - providerThread: { - ...requestedTurnInput.providerThread, - nativeThreadRef: { - driver: CLAUDE_PROVIDER, - nativeId: adoptedNativeId, - strength: "strong" as const, - }, - nativeConversationHeadRef: null, - }, - }; + const turnInput = followAdoptedNativeSession(requestedTurnInput); const startedAt = yield* DateTime.now; let nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); const nativeTurnId = `turn:${turnInput.attemptId}`; @@ -9662,7 +9682,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( skills: yield* userInvocableSkills(turnInput.runtimePolicy.cwd), uuid: promptUuid, }); - const querySession = yield* openQuery(turnInput, nativeThreadId); + const querySession = yield* openQuery(turnInput); const context: ActiveClaudeTurnContext = { input: turnInput, queryPolicy: querySession.queryPolicy, @@ -9712,10 +9732,18 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const attachTurn = Effect.gen(function* () { if (querySession.nativeThreadId !== context.nativeThreadId) { context.nativeThreadId = querySession.nativeThreadId; - const providerThread = (yield* Ref.get(lastProviderThreadByNativeThread)).get( - querySession.nativeThreadId, - ); - if (providerThread !== undefined) context.input = { ...turnInput, providerThread }; + context.input = { + ...turnInput, + providerThread: { + ...turnInput.providerThread, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: querySession.nativeThreadId, + strength: "strong", + }, + nativeConversationHeadRef: null, + }, + }; } nativeThreadId = context.nativeThreadId; querySession.lastTurn = context; diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 1d6dc6f8f19fe..2a6a10db4d432 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1673,6 +1673,74 @@ export const layerWithOptions = ( ), ); + const ingestRunlessEvent = Effect.fn("ProviderSessionManager.ingestRunlessEvent")(function* ( + entry: LiveSessionEntry, + threadId: ThreadId, + event: ProviderAdapter.ProviderAdapterV2Event, + ) { + yield* Effect.gen(function* () { + const current = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + if (current?.runtime !== entry.runtime) return; + yield* providerEventIngestor + .ingestNormalized({ + providerSessionId: entry.runtime.providerSessionId, + providerInstanceId: entry.runtime.instanceId, + threadId, + event, + }) + .pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: entry.runtime.driver, + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + ), + ); + }).pipe(entry.requestEventPermit.withPermits(1)); + }); + + // A run stops reading once it settles, but Claude can adopt a new native session + // (`/clear`) in events queued behind the terminal. Persist those provider thread + // updates when the last reader leaves, or a later resume reopens the old session. + const persistUnreadProviderThreadUpdates = Effect.fn( + "ProviderSessionManager.persistUnreadProviderThreadUpdates", + )( + function* ( + subscribers: Ref.Ref< + ReadonlyMap> + >, + unread: ReadonlyArray, + ) { + if ((yield* Ref.get(subscribers)).size > 0) return; + const entry = [...(yield* Ref.get(sessions)).values()].find( + (candidate) => candidate.eventSubscribers === subscribers, + ); + if (entry === undefined) return; + for (const signal of unread) { + if ( + signal.type === "event" && + signal.event.type === "provider_thread.updated" && + signal.event.providerThread.appThreadId !== null + ) { + yield* ingestRunlessEvent( + entry, + signal.event.providerThread.appThreadId, + signal.event, + ); + } + } + }, + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.unread-thread-update-failed", { + cause, + }), + ), + ); + const makeEventSubscription = Effect.fn("ProviderSessionManager.makeEventSubscription")( function* ( subscribers: Ref.Ref< @@ -1696,11 +1764,17 @@ export const layerWithOptions = ( }).pipe( Effect.flatMap((removed) => removed - ? Queue.clear(queue).pipe(Effect.andThen(Queue.end(queue)), Effect.asVoid) + ? Queue.clear(queue).pipe( + Effect.tap(() => Queue.end(queue)), + Effect.flatMap((unread) => + persistUnreadProviderThreadUpdates(subscribers, unread), + ), + ) : Effect.void, ), ); - const events = Stream.fromQueue(queue).pipe( + // One signal per pull, so what a reader never pulled is still queued at close. + const events = Stream.fromEffectRepeat(Queue.take(queue)).pipe( Stream.mapEffect((signal) => signal.type === "event" ? Effect.succeed(signal.event) @@ -2004,29 +2078,7 @@ export const layerWithOptions = ( ? event.providerThread.appThreadId : undefined); if (threadId !== undefined) { - yield* Effect.gen(function* () { - const current = (yield* Ref.get(sessions)).get( - sessionKey(entry.runtime.providerSessionId), - ); - if (current?.runtime !== entry.runtime) return; - yield* providerEventIngestor - .ingestNormalized({ - providerSessionId: entry.runtime.providerSessionId, - providerInstanceId: entry.runtime.instanceId, - threadId, - event, - }) - .pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterEventStreamError({ - driver: entry.runtime.driver, - providerSessionId: entry.runtime.providerSessionId, - cause, - }), - ), - ); - }).pipe(entry.requestEventPermit.withPermits(1)); + yield* ingestRunlessEvent(entry, threadId, event); return; } yield* publishToSubscribers(entry.eventSubscribers, { type: "event", event }); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 89ae2cc2c0c13..594f109607ef8 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1087,6 +1087,19 @@ export const layer: Layer.Layer< .filter((source) => source.nativeThreadId === undefined) .map((source) => source.runId), ); + // A run commits the native id it starts on with its attempt, so a current id no + // attempt names was adopted by the provider mid-turn (Claude's `/clear`), and the + // provider has already written that session. + const runningNativeThreadId = runningProviderThread.nativeThreadRef?.nativeId; + const providerAdoptedNativeThread = + sameNativeThread && + runningNativeThreadId != null && + acceptedAttempts.some((source) => source.nativeThreadId !== undefined) && + !projection.attempts.some( + (source) => + source.providerThreadId === providerThread.id && + source.nativeThreadId === runningNativeThreadId, + ); const legacyRecoveredRunIds = new Set( projection.runs .filter( @@ -1313,6 +1326,7 @@ export const layer: Layer.Layer< ) + 1, nativeThreadHasTurns: nativeInputRunIds.size > 0 || + providerAdoptedNativeThread || (legacyInputRunIds.size > 0 && sameNativeThread && !acceptedAttempts.some((source) => source.nativeThreadId !== undefined)), From b09c2b94d436c81c4406cd2d93c6499c6adb6836 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:20:06 +0000 Subject: [PATCH 08/29] test(claude): reproduce a model change during /clear reopening the pre-clear session (#24153) A turn that replaces the Claude process snapshots its native id before reading the append system prompt file. A /clear adopted during that read is lost: the replacement resumes the pre-clear session. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 6558b3a0d1708..2336468c89de5 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2584,6 +2584,73 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect( + "resumes the session a /clear selects while a model change reads the system prompt file", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const reading = yield* Deferred.make(); + const release = yield* Deferred.make(); + let gateRead = false; + const gatedFileSystem: FileSystem.FileSystem = { + ...fileSystem, + readFileString: (filePath) => + filePath.endsWith("rules.md") + ? gateRead + ? Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as("Rules"), + ) + : Effect.succeed("Rules") + : fileSystem.readFileString(filePath), + }; + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + settings: { + ...DEFAULT_CLAUDE_SETTINGS, + launchArgs: "--append-system-prompt-file rules.md", + }, + }).pipe(Effect.provideService(FileSystem.FileSystem, gatedFileSystem)); + const turn = (ordinal: number) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`clear-during-prompt-file-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: [], + providerTurnOrdinal: ordinal, + ...(ordinal === 1 ? {} : { modelSelection: CLAUDE_TEST_RESTART_MODEL_SELECTION }), + }); + yield* harness.runtime.startTurn(turn(1)); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + + gateRead = true; + const started = yield* harness.runtime.startTurn(turn(2)).pipe(Effect.forkScoped); + yield* Deferred.await(reading); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000531", + }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(started); + + assert.equal(harness.processQueues.length, 2); + assert.equal(harness.getOpenedOptions()?.resume, "cleared-native-session"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + describe("Claude settings restrictions", () => { it.effect("resolves cached managed policy from the configured home through the real SDK", () => Effect.gen(function* () { From 775b10141ecd9c8dfd1c17eafb7a01563659bbc8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:27:32 +0000 Subject: [PATCH 09/29] test(server): reproduce fork and rollback across Claude /clear using the adopted session (#24153) Co-Authored-By: Claude Opus 5.5 --- .../CheckpointRollbackService.test.ts | 277 ++++++++++++++++++ .../ThreadFork.execution.test.ts | 204 +++++++++++++ 2 files changed, 481 insertions(+) diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index 149c82452b605..12bbf096e8623 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -12,12 +12,14 @@ import { RunAttemptId, RunId, CheckpointScopeId, + ClaudeSettings, ProjectId, ProviderInstanceId, ProviderSessionId, ProviderThreadId, ThreadId, } from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; import * as DateTime from "effect/DateTime"; import * as SqlClient from "effect/sql/SqlClient"; @@ -27,6 +29,9 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Option from "effect/Option"; import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerConfig from "../config.ts"; @@ -34,6 +39,7 @@ import * as ServerSettings from "../serverSettings.ts"; import * as GitWorkflow from "../git/GitWorkflowService.ts"; import { withWorkspaceLease } from "../workspace/workspaceLease.ts"; import { ProviderWorkspaceMissingError } from "../provider/Errors.ts"; +import * as ClaudeAdapterV2 from "./Adapters/ClaudeAdapterV2.ts"; import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts"; import * as CheckpointService from "./CheckpointService.ts"; import * as CheckpointRollbackService from "./CheckpointRollbackService.ts"; @@ -46,6 +52,7 @@ import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAda import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as RuntimePolicy from "./RuntimePolicy.ts"; +const DEFAULT_CLAUDE_SETTINGS = Schema.decodeSync(ClaudeSettings)({}); const unrelatedProject = Option.some({ workspaceRoot: "/nonexistent/t3-rollback-project", } as never); @@ -697,6 +704,276 @@ it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMe }), ); + // regression: #24153 a rollback across Claude's /clear paired the adopted session with a + // pre-clear message cursor, so the next resume asked the new session for a missing message. + it.effect("rolls a Claude thread back into the session the target turn ran in", () => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("rollback-clear"); + const providerThreadId = ProviderThreadId.make("rollback-clear-provider"); + const providerSessionId = ProviderSessionId.make("rollback-clear-session"); + const providerInstanceId = ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID; + const driver = ClaudeAdapterV2.CLAUDE_PROVIDER; + const modelSelection = { instanceId: providerInstanceId, model: "claude-sonnet-4-6" }; + const cwd = "/nonexistent/t3-rollback-clear"; + const scopeId = CheckpointScopeId.make("rollback-clear-scope"); + const runId = (ordinal: number) => RunId.make(`rollback-clear-run-${ordinal}`); + const nodeId = (ordinal: number) => NodeId.make(`rollback-clear-node-${ordinal}`); + const checkpointId = (ordinal: number) => + CheckpointId.make(`rollback-clear-checkpoint-${ordinal}`); + let sequence = 0; + const apply = (event: Record) => + store.apply({ + ...event, + id: EventId.make(`rollback-clear-event-${sequence++}`), + threadId, + occurredAt: now, + } as OrchestrationV2DomainEvent); + yield* apply({ + type: "thread.created", + payload: { + id: threadId, + createdBy: "user", + creationSource: "web", + projectId: ProjectId.make("rollback-clear-project"), + title: "Rollback across /clear", + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + activeProviderThreadId: providerThreadId, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + }); + // Run 2 started in the original session, and its `/clear` adopted a new one. + yield* apply({ + type: "provider-thread.updated", + payload: { + id: providerThreadId, + driver, + providerInstanceId, + providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-after-clear", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }); + yield* apply({ + type: "checkpoint-scope.created", + payload: { + id: scopeId, + threadId, + runId: runId(1), + nodeId: nodeId(1), + parentScopeId: null, + providerThreadId, + kind: "root_run", + ordinalWithinParent: 0, + advancesAppRunCount: true, + cwd, + createdAt: now, + }, + }); + for (const ordinal of [1, 2]) { + const attemptId = RunAttemptId.make(`rollback-clear-attempt-${ordinal}`); + const turnId = ProviderTurnId.make(`rollback-clear-turn-${ordinal}`); + yield* apply({ + type: "run.created", + payload: { + id: runId(ordinal), + threadId, + ordinal, + providerInstanceId, + modelSelection, + providerThreadId, + userMessageId: MessageId.make(`rollback-clear-message-${ordinal}`), + rootNodeId: nodeId(ordinal), + activeAttemptId: attemptId, + status: "completed", + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: checkpointId(ordinal), + contextHandoffId: null, + }, + }); + yield* apply({ + type: "node.updated", + payload: { + id: nodeId(ordinal), + threadId, + runId: runId(ordinal), + parentNodeId: null, + rootNodeId: nodeId(ordinal), + kind: "root_turn", + status: "completed", + countsForRun: true, + providerThreadId, + providerTurnId: turnId, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: scopeId, + startedAt: now, + completedAt: now, + }, + }); + yield* apply({ + type: "run-attempt.created", + runId: runId(ordinal), + payload: { + id: attemptId, + nativeThreadId: "native-before-clear", + runId: runId(ordinal), + attemptOrdinal: 1, + rootNodeId: nodeId(ordinal), + providerInstanceId, + providerThreadId, + providerTurnId: turnId, + reason: "initial", + status: "completed", + startedAt: now, + completedAt: now, + }, + }); + yield* apply({ + type: "provider-turn.updated", + payload: { + id: turnId, + providerThreadId, + nodeId: nodeId(ordinal), + runAttemptId: attemptId, + nativeTurnRef: { driver, nativeId: `assistant-${ordinal}`, strength: "weak" }, + ordinal, + status: "completed", + startedAt: now, + completedAt: now, + }, + }); + } + for (const ordinal of [0, 1, 2]) { + yield* apply({ + type: "checkpoint.captured", + payload: { + id: checkpointId(ordinal), + threadId, + scopeId, + runId: ordinal === 0 ? null : runId(ordinal), + nodeId: nodeId(Math.max(1, ordinal)), + parentCheckpointId: null, + ordinalWithinScope: ordinal, + appRunOrdinal: ordinal === 0 ? null : ordinal, + ref: CheckpointRef.make(`refs/t3/rollback-clear/${ordinal}`), + status: "ready", + files: [], + capturedAt: now, + }, + }); + } + + const fileSystem = yield* FileSystem.FileSystem; + const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: providerInstanceId, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir: yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-rollback-clear-", + }), + fileSystem, + path: yield* Path.Path, + crypto: yield* Crypto.Crypto, + idAllocator: yield* IdAllocator.IdAllocatorV2, + queryRunner: { + allocateSessionId: Effect.die("unused allocateSessionId"), + open: () => Effect.die("unused open"), + forkSession: () => Effect.die("unused forkSession"), + subagentLaunchToolUseId: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy: { runtimeMode: "full-access", interactionMode: "default", cwd: null }, + }); + yield* runtime.events.pipe(Stream.runDrain, Effect.forkScoped); + const committed: OrchestrationV2DomainEvent[] = []; + yield* Effect.gen(function* () { + const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2; + yield* service.execute({ + threadId, + providerThreadId, + scopeId, + checkpointId: checkpointId(1), + restoreFiles: false, + }); + }).pipe( + Effect.provide( + layerCheckpointRollbackService.pipe( + Layer.provide( + Layer.mergeAll( + ThreadCommandExecutor.layer, + IdAllocator.layer, + Layer.mock(CheckpointService.CheckpointServiceV2)({ + restore: () => Effect.void, + deleteStaleRefs: () => Effect.void, + }), + Layer.mock(EventSink.EventSinkV2)({ + write: ({ events }) => + Effect.sync(() => { + committed.push(...events); + return []; + }), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + open: () => Effect.succeed(runtime as never), + }), + Layer.mock(RuntimePolicy.RuntimePolicyV2)({ + resolve: () => Effect.succeed({} as never), + }), + ), + ), + ), + ), + ); + + const providerThread = committed.find( + (event) => event.type === "provider-thread.updated", + )?.payload; + assert.deepEqual( + { + session: providerThread?.nativeThreadRef?.nativeId, + resumeAt: providerThread?.nativeConversationHeadRef?.nativeId, + }, + { session: "native-before-clear", resumeAt: "assistant-1" }, + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect( "rewinds stopped runs repeatedly without counting discarded provider turns again", () => diff --git a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts index f3cf10851ae9f..6446438b20b7c 100644 --- a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts +++ b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts @@ -1,5 +1,7 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { + ClaudeSettings, CommandId, EventId, MessageId, @@ -14,19 +16,28 @@ import { ThreadId, TurnItemId, } from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as ClaudeAdapterV2 from "./Adapters/ClaudeAdapterV2.ts"; import { ClaudeProviderCapabilitiesV2 } from "./Adapters/ClaudeAdapterV2.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +const DEFAULT_CLAUDE_SETTINGS = Schema.decodeSync(ClaudeSettings)({}); const forkCases = ( [ ["codex", "interrupted"], @@ -610,3 +621,196 @@ it.effect.each([ ); }, ); + +// regression: #24153 a fork across Claude's /clear asked the adopted session for a message +// that only exists in the session the source turn ran in. +it.effect("forks a Claude run from the session it ran in after /clear adopted another", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const forkCalls: Array = []; + const claude = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir: yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-fork-clear-" }), + fileSystem, + path: yield* Path.Path, + crypto: yield* Crypto.Crypto, + idAllocator: yield* IdAllocator.IdAllocatorV2, + queryRunner: { + allocateSessionId: Effect.succeed("unused-native-session"), + open: () => Effect.die("unused open"), + forkSession: (input) => + Effect.sync(() => forkCalls.push(input)).pipe( + Effect.andThen(Effect.die("fork inspected")), + ), + subagentLaunchToolUseId: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + const driver = claude.driver; + const instanceId = claude.instanceId; + const modelSelection = { instanceId, model: "claude-sonnet-4-6" }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const now = yield* DateTime.now; + const sourceThreadId = ThreadId.make("fork-clear-source"); + const providerThreadId = ProviderThreadId.make("fork-clear-provider-thread"); + const sourceRunId = RunId.make("fork-clear-source-run"); + const attemptId = RunAttemptId.make("fork-clear-source-attempt"); + const providerTurnId = ProviderTurnId.make("fork-clear-source-turn"); + const rootNodeId = NodeId.make("fork-clear-source-node"); + + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("fork-clear-create-source"), + threadId: sourceThreadId, + projectId: ProjectId.make("fork-clear-project"), + title: "Fork across /clear", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* eventSink.write({ + events: [ + // A later run's `/clear` adopted a new native session for this provider thread. + { + id: EventId.make("fork-clear-provider-thread"), + type: "provider-thread.updated", + threadId: sourceThreadId, + occurredAt: now, + payload: { + id: providerThreadId, + driver, + providerInstanceId: instanceId, + providerSessionId: null, + appThreadId: sourceThreadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-after-clear", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("fork-clear-source-run"), + type: "run.created", + threadId: sourceThreadId, + runId: sourceRunId, + occurredAt: now, + payload: { + id: sourceRunId, + threadId: sourceThreadId, + ordinal: 1, + providerInstanceId: instanceId, + modelSelection, + providerThreadId, + userMessageId: MessageId.make("fork-clear-source-message"), + rootNodeId, + activeAttemptId: attemptId, + status: "completed", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }, + }, + { + id: EventId.make("fork-clear-source-attempt"), + type: "run-attempt.created", + threadId: sourceThreadId, + runId: sourceRunId, + occurredAt: now, + payload: { + id: attemptId, + nativeThreadId: "native-before-clear", + runId: sourceRunId, + attemptOrdinal: 1, + rootNodeId, + providerInstanceId: instanceId, + providerThreadId, + providerTurnId, + reason: "initial", + status: "completed", + startedAt: now, + completedAt: now, + }, + }, + { + id: EventId.make("fork-clear-source-turn"), + type: "provider-turn.updated", + threadId: sourceThreadId, + occurredAt: now, + payload: { + id: providerTurnId, + providerThreadId, + nodeId: rootNodeId, + runAttemptId: attemptId, + nativeTurnRef: { driver, nativeId: "assistant-before-clear", strength: "weak" }, + ordinal: 1, + status: "completed", + startedAt: now, + completedAt: now, + }, + }, + ], + }); + const targetThreadId = ThreadId.make("fork-clear-target"); + yield* orchestrator.dispatch({ + type: "thread.fork", + commandId: CommandId.make("fork-clear-fork"), + sourceThreadId, + targetThreadId, + sourcePoint: { type: "run", runId: sourceRunId }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("fork-clear-message"), + threadId: targetThreadId, + messageId: MessageId.make("fork-clear-message"), + text: "Continue from here", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "fork-across-clear" }, + ProviderAdapterRegistry.layerFromAdapters([claude]), + ), + ), + ); + + assert.deepEqual( + forkCalls.map((call) => ({ + sessionId: call.sessionId, + upToMessageId: call.options.upToMessageId, + })), + [{ sessionId: "native-before-clear", upToMessageId: "assistant-before-clear" }], + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), +); From 89d0e9a3c4eed815bd0d310c3a145062cc08d819 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:27:53 +0000 Subject: [PATCH 10/29] fix(claude): a /clear adopted while a turn replaces the process is not lost (#24153) A turn that replaced the Claude process picked the native session before reading the append system prompt file, then closed and reopened outside the SDK lock. An init adopting a /clear during that read was overwritten by a resume of the pre-clear session. The reads now run first and the decision, close and open happen together under the SDK lock. A provider thread update routed while the last run subscriber closed could also be dropped. Close now ends the queue before draining it, so an update is either drained or refused, and a refused update is persisted runless. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.ts | 496 +++++++++--------- .../ProviderSessionManager.ts | 40 +- 2 files changed, 278 insertions(+), 258 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 0f3f5cdaa651a..789e2eec72920 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -9357,62 +9357,262 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); const queryPolicyKey = claudeEffectiveQueryPolicyKey(queryPolicy, mcpOverrides); const compiledSelection = compileClaudeModelSelection(requestedTurnInput.modelSelection); - // Adoption renames the live query under the SDK lock, so read it together with the - // alias it implies. A later adoption only renames this same process, which the turn - // picks up when it attaches. - const [existing, existingNativeThreadId, turnInput] = yield* sdkMessageLock.withPermits( - 1, - )( - Effect.map( - Ref.get(queryContext), - (live) => - [ - live, - live?.nativeThreadId, - followAdoptedNativeSession(requestedTurnInput), - ] as const, - ), - ); - const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); - const resumeSessionAt = yield* getNativeConversationHeadId(turnInput.providerThread); - // A continuation prompts nothing: it drains output the live process - // already produced, so it keeps that process whatever its selection. - if ( - existing !== null && - existingNativeThreadId === nativeThreadId && - isClaudeProviderContinuationTurn(turnInput) - ) { - return existing; - } - if ( - existing !== null && - existingNativeThreadId === nativeThreadId && - existing.queryPolicyKey === queryPolicyKey && - !existing.stopping && - existing.selection.queryIdentity !== compiledSelection.queryIdentity && - canApplyClaudeSelectionLive(existing.selection, compiledSelection) && - (existing.selection.apiModelId === compiledSelection.apiModelId || - existing.openedPermissionMode === queryPolicy.permissionMode) - ) { - const live = existing; - yield* live.query.setModel(compiledSelection.apiModelId).pipe( - Effect.andThen( - live.query.applyFlagSettings(claudeLiveSelectionFlagSettings(compiledSelection)), - ), - Effect.tapError(() => - Effect.sync(() => { - live.selection = { ...live.selection, queryIdentity: "" }; - }), - ), + // Replacing the process reads files first. Those reads run before the SDK lock and + // the decision is made again under it, so a `/clear` adopted meanwhile is followed. + let replacementPrep: + | { + readonly appendSystemPromptFileText: string | undefined; + readonly subagentPromptCacheTtl: typeof ClaudePromptCacheTtl.Type | undefined; + } + | undefined; + while (true) { + const prep = replacementPrep; + // Adoption renames the live query under the SDK lock, so the alias, the decision + // and any replacement happen under it too. A later adoption only renames the + // process this returns, which the turn picks up when it attaches. + const decision = yield* sdkMessageLock.withPermits(1)( + Effect.gen(function* () { + const existing = yield* Ref.get(queryContext); + const turnInput = followAdoptedNativeSession(requestedTurnInput); + const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); + const resumeSessionAt = yield* getNativeConversationHeadId( + turnInput.providerThread, + ); + const sameNativeThread = + existing !== null && existing.nativeThreadId === nativeThreadId; + // A continuation prompts nothing: it drains output the live process + // already produced, so it keeps that process whatever its selection. + if (sameNativeThread && isClaudeProviderContinuationTurn(turnInput)) { + return { _tag: "Continue", live: existing } as const; + } + if ( + sameNativeThread && + existing.queryPolicyKey === queryPolicyKey && + (existing.selection.queryIdentity === compiledSelection.queryIdentity || + (!existing.stopping && + canApplyClaudeSelectionLive(existing.selection, compiledSelection) && + (existing.selection.apiModelId === compiledSelection.apiModelId || + existing.openedPermissionMode === queryPolicy.permissionMode))) + ) { + return { _tag: "Reuse", live: existing } as const; + } + + // Background agents and shells run inside the CLI process, so a new selection + // would kill them and lose their results. Refuse until they finish or the user + // presses Stop, which closes the process. + if ( + sameNativeThread && + !existing.stopping && + (yield* liveProcessRunsBackgroundWork(existing)) + ) { + return yield* new ClaudeBackgroundWorkBlocksQueryReplacementError(); + } + if (prep === undefined) return { _tag: "Prepare", turnInput } as const; + + const closedExistingNativeThreadId = + existing !== null ? existing.nativeThreadId : null; + if (existing !== null) { + yield* existing.query.close.pipe(Effect.ignore); + if (existing.nativeThreadId !== nativeThreadId) { + yield* clearWakeStateForNativeThread(existing.nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(existing.nativeThreadId, { + status: "idle", + }); + } + } + const openedWithResume = (yield* Ref.get(openedNativeThreads)).has(nativeThreadId); + // openedNativeThreads is per session instance and is lost when the + // provider session is idle-released. A prior turn on this native id + // requires resume; sessionId would fail with "already in use". + const hasPersistedProviderTurn = + turnInput.nativeThreadHasTurns ?? turnInput.providerTurnOrdinal > 1; + const shouldResume = + resumeSessionAt !== undefined || openedWithResume || hasPersistedProviderTurn; + const queryOptions = makeClaudeQueryOptions({ + modelSelection: turnInput.modelSelection, + nativeThreadId, + resume: shouldResume, + ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), + cwd: turnInput.runtimePolicy.cwd, + attachmentsDir, + settings: adapterOptions.settings, + environment: claudeQueryEnvironment( + adapterOptions.environment, + turnInput.appThread, + turnInput.appThread.lineage.relationshipToParent === "subagent" + ? prep.subagentPromptCacheTtl + : undefined, + ), + tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, + ...(mcpOverrides.allowedTools === undefined + ? {} + : { allowedTools: mcpOverrides.allowedTools }), + ...(mcpOverrides.mcpServers === undefined + ? {} + : { mcpServers: mcpOverrides.mcpServers }), + permissionMode: queryPolicy.permissionMode, + ...(queryPolicy.allowDangerouslySkipPermissions === undefined + ? {} + : { + allowDangerouslySkipPermissions: + queryPolicy.allowDangerouslySkipPermissions, + }), + canUseTool, + onUserDialog, + supportedDialogKinds: ["resume_return"], + ...(prep.appendSystemPromptFileText === undefined + ? {} + : { appendSystemPromptFileText: prep.appendSystemPromptFileText }), + }); + const querySession = yield* queryRunner + .open({ + threadId: turnInput.threadId, + providerSessionId: input.providerSessionId, + options: queryOptions, + }) + .pipe( + Effect.onError(() => + closedExistingNativeThreadId === nativeThreadId + ? Effect.gen(function* () { + yield* clearWakeStateForNativeThread(nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "idle", + }); + }) + : Effect.void, + ), + ); + // Marked only after a successful open: a failed create must not + // leave the runtime believing the native session exists, or the + // retry would resume a session that was never created. + yield* Ref.update(openedNativeThreads, (current) => { + if (current.has(nativeThreadId)) { + return current; + } + const updated = new Set(current); + updated.add(nativeThreadId); + return updated; + }); + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "active", + }); + const closed = yield* Deferred.make(); + const context: ClaudeLiveQueryContext = { + queryPolicy, + nativeThreadId, + query: querySession, + queryPolicyKey, + selection: compiledSelection, + closed, + lastTurn: null, + promptEchoMode: "unknown", + openedPermissionMode: queryOptions.permissionMode, + receivedInit: false, + permissionMode: queryOptions.permissionMode, + stopping: false, + subagentsFromEarlierProcesses: new Set( + [...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter( + (subagent) => subagent.task.status === "running", + ), + ), + }; + yield* Ref.set(queryContext, context); + yield* querySession.messages.pipe( + Stream.runForEach((message) => + sdkMessageLock.withPermits(1)( + Effect.gen(function* () { + if ( + message.type === "system" && + message.subtype === "init" && + Reflect.get(message, "parent_tool_use_id") == null + ) { + yield* adoptNativeSessionId(context, message.session_id); + } + if ( + message.type === "system" && + message.subtype === "init" && + !context.receivedInit + ) { + context.openedPermissionMode = + message.permissionMode ?? context.openedPermissionMode; + context.receivedInit = true; + } + if ( + message.type === "system" && + (message.subtype === "init" || message.subtype === "status") && + message.permissionMode !== undefined + ) { + context.permissionMode = message.permissionMode; + } + yield* handleSdkMessage({ query: querySession, message }); + }), + ), + ), + Effect.exit, + Effect.flatMap( + Effect.fnUntraced(function* (exit: ClaudeQueryStreamExit) { + const heldContext = yield* Ref.get(activeTurn); + if ( + (yield* Ref.get(queryContext))?.query === querySession && + heldContext !== null && + heldContext.heldRootFrames.length > 0 + ) { + yield* releaseHeldRootFrames(heldContext); + } + const ownsLiveQuery = yield* Ref.modify(queryContext, (current) => + current?.query === querySession ? [true, null] : [false, current], + ); + if (ownsLiveQuery) { + yield* finalizeActiveTurnAfterQueryExit( + context, + exit._tag === "Failure" ? exit.cause : undefined, + ); + yield* endSubagentCallsOfClosedQuery( + context.nativeThreadId, + context.stopping ? "interrupted" : "failed", + ); + } + }), + ), + Effect.ensuring(Deferred.succeed(closed, undefined)), + Effect.forkIn(sessionScope), + ); + return { _tag: "Opened", live: context } as const; + }), ); - live.selection = compiledSelection; - } - if ( - existing !== null && - existingNativeThreadId === nativeThreadId && - existing.queryPolicyKey === queryPolicyKey && - existing.selection.queryIdentity === compiledSelection.queryIdentity - ) { + if (decision._tag === "Prepare") { + // Read before closing the live process so a bad path leaves it running. + replacementPrep = { + appendSystemPromptFileText: yield* readClaudeAppendSystemPromptFile({ + settings: adapterOptions.settings, + cwd: decision.turnInput.runtimePolicy.cwd, + fileSystem, + path, + }), + subagentPromptCacheTtl: + decision.turnInput.appThread.lineage.relationshipToParent === "subagent" + ? yield* resolveSubagentPromptCacheTtl(decision.turnInput.runtimePolicy.cwd) + : undefined, + }; + continue; + } + if (decision._tag !== "Reuse") return decision.live; + const existing = decision.live; + if (existing.selection.queryIdentity !== compiledSelection.queryIdentity) { + yield* existing.query.setModel(compiledSelection.apiModelId).pipe( + Effect.andThen( + existing.query.applyFlagSettings( + claudeLiveSelectionFlagSettings(compiledSelection), + ), + ), + Effect.tapError(() => + Effect.sync(() => { + existing.selection = { ...existing.selection, queryIdentity: "" }; + }), + ), + ); + existing.selection = compiledSelection; + } // A denied ExitPlanMode leaves Claude in plan mode. Restore only from plan: // Auto/default may reflect a policy restriction that forbids restoring bypass. if (existing.permissionMode === "plan" && existing.openedPermissionMode !== "plan") { @@ -9432,192 +9632,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return existing; } - - // Background agents and shells run inside the CLI process, so a new selection would - // kill them and lose their results. Refuse until they finish or the user presses Stop, - // which closes the process. - if ( - existing !== null && - existingNativeThreadId === nativeThreadId && - !existing.stopping && - (yield* liveProcessRunsBackgroundWork(existing)) - ) { - return yield* new ClaudeBackgroundWorkBlocksQueryReplacementError(); - } - - // Read before closing the live process so a bad path leaves it running. - const appendSystemPromptFileText = yield* readClaudeAppendSystemPromptFile({ - settings: adapterOptions.settings, - cwd: turnInput.runtimePolicy.cwd, - fileSystem, - path, - }); - const closedExistingNativeThreadId = existing !== null ? existing.nativeThreadId : null; - if (existing !== null) { - yield* existing.query.close.pipe(Effect.ignore); - if (existing.nativeThreadId !== nativeThreadId) { - yield* clearWakeStateForNativeThread(existing.nativeThreadId); - yield* resetBackgroundTaskStateForNativeThreadProcess(existing.nativeThreadId, { - status: "idle", - }); - } - } - - const openedWithResume = (yield* Ref.get(openedNativeThreads)).has(nativeThreadId); - // openedNativeThreads is per session instance and is lost when the - // provider session is idle-released. A prior turn on this native id - // requires resume; sessionId would fail with "already in use". - const hasPersistedProviderTurn = - turnInput.nativeThreadHasTurns ?? turnInput.providerTurnOrdinal > 1; - const shouldResume = - resumeSessionAt !== undefined || openedWithResume || hasPersistedProviderTurn; - const queryOptions = makeClaudeQueryOptions({ - modelSelection: turnInput.modelSelection, - nativeThreadId, - resume: shouldResume, - ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), - cwd: turnInput.runtimePolicy.cwd, - attachmentsDir, - settings: adapterOptions.settings, - environment: claudeQueryEnvironment( - adapterOptions.environment, - turnInput.appThread, - turnInput.appThread.lineage.relationshipToParent === "subagent" - ? yield* resolveSubagentPromptCacheTtl(turnInput.runtimePolicy.cwd) - : undefined, - ), - tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, - ...(mcpOverrides.allowedTools === undefined - ? {} - : { allowedTools: mcpOverrides.allowedTools }), - ...(mcpOverrides.mcpServers === undefined - ? {} - : { mcpServers: mcpOverrides.mcpServers }), - permissionMode: queryPolicy.permissionMode, - ...(queryPolicy.allowDangerouslySkipPermissions === undefined - ? {} - : { - allowDangerouslySkipPermissions: queryPolicy.allowDangerouslySkipPermissions, - }), - canUseTool, - onUserDialog, - supportedDialogKinds: ["resume_return"], - ...(appendSystemPromptFileText === undefined ? {} : { appendSystemPromptFileText }), - }); - const querySession = yield* queryRunner - .open({ - threadId: turnInput.threadId, - providerSessionId: input.providerSessionId, - options: queryOptions, - }) - .pipe( - Effect.onError(() => - closedExistingNativeThreadId === nativeThreadId - ? Effect.gen(function* () { - yield* clearWakeStateForNativeThread(nativeThreadId); - yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { - status: "idle", - }); - }) - : Effect.void, - ), - ); - // Marked only after a successful open: a failed create must not - // leave the runtime believing the native session exists, or the - // retry would resume a session that was never created. - yield* Ref.update(openedNativeThreads, (current) => { - if (current.has(nativeThreadId)) { - return current; - } - const updated = new Set(current); - updated.add(nativeThreadId); - return updated; - }); - yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { - status: "active", - }); - const closed = yield* Deferred.make(); - const context: ClaudeLiveQueryContext = { - queryPolicy, - nativeThreadId, - query: querySession, - queryPolicyKey, - selection: compiledSelection, - closed, - lastTurn: null, - promptEchoMode: "unknown", - openedPermissionMode: queryOptions.permissionMode, - receivedInit: false, - permissionMode: queryOptions.permissionMode, - stopping: false, - subagentsFromEarlierProcesses: new Set( - [...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter( - (subagent) => subagent.task.status === "running", - ), - ), - }; - yield* Ref.set(queryContext, context); - yield* querySession.messages.pipe( - Stream.runForEach((message) => - sdkMessageLock.withPermits(1)( - Effect.gen(function* () { - if ( - message.type === "system" && - message.subtype === "init" && - Reflect.get(message, "parent_tool_use_id") == null - ) { - yield* adoptNativeSessionId(context, message.session_id); - } - if ( - message.type === "system" && - message.subtype === "init" && - !context.receivedInit - ) { - context.openedPermissionMode = - message.permissionMode ?? context.openedPermissionMode; - context.receivedInit = true; - } - if ( - message.type === "system" && - (message.subtype === "init" || message.subtype === "status") && - message.permissionMode !== undefined - ) { - context.permissionMode = message.permissionMode; - } - yield* handleSdkMessage({ query: querySession, message }); - }), - ), - ), - Effect.exit, - Effect.flatMap( - Effect.fnUntraced(function* (exit: ClaudeQueryStreamExit) { - const heldContext = yield* Ref.get(activeTurn); - if ( - (yield* Ref.get(queryContext))?.query === querySession && - heldContext !== null && - heldContext.heldRootFrames.length > 0 - ) { - yield* releaseHeldRootFrames(heldContext); - } - const ownsLiveQuery = yield* Ref.modify(queryContext, (current) => - current?.query === querySession ? [true, null] : [false, current], - ); - if (ownsLiveQuery) { - yield* finalizeActiveTurnAfterQueryExit( - context, - exit._tag === "Failure" ? exit.cause : undefined, - ); - yield* endSubagentCallsOfClosedQuery( - context.nativeThreadId, - context.stopping ? "interrupted" : "failed", - ); - } - }), - ), - Effect.ensuring(Deferred.succeed(closed, undefined)), - Effect.forkIn(sessionScope), - ); - return context; }); const startTurn = Effect.fn("ClaudeAdapterV2.startTurn")( diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 2a6a10db4d432..92fce5a248888 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -519,10 +519,10 @@ export const layerWithOptions = ( ) => Ref.get(subscribers).pipe( Effect.flatMap((current) => - Effect.forEach(current.values(), (queue) => Queue.offer(queue, signal), { - discard: true, - }), + Effect.forEach(current.values(), (queue) => Queue.offer(queue, signal)), ), + // A closing subscriber's queue refuses offers, so false means no reader got it. + Effect.map((accepted) => accepted.includes(true)), ); const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* ( @@ -1764,8 +1764,10 @@ export const layerWithOptions = ( }).pipe( Effect.flatMap((removed) => removed - ? Queue.clear(queue).pipe( - Effect.tap(() => Queue.end(queue)), + ? // Ending first makes every later offer report it was refused, so a signal is + // either drained here or seen as undelivered by its publisher. + Queue.end(queue).pipe( + Effect.andThen(Queue.clear(queue)), Effect.flatMap((unread) => persistUnreadProviderThreadUpdates(subscribers, unread), ), @@ -2066,22 +2068,26 @@ export const layerWithOptions = ( // (project trust, login, or session-switch hooks). Persist // their runless request artifacts directly so the normal T3 // request UI can answer them and unblock session setup. - // A provider thread can also change while idle (Claude adopts - // a new session after `/clear`). With no run subscribed, that - // update would be dropped and a later resume would reopen the - // old native session. - const threadId = - sessionScopedRuntimeRequestThreadId(event) ?? - (event.type === "provider_thread.updated" && - event.providerThread.appThreadId !== null && - (yield* Ref.get(entry.eventSubscribers)).size === 0 - ? event.providerThread.appThreadId - : undefined); + const threadId = sessionScopedRuntimeRequestThreadId(event); if (threadId !== undefined) { yield* ingestRunlessEvent(entry, threadId, event); return; } - yield* publishToSubscribers(entry.eventSubscribers, { type: "event", event }); + const delivered = yield* publishToSubscribers(entry.eventSubscribers, { + type: "event", + event, + }); + // A provider thread can also change while idle (Claude adopts + // a new session after `/clear`). With no run reading it, that + // update would be dropped and a later resume would reopen the + // old native session. + if ( + !delivered && + event.type === "provider_thread.updated" && + event.providerThread.appThreadId !== null + ) { + yield* ingestRunlessEvent(entry, event.providerThread.appThreadId, event); + } yield* releaseLiveOutput; }), ), From 2a93ea0a3e271cbd1f691427431cf3a99fddbb94 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:33:28 +0000 Subject: [PATCH 11/29] fix(server): fork and rollback across Claude /clear use the session the turn ran in (#24153) Adopting a /clear moves the provider thread to the new native session, so forking or rolling back to a pre-clear turn asked the new session for a message only the old one has. Fork and rollback now use the native session their run attempt recorded, and a rollback into a session /clear left stops following the adoption alias. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.ts | 2 ++ .../CheckpointRollbackService.ts | 18 +++++++++-- .../src/orchestration-v2/ProjectionStore.ts | 30 +++++++++++-------- .../ProviderTurnStartService.ts | 21 +++++++++++-- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 789e2eec72920..a41629d058a0d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -10558,6 +10558,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const nativeThreadId = yield* getNativeThreadId(rollbackInput.providerThread); + // Rolling back into a session `/clear` left makes it current again. + adoptedNativeSessionIds.delete(nativeThreadId); yield* closeLiveQueryForNativeThread(nativeThreadId); const now = yield* DateTime.now; const resumeSessionAt = yield* resolveClaudeRollbackResumeSessionAt(rollbackInput); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 96a5e10c98927..4db4b17ac6570 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -142,8 +142,22 @@ export const layer: Layer.Layer< readonly restoreFiles?: boolean; }) { const context = yield* projections.getCheckpointRollbackContext(input.threadId, input); - const { providerThread, checkpoint, scope, targetAttempt, runsToRollback, staleCheckpoints } = - context; + const { checkpoint, scope, targetAttempt, runsToRollback, staleCheckpoints } = context; + // A provider can move its thread to another native session (Claude's `/clear`), so the + // target turn's messages live in the session its attempt ran in. + const providerThread = + context.providerThread?.nativeThreadRef == null || + targetAttempt?.nativeThreadId === undefined || + targetAttempt.nativeThreadId === context.providerThread.nativeThreadRef.nativeId + ? context.providerThread + : { + ...context.providerThread, + nativeThreadRef: { + ...context.providerThread.nativeThreadRef, + nativeId: targetAttempt.nativeThreadId, + }, + nativeConversationHeadRef: null, + }; if ( providerThread === undefined || providerThread.providerSessionId === null || diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8deb5bf7e93f0..a380cc5bc6cc6 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -249,21 +249,18 @@ export interface ProjectionCheckpointCaptureContext { readonly readyCheckpointOrdinals: ReadonlyArray; } -const RollbackTargetAttempts = Schema.Array( - OrchestrationV2RunAttemptJsonSchema.mapFields(({ id, providerTurnId }) => ({ - id, - providerTurnId, - })), -); -const decodeRollbackTargetAttempts = Schema.decodeUnknownEffect(RollbackTargetAttempts); - export interface ProjectionCheckpointRollbackContext { readonly thread: OrchestrationV2AppThread; readonly providerThread: OrchestrationV2ProviderThread | undefined; readonly providerSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined; readonly checkpoint: OrchestrationV2Checkpoint | undefined; readonly scope: OrchestrationV2CheckpointScope | undefined; - readonly targetAttempt: (typeof RollbackTargetAttempts.Type)[number] | undefined; + readonly targetAttempt: + | Pick< + typeof OrchestrationV2RunAttemptJsonSchema.Type, + "id" | "nativeThreadId" | "providerTurnId" + > + | undefined; readonly providerTurns: ReadonlyArray; readonly rolledBackAttemptIds: ReadonlyArray; readonly runsToRollback: ReadonlyArray; @@ -5116,7 +5113,7 @@ export const layer: Layer.Layer = const targetOrdinal = checkpoint.appRunOrdinal ?? 0; const [attemptRows, runRows, sessionRows, rolledBackAttempts, staleRows] = yield* Effect.all([ - sql`SELECT attempt.attempt_id AS id, attempt.provider_turn_id AS "providerTurnId" + sql`SELECT attempt.payload_json FROM orchestration_v2_projection_run_attempts AS attempt JOIN orchestration_v2_projection_runs AS run ON attempt.attempt_id = json_extract(run.payload_json, '$.activeAttemptId') @@ -5142,7 +5139,10 @@ export const layer: Layer.Layer = AND app_run_ordinal > ${targetOrdinal} ORDER BY ordinal_within_scope ASC`, ]); - const targetAttempt = (yield* decodeRollbackTargetAttempts(attemptRows))[0]; + const targetAttempt = (yield* decodeRows( + decodeRunAttemptPayload, + threadId, + )(attemptRows))[0]; const runsToRollback = yield* decodeRows(decodeRunPayload, threadId)(runRows); const [nodeRows, turnRows] = yield* Effect.all([ sql`SELECT payload_json FROM orchestration_v2_projection_nodes @@ -7177,7 +7177,13 @@ export const layerMemory: Layer.Layer = Layer.effect( targetAttempt: attempt === undefined ? undefined - : { id: attempt.id, providerTurnId: attempt.providerTurnId }, + : { + id: attempt.id, + ...(attempt.nativeThreadId === undefined + ? {} + : { nativeThreadId: attempt.nativeThreadId }), + providerTurnId: attempt.providerTurnId, + }, runsToRollback, rootNodes: projection.nodes.filter((node) => rootNodeIds.has(node.id)), rolledBackAttemptIds: projection.attempts diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 59fb5ca0bcda9..b396f8f8c31f2 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -704,12 +704,27 @@ export const layer: Layer.Layer< const sourceRun = sourceProjection.runs.find( (candidate) => candidate.id === nativeForkTransfer.sourcePoint.runId, ); - const sourceProviderThread = sourceProjection.providerThreads.find( - (candidate) => candidate.id === sourceRun?.providerThreadId, - ); const sourceAttempt = sourceProjection.attempts.find( (candidate) => candidate.id === sourceRun?.activeAttemptId, ); + const currentSourceProviderThread = sourceProjection.providerThreads.find( + (candidate) => candidate.id === sourceRun?.providerThreadId, + ); + // A provider can move its thread to another native session (Claude's `/clear`), so + // fork from the session the source attempt ran in. + const sourceProviderThread = + currentSourceProviderThread?.nativeThreadRef == null || + sourceAttempt?.nativeThreadId === undefined || + sourceAttempt.nativeThreadId === currentSourceProviderThread.nativeThreadRef.nativeId + ? currentSourceProviderThread + : { + ...currentSourceProviderThread, + nativeThreadRef: { + ...currentSourceProviderThread.nativeThreadRef, + nativeId: sourceAttempt.nativeThreadId, + }, + nativeConversationHeadRef: null, + }; const sourceProviderTurn = latestProviderTurnForAttempt(sourceProjection.providerTurns, sourceAttempt?.id) ?? sourceProjection.providerTurns.find( From 21e40c676f5d20e399cd50460a4aaef85e94286f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:43:40 +0000 Subject: [PATCH 12/29] test(claude): reproduce rollback across /clear leaving the cleared session live (#24153) Rolling back into the pre-clear session leaves the live query on the session /clear adopted, so its late background notification republishes that session over the rollback. resumeThread with the pre-clear thread also returns the pre-clear session instead of following the adoption. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 2336468c89de5..6741c86be73b6 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2469,6 +2469,100 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect( + "closes the cleared session's live query when rolling back into the pre-clear one", + () => + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }); + const preClearUuid = "00000000-0000-4000-8000-000000000521"; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make("rollback-across-clear-1"), + text: "Start the build in the background.", + attachments: [], + }), + ); + yield* harness.offerAndWait(wakeTaskStarted); + yield* harness.offerAndWait( + makeAssistantTextFrame({ uuid: preClearUuid, text: "Build started." }), + ); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + const preClearTurn = lastProviderTurn(harness.events); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000522", + }), + ); + const isClearedThreadUpdate = (event: ProviderAdapter.ProviderAdapterV2Event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === "cleared-native-session"; + yield* harness.until(() => harness.events.some(isClearedThreadUpdate)); + + // History rollback hands the adapter the pre-clear session the turn ran in. + const rolledBack = yield* harness.runtime.rollbackThread({ + providerThread: harness.providerThread, + target: { + type: "provider_turn", + checkpointId: CheckpointId.make("checkpoint-before-clear"), + appRunOrdinal: 1, + providerTurn: preClearTurn, + }, + providerThreadTurns: [preClearTurn], + }); + assert.equal( + rolledBack.providerThread.nativeThreadRef?.nativeId, + harness.providerThread.nativeThreadRef?.nativeId, + ); + assert.equal(rolledBack.providerThread.nativeConversationHeadRef?.nativeId, preClearUuid); + const eventsAfterRollback = harness.events.length; + + // The cleared session's process finishing its background task after the rollback. + const clearedQuery = harness.processQueues[0]!; + const clearedQueryStillLive = clearedQuery.state._tag === "Open"; + if (clearedQueryStillLive) yield* harness.offerAndWait(wakeNotification); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: rolledBack.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make("rollback-across-clear-2"), + text: "Continue from before the clear.", + attachments: [], + providerTurnOrdinal: 2, + }), + ); + assert.deepEqual( + harness.events + .slice(eventsAfterRollback) + .filter(isClearedThreadUpdate) + .map((event) => event.type), + [], + "the cleared session overwrote the rolled-back provider thread", + ); + assert.isFalse(clearedQueryStillLive, "rollback left the cleared session's query running"); + assert.equal( + harness.getOpenedOptions()?.resume, + harness.providerThread.nativeThreadRef?.nativeId, + ); + assert.equal(harness.getOpenedOptions()?.resumeSessionAt, preClearUuid); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("keeps the cleared session when the next turn carries the pre-clear thread", () => Effect.gen(function* () { const harness = yield* makeWakeHarnessWithOptions({ @@ -2496,6 +2590,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000000511", }), ); + const resumed = yield* harness.runtime.resumeThread({ + providerThread: harness.providerThread, + }); + assert.equal(resumed.nativeThreadRef?.nativeId, "cleared-native-session"); yield* harness.runtime.startTurn(turn(2)); assert.equal(harness.processQueues.length, 1); yield* harness.offerAndWait(turnOneResult); From f7828e7c2edf5fe421335c0a285b95cc40a1ad86 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 11:50:57 +0000 Subject: [PATCH 13/29] fix(claude): rollback across /clear stops the cleared session, resume follows the adoption (#24153) Rolling back into the pre-clear session now closes the live query still running the session /clear adopted, and forgets that session first so its exit cannot republish it over the rollback. resumeThread with a pre-clear thread follows the adoption the same way a turn does. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.ts | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index a41629d058a0d..12d29eeb2fba8 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -9318,29 +9318,39 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // A turn can be built from a projection read before an idle `/clear` adoption // landed. Opening its native ID would replace the live process with the // pre-clear session, so follow the adoption this adapter already made. - const followAdoptedNativeSession = ( - turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, - ): ProviderAdapter.ProviderAdapterV2TurnInput => { - const requestedNativeId = turnInput.providerThread.nativeThreadRef?.nativeId ?? undefined; - let adoptedNativeId = requestedNativeId; - while (adoptedNativeId !== undefined && adoptedNativeSessionIds.has(adoptedNativeId)) { - adoptedNativeId = adoptedNativeSessionIds.get(adoptedNativeId); + const resolveAdoptedNativeId = (nativeId: string): string => { + let adoptedNativeId = nativeId; + while (adoptedNativeSessionIds.has(adoptedNativeId)) { + adoptedNativeId = adoptedNativeSessionIds.get(adoptedNativeId)!; } - return adoptedNativeId === undefined || adoptedNativeId === requestedNativeId - ? turnInput + return adoptedNativeId; + }; + const followAdoptedProviderThread = ( + providerThread: OrchestrationV2ProviderThread, + ): OrchestrationV2ProviderThread => { + const requestedNativeId = providerThread.nativeThreadRef?.nativeId; + if (requestedNativeId === undefined || requestedNativeId === null) return providerThread; + const adoptedNativeId = resolveAdoptedNativeId(requestedNativeId); + return adoptedNativeId === requestedNativeId + ? providerThread : { - ...turnInput, - providerThread: { - ...turnInput.providerThread, - nativeThreadRef: { - driver: CLAUDE_PROVIDER, - nativeId: adoptedNativeId, - strength: "strong", - }, - nativeConversationHeadRef: null, + ...providerThread, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: adoptedNativeId, + strength: "strong", }, + nativeConversationHeadRef: null, }; }; + const followAdoptedNativeSession = ( + turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, + ): ProviderAdapter.ProviderAdapterV2TurnInput => { + const providerThread = followAdoptedProviderThread(turnInput.providerThread); + return providerThread === turnInput.providerThread + ? turnInput + : { ...turnInput, providerThread }; + }; const openQuery = Effect.fnUntraced(function* ( requestedTurnInput: ProviderAdapter.ProviderAdapterV2TurnInput, @@ -10352,7 +10362,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), resumeThread: Effect.fn("ClaudeAdapterV2.resumeThread")( function* (threadInput: { readonly providerThread: OrchestrationV2ProviderThread }) { - const nativeThreadId = threadInput.providerThread.nativeThreadRef?.nativeId; + // A projection read before an idle `/clear` adoption still names the pre-clear session. + const providerThread = followAdoptedProviderThread(threadInput.providerThread); + const nativeThreadId = providerThread.nativeThreadRef?.nativeId; if ( nativeThreadId !== undefined && nativeThreadId !== null && @@ -10370,7 +10382,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const updatedAt = yield* DateTime.now; return { - ...threadInput.providerThread, + ...providerThread, providerSessionId: input.providerSessionId, status: "idle" as const, updatedAt, @@ -10558,8 +10570,19 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const nativeThreadId = yield* getNativeThreadId(rollbackInput.providerThread); - // Rolling back into a session `/clear` left makes it current again. + // Rolling back into a session `/clear` left makes it current again. The process + // still running the adopted session must stop, and forgetting that session keeps + // its exit from republishing it over the rollback. + const adoptedNativeThreadId = resolveAdoptedNativeId(nativeThreadId); adoptedNativeSessionIds.delete(nativeThreadId); + if (adoptedNativeThreadId !== nativeThreadId) { + yield* Ref.update(lastProviderThreadByNativeThread, (entries) => { + const next = new Map(entries); + next.delete(adoptedNativeThreadId); + return next; + }); + yield* closeLiveQueryForNativeThread(adoptedNativeThreadId); + } yield* closeLiveQueryForNativeThread(nativeThreadId); const now = yield* DateTime.now; const resumeSessionAt = yield* resolveClaudeRollbackResumeSessionAt(rollbackInput); From 652d0465d954d53c584784610ab6fa9ed2cd0823 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 12:20:01 +0000 Subject: [PATCH 14/29] test(server): reproduce forking a Claude run that adopted a session mid-turn (#24153) A run's attempt records the native session it started in. When Claude's `/clear` adopts a new session inside that run, its assistant cursor lives in the new session, but fork (and rollback) still pair the cursor with the old one. Co-Authored-By: Claude Opus 5.5 --- .../testkit/ThreadFork.integration.test.ts | 134 ++++++++++++++++++ .../claude_transcript.ndjson | 26 ++++ 2 files changed, 160 insertions(+) create mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson diff --git a/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts index d90886835a234..46a2de65e2645 100644 --- a/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts @@ -44,6 +44,7 @@ const CLAUDE_MODEL_SELECTION = { const PRIOR_TURN_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_prior_turn/codex_transcript.ndjson`; const CLAUDE_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native/claude_transcript.ndjson`; const CLAUDE_PRIOR_TURN_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_prior_turn/claude_transcript.ndjson`; +const CLAUDE_MID_TURN_CLEAR_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson`; const CLAUDE_FORK_LOCAL_ROLLBACK_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_fork_local_rollback/claude_transcript.ndjson`; const CODEX_READ_ONLY_NEVER_POLICY = { approvalPolicy: "never", @@ -661,6 +662,139 @@ describe("orchestration V2 thread fork", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + // regression: #24153 a run whose `/clear` adopted a new native session mid-turn kept the + // session it started in, so forking it asked that session for a message only the new one has. + it.effect("forks a Claude run from the session it adopted mid-turn", () => + Effect.gen(function* () { + const rawTranscript = yield* readTranscript(CLAUDE_MID_TURN_CLEAR_TRANSCRIPT_PATH); + const transcript = yield* ClaudeOrchestratorReplayHarness.decodeTranscript(rawTranscript); + const cwd = yield* Effect.acquireRelease(makeCheckpointWorkspace, (directory) => + Effect.service(FileSystem.FileSystem).pipe( + Effect.flatMap((fs) => fs.remove(directory, { recursive: true, force: true })), + Effect.orDie, + ), + ); + + const materialized = yield* Effect.gen(function* () { + const ids = yield* IdAllocator.IdAllocatorV2; + const fixtureName = "thread-fork-native-mid-turn-clear"; + const projectId = yield* ids.allocate.project({ fixtureName }); + const sourceThreadId = yield* ids.allocate.thread({ + fixtureName: `${fixtureName}-source`, + projectId, + }); + const targetThreadId = ThreadId.make(`${fixtureName}-target`); + const sourceMessage = (commandName: string, text: string) => + Effect.map( + ids.allocate.command({ fixtureName, commandName }), + (commandId) => + ({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId, + threadId: sourceThreadId, + messageId: MessageId.make(`message-${fixtureName}-${commandName}`), + text, + attachments: [], + modelSelection: CLAUDE_MODEL_SELECTION, + dispatchMode: { type: "start_immediately" }, + }) satisfies OrchestrationV2Command, + ); + const commands = [ + { + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: yield* ids.allocate.command({ + fixtureName, + commandName: "thread-create-source", + }), + threadId: sourceThreadId, + projectId, + title: "Source thread", + modelSelection: CLAUDE_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }, + yield* sourceMessage("source-message-alpha", THREAD_FORK_NATIVE_PRIOR_TURN_ALPHA_PROMPT), + // The provider reports a new native session inside this run, as Claude does for `/clear`. + yield* sourceMessage("source-message-beta", THREAD_FORK_NATIVE_PRIOR_TURN_BETA_PROMPT), + { + type: "thread.fork", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`command-${fixtureName}`), + sourceThreadId, + targetThreadId, + sourcePoint: { + type: "run", + runId: ids.derive.run({ threadId: sourceThreadId, ordinal: 2 }), + }, + title: "Forked from the cleared run", + }, + { + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: yield* ids.allocate.command({ + fixtureName, + commandName: "target-message-repeat", + }), + threadId: targetThreadId, + messageId: MessageId.make(`message-${fixtureName}-repeat`), + text: THREAD_FORK_NATIVE_PRIOR_TURN_REPEAT_PROMPT, + attachments: [], + modelSelection: CLAUDE_MODEL_SELECTION, + dispatchMode: { type: "start_immediately" }, + }, + ] satisfies ReadonlyArray; + + return { sourceThreadId, targetThreadId, commands }; + }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime); + + // The transcript expects `session.fork` against the adopted session, so forking the + // session the run started in fails the replay. + const result = yield* runOrchestratorV2ProviderReplayScenario( + { + name: "thread_fork_native_mid_turn_clear/claude", + transcript, + commands: materialized.commands, + steps: [ + { type: "dispatch", command: materialized.commands[0]!, await: true }, + { type: "advance_clock", duration: "1 millis" }, + { type: "dispatch", command: materialized.commands[1]!, await: true }, + { type: "await_thread_idle", threadId: materialized.sourceThreadId }, + { type: "dispatch", command: materialized.commands[2]!, await: true }, + { type: "await_thread_idle", threadId: materialized.sourceThreadId }, + { type: "dispatch", command: materialized.commands[3]!, await: true }, + { type: "dispatch", command: materialized.commands[4]!, await: true }, + { type: "await_thread_idle", threadId: materialized.targetThreadId }, + ], + projectionThreadIds: [materialized.sourceThreadId, materialized.targetThreadId], + runtimePolicyOverride: { cwd }, + }, + ClaudeOrchestratorReplayHarness, + ).pipe(provideDeterministicTestRuntime); + + const sourceProjection = result.projections.get(materialized.sourceThreadId); + const targetProjection = result.projections.get(materialized.targetThreadId); + assert.isDefined(sourceProjection); + assert.isDefined(targetProjection); + assert.equal( + sourceProjection.providerThreads[0]?.nativeThreadRef?.nativeId, + metadataString(transcript, "clearedNativeSessionId"), + ); + assert.equal( + targetProjection.providerThreads[0]?.nativeThreadRef?.nativeId, + metadataString(transcript, "forkedNativeSessionId"), + ); + assert.equal(targetProjection.contextTransfers[0]?.resolution?.strategy, "native_fork"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("rolls back a Claude native fork to an earlier fork-local turn", () => Effect.gen(function* () { const rawTranscript = yield* readTranscript(CLAUDE_FORK_LOCAL_ROLLBACK_TRANSCRIPT_PATH); diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson new file mode 100644 index 0000000000000..3f227ef4a90a0 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson @@ -0,0 +1,26 @@ +{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"thread_fork_native_prior_turn","metadata":{"prompts":["For this fork-boundary fixture, respond with exactly: fork boundary alpha","For this fork-boundary fixture, respond with exactly: fork boundary beta","Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."],"model":"claude-sonnet-4-6","nativeSessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","queryMode":"fork_session_prior_turn","tools":"claude_code","permissionMode":"bypassPermissions","generatedBy":"recordClaudeAgentSdkReplayTranscript","sourceAssistantMessageUuids":["be0e3276-8c27-44bf-8ed0-0b4424e9f902","c517b74b-9537-4260-afcb-186c4056c0a4"],"forkUpToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4","forkedNativeSessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","clearedNativeSessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} +{"type":"expect_outbound","label":"query.open:source","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"sessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","settings":{"showThinkingSummaries":true}}}} +{"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"For this fork-boundary fixture, respond with exactly: fork boundary alpha"},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"25b085e7-a6e9-4f30-9708-b189c961d06f","hook_name":"SessionStart:startup","hook_event":"SessionStart","uuid":"42c40147-86c8-4294-bdb4-f1815f48808e","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"25b085e7-a6e9-4f30-9708-b189c961d06f","hook_name":"SessionStart:startup","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"9804deb1-9abf-4e58-9d04-6e163dc7b0af","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_prior_turn","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"b9412e24-eefc-4838-9ede-bb6e00951d15","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_014hdmSJdwbpJ5NkFgqbhfEe","type":"message","role":"assistant","content":[{"type":"text","text":"fork boundary alpha"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":2342,"cache_read_input_tokens":9455,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2342},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","uuid":"be0e3276-8c27-44bf-8ed0-0b4424e9f902"}} +{"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"0c649726-b7a2-4e89-a62a-a9ae534905c7","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":1174,"duration_api_ms":897,"num_turns":1,"result":"fork boundary alpha","stop_reason":"end_turn","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","total_cost_usd":0.011718000000000001,"usage":{"input_tokens":3,"cache_creation_input_tokens":2342,"cache_read_input_tokens":9455,"output_tokens":6,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2342,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":6,"cache_read_input_tokens":9455,"cache_creation_input_tokens":2342,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2342},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":6,"cacheReadInputTokens":9455,"cacheCreationInputTokens":2342,"webSearchRequests":0,"costUSD":0.011718000000000001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"d384185c-9b38-44a4-8917-53d7a8765a5a"}} +{"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"For this fork-boundary fixture, respond with exactly: fork boundary beta"},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_prior_turn","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"5fd1f879-e742-4911-95e3-3dfb0141dc52","session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_0181K4CdzyVanM6Px26CQP7L","type":"message","role":"assistant","content":[{"type":"text","text":"fork boundary beta"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea","uuid":"c517b74b-9537-4260-afcb-186c4056c0a4"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2545,"duration_api_ms":3001,"num_turns":1,"result":"fork boundary beta","stop_reason":"end_turn","session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea","total_cost_usd":0.01544235,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"output_tokens":6,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":23,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":6,"cache_read_input_tokens":11797,"cache_creation_input_tokens":23,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":6,"outputTokens":12,"cacheReadInputTokens":21252,"cacheCreationInputTokens":2365,"webSearchRequests":0,"costUSD":0.01544235,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"e4f26af4-3a60-45f3-8d5c-c0f8b968c400"}} +{"type":"runtime_exit","status":"success"} +{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea","options":{"dir":"/tmp/claude-replay-thread_fork_native_prior_turn","upToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4"}}} +{"type":"emit_inbound","label":"session.forked","frame":{"type":"session.forked","sessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} +{"type":"expect_outbound","label":"query.open:fork","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"resume":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","settings":{"showThinkingSummaries":true}}}} +{"type":"expect_outbound","label":"prompt.offer:3","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"6ce9327c-4534-484c-96e5-b47fa1a3143f","hook_name":"SessionStart:resume","hook_event":"SessionStart","uuid":"d4280b57-8334-4021-971d-93bba0e38b4c","session_id":"1dc8dab2-9e1b-4475-bd46-f40e2ebca07d"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"6ce9327c-4534-484c-96e5-b47fa1a3143f","hook_name":"SessionStart:resume","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"ac1dee1e-63c5-4db8-81b7-e1f68ecadf01","session_id":"1dc8dab2-9e1b-4475-bd46-f40e2ebca07d"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_prior_turn","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"65e2ff1d-ded8-42d2-a491-70fed78b2368","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01NP2pmrWJHY1AZ8HWqgsCQS","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"The user wants me to repeat the user-visible conversation so far verbatim, including only user and assistant messages.","signature":"ErQCClsIDRgCKkAHbcjlRRK3rHsyqzZn5jI5+wWIl32x7k4E5feJpl1bBZVM6FJVJDBBV5ZgPlHGZMMALoCcDsULDBqzwNbRQ8o0MhFjbGF1ZGUtc29ubmV0LTQtNjgAEgzxdU+J4dE9HPjPJBwaDFLfMOU1oTIY4zJlVSIwkqZMdRU0W/uaPIztqHfQSORVwpEKdR6RAOJb0t5E+IOBu+tFmopmwfCbQjON3NoEKoYBs8E4pkuYLNz2Q+Qp7XeZOdCvvXUTYVIBJ90r2tl9NtEyDgCKce2VtEShQxnPZH9OhcyuEuKrL8c+Fsc8rTj6vYelMB2UJfj9KI1bREqSwoBf1oD7uwiuOz+S40+1Tx7LprJUcmw+JzkTO73YmVr8EHYGCHbQlLSeUHm7NgypEkTcOAaETiIYAQ=="}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"output_tokens":7,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","uuid":"3a81c9d0-22dd-4bbd-8f27-103c75e2878c"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01NP2pmrWJHY1AZ8HWqgsCQS","type":"message","role":"assistant","content":[{"type":"text","text":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary alpha\n\n**Assistant:** fork boundary alpha\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"output_tokens":7,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","uuid":"90f4c83b-8fff-454a-a95d-aa0cf87ff967"}} +{"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"74ba363a-dc66-4335-b18c-b335f75d4c9b","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2557,"duration_api_ms":2194,"num_turns":1,"result":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary alpha\n\n**Assistant:** fork boundary alpha\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content.","stop_reason":"end_turn","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","total_cost_usd":0.0051756,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"output_tokens":99,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":38,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":99,"cache_read_input_tokens":11797,"cache_creation_input_tokens":38,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":99,"cacheReadInputTokens":11797,"cacheCreationInputTokens":38,"webSearchRequests":0,"costUSD":0.0051756,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"6fe96da3-b212-41e9-be7b-12d5aee4f191"}} +{"type":"runtime_exit","status":"success"} From 91b52c8aee2c08056372be585f5be59b70529b09 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 12:30:01 +0000 Subject: [PATCH 15/29] fix(server): fork and rollback follow a session Claude adopted mid-run (#24153) A run's attempt kept the native session it started in, so after `/clear` adopted a new session inside the run, fork and rollback paired the run's cursor with the cleared session. Finalizing the attempt now records the native session the run last reported. Closing an event subscription drained unread provider thread updates and then waited for the session's event permit, so a newer update the pump saw refused could commit first and be overwritten by the older drained one. Close now holds the permit from ending the queue through the drained writes. Co-Authored-By: Claude Opus 5.5 --- .../ProviderSessionManager.ts | 95 +++++++++++-------- .../orchestration-v2/RunExecutionService.ts | 9 ++ 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 92fce5a248888..1716909350348 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1673,12 +1673,13 @@ export const layerWithOptions = ( ), ); - const ingestRunlessEvent = Effect.fn("ProviderSessionManager.ingestRunlessEvent")(function* ( - entry: LiveSessionEntry, - threadId: ThreadId, - event: ProviderAdapter.ProviderAdapterV2Event, - ) { - yield* Effect.gen(function* () { + // Callers hold `entry.requestEventPermit`. + const ingestRunlessEventHeld = Effect.fn("ProviderSessionManager.ingestRunlessEventHeld")( + function* ( + entry: LiveSessionEntry, + threadId: ThreadId, + event: ProviderAdapter.ProviderAdapterV2Event, + ) { const current = (yield* Ref.get(sessions)).get( sessionKey(entry.runtime.providerSessionId), ); @@ -1700,8 +1701,17 @@ export const layerWithOptions = ( }), ), ); - }).pipe(entry.requestEventPermit.withPermits(1)); - }); + }, + ); + + const ingestRunlessEvent = ( + entry: LiveSessionEntry, + threadId: ThreadId, + event: ProviderAdapter.ProviderAdapterV2Event, + ) => + ingestRunlessEventHeld(entry, threadId, event).pipe( + entry.requestEventPermit.withPermits(1), + ); // A run stops reading once it settles, but Claude can adopt a new native session // (`/clear`) in events queued behind the terminal. Persist those provider thread @@ -1709,24 +1719,15 @@ export const layerWithOptions = ( const persistUnreadProviderThreadUpdates = Effect.fn( "ProviderSessionManager.persistUnreadProviderThreadUpdates", )( - function* ( - subscribers: Ref.Ref< - ReadonlyMap> - >, - unread: ReadonlyArray, - ) { - if ((yield* Ref.get(subscribers)).size > 0) return; - const entry = [...(yield* Ref.get(sessions)).values()].find( - (candidate) => candidate.eventSubscribers === subscribers, - ); - if (entry === undefined) return; + function* (entry: LiveSessionEntry, unread: ReadonlyArray) { + if ((yield* Ref.get(entry.eventSubscribers)).size > 0) return; for (const signal of unread) { if ( signal.type === "event" && signal.event.type === "provider_thread.updated" && signal.event.providerThread.appThreadId !== null ) { - yield* ingestRunlessEvent( + yield* ingestRunlessEventHeld( entry, signal.event.providerThread.appThreadId, signal.event, @@ -1754,25 +1755,41 @@ export const layerWithOptions = ( updated.set(subscriberId, queue); return updated; }); - const close = Ref.modify(subscribers, (current) => { - if (!current.has(subscriberId)) { - return [false, current] as const; - } - const updated = new Map(current); - updated.delete(subscriberId); - return [true, updated] as const; - }).pipe( - Effect.flatMap((removed) => - removed - ? // Ending first makes every later offer report it was refused, so a signal is - // either drained here or seen as undelivered by its publisher. - Queue.end(queue).pipe( - Effect.andThen(Queue.clear(queue)), - Effect.flatMap((unread) => - persistUnreadProviderThreadUpdates(subscribers, unread), - ), - ) - : Effect.void, + const removeAndDrain = (entry: LiveSessionEntry | undefined) => + Ref.modify(subscribers, (current) => { + if (!current.has(subscriberId)) { + return [false, current] as const; + } + const updated = new Map(current); + updated.delete(subscriberId); + return [true, updated] as const; + }).pipe( + Effect.flatMap((removed) => + removed + ? // Ending first makes every later offer report it was refused, so a signal is + // either drained here or seen as undelivered by its publisher. + Queue.end(queue).pipe( + Effect.andThen(Queue.clear(queue)), + Effect.flatMap((unread) => + entry === undefined + ? Effect.void + : persistUnreadProviderThreadUpdates(entry, unread), + ), + ) + : Effect.void, + ), + ); + // The pump persists a refused update under the same permit. Holding it from + // the end of the queue through the drained writes keeps an older drained + // update from committing over a newer refused one. + const close = Ref.get(sessions).pipe( + Effect.map((current) => + [...current.values()].find((candidate) => candidate.eventSubscribers === subscribers), + ), + Effect.flatMap((entry) => + entry === undefined + ? removeAndDrain(undefined) + : removeAndDrain(entry).pipe(entry.requestEventPermit.withPermits(1)), ), ); // One signal per pull, so what a reader never pulled is still queued at close. diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 11ab94cefa69b..5e0b5978d940e 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -635,8 +635,17 @@ export const layer: Layer.Layer< }; }) { const completedAt = input.completedAt ?? (yield* DateTime.now); + // Claude's `/clear` can move a run to a new native session after its attempt was + // written; rollback and fork must pair this run's cursor with that session. + const reportedNativeThreadId = + input.providerThread.id === input.attempt.providerThreadId + ? input.providerThread.nativeThreadRef?.nativeId + : undefined; const finalizedAttempt: OrchestrationV2RunAttempt | null = { ...input.attempt, + ...(input.attempt.nativeThreadId === undefined || reportedNativeThreadId == null + ? {} + : { nativeThreadId: reportedNativeThreadId }), status: input.terminal.status, completedAt, }; From a10d7c565296904984f883ef064e5ff957d65a76 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:42:39 +0100 Subject: [PATCH 16/29] fix(claude): preserve adopted sessions across unread updates and release --- .../ProviderEventIngestor.test.ts | 94 +++++-- .../orchestration-v2/ProviderEventIngestor.ts | 20 +- .../ProviderSessionManager.test.ts | 205 ++++++++++++++ .../ProviderSessionManager.ts | 259 +++++++++++++----- .../RunExecutionService.test.ts | 25 ++ .../orchestration-v2/RunExecutionService.ts | 79 +++--- .../testkit/ThreadFork.integration.test.ts | 17 ++ .../claude_transcript.ndjson | 14 +- .../src/server/ProviderAdapter.ts | 10 + 9 files changed, 571 insertions(+), 152 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 849cedded9292..c999a0d529035 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -417,7 +417,7 @@ layer("ProviderEventIngestorV2", (it) => { }), ); - it.effect("rolls back runs a provider rewound off its active branch", () => + it.effect.each([false, true])("rewinds runs (skip snapshot: %s)", (skipSnapshot) => Effect.gen(function* () { const now = yield* DateTime.now; const eventSink = yield* EventSink.EventSinkV2; @@ -602,7 +602,63 @@ layer("ProviderEventIngestorV2", (it) => { nativeTurnId: null, }); - yield* ingestor.ingestNormalized({ + const reportedProviderThread: OrchestrationV2ProviderThread = { + id: rewoundThreadId, + driver: CODEX_DRIVER, + providerInstanceId: modelSelection.instanceId, + providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "rewound-thread", + strength: "strong", + }, + nativeConversationHeadRef: { + driver: CODEX_DRIVER, + nativeId: "kept-turn-reply", + strength: "strong", + }, + status: "idle", + firstRunOrdinal: 2, + lastRunOrdinal: 5, + handoffIds: [], + forkedFrom: null, + pendingBackgroundTasks: [], + contextUsage: null, + nativeMetadata: null, + goal: null, + createdAt: now, + updatedAt: now, + }; + const currentProviderThread: OrchestrationV2ProviderThread = { + ...reportedProviderThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "current-native-session", + strength: "strong", + }, + nativeConversationHeadRef: { + driver: CODEX_DRIVER, + nativeId: "current-head", + strength: "strong", + }, + lastRunOrdinal: 6, + }; + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: currentProviderThread, + }, + ], + }); + + const storedEvents = yield* ingestor.ingestNormalized({ + skipProviderThreadSnapshot: skipSnapshot, providerSessionId, providerInstanceId: modelSelection.instanceId, threadId, @@ -610,31 +666,7 @@ layer("ProviderEventIngestorV2", (it) => { event: { type: "provider_thread.updated", driver: CODEX_DRIVER, - providerThread: { - id: rewoundThreadId, - driver: CODEX_DRIVER, - providerInstanceId: modelSelection.instanceId, - providerSessionId, - appThreadId: threadId, - ownerNodeId: null, - nativeThreadRef: { - driver: CODEX_DRIVER, - nativeId: "rewound-thread", - strength: "strong", - }, - nativeConversationHeadRef: { - driver: CODEX_DRIVER, - nativeId: "kept-turn-reply", - strength: "strong", - }, - status: "idle", - firstRunOrdinal: 2, - lastRunOrdinal: 5, - handoffIds: [], - forkedFrom: null, - createdAt: now, - updatedAt: now, - }, + providerThread: reportedProviderThread, retainedNativeTurnIds: ["turn-started-outside-t3", "kept-turn"], }, }); @@ -656,9 +688,13 @@ layer("ProviderEventIngestorV2", (it) => { ), ["ready", "ready", "stale"], ); + assert.deepEqual( + projection.providerThreads.find((thread) => thread.id === rewoundThreadId), + skipSnapshot ? currentProviderThread : reportedProviderThread, + ); assert.equal( - projection.providerThreads.find((thread) => thread.id === rewoundThreadId)?.lastRunOrdinal, - 5, + storedEvents.some((stored) => stored.event.type === "provider-thread.updated"), + !skipSnapshot, ); }), ); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 8e916ab79319b..226d1a5b9d9ea 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -247,6 +247,7 @@ function runsOffNativeBranch(input: { } export interface ProviderEventIngestInput { + readonly skipProviderThreadSnapshot?: boolean; readonly providerSessionId: ProviderSessionId; readonly providerInstanceId: ProviderInstanceId; readonly commandId?: CommandId; @@ -564,15 +565,20 @@ export const layer: Layer.Layer< ]; case "provider_thread.updated": { const threadId = input.event.providerThread.appThreadId ?? input.threadId; - const updated = yield* makeDomainEvent(input, { - type: "provider-thread.updated", - threadId, - payload: input.event.providerThread, - }); + const updates = + input.skipProviderThreadSnapshot === true + ? [] + : [ + yield* makeDomainEvent(input, { + type: "provider-thread.updated", + threadId, + payload: input.event.providerThread, + }), + ]; return input.event.retainedNativeTurnIds === undefined - ? [updated] + ? updates : [ - updated, + ...updates, ...(yield* rollBackRewoundRuns(input, { threadId, providerThreadId: input.event.providerThread.id, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 56f6ab47cdab8..09f4c86e56ce9 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3123,6 +3123,211 @@ it.effect( }), ); +it.effect.each([ + "late subscriber", + "last recipient", + "runtime failure", + "newer received update", + "unrelated reader", + "coalesced updates", + "rejected write", + "released rewind", +] as const)("ProviderSessionManagerV2 preserves unread thread updates with %s", (scenario) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: `unread-update-${scenario}`, + projectId: yield* idAllocator.allocate.project({ fixtureName: "unread-updates" }), + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const first = yield* runtime.subscribeEvents!; + const secondBefore = + scenario === "last recipient" || scenario === "unrelated reader" + ? yield* runtime.subscribeEvents! + : undefined; + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + const sessionUpdates = yield* Queue.unbounded(); + const afterSequence = yield* eventSink.latestSequence({ threadId }); + yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter((stored) => stored.event.type === "provider-session.updated"), + Stream.runForEach(() => Queue.offer(sessionUpdates, undefined)), + Effect.forkScoped, + ); + const update = (nativeId: string) => ({ + type: "provider_thread.updated" as const, + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { driver: CODEX_DRIVER, nativeId, strength: "strong" as const }, + }, + }); + const routed = Effect.gen(function* () { + yield* Queue.offer(adapterEvents, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + yield* Queue.take(sessionUpdates); + }); + const persist = + (event: ProviderAdapter.ProviderAdapterV2Event) => (snapshotCurrent: boolean) => + !snapshotCurrent || event.type !== "provider_thread.updated" + ? Effect.succeed([]) + : eventSink.write({ + events: [ + { + id: yieldEventId, + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: event.providerThread, + }, + ], + }); + const yieldEventId = yield* idAllocator.allocate.event({ threadId }); + if (scenario === "runtime failure") { + yield* Queue.offer(adapterEvents, update("old-session")); + yield* Queue.offer(adapterEvents, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "release-terminal", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + } + if (scenario === "released rewind") + yield* Queue.offer(adapterEvents, { + ...update("rewound-session"), + retainedNativeTurnIds: [], + }); + yield* Queue.offer(adapterEvents, update("cleared-session")); + if (scenario === "coalesced updates") + yield* Queue.offer(adapterEvents, update("newer-session")); + yield* routed; + if (scenario === "runtime failure" || scenario === "released rewind") { + const released = yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-session.updated" && + stored.event.payload.status === "error", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Queue.end(adapterEvents); + yield* Fiber.join(released); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + yield* first.events.pipe( + Stream.mapEffect((event) => + first.persistEvent!(event, (snapshotCurrent) => { + assert.isFalse( + event.type === "provider_thread.updated" && + event.retainedNativeTurnIds !== undefined, + "A committed rewind must not run again after release", + ); + return persist(event)(snapshotCurrent); + }), + ), + Stream.take(2), + Stream.runDrain, + ); + } else if (scenario === "newer received update") { + const second = yield* runtime.subscribeEvents!; + const newer = update("newer-session"); + yield* Queue.offer(adapterEvents, newer); + yield* routed; + yield* second.events.pipe( + Stream.mapEffect((event) => second.persistEvent!(event, persist(event))), + Stream.take(1), + Stream.runDrain, + ); + yield* first.close; + } else if (scenario === "unrelated reader") { + const discarded = yield* Deferred.make(); + const reader = yield* secondBefore!.events.pipe( + Stream.tap((event) => + secondBefore!.discardEvent!(event).pipe( + Effect.andThen( + event.type === "provider_thread.updated" + ? Deferred.succeed(discarded, undefined) + : Effect.void, + ), + ), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Deferred.await(discarded); + yield* first.close; + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0]?.nativeThreadRef + ?.nativeId, + "cleared-session", + ); + yield* Fiber.interrupt(reader); + } else if (scenario === "rejected write") { + yield* first.events.pipe( + Stream.mapEffect((event) => first.persistEvent!(event, () => Effect.succeed([]))), + Stream.take(1), + Stream.runDrain, + ); + } else { + const second = secondBefore ?? (yield* runtime.subscribeEvents!); + yield* first.close; + if (scenario === "last recipient") { + assert.isEmpty((yield* projectionStore.getThreadProjection(threadId)).providerThreads); + } + yield* second.close; + } + const eventStore = yield* EventStore.EventStoreV2; + const persistedUpdates = yield* eventStore + .read({ threadId, afterSequence, eventType: "provider-thread.updated" }) + .pipe(Stream.runCollect); + assert.lengthOf(persistedUpdates, 1); + const projection = yield* projectionStore.getThreadProjection(threadId); + assert.deepEqual( + projection.providerThreads.map((thread) => thread.nativeThreadRef?.nativeId), + [ + scenario === "newer received update" || scenario === "coalesced updates" + ? "newer-session" + : "cleared-session", + ], + ); + }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 }))); + }), +); + it.effect( "ProviderSessionManagerV2 opens one shared runtime, broadcasts events, and detaches threads independently", () => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 1716909350348..7cb6695e20966 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -203,6 +203,12 @@ interface LiveSessionEntry { readonly eventSubscribers: Ref.Ref< ReadonlyMap> >; + readonly latestThreadUpdates: Map; + readonly threadUpdateDeliveries: WeakMap< + ProviderAdapter.ProviderAdapterV2Event, + ProviderThreadUpdateDelivery + >; + readonly pendingRewinds: Map; readonly requestEventPermit: Semaphore.Semaphore; readonly scope: Scope.Closeable; readonly idleGeneration: number; @@ -219,8 +225,21 @@ interface IdleThreadUnload { readonly fiber: Fiber.Fiber | null; } +interface ProviderThreadUpdateDelivery { + readonly event: Extract< + ProviderAdapter.ProviderAdapterV2Event, + { readonly type: "provider_thread.updated" } + >; + readonly recipients: Set; + received: boolean; +} + type ProviderSessionEventSignal = - | { readonly type: "event"; readonly event: ProviderAdapter.ProviderAdapterV2Event } + | { + readonly type: "event"; + readonly event: ProviderAdapter.ProviderAdapterV2Event; + readonly threadUpdate?: ProviderThreadUpdateDelivery; + } | { readonly type: "failure"; readonly cause: Cause.Cause; @@ -511,19 +530,36 @@ export const layerWithOptions = ( ), ); - const publishToSubscribers = ( - subscribers: Ref.Ref< - ReadonlyMap> - >, - signal: ProviderSessionEventSignal, - ) => - Ref.get(subscribers).pipe( - Effect.flatMap((current) => - Effect.forEach(current.values(), (queue) => Queue.offer(queue, signal)), - ), - // A closing subscriber's queue refuses offers, so false means no reader got it. - Effect.map((accepted) => accepted.includes(true)), - ); + const publishToSubscribers = Effect.fn("ProviderSessionManager.publishToSubscribers")( + function* (entry: LiveSessionEntry, signal: ProviderSessionEventSignal) { + const current = yield* Ref.get(entry.eventSubscribers); + const threadUpdate = + signal.type === "event" && signal.event.type === "provider_thread.updated" + ? { event: signal.event, recipients: new Set(), received: false } + : undefined; + if (threadUpdate !== undefined) { + entry.latestThreadUpdates.set(threadUpdate.event.providerThread.id, threadUpdate); + entry.threadUpdateDeliveries.set(threadUpdate.event, threadUpdate); + if (threadUpdate.event.retainedNativeTurnIds !== undefined) { + entry.pendingRewinds.set(threadUpdate.event, threadUpdate); + } + } + let delivered = false; + for (const [id, queue] of current) { + const accepted = yield* Queue.offer( + queue, + threadUpdate === undefined + ? signal + : { type: "event", event: threadUpdate.event, threadUpdate }, + ); + if (accepted) { + delivered = true; + threadUpdate?.recipients.add(id); + } + } + return delivered; + }, + ); const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* ( entry: LiveSessionEntry, @@ -927,25 +963,40 @@ export const layerWithOptions = ( if (candidate === undefined) { return [Option.none(), yield* DateTime.now] as const; } - const removed = yield* Effect.zip( - Ref.modify(sessions, (current) => { - const existing = current.get(key); - if (existing !== candidate) { - return [existing === undefined ? "gone" : "changed", current] as const; - } - if ( - input.onlyIfIdleGeneration !== undefined && - (existing.busyTurns.size > 0 || - existing.idleGeneration !== input.onlyIfIdleGeneration) - ) { - return ["kept", current] as const; - } - const updated = new Map(current); - updated.delete(key); - return ["removed", updated] as const; - }), - DateTime.now, - ).pipe(candidate.requestEventPermit.withPermits(1)); + const removed = yield* Effect.gen(function* () { + const latest = (yield* Ref.get(sessions)).get(key); + if (latest !== candidate) { + return [latest === undefined ? "gone" : "changed", yield* DateTime.now] as const; + } + if ( + input.onlyIfIdleGeneration !== undefined && + (latest.busyTurns.size > 0 || latest.idleGeneration !== input.onlyIfIdleGeneration) + ) { + return ["kept", yield* DateTime.now] as const; + } + // Unread updates still belong to this runtime. Save them before removal makes + // subscription cleanup unable to find it, including on an event-stream failure. + yield* persistPendingThreadUpdates(candidate, true); + return yield* Effect.zip( + Ref.modify(sessions, (current) => { + const existing = current.get(key); + if (existing !== candidate) { + return [existing === undefined ? "gone" : "changed", current] as const; + } + if ( + input.onlyIfIdleGeneration !== undefined && + (existing.busyTurns.size > 0 || + existing.idleGeneration !== input.onlyIfIdleGeneration) + ) { + return ["kept", current] as const; + } + const updated = new Map(current); + updated.delete(key); + return ["removed", updated] as const; + }), + DateTime.now, + ); + }).pipe(candidate.requestEventPermit.withPermits(1)); const [outcome, releasedAt] = removed; if (outcome === "changed") return yield* removeLiveEntry(input); return [ @@ -1679,6 +1730,7 @@ export const layerWithOptions = ( entry: LiveSessionEntry, threadId: ThreadId, event: ProviderAdapter.ProviderAdapterV2Event, + skipProviderThreadSnapshot = false, ) { const current = (yield* Ref.get(sessions)).get( sessionKey(entry.runtime.providerSessionId), @@ -1690,6 +1742,7 @@ export const layerWithOptions = ( providerInstanceId: entry.runtime.instanceId, threadId, event, + ...(skipProviderThreadSnapshot ? { skipProviderThreadSnapshot: true } : {}), }) .pipe( Effect.mapError( @@ -1713,26 +1766,25 @@ export const layerWithOptions = ( entry.requestEventPermit.withPermits(1), ); - // A run stops reading once it settles, but Claude can adopt a new native session - // (`/clear`) in events queued behind the terminal. Persist those provider thread - // updates when the last reader leaves, or a later resume reopens the old session. - const persistUnreadProviderThreadUpdates = Effect.fn( - "ProviderSessionManager.persistUnreadProviderThreadUpdates", + // Keep only the newest snapshot per provider thread. A subscriber joining later + // is not a recipient of an earlier signal, and must not prevent its persistence. + // Callers hold requestEventPermit, including the release cutoff. + const persistPendingThreadUpdates = Effect.fn( + "ProviderSessionManager.persistPendingThreadUpdates", )( - function* (entry: LiveSessionEntry, unread: ReadonlyArray) { - if ((yield* Ref.get(entry.eventSubscribers)).size > 0) return; - for (const signal of unread) { - if ( - signal.type === "event" && - signal.event.type === "provider_thread.updated" && - signal.event.providerThread.appThreadId !== null - ) { - yield* ingestRunlessEventHeld( - entry, - signal.event.providerThread.appThreadId, - signal.event, - ); - } + function* (entry: LiveSessionEntry, releasing = false) { + const pending = new Set([ + ...entry.pendingRewinds.values(), + ...entry.latestThreadUpdates.values(), + ]); + for (const update of pending) { + if (update.received || (!releasing && update.recipients.size > 0)) continue; + const threadId = update.event.providerThread.appThreadId; + if (threadId === null) continue; + const latest = entry.latestThreadUpdates.get(update.event.providerThread.id); + yield* ingestRunlessEventHeld(entry, threadId, update.event, latest !== update); + update.received = true; + entry.pendingRewinds.delete(update.event); } }, Effect.catchCause((cause) => @@ -1771,9 +1823,21 @@ export const layerWithOptions = ( Queue.end(queue).pipe( Effect.andThen(Queue.clear(queue)), Effect.flatMap((unread) => - entry === undefined - ? Effect.void - : persistUnreadProviderThreadUpdates(entry, unread), + Effect.gen(function* () { + for (const signal of unread) { + if (signal.type === "event") + signal.threadUpdate?.recipients.delete(subscriberId); + } + if (entry !== undefined) { + for (const update of new Set([ + ...entry.latestThreadUpdates.values(), + ...entry.pendingRewinds.values(), + ])) { + update.recipients.delete(subscriberId); + } + yield* persistPendingThreadUpdates(entry); + } + }), ), ) : Effect.void, @@ -1801,7 +1865,49 @@ export const layerWithOptions = ( ), Stream.ensuring(close), ); - return { events, close } satisfies ProviderAdapter.ProviderAdapterV2EventSubscription; + const entry = [...(yield* Ref.get(sessions)).values()].find( + (candidate) => candidate.eventSubscribers === subscribers, + ); + const persistEvent: NonNullable< + ProviderAdapter.ProviderAdapterV2EventSubscription["persistEvent"] + > = (event, persist) => + event.type !== "provider_thread.updated" || entry === undefined + ? persist(true) + : Effect.gen(function* () { + const current = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + const latest = entry.latestThreadUpdates.get(event.providerThread.id); + const delivery = entry.threadUpdateDeliveries.get(event); + if (delivery?.received === true) return []; + const snapshotCurrent = + current?.runtime === entry.runtime && + (latest?.event === event || latest?.received !== true); + const stored = yield* persist(snapshotCurrent); + if ( + delivery !== undefined && + (stored.some((record) => record.event.type === "provider-thread.updated") || + (event.retainedNativeTurnIds !== undefined && stored.length > 0)) + ) { + delivery.received = true; + } + if (delivery?.received === true) entry.pendingRewinds.delete(event); + return stored; + }).pipe(entry.requestEventPermit.withPermits(1)); + const discardEvent = (event: ProviderAdapter.ProviderAdapterV2Event) => + entry === undefined || event.type !== "provider_thread.updated" + ? Effect.void + : Effect.gen(function* () { + const delivery = entry.threadUpdateDeliveries.get(event); + delivery?.recipients.delete(subscriberId); + yield* persistPendingThreadUpdates(entry); + }).pipe(entry.requestEventPermit.withPermits(1)); + return { + events, + close, + persistEvent, + discardEvent, + } satisfies ProviderAdapter.ProviderAdapterV2EventSubscription; }, ); @@ -2090,21 +2196,26 @@ export const layerWithOptions = ( yield* ingestRunlessEvent(entry, threadId, event); return; } - const delivered = yield* publishToSubscribers(entry.eventSubscribers, { - type: "event", - event, - }); - // A provider thread can also change while idle (Claude adopts - // a new session after `/clear`). With no run reading it, that - // update would be dropped and a later resume would reopen the - // old native session. - if ( - !delivered && - event.type === "provider_thread.updated" && - event.providerThread.appThreadId !== null - ) { - yield* ingestRunlessEvent(entry, event.providerThread.appThreadId, event); - } + yield* Effect.gen(function* () { + const delivered = yield* publishToSubscribers(entry, { + type: "event", + event, + }); + // A provider thread can also change while idle (Claude adopts + // a new session after `/clear`). With no run reading it, that + // update would be dropped and a later resume would reopen the + // old native session. + if ( + !delivered && + event.type === "provider_thread.updated" && + event.providerThread.appThreadId !== null + ) { + yield* ingestRunlessEventHeld(entry, event.providerThread.appThreadId, event); + const update = entry.latestThreadUpdates.get(event.providerThread.id); + if (update !== undefined) update.received = true; + entry.pendingRewinds.delete(event); + } + }).pipe(entry.requestEventPermit.withPermits(1)); yield* releaseLiveOutput; }), ), @@ -2139,11 +2250,10 @@ export const layerWithOptions = ( cause: "Provider event stream ended unexpectedly.", }), ); - yield* publishToSubscribers(entry.eventSubscribers, { + yield* publishToSubscribers(entry, { type: "failure", cause, }); - yield* Ref.set(entry.eventSubscribers, new Map()); yield* releaseEntry({ providerSessionId: entry.runtime.providerSessionId, reason: "runtime_error", @@ -2318,6 +2428,9 @@ export const layerWithOptions = ( runtime, exposedRuntime, eventSubscribers, + latestThreadUpdates: new Map(), + threadUpdateDeliveries: new WeakMap(), + pendingRewinds: new Map(), requestEventPermit: yield* Semaphore.make(1), scope: sessionScope, idleGeneration: 0, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index e7937f8a1fc9f..58d5954009790 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2349,6 +2349,23 @@ it.effect( }), ); +it.effect.each([undefined, "native-session-before-clear"])( + "records the reported native session when finalizing an attempt initialized with %s", + (nativeThreadId) => + Effect.gen(function* () { + const { finalEvents } = yield* captureRootRunTermination({ + key: "finalize-reported-native-session", + shouldFinalizeRun: () => Effect.succeed(true), + ...(nativeThreadId === undefined ? {} : { nativeThreadId }), + reportedNativeThreadId: "native-session-after-clear", + }); + const attempt = finalEvents.find((event) => event.type === "run-attempt.updated"); + assert.isDefined(attempt); + assert.equal(attempt.payload.nativeThreadId, "native-session-after-clear"); + assert.equal(attempt.payload.status, "interrupted"); + }), +); + it.effect("omits interrupt results and subagent cascade for a superseded attempt", () => Effect.gen(function* () { const { written, observed } = yield* captureRootRunTermination({ @@ -2712,6 +2729,8 @@ function captureRootRunTermination(input: { readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"]; readonly interruptTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["interruptTurn"]; readonly initialProviderTurnId?: ProviderTurnId | null; + readonly nativeThreadId?: string; + readonly reportedNativeThreadId?: string; readonly onStartFailure?: ( error: RunExecutionService.RunExecutionServiceV2Error, ) => Effect.Effect; @@ -2870,9 +2889,15 @@ function captureRootRunTermination(input: { providerThread: { id: ids.providerThreadId, driver, + nativeThreadRef: + input.reportedNativeThreadId === undefined + ? null + : { driver, nativeId: input.reportedNativeThreadId, strength: "strong" }, } as OrchestrationV2ProviderThread, attempt: { id: ids.attemptId, + providerThreadId: ids.providerThreadId, + ...(input.nativeThreadId === undefined ? {} : { nativeThreadId: input.nativeThreadId }), providerTurnId: input.initialProviderTurnId === undefined ? ids.rootProviderTurnId diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 5e0b5978d940e..a4bfef12df34b 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -643,9 +643,7 @@ export const layer: Layer.Layer< : undefined; const finalizedAttempt: OrchestrationV2RunAttempt | null = { ...input.attempt, - ...(input.attempt.nativeThreadId === undefined || reportedNativeThreadId == null - ? {} - : { nativeThreadId: reportedNativeThreadId }), + ...(reportedNativeThreadId == null ? {} : { nativeThreadId: reportedNativeThreadId }), status: input.terminal.status, completedAt, }; @@ -1009,7 +1007,7 @@ export const layer: Layer.Layer< attemptId: input.attempt.id, providerThreadId: input.providerThread.id, }; - const eventSubscription = + const eventSubscription: ProviderAdapter.ProviderAdapterV2EventSubscription = input.session.subscribeEvents === undefined ? { events: input.session.events, close: Effect.void } : yield* input.session.subscribeEvents; @@ -1361,7 +1359,11 @@ export const layer: Layer.Layer< ), Stream.tap(({ event, accepted }) => Effect.gen(function* () { - if (!accepted) return; + if (!accepted) { + if (eventSubscription.discardEvent !== undefined) + yield* eventSubscription.discardEvent(event); + return; + } if ( event.type === "provider_turn.updated" && event.providerTurn.runAttemptId === input.attempt.id && @@ -1379,37 +1381,42 @@ export const layer: Layer.Layer< const isRootProviderThreadUpdate = event.type === "provider_thread.updated" && event.providerThread.id === input.providerThread.id; - const storedEvents = yield* providerEventIngestor.ingestNormalized({ - analyticsContext: { - modelSelection: input.modelSelection, - runtimeMode: input.runtimePolicy.runtimeMode, - interactionMode: input.runtimePolicy.interactionMode, - }, - providerSessionId: input.providerSessionId, - providerInstanceId: input.run.providerInstanceId, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - event: deliveredEvent, - ...(isRootProviderThreadUpdate - ? rootTerminalAlreadySeen - ? { - writeIfProviderThreadOwner: { - providerThreadId: input.providerThread.id, - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedLastRunOrdinal: input.run.ordinal, - }, - } - : { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } - : {}), - }); + const persist = (snapshotCurrent: boolean) => + providerEventIngestor.ingestNormalized({ + ...(!snapshotCurrent ? { skipProviderThreadSnapshot: true } : {}), + analyticsContext: { + modelSelection: input.modelSelection, + runtimeMode: input.runtimePolicy.runtimeMode, + interactionMode: input.runtimePolicy.interactionMode, + }, + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: deliveredEvent, + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } + : {}), + }); + const storedEvents = yield* eventSubscription.persistEvent === undefined + ? persist(true) + : eventSubscription.persistEvent(event, persist); storedEventCount = storedEvents.length; if ( isRootProviderThreadUpdate && diff --git a/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts index 46a2de65e2645..7f6b364a3ce69 100644 --- a/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts @@ -792,6 +792,23 @@ describe("orchestration V2 thread fork", () => { metadataString(transcript, "forkedNativeSessionId"), ); assert.equal(targetProjection.contextTransfers[0]?.resolution?.strategy, "native_fork"); + const targetAssistantText = targetProjection.turnItems + .filter((item) => item.type === "assistant_message") + .map((item) => item.text) + .join("\n"); + assert.include(targetAssistantText, "**Assistant:** fork boundary beta"); + assert.notInclude( + targetAssistantText, + "fork boundary alpha", + "the cleared native session must not recover pre-clear conversation context", + ); + const visibleText = targetProjection.visibleTurnItems + .map((row) => row.item) + .filter((item) => item.type === "user_message" || item.type === "assistant_message") + .map((item) => item.text) + .join("\n"); + assert.include(visibleText, "fork boundary alpha"); + assert.include(visibleText, "fork boundary beta"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson index 3f227ef4a90a0..17b44ab629c36 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson @@ -1,26 +1,26 @@ -{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"thread_fork_native_prior_turn","metadata":{"prompts":["For this fork-boundary fixture, respond with exactly: fork boundary alpha","For this fork-boundary fixture, respond with exactly: fork boundary beta","Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."],"model":"claude-sonnet-4-6","nativeSessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","queryMode":"fork_session_prior_turn","tools":"claude_code","permissionMode":"bypassPermissions","generatedBy":"recordClaudeAgentSdkReplayTranscript","sourceAssistantMessageUuids":["be0e3276-8c27-44bf-8ed0-0b4424e9f902","c517b74b-9537-4260-afcb-186c4056c0a4"],"forkUpToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4","forkedNativeSessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","clearedNativeSessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} +{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"thread_fork_native_mid_turn_clear","metadata":{"prompts":["For this fork-boundary fixture, respond with exactly: fork boundary alpha","For this fork-boundary fixture, respond with exactly: fork boundary beta","Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."],"model":"claude-sonnet-4-6","nativeSessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","queryMode":"fork_session_prior_turn","tools":"claude_code","permissionMode":"bypassPermissions","generatedBy":"synthetic adaptation of thread_fork_native_prior_turn/claude_transcript.ndjson","sourceAssistantMessageUuids":["be0e3276-8c27-44bf-8ed0-0b4424e9f902","c517b74b-9537-4260-afcb-186c4056c0a4"],"forkUpToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4","forkedNativeSessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","clearedNativeSessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} {"type":"expect_outbound","label":"query.open:source","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"sessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","settings":{"showThinkingSummaries":true}}}} {"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"For this fork-boundary fixture, respond with exactly: fork boundary alpha"},"parent_tool_use_id":null}}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"25b085e7-a6e9-4f30-9708-b189c961d06f","hook_name":"SessionStart:startup","hook_event":"SessionStart","uuid":"42c40147-86c8-4294-bdb4-f1815f48808e","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"25b085e7-a6e9-4f30-9708-b189c961d06f","hook_name":"SessionStart:startup","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"9804deb1-9abf-4e58-9d04-6e163dc7b0af","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} -{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_prior_turn","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"b9412e24-eefc-4838-9ede-bb6e00951d15","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_mid_turn_clear","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"b9412e24-eefc-4838-9ede-bb6e00951d15","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} {"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_014hdmSJdwbpJ5NkFgqbhfEe","type":"message","role":"assistant","content":[{"type":"text","text":"fork boundary alpha"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":2342,"cache_read_input_tokens":9455,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2342},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","uuid":"be0e3276-8c27-44bf-8ed0-0b4424e9f902"}} {"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"0c649726-b7a2-4e89-a62a-a9ae534905c7","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} {"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":1174,"duration_api_ms":897,"num_turns":1,"result":"fork boundary alpha","stop_reason":"end_turn","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","total_cost_usd":0.011718000000000001,"usage":{"input_tokens":3,"cache_creation_input_tokens":2342,"cache_read_input_tokens":9455,"output_tokens":6,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2342,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":6,"cache_read_input_tokens":9455,"cache_creation_input_tokens":2342,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2342},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":6,"cacheReadInputTokens":9455,"cacheCreationInputTokens":2342,"webSearchRequests":0,"costUSD":0.011718000000000001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"d384185c-9b38-44a4-8917-53d7a8765a5a"}} {"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"For this fork-boundary fixture, respond with exactly: fork boundary beta"},"parent_tool_use_id":null}}} -{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_prior_turn","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"5fd1f879-e742-4911-95e3-3dfb0141dc52","session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_mid_turn_clear","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"5fd1f879-e742-4911-95e3-3dfb0141dc52","session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} {"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_0181K4CdzyVanM6Px26CQP7L","type":"message","role":"assistant","content":[{"type":"text","text":"fork boundary beta"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea","uuid":"c517b74b-9537-4260-afcb-186c4056c0a4"}} {"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2545,"duration_api_ms":3001,"num_turns":1,"result":"fork boundary beta","stop_reason":"end_turn","session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea","total_cost_usd":0.01544235,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"output_tokens":6,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":23,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":6,"cache_read_input_tokens":11797,"cache_creation_input_tokens":23,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":6,"outputTokens":12,"cacheReadInputTokens":21252,"cacheCreationInputTokens":2365,"webSearchRequests":0,"costUSD":0.01544235,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"e4f26af4-3a60-45f3-8d5c-c0f8b968c400"}} {"type":"runtime_exit","status":"success"} -{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea","options":{"dir":"/tmp/claude-replay-thread_fork_native_prior_turn","upToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4"}}} +{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea","options":{"dir":"/tmp/claude-replay-thread_fork_native_mid_turn_clear","upToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4"}}} {"type":"emit_inbound","label":"session.forked","frame":{"type":"session.forked","sessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} {"type":"expect_outbound","label":"query.open:fork","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"resume":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","settings":{"showThinkingSummaries":true}}}} {"type":"expect_outbound","label":"prompt.offer:3","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."},"parent_tool_use_id":null}}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"6ce9327c-4534-484c-96e5-b47fa1a3143f","hook_name":"SessionStart:resume","hook_event":"SessionStart","uuid":"d4280b57-8334-4021-971d-93bba0e38b4c","session_id":"1dc8dab2-9e1b-4475-bd46-f40e2ebca07d"}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"6ce9327c-4534-484c-96e5-b47fa1a3143f","hook_name":"SessionStart:resume","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"ac1dee1e-63c5-4db8-81b7-e1f68ecadf01","session_id":"1dc8dab2-9e1b-4475-bd46-f40e2ebca07d"}} -{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_prior_turn","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"65e2ff1d-ded8-42d2-a491-70fed78b2368","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_mid_turn_clear","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"65e2ff1d-ded8-42d2-a491-70fed78b2368","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} {"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01NP2pmrWJHY1AZ8HWqgsCQS","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"The user wants me to repeat the user-visible conversation so far verbatim, including only user and assistant messages.","signature":"ErQCClsIDRgCKkAHbcjlRRK3rHsyqzZn5jI5+wWIl32x7k4E5feJpl1bBZVM6FJVJDBBV5ZgPlHGZMMALoCcDsULDBqzwNbRQ8o0MhFjbGF1ZGUtc29ubmV0LTQtNjgAEgzxdU+J4dE9HPjPJBwaDFLfMOU1oTIY4zJlVSIwkqZMdRU0W/uaPIztqHfQSORVwpEKdR6RAOJb0t5E+IOBu+tFmopmwfCbQjON3NoEKoYBs8E4pkuYLNz2Q+Qp7XeZOdCvvXUTYVIBJ90r2tl9NtEyDgCKce2VtEShQxnPZH9OhcyuEuKrL8c+Fsc8rTj6vYelMB2UJfj9KI1bREqSwoBf1oD7uwiuOz+S40+1Tx7LprJUcmw+JzkTO73YmVr8EHYGCHbQlLSeUHm7NgypEkTcOAaETiIYAQ=="}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"output_tokens":7,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","uuid":"3a81c9d0-22dd-4bbd-8f27-103c75e2878c"}} -{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01NP2pmrWJHY1AZ8HWqgsCQS","type":"message","role":"assistant","content":[{"type":"text","text":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary alpha\n\n**Assistant:** fork boundary alpha\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"output_tokens":7,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","uuid":"90f4c83b-8fff-454a-a95d-aa0cf87ff967"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01NP2pmrWJHY1AZ8HWqgsCQS","type":"message","role":"assistant","content":[{"type":"text","text":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary beta\n\n**Assistant:** fork boundary beta\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"output_tokens":7,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","uuid":"90f4c83b-8fff-454a-a95d-aa0cf87ff967"}} {"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"74ba363a-dc66-4335-b18c-b335f75d4c9b","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} -{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2557,"duration_api_ms":2194,"num_turns":1,"result":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary alpha\n\n**Assistant:** fork boundary alpha\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content.","stop_reason":"end_turn","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","total_cost_usd":0.0051756,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"output_tokens":99,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":38,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":99,"cache_read_input_tokens":11797,"cache_creation_input_tokens":38,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":99,"cacheReadInputTokens":11797,"cacheCreationInputTokens":38,"webSearchRequests":0,"costUSD":0.0051756,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"6fe96da3-b212-41e9-be7b-12d5aee4f191"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2557,"duration_api_ms":2194,"num_turns":1,"result":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary beta\n\n**Assistant:** fork boundary beta\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content.","stop_reason":"end_turn","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","total_cost_usd":0.0051756,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"output_tokens":99,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":38,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":99,"cache_read_input_tokens":11797,"cache_creation_input_tokens":38,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":99,"cacheReadInputTokens":11797,"cacheCreationInputTokens":38,"webSearchRequests":0,"costUSD":0.0051756,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"6fe96da3-b212-41e9-be7b-12d5aee4f191"}} {"type":"runtime_exit","status":"success"} diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 314705eca3afc..04da44144043f 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -14,6 +14,7 @@ import { OrchestrationV2ProviderFailure, OrchestrationV2ProviderRetry, OrchestrationV2ProviderThread, + OrchestrationV2StoredEvent, OrchestrationV2ProviderTurn, OrchestrationV2RuntimeRequest, OrchestrationV2Subagent, @@ -504,6 +505,15 @@ export interface ProviderAdapterV2ForkThreadInput { export interface ProviderAdapterV2EventSubscription { readonly events: Stream.Stream; readonly close: Effect.Effect; + // Managed sessions serialize snapshot persistence with delivery and release. Rewind + // side effects may still apply when the accompanying thread snapshot is stale. + readonly discardEvent?: (event: ProviderAdapterV2Event) => Effect.Effect; + readonly persistEvent?: ( + event: ProviderAdapterV2Event, + persist: ( + snapshotCurrent: boolean, + ) => Effect.Effect, E, R>, + ) => Effect.Effect, E, R>; } export interface ProviderAdapterV2HistoricalContext { From b10bab61d6762e103a1b086020e111e909ef0a51 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Sun, 25 Oct 2026 00:56:26 +0100 Subject: [PATCH 17/29] fix(server): keep stale session updates from undoing selection restarts --- apps/server/src/orchestration-v2/EventSink.ts | 25 ++++-- .../ProviderSessionManager.test.ts | 69 ++++++++++++++- .../ProviderSessionManager.ts | 88 +++++++++++++------ 3 files changed, 145 insertions(+), 37 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 3d2bc8e4b45a8..27e24f1e767bb 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -135,10 +135,17 @@ export interface EventSinkV2Shape { * cannot overwrite an update published in between. The whole row is * compared because not every writer moves `updatedAt`. */ - readonly writeIfProviderThreadUnchanged: (input: { - readonly expected: OrchestrationV2ProviderThread; - readonly events: ReadonlyArray; - }) => Effect.Effect< + readonly writeIfProviderThreadUnchanged: ( + input: { + readonly events: ReadonlyArray; + } & ( + | { readonly expected: OrchestrationV2ProviderThread } + | { + readonly expected: null; + readonly providerThreadId: ProviderThreadId; + } + ), + ) => Effect.Effect< { readonly committed: boolean; readonly storedEvents: ReadonlyArray; @@ -588,18 +595,22 @@ const layerBase: Layer.Layer< const writeIfProviderThreadUnchangedEffect = Effect.fn( "orchestrationV2.EventSink.writeIfProviderThreadUnchanged", )(function* (input: Parameters[0]) { + const providerThreadId = input.expected === null ? input.providerThreadId : input.expected.id; yield* Effect.annotateCurrentSpan({ "orchestration_v2.event_count": input.events.length, - "orchestration_v2.provider_thread_id": input.expected.id, + "orchestration_v2.provider_thread_id": providerThreadId, }); - const expectedJson = yield* ProjectionStore.encodeProviderThreadPayload(input.expected); + const expectedJson = + input.expected === null + ? undefined + : yield* ProjectionStore.encodeProviderThreadPayload(input.expected); return yield* commitThenPublish( Effect.gen(function* () { const rows = yield* sql<{ readonly payload_json: string }>` SELECT payload_json FROM orchestration_v2_projection_provider_threads - WHERE provider_thread_id = ${input.expected.id} + WHERE provider_thread_id = ${providerThreadId} LIMIT 1 `; if (rows[0]?.payload_json !== expectedJson) { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 09f4c86e56ce9..76d93397a5439 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3132,9 +3132,14 @@ it.effect.each([ "coalesced updates", "rejected write", "released rewind", + "replacement owner", + "concurrent owner switch", + "replacement insert", + "same-owner write", ] as const)("ProviderSessionManagerV2 preserves unread thread updates with %s", (scenario) => Effect.gen(function* () { const state = yield* Ref.make(emptyState); + const beforeUnchangedWrite = yield* Ref.make>(Effect.void); yield* Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const idAllocator = yield* IdAllocator.IdAllocatorV2; @@ -3171,6 +3176,23 @@ it.effect.each([ : undefined; const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; const sessionUpdates = yield* Queue.unbounded(); + if ( + scenario === "replacement owner" || + scenario === "concurrent owner switch" || + scenario === "same-owner write" + ) { + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: providerThread, + }, + ], + }); + } const afterSequence = yield* eventSink.latestSequence({ threadId }); yield* eventSink.stream({ threadId, afterSequence }).pipe( Stream.filter((stored) => stored.event.type === "provider-session.updated"), @@ -3296,6 +3318,43 @@ it.effect.each([ "cleared-session", ); yield* Fiber.interrupt(reader); + } else if ( + scenario === "replacement owner" || + scenario === "concurrent owner switch" || + scenario === "replacement insert" || + scenario === "same-owner write" + ) { + const replacement = { + ...update("newer-session").providerThread, + providerSessionId: + scenario === "same-owner write" + ? providerSessionId + : yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }), + }; + const switchOwner = eventSink + .write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: replacement, + }, + ], + }) + .pipe(Effect.asVoid, Effect.orDie); + if (scenario !== "replacement owner") yield* Ref.set(beforeUnchangedWrite, switchOwner); + else yield* switchOwner; + yield* first.close; + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0] + ?.providerSessionId, + replacement.providerSessionId, + ); } else if (scenario === "rejected write") { yield* first.events.pipe( Stream.mapEffect((event) => first.persistEvent!(event, () => Effect.succeed([]))), @@ -3314,17 +3373,21 @@ it.effect.each([ const persistedUpdates = yield* eventStore .read({ threadId, afterSequence, eventType: "provider-thread.updated" }) .pipe(Stream.runCollect); - assert.lengthOf(persistedUpdates, 1); + assert.lengthOf(persistedUpdates, scenario === "same-owner write" ? 2 : 1); const projection = yield* projectionStore.getThreadProjection(threadId); assert.deepEqual( projection.providerThreads.map((thread) => thread.nativeThreadRef?.nativeId), [ - scenario === "newer received update" || scenario === "coalesced updates" + scenario === "newer received update" || + scenario === "coalesced updates" || + scenario === "replacement owner" || + scenario === "concurrent owner switch" || + scenario === "replacement insert" ? "newer-session" : "cleared-session", ], ); - }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 }))); + }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, beforeUnchangedWrite }))); }), ); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 7cb6695e20966..708d56e08c2b1 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1732,29 +1732,55 @@ export const layerWithOptions = ( event: ProviderAdapter.ProviderAdapterV2Event, skipProviderThreadSnapshot = false, ) { - const current = (yield* Ref.get(sessions)).get( - sessionKey(entry.runtime.providerSessionId), - ); - if (current?.runtime !== entry.runtime) return; - yield* providerEventIngestor - .ingestNormalized({ - providerSessionId: entry.runtime.providerSessionId, - providerInstanceId: entry.runtime.instanceId, - threadId, - event, - ...(skipProviderThreadSnapshot ? { skipProviderThreadSnapshot: true } : {}), - }) - .pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterEventStreamError({ - driver: entry.runtime.driver, - providerSessionId: entry.runtime.providerSessionId, - cause, - }), - ), + const input = { + providerSessionId: entry.runtime.providerSessionId, + providerInstanceId: entry.runtime.instanceId, + threadId, + event, + ...(skipProviderThreadSnapshot ? { skipProviderThreadSnapshot: true } : {}), + }; + for (let attempt = 0; attempt < 3; attempt++) { + const current = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + if (current?.runtime !== entry.runtime) return true; + if (event.type !== "provider_thread.updated") { + yield* providerEventIngestor.ingestNormalized(input); + return true; + } + const projection = yield* projectionStore.getThreadRecords(threadId, [ + "providerThreads", + ]); + const owner = projection.providerThreads.find( + (thread) => thread.id === event.providerThread.id, ); + // A pooled old runtime can outlive a selection restart. Its unread snapshot + // must not reclaim a thread that has moved to the replacement session. + if (owner !== undefined && owner.providerSessionId !== entry.runtime.providerSessionId) + return true; + const events = yield* providerEventIngestor.normalize(input); + if (events.length === 0) return true; + const written = yield* eventSink.writeIfProviderThreadUnchanged({ + ...(owner === undefined + ? { expected: null, providerThreadId: event.providerThread.id } + : { expected: owner }), + events, + }); + if (written.committed) return true; + } + return false; }, + (effect, entry) => + effect.pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: entry.runtime.driver, + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + ), + ), ); const ingestRunlessEvent = ( @@ -1782,9 +1808,13 @@ export const layerWithOptions = ( const threadId = update.event.providerThread.appThreadId; if (threadId === null) continue; const latest = entry.latestThreadUpdates.get(update.event.providerThread.id); - yield* ingestRunlessEventHeld(entry, threadId, update.event, latest !== update); - update.received = true; - entry.pendingRewinds.delete(update.event); + update.received = yield* ingestRunlessEventHeld( + entry, + threadId, + update.event, + latest !== update, + ); + if (update.received) entry.pendingRewinds.delete(update.event); } }, Effect.catchCause((cause) => @@ -2210,10 +2240,14 @@ export const layerWithOptions = ( event.type === "provider_thread.updated" && event.providerThread.appThreadId !== null ) { - yield* ingestRunlessEventHeld(entry, event.providerThread.appThreadId, event); + const received = yield* ingestRunlessEventHeld( + entry, + event.providerThread.appThreadId, + event, + ); const update = entry.latestThreadUpdates.get(event.providerThread.id); - if (update !== undefined) update.received = true; - entry.pendingRewinds.delete(event); + if (update !== undefined) update.received = received; + if (received) entry.pendingRewinds.delete(event); } }).pipe(entry.requestEventPermit.withPermits(1)); yield* releaseLiveOutput; From 1c5b8fb0cef3617a9042786f9f36f7bf4c758a81 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:17:14 +0100 Subject: [PATCH 18/29] fix(server): address review findings for routing and unread updates Route a superseded plan from an earlier run when it belongs to the current thread, so the superseded status reaches the projection instead of being discarded. Catch a failed runless provider_thread.updated fallback write, log it, and leave the update pending instead of ending the event pump and releasing the session. Both changes have regression tests that fail on the previous code. --- .../ProviderSessionManager.test.ts | 97 +++++++++++++++++++ .../ProviderSessionManager.ts | 12 ++- .../RunExecutionService.test.ts | 50 ++++++++++ .../orchestration-v2/RunExecutionService.ts | 10 +- 4 files changed, 167 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 76d93397a5439..433b71df258f3 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3015,6 +3015,103 @@ it.effect("ProviderSessionManagerV2 persists a provider thread update without a }), ); +it.effect( + "ProviderSessionManagerV2 keeps the event pump alive when a runless thread update fails", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const beforeUnchangedWrite = yield* Ref.make>(Effect.void); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-runless-update-failure", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-runless-update-failure", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const afterOpen = yield* eventSink.latestSequence({ threadId }); + const persistedFiber = yield* eventSink.stream({ threadId, afterSequence: afterOpen }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-thread.updated" && + stored.event.payload.nativeThreadRef?.nativeId === "next-session", + ), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterEvents); + // Fail the first unread-snapshot write, then let later writes through. + yield* Ref.set( + beforeUnchangedWrite, + Effect.die(new Error("simulated unread snapshot write failure")), + ); + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "cleared-session", + strength: "strong", + }, + }, + }); + // The pump must keep processing after the failed fallback write. + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "next-session", + strength: "strong", + }, + }, + }); + yield* Fiber.join(persistedFiber); + + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0]?.nativeThreadRef + ?.nativeId, + "next-session", + ); + }); + + yield* effect.pipe( + Effect.provide(layerTest({ state, idleTimeoutMs: 1000, beforeUnchangedWrite })), + ); + }), +); + it.effect( "ProviderSessionManagerV2 persists a provider thread update queued behind a closing run", () => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 708d56e08c2b1..b972ca7f1c0a2 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2240,12 +2240,22 @@ export const layerWithOptions = ( event.type === "provider_thread.updated" && event.providerThread.appThreadId !== null ) { + // Keep the update pending when persistence fails. Letting the + // failure escape would end the event pump and tear down the + // session for an unread snapshot that a later retry can save. const received = yield* ingestRunlessEventHeld( entry, event.providerThread.appThreadId, event, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "orchestration-v2.driver-session.unread-thread-update-failed", + { providerSessionId: entry.runtime.providerSessionId, cause }, + ).pipe(Effect.as(false)), + ), ); - const update = entry.latestThreadUpdates.get(event.providerThread.id); + const update = entry.threadUpdateDeliveries.get(event); if (update !== undefined) update.received = received; if (received) entry.pendingRewinds.delete(event); } diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 2d304a044b977..55cad2bcb2c0e 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -6,6 +6,7 @@ import { CommandId, MessageId, NodeId, + PlanId, type OrchestrationV2AppThread, type OrchestrationV2CheckpointScope, type OrchestrationV2DomainEvent, @@ -169,6 +170,55 @@ it.effect("routes shared-runtime events only to their owning root run", () => }), ); +it("routes a superseded plan from an earlier run on the same thread", () => { + const threadId = ThreadId.make("thread:superseded-plan"); + const identity: RunExecutionService.ProviderEventRouteIdentity = { + threadId, + runId: RunId.make("run:superseded-plan:current"), + attemptId: RunAttemptId.make("attempt:superseded-plan:current"), + providerThreadId: ProviderThreadId.make("provider-thread:superseded-plan"), + }; + const state = RunExecutionService.makeProviderEventRoutingState({ + identity, + providerTurnId: null, + }); + const plan = { + id: PlanId.make("plan:superseded-plan"), + threadId, + runId: RunId.make("run:superseded-plan:earlier"), + nodeId: NodeId.make("node:superseded-plan"), + kind: "proposed_plan" as const, + status: "superseded" as const, + markdown: "# Earlier plan", + }; + + assert.isTrue( + RunExecutionService.routeProviderEvent( + { type: "plan.updated", driver, plan }, + identity, + state, + )[0], + ); + assert.isFalse( + RunExecutionService.routeProviderEvent( + { type: "plan.updated", driver, plan: { ...plan, status: "active" } }, + identity, + state, + )[0], + ); + assert.isFalse( + RunExecutionService.routeProviderEvent( + { + type: "plan.updated", + driver, + plan: { ...plan, threadId: ThreadId.make("thread:superseded-plan:other") }, + }, + identity, + state, + )[0], + ); +}); + it("routes only enrolled workflow members after the root turn ends", () => { const threadId = ThreadId.make("thread:late-child"); const rootProviderTurnId = ProviderTurnId.make("provider-turn:late-child"); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index e952465209413..faf958eb963b5 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -510,7 +510,15 @@ export function routeProviderEvent( return [true, { ...state, inheritedBackgroundTurnItems }]; } case "plan.updated": - return [ownsRun(event.plan.runId) || ownsChildThread(event.plan.threadId), state]; + return [ + ownsRun(event.plan.runId) || + ownsChildThread(event.plan.threadId) || + // A replacement plan supersedes the previous plan, which still carries + // the run that created it. Accept that update for the current thread so + // the superseded status reaches the projection instead of being dropped. + (event.plan.status === "superseded" && event.plan.threadId === input.threadId), + state, + ]; case "runtime_request.updated": return [ (event.threadId !== undefined && ownsChildThread(event.threadId)) || From 5e1c4d9cdd65e8c5fbe6e2a6f24062da42af918c Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:28:54 +0100 Subject: [PATCH 19/29] fix(server): keep finalization on the persisted provider thread When persistEvent skipped an update already persisted during release, it returned no stored events, so the run's latestProviderThread stayed stale and finalization could emit the old native session. The subscription now reports the committed snapshot when it skips, and RunExecutionService only marks the owner lost for a genuine rejection. Adds a regression test for finalization with a skipped event write. --- .../ProviderSessionManager.ts | 23 +++++++++--- .../RunExecutionService.test.ts | 36 +++++++++++++++++++ .../orchestration-v2/RunExecutionService.ts | 25 +++++++++---- .../src/server/ProviderAdapter.ts | 9 ++++- 4 files changed, 81 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index b972ca7f1c0a2..043a6fba5e82c 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1902,27 +1902,42 @@ export const layerWithOptions = ( ProviderAdapter.ProviderAdapterV2EventSubscription["persistEvent"] > = (event, persist) => event.type !== "provider_thread.updated" || entry === undefined - ? persist(true) + ? persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))) : Effect.gen(function* () { const current = (yield* Ref.get(sessions)).get( sessionKey(entry.runtime.providerSessionId), ); const latest = entry.latestThreadUpdates.get(event.providerThread.id); const delivery = entry.threadUpdateDeliveries.get(event); - if (delivery?.received === true) return []; + if (delivery?.received === true) { + // The release path may persist the update before this subscriber reads it. + // Report the committed snapshot so finalization cannot emit the old session. + return { + storedEvents: [], + ...(current?.runtime === entry.runtime && latest?.received === true + ? { persistedProviderThread: latest.event.providerThread } + : {}), + }; + } const snapshotCurrent = current?.runtime === entry.runtime && (latest?.event === event || latest?.received !== true); const stored = yield* persist(snapshotCurrent); + const persistedSnapshot = stored.some( + (record) => record.event.type === "provider-thread.updated", + ); if ( delivery !== undefined && - (stored.some((record) => record.event.type === "provider-thread.updated") || + (persistedSnapshot || (event.retainedNativeTurnIds !== undefined && stored.length > 0)) ) { delivery.received = true; } if (delivery?.received === true) entry.pendingRewinds.delete(event); - return stored; + return { + storedEvents: stored, + ...(persistedSnapshot ? { persistedProviderThread: event.providerThread } : {}), + }; }).pipe(entry.requestEventPermit.withPermits(1)); const discardEvent = (event: ProviderAdapter.ProviderAdapterV2Event) => entry === undefined || event.type !== "provider_thread.updated" diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 55cad2bcb2c0e..9321a0987889f 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2788,12 +2788,47 @@ it.effect("keeps completed runs completed when pull request refresh fails", () = }), ); +it.effect("finalizes with a provider snapshot already persisted by a skipped event write", () => + Effect.gen(function* () { + const { finalEvents } = yield* captureRootRunTermination({ + key: "skipped-provider-thread-persist", + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.make( + { + type: "provider_thread.updated", + driver, + providerThread: { + id: ids.providerThreadId, + driver, + nativeThreadRef: { driver, nativeId: "adopted-session", strength: "strong" }, + } as unknown as OrchestrationV2ProviderThread, + } as ProviderAdapter.ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + ), + persistEvent: (event, persist) => + event.type === "provider_thread.updated" + ? Effect.succeed({ + storedEvents: [], + persistedProviderThread: event.providerThread, + }) + : persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))), + }); + assert.equal( + finalEvents.find((event) => event.type === "provider-thread.updated")?.payload.nativeThreadRef + ?.nativeId, + "adopted-session", + ); + }), +); + function captureRootRunTermination(input: { readonly key: string; readonly shouldFinalizeRun: () => Effect.Effect; readonly rejectTerminalWrite?: boolean; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly seedOpenSubagent?: boolean; + readonly persistEvent?: ProviderAdapter.ProviderAdapterV2EventSubscription["persistEvent"]; readonly events?: ( ids: BackgroundScenarioIds, ) => Stream.Stream< @@ -2943,6 +2978,7 @@ function captureRootRunTermination(input: { close: Deferred.succeed(ingestionDone, undefined).pipe( Effect.andThen(input.onSubscriptionClose ?? Effect.void), ), + ...(input.persistEvent === undefined ? {} : { persistEvent: input.persistEvent }), }), startTurn: input.startTurn ?? (() => Effect.void), interruptTurn: input.interruptTurn ?? (() => Effect.void), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index faf958eb963b5..a7fa0b592f9fb 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1394,6 +1394,7 @@ export const layer: Layer.Layer< yield* Ref.set(latestRootProviderTurn, event.providerTurn); } let storedEventCount = 0; + let persistedProviderThread: OrchestrationV2ProviderThread | undefined; const deliveredEvent = filterAssistantEvent( event, DateTime.toEpochMillis(yield* DateTime.now), @@ -1441,14 +1442,17 @@ export const layer: Layer.Layer< } : {}), }); - const storedEvents = yield* eventSubscription.persistEvent === undefined - ? persist(true) - : eventSubscription.persistEvent(event, persist); - storedEventCount = storedEvents.length; + const persisted: ProviderAdapter.ProviderAdapterV2PersistEventResult = + yield* eventSubscription.persistEvent === undefined + ? persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))) + : eventSubscription.persistEvent(event, persist); + storedEventCount = persisted.storedEvents.length; + persistedProviderThread = persisted.persistedProviderThread; if ( isRootProviderThreadUpdate && rootTerminalAlreadySeen && - storedEventCount === 0 + storedEventCount === 0 && + persisted.persistedProviderThread === undefined ) { yield* Ref.set(providerThreadOwnerLost, true); } @@ -1467,8 +1471,15 @@ export const layer: Layer.Layer< reservedHeldTurnItemIds.add(event.turnItem.id); } if (event.type === "provider_thread.updated") { - if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { - yield* Ref.set(latestProviderThread, event.providerThread); + if (event.providerThread.id === input.providerThread.id) { + // A skipped write (already persisted during release) still reports the + // committed snapshot; a rejected write must not move the session forward. + const providerThread = + persistedProviderThread ?? + (storedEventCount > 0 ? event.providerThread : undefined); + if (providerThread !== undefined) { + yield* Ref.set(latestProviderThread, providerThread); + } } } if ( diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 04da44144043f..c634ddc99f88d 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -502,6 +502,13 @@ export interface ProviderAdapterV2ForkThreadInput { readonly runtimePolicy?: ProviderAdapterV2RuntimePolicy; } +export interface ProviderAdapterV2PersistEventResult { + readonly storedEvents: ReadonlyArray; + // Set when the snapshot is already persisted, so a skipped write still lets + // consumers (finalization) use the committed provider thread. + readonly persistedProviderThread?: OrchestrationV2ProviderThread; +} + export interface ProviderAdapterV2EventSubscription { readonly events: Stream.Stream; readonly close: Effect.Effect; @@ -513,7 +520,7 @@ export interface ProviderAdapterV2EventSubscription { persist: ( snapshotCurrent: boolean, ) => Effect.Effect, E, R>, - ) => Effect.Effect, E, R>; + ) => Effect.Effect; } export interface ProviderAdapterV2HistoricalContext { From ab3fea5df17e9688c34645f8b43ec48acad4b66e Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:39:15 +0100 Subject: [PATCH 20/29] fix(server): only adopt a snapshot that actually persisted The finalization fallback treated any stored event as proof that the provider-thread snapshot persisted, so a skipped stale snapshot with rewind events could still overwrite the adopted session. Check the stored records for this provider thread instead. Adds a regression test. --- .../RunExecutionService.test.ts | 37 +++++++++++++++++++ .../orchestration-v2/RunExecutionService.ts | 28 +++++++++----- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 9321a0987889f..5713945cd3915 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2822,6 +2822,43 @@ it.effect("finalizes with a provider snapshot already persisted by a skipped eve }), ); +it.effect("does not adopt a stale snapshot that only stored rewind events", () => + Effect.gen(function* () { + const rewind = { + sequence: 1, + commandId: null, + event: { type: "turn-item.updated", payload: { id: "rewind" } }, + } as unknown as ProviderAdapter.ProviderAdapterV2PersistEventResult["storedEvents"][number]; + const { finalEvents } = yield* captureRootRunTermination({ + key: "stale-provider-thread-persist", + shouldFinalizeRun: () => Effect.succeed(true), + reportedNativeThreadId: "original-session", + events: (ids) => + Stream.make( + { + type: "provider_thread.updated", + driver, + providerThread: { + id: ids.providerThreadId, + driver, + nativeThreadRef: { driver, nativeId: "stale-session", strength: "strong" }, + } as unknown as OrchestrationV2ProviderThread, + } as ProviderAdapter.ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + ), + persistEvent: (event, persist) => + event.type === "provider_thread.updated" + ? Effect.succeed({ storedEvents: [rewind] }) + : persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))), + }); + assert.equal( + finalEvents.find((event) => event.type === "provider-thread.updated")?.payload.nativeThreadRef + ?.nativeId, + "original-session", + ); + }), +); + function captureRootRunTermination(input: { readonly key: string; readonly shouldFinalizeRun: () => Effect.Effect; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index a7fa0b592f9fb..a370a00693358 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1395,6 +1395,7 @@ export const layer: Layer.Layer< } let storedEventCount = 0; let persistedProviderThread: OrchestrationV2ProviderThread | undefined; + let persistedProviderThreadSnapshot = false; const deliveredEvent = filterAssistantEvent( event, DateTime.toEpochMillis(yield* DateTime.now), @@ -1448,6 +1449,14 @@ export const layer: Layer.Layer< : eventSubscription.persistEvent(event, persist); storedEventCount = persisted.storedEvents.length; persistedProviderThread = persisted.persistedProviderThread; + persistedProviderThreadSnapshot = + persisted.persistedProviderThread !== undefined || + (event.type === "provider_thread.updated" && + persisted.storedEvents.some( + (record) => + record.event.type === "provider-thread.updated" && + record.event.payload.id === event.providerThread.id, + )); if ( isRootProviderThreadUpdate && rootTerminalAlreadySeen && @@ -1471,15 +1480,16 @@ export const layer: Layer.Layer< reservedHeldTurnItemIds.add(event.turnItem.id); } if (event.type === "provider_thread.updated") { - if (event.providerThread.id === input.providerThread.id) { - // A skipped write (already persisted during release) still reports the - // committed snapshot; a rejected write must not move the session forward. - const providerThread = - persistedProviderThread ?? - (storedEventCount > 0 ? event.providerThread : undefined); - if (providerThread !== undefined) { - yield* Ref.set(latestProviderThread, providerThread); - } + if ( + event.providerThread.id === input.providerThread.id && + persistedProviderThreadSnapshot + ) { + // Only a snapshot that actually persisted may replace the session. + // Rewind events stored alongside a skipped snapshot are not evidence. + yield* Ref.set( + latestProviderThread, + persistedProviderThread ?? event.providerThread, + ); } } if ( From b30eaef2a6fa36b074b1435d8ff6938f7ebb0893 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:49:10 +0000 Subject: [PATCH 21/29] fix(claude): retain awaited task marks when adopting a cleared session --- .../Adapters/ClaudeAdapterV2.test.ts | 73 +++++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 1 + 2 files changed, 74 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 9605b35f6027b..024eb5f4763dd 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -6976,6 +6976,79 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("keeps awaited background task marks when /clear adopts a new session", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-clear-awaited-task"), + text: "Wait for the background task.", + attachments: [], + }), + ); + const nextThread = Effect.fn("nextAwaitedThread")(function* ( + nativeId: string, + awaited?: boolean, + ) { + while (true) { + const { providerThread } = yield* Queue.take(harness.providerThreadReceipts); + const task = providerThread.pendingBackgroundTasks?.find( + (task) => task.taskId === WAKE_TASK_ID, + ); + if ( + providerThread.nativeThreadRef?.nativeId === nativeId && + task !== undefined && + (awaited === undefined || task.awaited === awaited) + ) + return providerThread; + } + }); + const roster = (sessionId: string, uuid: string) => + claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [{ task_id: WAKE_TASK_ID, task_type: "local_bash", description: "test task" }], + session_id: sessionId, + uuid, + }); + yield* harness.offerAndWait( + roster(WAKE_NATIVE_SESSION, "00000000-0000-4000-8000-000000000610"), + ); + yield* nextThread(WAKE_NATIVE_SESSION); + yield* harness.runtime.awaitBackgroundTasks!({ + providerThread: harness.providerThread, + taskIds: [WAKE_TASK_ID], + awaited: true, + }); + yield* nextThread(WAKE_NATIVE_SESSION, true); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-awaited-session", + uuid: "00000000-0000-4000-8000-000000000611", + }), + ); + yield* nextThread("cleared-awaited-session"); + yield* harness.offerAndWait( + roster("cleared-awaited-session", "00000000-0000-4000-8000-000000000612"), + ); + const updated = yield* nextThread("cleared-awaited-session"); + assert.isTrue( + updated.pendingBackgroundTasks?.find((task) => task.taskId === WAKE_TASK_ID)?.awaited, + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("holds only the background tasks the agent marks awaited, until they end", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 75650a2f163b0..7ad7227907fdf 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4513,6 +4513,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // /clear replaces the transcript, not the live process and its background routing. yield* Ref.update(lastTurnRouteByNativeThread, rekey); yield* Ref.update(pendingBackgroundTasksByNativeThread, rekey); + yield* Ref.update(awaitedBackgroundTasksByNativeThread, rekey); yield* Ref.update(wakeEligibleBackgroundTasksByNativeThread, rekey); yield* Ref.update(opaqueBackgroundTaskReplayTombstonesByNativeThread, rekey); // Claude keys its task store by session unless CLAUDE_CODE_TASK_LIST_ID is set, so From 7033d666fdaf351afcd09ce0833d72d1470f18f7 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:39:25 +0000 Subject: [PATCH 22/29] fix(claude): preserve launch settings during session preparation --- .../src/orchestration-v2/Adapters/ClaudeAdapterV2.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3df7fe57e0b01..9e8e55a2304ea 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -9886,6 +9886,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( let replacementPrep: | { readonly appendSystemPromptFileText: string | undefined; + readonly launchArgSettings: ClaudeSdkSettings | undefined; readonly toolSearchOverride: "true" | undefined; readonly subagentPromptCacheTtl: typeof ClaudePromptCacheTtl.Type | undefined; } @@ -9962,6 +9963,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( cwd: turnInput.runtimePolicy.cwd, attachmentsDir, settings: adapterOptions.settings, + ...(prep.launchArgSettings === undefined + ? {} + : { sdkSettings: prep.launchArgSettings }), ...(prep.toolSearchOverride === undefined ? {} : { toolSearchOverride: prep.toolSearchOverride }), @@ -10111,9 +10115,17 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); if (decision._tag === "Prepare") { // Read before closing the live process so a bad path leaves it running. + const launchArgSettings = yield* readClaudeLaunchArgSettings({ + settings: adapterOptions.settings, + cwd: decision.turnInput.runtimePolicy.cwd, + fileSystem, + path, + }); replacementPrep = { + launchArgSettings, toolSearchOverride: yield* resolveToolSearchOverride( decision.turnInput.runtimePolicy.cwd, + launchArgSettings, ), appendSystemPromptFileText: yield* readClaudeAppendSystemPromptFile({ settings: adapterOptions.settings, From d2d114f753bd8b7e3883582af90148126d70497e Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:43:16 +0000 Subject: [PATCH 23/29] fix(claude): preserve launch features and release frame lock during mod closure --- .../orchestration-v2/Adapters/ClaudeAdapterV2.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index f5d7da2638bd7..55cdcde254070 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -8466,7 +8466,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); // Closing ends the CLI process and abandons the held tool call // without answering it. Not interrupt: the held CLI may never ack it. - yield* liveQuery.query.close.pipe(Effect.ignore); + // Closing can wait for SDK teardown; do not hold the frame lock while it finishes. + yield* liveQuery.query.close.pipe(Effect.ignore, Effect.forkIn(sessionScope)); return; } yield* trackClaudeGoal({ nativeThreadId: liveQuery.nativeThreadId, context, message }); @@ -10268,6 +10269,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( let replacementPrep: | { readonly appendSystemPromptFileText: string | undefined; + readonly resolvedTurnCwd: string | null; readonly launchArgSettings: ClaudeSdkSettings | undefined; readonly toolSearchOverride: "true" | undefined; readonly subagentPromptCacheTtl: typeof ClaudePromptCacheTtl.Type | undefined; @@ -10352,7 +10354,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( settings: adapterOptions.settings, ...(prep.launchArgSettings === undefined ? {} - : { sdkSettings: prep.launchArgSettings }), + : { sdkSettings: claudeSdkLaunchArgSettings(prep.launchArgSettings) }), ...(prep.toolSearchOverride === undefined ? {} : { toolSearchOverride: prep.toolSearchOverride }), @@ -10380,6 +10382,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), canUseTool, onUserDialog, + onElicitation, supportedDialogKinds: ["resume_return"], ...(prep.appendSystemPromptFileText === undefined ? {} @@ -10419,6 +10422,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); const closed = yield* Deferred.make(); const context: ClaudeLiveQueryContext = { + taskOutputRoot: claudeTaskOutputRoot( + adapterOptions.environment, + path, + prep.resolvedTurnCwd, + hostPlatform, + ), queryPolicy, nativeThreadId, query: querySession, @@ -10510,6 +10519,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( path, }); replacementPrep = { + resolvedTurnCwd: yield* fileSystem + .realPath(decision.turnInput.runtimePolicy.cwd ?? ".") + .pipe(Effect.orElseSucceed(() => decision.turnInput.runtimePolicy.cwd)), launchArgSettings, toolSearchOverride: yield* resolveToolSearchOverride( decision.turnInput.runtimePolicy.cwd, From cda6fb6ad773a100c616832dc1bd9ae129dae639 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:05:08 +0000 Subject: [PATCH 24/29] fix(server): avoid persisting repeated runless thread snapshots --- .../ProviderSessionManager.test.ts | 9 +++++++++ .../ProviderSessionManager.ts | 20 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index b38388748de4d..d21cfd35c5838 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3628,6 +3628,7 @@ it.effect.each([ "newer received update", "unrelated reader", "coalesced updates", + "repeated committed snapshot", "rejected write", "released rewind", "replacement owner", @@ -3782,6 +3783,14 @@ it.effect.each([ Stream.take(2), Stream.runDrain, ); + } else if (scenario === "repeated committed snapshot") { + yield* first.events.pipe( + Stream.mapEffect((event) => first.persistEvent!(event, persist(event))), + Stream.take(1), + Stream.runDrain, + ); + yield* Queue.offer(adapterEvents, update("cleared-session")); + yield* routed; } else if (scenario === "newer received update") { const second = yield* runtime.subscribeEvents!; const newer = update("newer-session"); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 1ce227a7d7ee0..d1e71c4d25fe1 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -6,7 +6,7 @@ import { ModelSelection, OrchestrationV2DomainEvent, OrchestrationV2ProviderSession, - type OrchestrationV2ProviderThread, + OrchestrationV2ProviderThread, OrchestrationV2RuntimeRequest, ProviderInstanceId, ProviderSessionId, @@ -54,6 +54,8 @@ import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +const providerThreadsEqual = Schema.toEquivalence(OrchestrationV2ProviderThread); + const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; @@ -1736,7 +1738,21 @@ export const layerWithOptions = ( // must not reclaim a thread that has moved to the replacement session. if (owner !== undefined && owner.providerSessionId !== entry.runtime.providerSessionId) return true; - const events = yield* providerEventIngestor.normalize(input); + // A provider may repeat an already committed snapshot after the run + // closes. Keep rewind effects, but do not store that snapshot a second time. + const events = yield* providerEventIngestor.normalize({ + ...input, + skipProviderThreadSnapshot: + skipProviderThreadSnapshot || + (owner !== undefined && + providerThreadsEqual(owner, { + ...event.providerThread, + pendingBackgroundTasks: event.providerThread.pendingBackgroundTasks ?? [], + contextUsage: event.providerThread.contextUsage ?? null, + nativeMetadata: event.providerThread.nativeMetadata ?? null, + goal: event.providerThread.goal ?? null, + })), + }); if (events.length === 0) return true; const written = yield* eventSink.writeIfProviderThreadUnchanged({ ...(owner === undefined From 69d9b27d2e0dc31a4f8089ba4f8f03a06d2b6bea Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:22:34 +0000 Subject: [PATCH 25/29] fix(server): preserve native sessions across no-op rollback and failed release --- .../orchestration-v2/AttemptNativeSession.ts | 19 +++++++ .../CheckpointRollbackService.test.ts | 10 ++-- .../CheckpointRollbackService.ts | 22 +++----- .../ProviderSessionManager.test.ts | 50 +++++++++++++++++++ .../ProviderSessionManager.ts | 47 +++++++++-------- .../ProviderTurnStartService.ts | 21 +++----- 6 files changed, 115 insertions(+), 54 deletions(-) create mode 100644 apps/server/src/orchestration-v2/AttemptNativeSession.ts diff --git a/apps/server/src/orchestration-v2/AttemptNativeSession.ts b/apps/server/src/orchestration-v2/AttemptNativeSession.ts new file mode 100644 index 0000000000000..1fae2eb69c84d --- /dev/null +++ b/apps/server/src/orchestration-v2/AttemptNativeSession.ts @@ -0,0 +1,19 @@ +import type { OrchestrationV2ProviderThread } from "@t3tools/contracts"; + +// Historical attempts can belong to a session from before Claude's /clear. +export function withAttemptNativeSession( + providerThread: OrchestrationV2ProviderThread, + nativeThreadId: string | undefined, +): OrchestrationV2ProviderThread { + if ( + providerThread.nativeThreadRef === null || + nativeThreadId === undefined || + providerThread.nativeThreadRef.nativeId === nativeThreadId + ) + return providerThread; + return { + ...providerThread, + nativeThreadRef: { ...providerThread.nativeThreadRef, nativeId: nativeThreadId }, + nativeConversationHeadRef: null, + }; +} diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index 4cb14cf756453..4b7f5ba9cbaa5 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -706,7 +706,7 @@ it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMe // regression: #24153 a rollback across Claude's /clear paired the adopted session with a // pre-clear message cursor, so the next resume asked the new session for a missing message. - it.effect("rolls a Claude thread back into the session the target turn ran in", () => + it.effect.each([1, 2] as const)("keeps Claude session at turn %s", (targetOrdinal) => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; const now = yield* DateTime.now; @@ -767,7 +767,7 @@ it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMe appThreadId: threadId, ownerNodeId: null, nativeThreadRef: { driver, nativeId: "native-after-clear", strength: "strong" }, - nativeConversationHeadRef: null, + nativeConversationHeadRef: { driver, nativeId: "current-head", strength: "weak" }, status: "idle", firstRunOrdinal: 1, lastRunOrdinal: 2, @@ -923,7 +923,7 @@ it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMe threadId, providerThreadId, scopeId, - checkpointId: checkpointId(1), + checkpointId: checkpointId(targetOrdinal), restoreFiles: false, }); }).pipe( @@ -964,7 +964,9 @@ it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMe session: providerThread?.nativeThreadRef?.nativeId, resumeAt: providerThread?.nativeConversationHeadRef?.nativeId, }, - { session: "native-before-clear", resumeAt: "assistant-1" }, + targetOrdinal === 1 + ? { session: "native-before-clear", resumeAt: "assistant-1" } + : { session: "native-after-clear", resumeAt: "current-head" }, ); }).pipe( Effect.scoped, diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 26a0b67f7f816..9ee3eccb8aea0 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -1,3 +1,4 @@ +import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { CheckpointId, CheckpointScopeId, @@ -145,21 +146,7 @@ export const layer: Layer.Layer< }) { const context = yield* projections.getCheckpointRollbackContext(input.threadId, input); const { checkpoint, scope, targetAttempt, runsToRollback, staleCheckpoints } = context; - // A provider can move its thread to another native session (Claude's `/clear`), so the - // target turn's messages live in the session its attempt ran in. - const providerThread = - context.providerThread?.nativeThreadRef == null || - targetAttempt?.nativeThreadId === undefined || - targetAttempt.nativeThreadId === context.providerThread.nativeThreadRef.nativeId - ? context.providerThread - : { - ...context.providerThread, - nativeThreadRef: { - ...context.providerThread.nativeThreadRef, - nativeId: targetAttempt.nativeThreadId, - }, - nativeConversationHeadRef: null, - }; + const providerThread = context.providerThread; if ( providerThread === undefined || providerThread.providerSessionId === null || @@ -299,7 +286,10 @@ export const layer: Layer.Layer< runsToRollback.length === 0 ? { providerThread } : yield* session.rollbackThread({ - providerThread, + providerThread: withAttemptNativeSession( + providerThread, + targetAttempt?.nativeThreadId, + ), target: rollbackTarget, providerThreadTurns, }); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index d21cfd35c5838..a690f41d11b3b 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -16,6 +16,7 @@ import { ProviderDriverKind, ProviderInstanceId, type ProviderSessionId, + ProviderThreadId, ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -3630,6 +3631,8 @@ it.effect.each([ "coalesced updates", "repeated committed snapshot", "rejected write", + "transient release failure", + "persistent release failure", "released rewind", "replacement owner", "concurrent owner switch", @@ -3783,6 +3786,53 @@ it.effect.each([ Stream.take(2), Stream.runDrain, ); + } else if ( + scenario === "transient release failure" || + scenario === "persistent release failure" + ) { + if (scenario === "persistent release failure") { + yield* Queue.offer(adapterEvents, { + ...update("later-session"), + providerThread: { + ...update("later-session").providerThread, + id: ProviderThreadId.make("later-unread-provider-thread"), + }, + }); + yield* routed; + } + let attempts = 0; + const failWrite: Effect.Effect = Effect.suspend(() => { + attempts++; + return ( + scenario === "persistent release failure" && attempts < 3 + ? Ref.set(beforeUnchangedWrite, failWrite) + : Effect.void + ).pipe(Effect.andThen(Effect.die("simulated release save failure"))); + }); + yield* Ref.set(beforeUnchangedWrite, failWrite); + yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + if (scenario === "persistent release failure") { + assert.equal(attempts, 3); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.deepEqual( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads.map( + (thread) => thread.nativeThreadRef?.nativeId, + ), + ["later-session"], + ); + yield* Ref.set(beforeUnchangedWrite, Effect.void); + yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + } else assert.equal(attempts, 1); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + if (scenario === "persistent release failure") { + assert.deepEqual( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads + .map((thread) => thread.nativeThreadRef?.nativeId) + .sort(), + ["cleared-session", "later-session"], + ); + return; + } } else if (scenario === "repeated committed snapshot") { yield* first.events.pipe( Stream.mapEffect((event) => first.persistEvent!(event, persist(event))), diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index d1e71c4d25fe1..57242c20fb75d 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -980,7 +980,9 @@ export const layerWithOptions = ( } // Unread updates still belong to this runtime. Save them before removal makes // subscription cleanup unable to find it, including on an event-stream failure. - yield* persistPendingThreadUpdates(candidate, true); + if (!(yield* persistPendingThreadUpdates(candidate, true))) { + return ["kept", yield* DateTime.now] as const; + } return yield* Effect.zip( Ref.modify(sessions, (current) => { const existing = current.get(key); @@ -1791,32 +1793,37 @@ export const layerWithOptions = ( // Callers hold requestEventPermit, including the release cutoff. const persistPendingThreadUpdates = Effect.fn( "ProviderSessionManager.persistPendingThreadUpdates", - )( - function* (entry: LiveSessionEntry, releasing = false) { - const pending = new Set([ - ...entry.pendingRewinds.values(), - ...entry.latestThreadUpdates.values(), - ]); - for (const update of pending) { - if (update.received || (!releasing && update.recipients.size > 0)) continue; - const threadId = update.event.providerThread.appThreadId; - if (threadId === null) continue; - const latest = entry.latestThreadUpdates.get(update.event.providerThread.id); + )(function* (entry: LiveSessionEntry, releasing = false) { + const pending = new Set([ + ...entry.pendingRewinds.values(), + ...entry.latestThreadUpdates.values(), + ]); + let saved = true; + for (const update of pending) { + if (update.received || (!releasing && update.recipients.size > 0)) continue; + const threadId = update.event.providerThread.appThreadId; + if (threadId === null) continue; + const latest = entry.latestThreadUpdates.get(update.event.providerThread.id); + for (let attempt = 0; attempt < (releasing ? 3 : 1); attempt++) { update.received = yield* ingestRunlessEventHeld( entry, threadId, update.event, latest !== update, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.unread-thread-update-failed", { + cause, + }).pipe(Effect.as(false)), + ), ); - if (update.received) entry.pendingRewinds.delete(update.event); + if (update.received) break; } - }, - Effect.catchCause((cause) => - Effect.logWarning("orchestration-v2.driver-session.unread-thread-update-failed", { - cause, - }), - ), - ); + if (update.received) entry.pendingRewinds.delete(update.event); + else saved = false; + } + return saved; + }); const makeEventSubscription = Effect.fn("ProviderSessionManager.makeEventSubscription")( function* ( diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index bb9a4ddcfe1a3..bcb460f62de8f 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1,3 +1,4 @@ +import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { modelSelectionsEqual } from "@t3tools/shared/model"; import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import { @@ -1133,21 +1134,13 @@ export const layer: Layer.Layer< const currentSourceProviderThread = sourceProjection.providerThreads.find( (candidate) => candidate.id === sourceRun?.providerThreadId, ); - // A provider can move its thread to another native session (Claude's `/clear`), so - // fork from the session the source attempt ran in. const sourceProviderThread = - currentSourceProviderThread?.nativeThreadRef == null || - sourceAttempt?.nativeThreadId === undefined || - sourceAttempt.nativeThreadId === currentSourceProviderThread.nativeThreadRef.nativeId - ? currentSourceProviderThread - : { - ...currentSourceProviderThread, - nativeThreadRef: { - ...currentSourceProviderThread.nativeThreadRef, - nativeId: sourceAttempt.nativeThreadId, - }, - nativeConversationHeadRef: null, - }; + currentSourceProviderThread === undefined + ? undefined + : withAttemptNativeSession( + currentSourceProviderThread, + sourceAttempt?.nativeThreadId, + ); const sourceProviderTurn = latestProviderTurnForAttempt(sourceProjection.providerTurns, sourceAttempt?.id) ?? sourceProjection.providerTurns.find( From cacb482bf8ed742e14d11f12a478a8b4e85b8099 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:37:25 +0000 Subject: [PATCH 26/29] fix(server): report pending-save failures on explicit release --- .../ProviderSessionManager.test.ts | 6 +++++- .../ProviderSessionManager.ts | 21 +++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index a690f41d11b3b..418092aa41e7d 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3810,7 +3810,11 @@ it.effect.each([ ).pipe(Effect.andThen(Effect.die("simulated release save failure"))); }); yield* Ref.set(beforeUnchangedWrite, failWrite); - yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + const release = manager.release({ providerSessionId, reason: "manual_shutdown" }); + if (scenario === "persistent release failure") { + const error = yield* release.pipe(Effect.flip); + assert.equal(error._tag, "ProviderSessionReleaseError"); + } else yield* release; if (scenario === "persistent release failure") { assert.equal(attempts, 3); assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 57242c20fb75d..0d7f4006b56d4 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -961,11 +961,11 @@ export const layerWithOptions = ( readonly providerSessionId: ProviderSessionId; readonly onlyIfIdleGeneration?: number; readonly keepIfBusyOtherThan?: ReadonlyArray; - }): Effect.fn.Return, DateTime.Utc]> { + }): Effect.fn.Return, DateTime.Utc, boolean]> { const key = sessionKey(input.providerSessionId); const candidate = (yield* Ref.get(sessions)).get(key); if (candidate === undefined) { - return [Option.none(), yield* DateTime.now] as const; + return [Option.none(), yield* DateTime.now, false] as const; } const removed = yield* Effect.gen(function* () { const latest = (yield* Ref.get(sessions)).get(key); @@ -981,7 +981,7 @@ export const layerWithOptions = ( // Unread updates still belong to this runtime. Save them before removal makes // subscription cleanup unable to find it, including on an event-stream failure. if (!(yield* persistPendingThreadUpdates(candidate, true))) { - return ["kept", yield* DateTime.now] as const; + return ["persist-failed", yield* DateTime.now] as const; } return yield* Effect.zip( Ref.modify(sessions, (current) => { @@ -1017,6 +1017,7 @@ export const layerWithOptions = ( return [ outcome === "removed" ? Option.some(candidate) : Option.none(), releasedAt, + outcome === "persist-failed", ] as const; }, ); @@ -1068,7 +1069,19 @@ export const layerWithOptions = ( readonly gracefulSubscribers?: boolean; }) => Effect.acquireUseRelease( - removeLiveEntry(input), + removeLiveEntry(input).pipe( + Effect.flatMap(([entry, releasedAt, persistFailed]) => + persistFailed && input.onlyIfIdleGeneration === undefined + ? Effect.fail( + new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: input.reason, + cause: "Pending provider-thread updates could not be saved before release.", + }), + ) + : Effect.succeed([entry, releasedAt] as const), + ), + ), ([entry, releasedAt]) => Option.match(entry, { onNone: () => Effect.void, From df27c586f5b726ce1831a208a740b434a941b86b Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:57:16 +0000 Subject: [PATCH 27/29] fix(server): retry idle saves and reject stopped event pumps --- .../ProviderSessionManager.test.ts | 42 +++++++++++++++++++ .../ProviderSessionManager.ts | 28 ++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 418092aa41e7d..a4442cf0023ee 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3633,6 +3633,8 @@ it.effect.each([ "rejected write", "transient release failure", "persistent release failure", + "idle release retry", + "stopped pump release failure", "released rewind", "replacement owner", "concurrent owner switch", @@ -3758,6 +3760,46 @@ it.effect.each([ if (scenario === "coalesced updates") yield* Queue.offer(adapterEvents, update("newer-session")); yield* routed; + if (scenario === "idle release retry" || scenario === "stopped pump release failure") { + const failedWrites = yield* Queue.unbounded(); + let attempts = 0; + const failWrite: Effect.Effect = Effect.suspend(() => { + attempts++; + return Ref.set(beforeUnchangedWrite, failWrite).pipe( + Effect.andThen(Queue.offer(failedWrites, undefined)), + Effect.andThen(Effect.die("simulated pending snapshot failure")), + ); + }); + yield* Ref.set(beforeUnchangedWrite, failWrite); + if (scenario === "idle release retry") yield* TestClock.adjust("60 seconds"); + else yield* Queue.end(adapterEvents); + for (let attempt = 0; attempt < 3; attempt++) yield* Queue.take(failedWrites); + assert.equal(attempts, 3); + assert.isTrue(yield* manager.isLive(providerSessionId)); + if (scenario === "stopped pump release failure") { + const openError = yield* manager + .open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal(openError._tag, "ProviderSessionOpenError"); + const lookupError = yield* manager.get(providerSessionId).pipe(Effect.flip); + assert.equal(lookupError._tag, "ProviderSessionLookupError"); + } + yield* Ref.set(beforeUnchangedWrite, Effect.void); + if (scenario === "idle release retry") yield* TestClock.adjust("60 seconds"); + else yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + assert.isFalse(yield* manager.isLive(providerSessionId)); + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0]?.nativeThreadRef + ?.nativeId, + "cleared-session", + ); + return; + } if (scenario === "runtime failure" || scenario === "released rewind") { const released = yield* eventSink.stream({ threadId, afterSequence }).pipe( Stream.filter( diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 0d7f4006b56d4..8fd35b22449f3 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -228,6 +228,7 @@ interface LiveSessionEntry { readonly eventSubscribers: Ref.Ref< ReadonlyMap> >; + readonly eventPumpAlive: Ref.Ref; readonly latestThreadUpdates: Map; readonly threadUpdateDeliveries: WeakMap< ProviderAdapter.ProviderAdapterV2Event, @@ -1071,7 +1072,7 @@ export const layerWithOptions = ( Effect.acquireUseRelease( removeLiveEntry(input).pipe( Effect.flatMap(([entry, releasedAt, persistFailed]) => - persistFailed && input.onlyIfIdleGeneration === undefined + persistFailed ? Effect.fail( new ProviderSessionReleaseError({ providerSessionId: input.providerSessionId, @@ -1247,7 +1248,10 @@ export const layerWithOptions = ( Effect.logWarning("orchestration-v2.driver-session.idle-release-failed", { providerSessionId: input.providerSessionId, cause, - }), + }).pipe( + Effect.andThen(Effect.sleep(Duration.millis(idleTimeoutMs))), + Effect.andThen(Effect.suspend(() => releaseIfStillIdle(input))), + ), ), ); }, @@ -1497,6 +1501,8 @@ export const layerWithOptions = ( providerSessionId, Effect.gen(function* () { const key = sessionKey(providerSessionId); + const live = (yield* Ref.get(sessions)).get(key); + if (live === undefined || !(yield* Ref.get(live.eventPumpAlive))) return false; const marked = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); if (entry === undefined || entry.runtime !== runtime) { @@ -2331,6 +2337,7 @@ export const layerWithOptions = ( Effect.exit, Effect.flatMap((exit) => Effect.gen(function* () { + yield* Ref.set(entry.eventPumpAlive, false); if (shutdownSignal.received) return; const current = (yield* Ref.get(sessions)).get( sessionKey(entry.runtime.providerSessionId), @@ -2428,6 +2435,14 @@ export const layerWithOptions = ( const key = sessionKey(input.providerSessionId); const existing = (yield* Ref.get(sessions)).get(key); if (existing !== undefined) { + if (!(yield* Ref.get(existing.eventPumpAlive))) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: + "The provider event stream stopped; pending session updates must be saved before reopening.", + }); + } const attached = existing.attachedThreadIds.has(input.threadId); if (!attached && !existing.supportsMultipleProviderThreads) { return yield* new ProviderSessionOpenError({ @@ -2537,6 +2552,7 @@ export const layerWithOptions = ( runtime, exposedRuntime, eventSubscribers, + eventPumpAlive: yield* Ref.make(true), latestThreadUpdates: new Map(), threadUpdateDeliveries: new WeakMap(), pendingRewinds: new Map(), @@ -2595,6 +2611,14 @@ export const layerWithOptions = ( if (entry === undefined) { return Option.none(); } + if (!(yield* Ref.get(entry.eventPumpAlive))) { + return yield* new ProviderSessionOpenError({ + instanceId: entry.runtime.instanceId, + providerSessionId, + cause: + "The provider event stream stopped; pending session updates must be saved before reopening.", + }); + } yield* touchActivity(providerSessionId); return Option.some(entry.exposedRuntime); }, From 612cb7224e15da794a2757b6e21a561eb203bbd2 Mon Sep 17 00:00:00 2001 From: luke2x <97411838+luke2x@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:15:58 +0000 Subject: [PATCH 28/29] fix(server): retry busy runtime cleanup and close live rollback queries --- .../Adapters/ClaudeAdapterV2.test.ts | 43 +++++++++++-------- .../Adapters/ClaudeAdapterV2.ts | 5 ++- .../ProviderSessionManager.test.ts | 25 ++++++++++- .../ProviderSessionManager.ts | 13 +++++- 4 files changed, 64 insertions(+), 22 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 48f94b2354557..b09c066f3eb3b 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3428,9 +3428,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect( - "closes the cleared session's live query when rolling back into the pre-clear one", - () => + it.effect.each([true, false])( + "closes the current query during rollback with native session adoption %s", + (adopted) => Effect.gen(function* () { const harness = yield* makeWakeHarnessWithOptions({ freshQueueOnReopen: true, @@ -3454,22 +3454,32 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* harness.offerAndWait(turnOneResult); yield* Queue.take(harness.terminalReceipts); const preClearTurn = lastProviderTurn(harness.events); - yield* harness.offerAndWait( - claudeSdkFrame({ - type: "system", - subtype: "init", - session_id: "cleared-native-session", - uuid: "00000000-0000-4000-8000-000000000522", - }), - ); + if (adopted) + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000522", + }), + ); const isClearedThreadUpdate = (event: ProviderAdapter.ProviderAdapterV2Event) => event.type === "provider_thread.updated" && event.providerThread.nativeThreadRef?.nativeId === "cleared-native-session"; - yield* harness.until(() => harness.events.some(isClearedThreadUpdate)); + if (adopted) yield* harness.until(() => harness.events.some(isClearedThreadUpdate)); + const targetThread = adopted + ? harness.providerThread + : { + ...harness.providerThread, + nativeThreadRef: { + ...harness.providerThread.nativeThreadRef!, + nativeId: "historical-native-session", + }, + }; // History rollback hands the adapter the pre-clear session the turn ran in. const rolledBack = yield* harness.runtime.rollbackThread({ - providerThread: harness.providerThread, + providerThread: targetThread, target: { type: "provider_turn", checkpointId: CheckpointId.make("checkpoint-before-clear"), @@ -3480,7 +3490,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); assert.equal( rolledBack.providerThread.nativeThreadRef?.nativeId, - harness.providerThread.nativeThreadRef?.nativeId, + targetThread.nativeThreadRef?.nativeId, ); assert.equal(rolledBack.providerThread.nativeConversationHeadRef?.nativeId, preClearUuid); const eventsAfterRollback = harness.events.length; @@ -3509,10 +3519,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { "the cleared session overwrote the rolled-back provider thread", ); assert.isFalse(clearedQueryStillLive, "rollback left the cleared session's query running"); - assert.equal( - harness.getOpenedOptions()?.resume, - harness.providerThread.nativeThreadRef?.nativeId, - ); + assert.equal(harness.getOpenedOptions()?.resume, targetThread.nativeThreadRef?.nativeId); assert.equal(harness.getOpenedOptions()?.resumeSessionAt, preClearUuid); }).pipe( Effect.scoped, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6da1b462d1f5d..8bc85b46200ac 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -11936,7 +11936,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); yield* closeLiveQueryForNativeThread(adoptedNativeThreadId); } - yield* closeLiveQueryForNativeThread(nativeThreadId); + const liveNativeThreadId = (yield* Ref.get(queryContext))?.nativeThreadId; + if (liveNativeThreadId !== undefined) { + yield* closeLiveQueryForNativeThread(liveNativeThreadId); + } const now = yield* DateTime.now; const resumeSessionAt = yield* resolveClaudeRollbackResumeSessionAt(rollbackInput); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index a4442cf0023ee..b2c9e1deed1a4 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3761,6 +3761,28 @@ it.effect.each([ yield* Queue.offer(adapterEvents, update("newer-session")); yield* routed; if (scenario === "idle release retry" || scenario === "stopped pump release failure") { + if (scenario === "stopped pump release failure") { + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Start a turn before the pump fails.", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + } const failedWrites = yield* Queue.unbounded(); let attempts = 0; const failWrite: Effect.Effect = Effect.suspend(() => { @@ -3790,8 +3812,7 @@ it.effect.each([ assert.equal(lookupError._tag, "ProviderSessionLookupError"); } yield* Ref.set(beforeUnchangedWrite, Effect.void); - if (scenario === "idle release retry") yield* TestClock.adjust("60 seconds"); - else yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + yield* TestClock.adjust("60 seconds"); assert.isFalse(yield* manager.isLive(providerSessionId)); assert.equal( (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0]?.nativeThreadRef diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 8fd35b22449f3..552c834149930 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2338,6 +2338,14 @@ export const layerWithOptions = ( Effect.flatMap((exit) => Effect.gen(function* () { yield* Ref.set(entry.eventPumpAlive, false); + // No more terminal events can clear turns owned by this pump. + yield* Ref.update(sessions, (current) => { + const key = sessionKey(entry.runtime.providerSessionId); + const live = current.get(key); + return live?.runtime === entry.runtime && live.busyTurns.size > 0 + ? new Map(current).set(key, { ...live, busyTurns: new Set() }) + : current; + }); if (shutdownSignal.received) return; const current = (yield* Ref.get(sessions)).get( sessionKey(entry.runtime.providerSessionId), @@ -2379,7 +2387,10 @@ export const layerWithOptions = ( providerSessionId: entry.runtime.providerSessionId, reason: "runtime_error", detail: makeProviderFailure({ cause }).message, - }).pipe(logReleaseFailure(entry.runtime.providerSessionId)); + }).pipe( + Effect.tapError(() => scheduleIdleRelease(entry.runtime.providerSessionId)), + logReleaseFailure(entry.runtime.providerSessionId), + ); }), ), Effect.forkIn(layerScope), From eb2b36251693a287c98c807c87fd37fdf39a1273 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 14 Nov 2026 06:49:23 +0000 Subject: [PATCH 29/29] fix(server): drop unrelated plan routing and a stale duplicate test The superseded-plan routing change fixes a separate bug and leaves with its test. An earlier merge left a second copy of main's task_started roster test. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 141 ------------------ .../CheckpointRollbackService.ts | 6 +- .../ProviderTurnStartService.ts | 2 +- .../RunExecutionService.test.ts | 50 ------- .../orchestration-v2/RunExecutionService.ts | 10 +- 5 files changed, 5 insertions(+), 204 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index bdaabcc79132f..8121523959ee4 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -11554,147 +11554,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { > => event.type === "provider_thread.updated", ); - it.effect( - "uses task_started as an incremental roster fallback, keeps its tool call, and clears internal work on empty snapshot", - () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeWakeHarness; - const now = yield* DateTime.now; - const emptyRoster = claudeSdkFrame({ - type: "system", - subtype: "background_tasks_changed", - tasks: [], - uuid: "00000000-0000-4000-8000-000000000202", - session_id: WAKE_NATIVE_SESSION, - }); - - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now, - attemptId: RunAttemptId.make("attempt-claude-roster-fallback"), - text: "Run the build in the background.", - attachments: [], - }), - ); - yield* Queue.offer( - harness.sdkMessages, - claudeSdkFrame({ - type: "assistant", - message: { - model: "claude-sonnet-4-6", - id: "msg_background_command", - type: "message", - role: "assistant", - content: [ - { - type: "tool_use", - id: WAKE_TOOL_USE_ID, - name: "Bash", - input: { - command: "python3 audit.py > audit.log 2>&1", - run_in_background: true, - }, - }, - ], - }, - parent_tool_use_id: null, - uuid: "background-command-launch", - session_id: WAKE_NATIVE_SESSION, - }), - ); - yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); - yield* Queue.offer(harness.sdkMessages, turnOneResult); - yield* harness.awaitTerminal(1); - - const afterStart = providerThreadRosterEvents(harness.events).filter( - (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, - ); - assert.isAtLeast(afterStart.length, 1); - assert.equal( - (afterStart.at(-1)?.providerThread.pendingBackgroundTasks ?? [])[0]?.taskId, - WAKE_TASK_ID, - ); - - const command = harness.events.find( - (event) => - event.type === "turn_item.updated" && - event.turnItem.type === "command_execution" && - event.turnItem.nativeItemRef?.nativeId === WAKE_TOOL_USE_ID, - ); - assert.equal(command?.type, "turn_item.updated"); - if (command?.type !== "turn_item.updated") return; - const details = { - command: "python3 audit.py > audit.log 2>&1", - itemId: command.turnItem.id, - }; - assert.deepInclude( - afterStart.at(-1)?.providerThread.pendingBackgroundTasks?.[0], - details, - ); - yield* harness.offerAndWait( - claudeSdkFrame({ - type: "system", - subtype: "background_tasks_changed", - tasks: [{ task_id: WAKE_TASK_ID, task_type: "local_bash", description: "Audit" }], - uuid: "background-command-snapshot", - session_id: WAKE_NATIVE_SESSION, - }), - ); - assert.deepInclude( - providerThreadRosterEvents(harness.events).at(-1)?.providerThread - .pendingBackgroundTasks?.[0], - details, - ); - - yield* Queue.offer( - harness.sdkMessages, - claudeSdkFrame({ - type: "system", - subtype: "background_tasks_changed", - tasks: [ - { - task_id: "subagent-shell", - task_type: "local_bash", - description: "sleep 150", - parent_task_id: "agent-1", - }, - ], - uuid: "00000000-0000-4000-8000-000000000203", - session_id: WAKE_NATIVE_SESSION, - }), - ); - yield* harness.until(() => - providerThreadRosterEvents(harness.events).some( - (event) => - event.providerThread.pendingBackgroundTasks?.[0]?.parentTaskId === "agent-1", - ), - ); - - yield* harness.offerAndWait(emptyRoster); - assert.include( - ( - providerThreadRosterEvents(harness.events).at(-1)?.providerThread - .pendingBackgroundTasks ?? [] - ).map((task) => task.taskId), - WAKE_TASK_ID, - ); - assert.isFalse(yield* harness.hasPendingBackgroundWork); - }).pipe( - Effect.provide( - Layer.mergeAll( - IdAllocator.layer, - McpProviderSessions.layer, - NodeServices.layer, - NodeCrypto.layer, - ), - ), - ), - ), - ); - it.effect("keeps awaited background task marks when /clear adopts a new session", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 0f38bbc9d457b..8cfda1c6f3430 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -1,4 +1,3 @@ -import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { CheckpointId, CheckpointScopeId, @@ -30,6 +29,7 @@ import { isCheckpointRestoreIsolated, SHARED_WORKSPACE_RESTORE_MESSAGE, } from "./CheckpointRestoreSafety.ts"; +import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { CheckpointRestoreError, CheckpointServiceV2 } from "./CheckpointService.ts"; import { EventSinkV2 } from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; @@ -149,8 +149,8 @@ export const layer: Layer.Layer< readonly restoreFiles?: boolean; }) { const context = yield* projections.getCheckpointRollbackContext(input.threadId, input); - const { checkpoint, scope, targetAttempt, runsToRollback, staleCheckpoints } = context; - const providerThread = context.providerThread; + const { providerThread, checkpoint, scope, targetAttempt, runsToRollback, staleCheckpoints } = + context; if ( providerThread === undefined || providerThread.providerSessionId === null || diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 07c7b030a3501..8c7d5b2ee81b1 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1,4 +1,3 @@ -import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { modelSelectionsEqual } from "@t3tools/shared/model"; import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import { @@ -27,6 +26,7 @@ import * as Predicate from "effect/Predicate"; import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; +import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { CLAUDE_PROVIDER, CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index cbcf3259ea1fb..fbb4e0a237afc 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -6,7 +6,6 @@ import { CommandId, MessageId, NodeId, - PlanId, type OrchestrationV2AppThread, type OrchestrationV2CheckpointScope, type OrchestrationV2DomainEvent, @@ -197,55 +196,6 @@ it.effect("routes shared-runtime events only to their owning root run", () => }), ); -it("routes a superseded plan from an earlier run on the same thread", () => { - const threadId = ThreadId.make("thread:superseded-plan"); - const identity: RunExecutionService.ProviderEventRouteIdentity = { - threadId, - runId: RunId.make("run:superseded-plan:current"), - attemptId: RunAttemptId.make("attempt:superseded-plan:current"), - providerThreadId: ProviderThreadId.make("provider-thread:superseded-plan"), - }; - const state = RunExecutionService.makeProviderEventRoutingState({ - identity, - providerTurnId: null, - }); - const plan = { - id: PlanId.make("plan:superseded-plan"), - threadId, - runId: RunId.make("run:superseded-plan:earlier"), - nodeId: NodeId.make("node:superseded-plan"), - kind: "proposed_plan" as const, - status: "superseded" as const, - markdown: "# Earlier plan", - }; - - assert.isTrue( - RunExecutionService.routeProviderEvent( - { type: "plan.updated", driver, plan }, - identity, - state, - )[0], - ); - assert.isFalse( - RunExecutionService.routeProviderEvent( - { type: "plan.updated", driver, plan: { ...plan, status: "active" } }, - identity, - state, - )[0], - ); - assert.isFalse( - RunExecutionService.routeProviderEvent( - { - type: "plan.updated", - driver, - plan: { ...plan, threadId: ThreadId.make("thread:superseded-plan:other") }, - }, - identity, - state, - )[0], - ); -}); - it("routes only enrolled workflow members after the root turn ends", () => { const threadId = ThreadId.make("thread:late-child"); const rootProviderTurnId = ProviderTurnId.make("provider-turn:late-child"); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index ea44f7156b7b0..c77d12bc9fbe3 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -512,15 +512,7 @@ export function routeProviderEvent( return [true, { ...state, inheritedBackgroundTurnItems }]; } case "plan.updated": - return [ - ownsRun(event.plan.runId) || - ownsChildThread(event.plan.threadId) || - // A replacement plan supersedes the previous plan, which still carries - // the run that created it. Accept that update for the current thread so - // the superseded status reaches the projection instead of being dropped. - (event.plan.status === "superseded" && event.plan.threadId === input.threadId), - state, - ]; + return [ownsRun(event.plan.runId) || ownsChildThread(event.plan.threadId), state]; case "runtime_request.updated": return [ (event.threadId !== undefined && ownsChildThread(event.threadId)) ||