diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index dd350fb736216..8121523959ee4 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3648,6 +3648,203 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + it.effect.each(["active", "idle"] as const)( + "persists a session cleared while %s and resumes it after process replacement", + (phase) => + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }); + const turn = (ordinal: number, providerThread: OrchestrationV2ProviderThread) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`clear-session-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: [], + providerTurnOrdinal: ordinal, + }); + yield* harness.runtime.startTurn(turn(1, harness.providerThread)); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "subagent-native-session", + parent_tool_use_id: "subagent-launch", + uuid: "00000000-0000-4000-8000-000000000500", + }), + ); + if (phase === "idle") { + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + } + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000501", + }), + ); + if (phase === "active") { + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + } else { + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === "cleared-native-session", + ), + ); + } + assert.isFalse( + harness.events.some( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === "subagent-native-session", + ), + ); + const completed = harness.events.findLast( + (event) => event.type === "provider_thread.updated", + ); + assert.equal(completed?.type, "provider_thread.updated"); + if (completed?.type !== "provider_thread.updated") return; + assert.equal(completed.providerThread.nativeThreadRef?.nativeId, "cleared-native-session"); + assert.equal(completed.providerThread.id, harness.providerThread.id); + yield* harness.runtime.startTurn(turn(2, completed.providerThread)); + assert.equal(harness.processQueues.length, 1); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + yield* harness.runtime.interruptTurn({ + providerThread: completed.providerThread, + providerTurnId: lastProviderTurn(harness.events).id, + requestRuntimeRestart: true, + }); + yield* harness.runtime.startTurn(turn(3, completed.providerThread)); + assert.equal(harness.processQueues.length, 2); + assert.equal(harness.getOpenedOptions()?.resume, "cleared-native-session"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect.each([true, false])( + "closes the current query during rollback with native session adoption %s", + (adopted) => + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }); + const preClearUuid = "00000000-0000-4000-8000-000000000521"; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make("rollback-across-clear-1"), + text: "Start the build in the background.", + attachments: [], + }), + ); + yield* harness.offerAndWait(wakeTaskStarted); + yield* harness.offerAndWait( + makeAssistantTextFrame({ uuid: preClearUuid, text: "Build started." }), + ); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + const preClearTurn = lastProviderTurn(harness.events); + if (adopted) + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000522", + }), + ); + const isClearedThreadUpdate = (event: ProviderAdapter.ProviderAdapterV2Event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === "cleared-native-session"; + if (adopted) yield* harness.until(() => harness.events.some(isClearedThreadUpdate)); + const targetThread = adopted + ? harness.providerThread + : { + ...harness.providerThread, + nativeThreadRef: { + ...harness.providerThread.nativeThreadRef!, + nativeId: "historical-native-session", + }, + }; + + // History rollback hands the adapter the pre-clear session the turn ran in. + const rolledBack = yield* harness.runtime.rollbackThread({ + providerThread: targetThread, + target: { + type: "provider_turn", + checkpointId: CheckpointId.make("checkpoint-before-clear"), + appRunOrdinal: 1, + providerTurn: preClearTurn, + }, + providerThreadTurns: [preClearTurn], + }); + assert.equal( + rolledBack.providerThread.nativeThreadRef?.nativeId, + targetThread.nativeThreadRef?.nativeId, + ); + assert.equal(rolledBack.providerThread.nativeConversationHeadRef?.nativeId, preClearUuid); + const eventsAfterRollback = harness.events.length; + + // The cleared session's process finishing its background task after the rollback. + const clearedQuery = harness.processQueues[0]!; + const clearedQueryStillLive = clearedQuery.state._tag === "Open"; + if (clearedQueryStillLive) yield* harness.offerAndWait(wakeNotification); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: rolledBack.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make("rollback-across-clear-2"), + text: "Continue from before the clear.", + attachments: [], + providerTurnOrdinal: 2, + }), + ); + assert.deepEqual( + harness.events + .slice(eventsAfterRollback) + .filter(isClearedThreadUpdate) + .map((event) => event.type), + [], + "the cleared session overwrote the rolled-back provider thread", + ); + assert.isFalse(clearedQueryStillLive, "rollback left the cleared session's query running"); + assert.equal(harness.getOpenedOptions()?.resume, targetThread.nativeThreadRef?.nativeId); + assert.equal(harness.getOpenedOptions()?.resumeSessionAt, preClearUuid); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("resumes the session a /clear selected after the Claude process is replaced", () => Effect.gen(function* () { const harness = yield* makeWakeHarnessWithOptions({ @@ -3706,6 +3903,207 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("keeps the cleared session when the next turn carries the pre-clear thread", () => + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }); + const turn = (ordinal: number) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`stale-clear-session-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: [], + providerTurnOrdinal: ordinal, + }); + yield* harness.runtime.startTurn(turn(1)); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000511", + }), + ); + const resumed = yield* harness.runtime.resumeThread({ + providerThread: harness.providerThread, + }); + assert.equal(resumed.nativeThreadRef?.nativeId, "cleared-native-session"); + yield* harness.runtime.startTurn(turn(2)); + assert.equal(harness.processQueues.length, 1); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + assert.equal(lastProviderTurn(harness.events).providerThreadId, harness.providerThread.id); + const thread = harness.events.findLast((event) => event.type === "provider_thread.updated"); + assert.equal( + thread?.type === "provider_thread.updated" + ? thread.providerThread.nativeThreadRef?.nativeId + : undefined, + "cleared-native-session", + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect("keeps the session a /clear selects while the next turn reads its attachments", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const reading = yield* Deferred.make(); + const release = yield* Deferred.make(); + const attachment = ChatImageAttachment.make({ + type: "image", + id: ChatAttachmentId.make("thread-claude-wake-12345678-1234-1234-1234-123456789abc"), + name: "diagram.png", + mimeType: "image/png", + sizeBytes: 4, + }); + const attachmentPath = attachmentRelativePath(attachment)!; + const gatedFileSystem: FileSystem.FileSystem = { + ...fileSystem, + readFile: (filePath) => + filePath.endsWith(attachmentPath) + ? Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(fileSystem.readFile(filePath)), + ) + : fileSystem.readFile(filePath), + }; + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + }).pipe(Effect.provideService(FileSystem.FileSystem, gatedFileSystem)); + yield* fileSystem.writeFile( + NodePath.join(harness.attachmentsDir, attachmentPath), + Uint8Array.from([1, 2, 3, 4]), + ); + const turn = (ordinal: number) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`clear-during-prep-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: ordinal === 1 ? [] : [attachment], + providerTurnOrdinal: ordinal, + }); + yield* harness.runtime.startTurn(turn(1)); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + + const started = yield* harness.runtime.startTurn(turn(2)).pipe(Effect.forkScoped); + yield* Deferred.await(reading); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000521", + }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(started); + + assert.equal(harness.processQueues.length, 1); + assert.notEqual(harness.getOpenedOptions()?.resume, WAKE_NATIVE_SESSION); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect( + "resumes the session a /clear selects while a model change reads the system prompt file", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const reading = yield* Deferred.make(); + const release = yield* Deferred.make(); + let gateRead = false; + const gatedFileSystem: FileSystem.FileSystem = { + ...fileSystem, + readFileString: (filePath) => + filePath.endsWith("rules.md") + ? gateRead + ? Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as("Rules"), + ) + : Effect.succeed("Rules") + : fileSystem.readFileString(filePath), + }; + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (queue) => Queue.shutdown(queue), + settings: { + ...DEFAULT_CLAUDE_SETTINGS, + launchArgs: "--append-system-prompt-file rules.md", + }, + }).pipe(Effect.provideService(FileSystem.FileSystem, gatedFileSystem)); + const turn = (ordinal: number) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe(0), + attemptId: RunAttemptId.make(`clear-during-prompt-file-${ordinal}`), + text: ordinal === 1 ? "/clear" : "Continue the cleared conversation", + attachments: [], + providerTurnOrdinal: ordinal, + ...(ordinal === 1 ? {} : { modelSelection: CLAUDE_TEST_RESTART_MODEL_SELECTION }), + }); + yield* harness.runtime.startTurn(turn(1)); + yield* harness.offerAndWait(turnOneResult); + yield* Queue.take(harness.terminalReceipts); + + gateRead = true; + const started = yield* harness.runtime.startTurn(turn(2)).pipe(Effect.forkScoped); + yield* Deferred.await(reading); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: "00000000-0000-4000-8000-000000000531", + }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(started); + + assert.equal(harness.processQueues.length, 2); + assert.equal(harness.getOpenedOptions()?.resume, "cleared-native-session"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect.each(["release-after", "cancel-after", "release-before", "resume-pending"] as const)( "keeps attachment reservations across compaction: %s boundary", (outcome) => @@ -10431,7 +10829,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { session_id: WAKE_NATIVE_SESSION, }), ); - return { configDir, startTurn, call, launchSubagent, finishTurn, plans }; + return { + configDir, + startTurn, + call, + launchSubagent, + finishTurn, + plans, + offer: harness.offerAndWait, + nextUuid, + }; }); const makeTaskToolHarness = makeTaskToolHarnessWithOptions(); @@ -10569,6 +10976,75 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("starts a fresh task list when /clear selects a new native session", () => + Effect.gen(function* () { + const harness = yield* makeTaskToolHarness; + yield* harness.startTurn(); + yield* harness.call(createTask("1", "Before clear")); + yield* harness.call(createTask("2", "Also before clear")); + yield* harness.offer( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: harness.nextUuid(), + }), + ); + yield* harness.call(createTask("1", "After clear")); + yield* harness.finishTurn(); + assert.deepEqual(harness.plans().at(-1)?.steps, [ + { id: "task-1", text: "After clear", status: "pending" }, + ]); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect.each([ + { listId: undefined, status: "superseded" }, + { listId: "shared-list", status: "active" }, + ] as const)( + "retires a session task list after an idle /clear (shared list: $listId)", + ({ listId, status }) => + Effect.gen(function* () { + const harness = yield* makeTaskToolHarnessWithOptions({ CLAUDE_CODE_TASK_LIST_ID: listId }); + yield* harness.startTurn(); + yield* harness.call(createTask("1", "Before clear")); + yield* harness.finishTurn(); + yield* harness.offer( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-native-session", + uuid: harness.nextUuid(), + }), + ); + yield* harness.startTurn(); + yield* harness.finishTurn(); + const plans = harness.plans(); + assert.equal(plans.at(-1)?.id, plans[0]?.id); + assert.equal(plans.at(-1)?.status, status); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("removes deleted tasks without resurrecting them on later updates or turns", () => Effect.gen(function* () { const harness = yield* makeTaskToolHarness; @@ -11078,6 +11554,84 @@ describe("ClaudeAdapterV2 background wake turns", () => { > => event.type === "provider_thread.updated", ); + it.effect("keeps awaited background task marks when /clear adopts a new session", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-clear-awaited-task"), + text: "Wait for the background task.", + attachments: [], + }), + ); + const nextThread = Effect.fn("nextAwaitedThread")(function* ( + nativeId: string, + awaited?: boolean, + ) { + while (true) { + const { providerThread } = yield* Queue.take(harness.providerThreadReceipts); + const task = providerThread.pendingBackgroundTasks?.find( + (task) => task.taskId === WAKE_TASK_ID, + ); + if ( + providerThread.nativeThreadRef?.nativeId === nativeId && + task !== undefined && + (awaited === undefined || task.awaited === awaited) + ) + return providerThread; + } + }); + const roster = (sessionId: string, uuid: string) => + claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [{ task_id: WAKE_TASK_ID, task_type: "local_bash", description: "test task" }], + session_id: sessionId, + uuid, + }); + yield* harness.offerAndWait( + roster(WAKE_NATIVE_SESSION, "00000000-0000-4000-8000-000000000610"), + ); + yield* nextThread(WAKE_NATIVE_SESSION); + yield* harness.runtime.awaitBackgroundTasks!({ + providerThread: harness.providerThread, + taskIds: [WAKE_TASK_ID], + awaited: true, + }); + yield* nextThread(WAKE_NATIVE_SESSION, true); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + session_id: "cleared-awaited-session", + uuid: "00000000-0000-4000-8000-000000000611", + }), + ); + yield* nextThread("cleared-awaited-session"); + yield* harness.offerAndWait( + roster("cleared-awaited-session", "00000000-0000-4000-8000-000000000612"), + ); + const updated = yield* nextThread("cleared-awaited-session"); + assert.isTrue( + updated.pendingBackgroundTasks?.find((task) => task.taskId === WAKE_TASK_ID)?.awaited, + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect( "uses task_started as an incremental roster fallback, keeps its tool call, and clears internal work on empty snapshot", () => diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index f1ae8c58e7ae9..81454a046936a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4719,9 +4719,9 @@ function formatClaudeUsageLimitWait(waitMs: number): string { interface ActiveClaudeTurnContext { readonly queryPolicy: ClaudeRuntimeQueryPolicy; - readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; + input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly autoCompactWindow: string; - readonly nativeThreadId: string; + nativeThreadId: string; readonly nativeTurnId: string; nativeMessageCursor: string | null; readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; @@ -4855,7 +4855,7 @@ interface ActiveClaudeSubagent { interface ClaudeLiveQueryContext { readonly queryPolicy: ClaudeRuntimeQueryPolicy; - readonly nativeThreadId: string; + nativeThreadId: string; readonly settingsCwd: string | null; readonly launchArgSettings: ClaudeSdkSettings | undefined; readonly query: ClaudeAgentSdkQuerySession; @@ -5984,8 +5984,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); const absorbedWakes = yield* Ref.make(new Set()); const heldProposedPlansByToolUseId = new Map(); - // /clear moves a live process to a new native session; persist that one so a reopen resumes it. - const clearedNativeSessions = new Map(); const runtimeContext = yield* Effect.context(); const runPromise = Effect.runPromiseWith(runtimeContext); @@ -6007,23 +6005,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, }; } - const nativeSessionId = clearedNativeSessions.get( - event.providerThread.nativeThreadRef?.nativeId ?? "", - ); - if (nativeSessionId !== undefined) { - event = { - ...event, - providerThread: { - ...event.providerThread, - nativeThreadRef: { - driver: CLAUDE_PROVIDER, - nativeId: nativeSessionId, - strength: "strong", - }, - nativeConversationHeadRef: null, - }, - }; - } } return Queue.offer(events, event).pipe(Effect.asVoid); }; @@ -6149,6 +6130,132 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); }); + // Old native ID -> the session `/clear` replaced it with, for turns built from a + // projection that has not caught up yet. Adoption removes the new ID as a key + // first, so following the chain cannot loop. + const adoptedNativeSessionIds = new Map(); + // Todo lists retired by an idle `/clear`. Nothing persists adapter events while no + // run is subscribed, so the next turn publishes them. + const supersededTodoPlans: Array = []; + const adoptNativeSessionId = Effect.fnUntraced(function* ( + query: ClaudeLiveQueryContext, + nativeThreadId: string, + ) { + if ((yield* Ref.get(queryContext)) !== query || query.stopping) return; + const previousId = query.nativeThreadId; + if (nativeThreadId.length === 0 || nativeThreadId === previousId) return; + const previous = (yield* Ref.get(lastProviderThreadByNativeThread)).get(previousId); + if (previous === undefined) return; + const providerThread: OrchestrationV2ProviderThread = { + ...previous, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: nativeThreadId, + strength: "strong", + }, + nativeConversationHeadRef: null, + updatedAt: yield* DateTime.now, + }; + const rekey = (entries: ReadonlyMap): Map => { + const next = new Map(entries); + if (next.has(previousId)) { + const value = next.get(previousId)!; + next.delete(previousId); + next.set(nativeThreadId, value); + } + return next; + }; + // /clear replaces the transcript, not the live process and its background routing. + yield* Ref.update(lastTurnRouteByNativeThread, rekey); + yield* Ref.update(pendingBackgroundTasksByNativeThread, rekey); + yield* Ref.update(awaitedBackgroundTasksByNativeThread, rekey); + yield* Ref.update(wakeEligibleBackgroundTasksByNativeThread, rekey); + yield* Ref.update(opaqueBackgroundTaskReplayTombstonesByNativeThread, rekey); + // Claude keys its task store by session unless CLAUDE_CODE_TASK_LIST_ID is set, so + // drop the cached list and let the next task tool reload the new session's store. + yield* Ref.update(taskListsByNativeThread, (entries) => { + const next = new Map(entries); + next.delete(previousId); + return next; + }); + yield* Ref.update(wakeBuffers, rekey); + yield* Ref.update(wakeReportsByNativeThread, (state) => ({ + userTurns: rekey(state.userTurns), + reports: rekey(state.reports), + })); + yield* Ref.update(lastProviderThreadByNativeThread, (entries) => + rekey(entries).set(nativeThreadId, providerThread), + ); + yield* Ref.update(openedNativeThreads, (entries) => { + const next = new Set(entries); + next.delete(previousId); + next.add(nativeThreadId); + return next; + }); + yield* Ref.update(requestedContinuations, rekey); + yield* Ref.update(claudeMonitors, (state) => ({ + ...state, + tasks: new Map( + [...state.tasks].map(([id, task]) => [ + id, + task.nativeThreadId === previousId ? { ...task, nativeThreadId } : task, + ]), + ), + })); + if (goalsByNativeThread.has(previousId)) { + goalsByNativeThread.set(nativeThreadId, goalsByNativeThread.get(previousId) ?? null); + goalsByNativeThread.delete(previousId); + } + const settled = settledTurnByNativeThread.get(previousId); + if (settled !== undefined) { + settledTurnByNativeThread.delete(previousId); + settledTurnByNativeThread.set(nativeThreadId, settled); + } + for (const turn of new Set([yield* Ref.get(activeTurn), query.lastTurn, settled])) { + if (turn?.nativeThreadId !== previousId) continue; + turn.nativeThreadId = nativeThreadId; + turn.input = { + ...turn.input, + providerThread: { + ...turn.input.providerThread, + nativeThreadRef: providerThread.nativeThreadRef, + nativeConversationHeadRef: null, + }, + }; + } + // Without a shared CLAUDE_CODE_TASK_LIST_ID the new session starts with no tasks, + // so the pre-clear todo list must stop being the thread's current one. + const todoKey = `${previous.appThreadId}:todo_list`; + const todoPlan = (yield* Ref.get(latestPlanByKind)).get(todoKey); + if ( + !adapterOptions.environment.CLAUDE_CODE_TASK_LIST_ID?.trim() && + previous.appThreadId !== null && + todoPlan !== undefined + ) { + yield* Ref.update(latestPlanByKind, (current) => { + const updated = new Map(current); + updated.delete(todoKey); + return updated; + }); + const superseded = { ...todoPlan, status: "superseded" as const }; + if ((yield* Ref.get(activeTurn)) === null) supersededTodoPlans.push(superseded); + else + yield* emitProviderEvent({ + type: "plan.updated", + driver: CLAUDE_PROVIDER, + plan: superseded, + }); + } + adoptedNativeSessionIds.delete(nativeThreadId); + adoptedNativeSessionIds.set(previousId, nativeThreadId); + query.nativeThreadId = nativeThreadId; + yield* emitProviderEvent({ + type: "provider_thread.updated", + driver: CLAUDE_PROVIDER, + providerThread, + }); + }); + const rosterForNativeThread = ( all: ReadonlyMap>, nativeThreadId: string, @@ -10142,7 +10249,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); // Closing ends the CLI process and abandons the held tool call // without answering it. Not interrupt: the held CLI may never ack it. - yield* liveQuery.query.close.pipe(Effect.ignore); + // Closing can wait for SDK teardown; do not hold the frame lock while it finishes. + yield* liveQuery.query.close.pipe(Effect.ignore, Effect.forkIn(sessionScope)); return; } yield* trackClaudeGoal({ nativeThreadId: liveQuery.nativeThreadId, context, message }); @@ -12128,55 +12236,404 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( live.selection = selection; }); - const openQuery = Effect.fnUntraced(function* ( + // A turn can be built from a projection read before an idle `/clear` adoption + // landed. Opening its native ID would replace the live process with the + // pre-clear session, so follow the adoption this adapter already made. + const resolveAdoptedNativeId = (nativeId: string): string => { + let adoptedNativeId = nativeId; + while (adoptedNativeSessionIds.has(adoptedNativeId)) { + adoptedNativeId = adoptedNativeSessionIds.get(adoptedNativeId)!; + } + return adoptedNativeId; + }; + const followAdoptedProviderThread = ( + providerThread: OrchestrationV2ProviderThread, + ): OrchestrationV2ProviderThread => { + const requestedNativeId = providerThread.nativeThreadRef?.nativeId; + if (requestedNativeId === undefined || requestedNativeId === null) return providerThread; + const adoptedNativeId = resolveAdoptedNativeId(requestedNativeId); + return adoptedNativeId === requestedNativeId + ? providerThread + : { + ...providerThread, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: adoptedNativeId, + strength: "strong", + }, + nativeConversationHeadRef: null, + }; + }; + const followAdoptedNativeSession = ( turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, - nativeThreadId: string, + ): ProviderAdapter.ProviderAdapterV2TurnInput => { + const providerThread = followAdoptedProviderThread(turnInput.providerThread); + return providerThread === turnInput.providerThread + ? turnInput + : { ...turnInput, providerThread }; + }; + + const openQuery = Effect.fnUntraced(function* ( + requestedTurnInput: ProviderAdapter.ProviderAdapterV2TurnInput, queryPolicy: Effect.Success>, ) { - const mcpSession = yield* mcpSessions.read(turnInput.threadId); + const mcpSession = yield* mcpSessions.read(requestedTurnInput.threadId); const mcpOverrides = claudeMcpQueryOverrides({ mcpSession, readOnlySandbox: - sandboxPolicyKindForClaudeRuntimePolicy(turnInput.runtimePolicy) === "readOnly", + sandboxPolicyKindForClaudeRuntimePolicy(requestedTurnInput.runtimePolicy) === + "readOnly", ...(queryPolicy.allowedTools === undefined ? {} : { allowedTools: queryPolicy.allowedTools }), }); const queryPolicyKey = claudeEffectiveQueryPolicyKey(queryPolicy, mcpOverrides); const compiledSelection = compileClaudeModelSelection( - turnInput.modelSelection, + requestedTurnInput.modelSelection, modelCatalog, ); - const resumeSessionAt = yield* getNativeConversationHeadId(turnInput.providerThread); - const existing = yield* Ref.get(queryContext); - // Continuations drain existing output without prompting; retain the live process regardless of selection. - if ( - existing !== null && - existing.nativeThreadId === nativeThreadId && - !existing.stopping && - isClaudeProviderContinuationTurn(turnInput) - ) { - return existing; - } - if ( - existing !== null && - existing.nativeThreadId === nativeThreadId && - existing.queryPolicyKey === queryPolicyKey && - !existing.stopping && - existing.selection.queryIdentity !== compiledSelection.queryIdentity && - canApplyClaudeSelectionLive(existing.selection, compiledSelection) && - (existing.selection.apiModelId === compiledSelection.apiModelId || - existing.openedPermissionMode === queryPolicy.permissionMode) - ) { - yield* applySelectionLive(existing, compiledSelection); - } - if ( - existing !== null && - existing.nativeThreadId === nativeThreadId && - existing.queryPolicyKey === queryPolicyKey && - !existing.stopping && - existing.selection.queryIdentity === compiledSelection.queryIdentity - ) { + // Replacing the process reads files first. Those reads run before the SDK lock and + // the decision is made again under it, so a `/clear` adopted meanwhile is followed. + let replacementPrep: + | { + readonly appendSystemPromptFileText: string | undefined; + readonly resolvedTurnCwd: string | null; + readonly launchArgSettings: ClaudeSdkSettings | undefined; + readonly toolSearchOverride: "true" | undefined; + readonly subagentPromptCacheTtl: typeof ClaudePromptCacheTtl.Type | undefined; + } + | undefined; + while (true) { + const prep = replacementPrep; + // Adoption renames the live query under the SDK lock, so the alias, the decision + // and any replacement happen under it too. A later adoption only renames the + // process this returns, which the turn picks up when it attaches. + const decision = yield* sdkMessageLock.withPermits(1)( + Effect.gen(function* () { + const existing = yield* Ref.get(queryContext); + const turnInput = followAdoptedNativeSession(requestedTurnInput); + const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); + const resumeSessionAt = yield* getNativeConversationHeadId( + turnInput.providerThread, + ); + const sameNativeThread = + existing !== null && existing.nativeThreadId === nativeThreadId; + // A continuation prompts nothing: it drains output the live process + // already produced, so it keeps that process whatever its selection. + if ( + sameNativeThread && + !existing.stopping && + isClaudeProviderContinuationTurn(turnInput) + ) { + return { _tag: "Continue", live: existing } as const; + } + if ( + sameNativeThread && + !existing.stopping && + existing.queryPolicyKey === queryPolicyKey && + (existing.selection.queryIdentity === compiledSelection.queryIdentity || + (!existing.stopping && + canApplyClaudeSelectionLive(existing.selection, compiledSelection) && + (existing.selection.apiModelId === compiledSelection.apiModelId || + existing.openedPermissionMode === queryPolicy.permissionMode))) + ) { + return { _tag: "Reuse", live: existing } as const; + } + + // Background agents and shells run inside the CLI process, so a new selection + // would kill them and lose their results. Refuse until they finish or the user + // presses Stop, which closes the process. + if ( + existing !== null && + (sameNativeThread || existing.lastTurn?.input.threadId === turnInput.threadId) && + !existing.stopping && + (yield* liveProcessRunsBackgroundWork(existing)) + ) { + return yield* new ClaudeBackgroundWorkBlocksQueryReplacementError(); + } + if (prep === undefined) return { _tag: "Prepare", turnInput } as const; + + const closedExistingNativeThreadId = + existing !== null ? existing.nativeThreadId : null; + if (existing !== null) { + yield* existing.query.close.pipe(Effect.ignore); + if (existing.nativeThreadId !== nativeThreadId) { + yield* clearWakeStateForNativeThread(existing.nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(existing.nativeThreadId, { + status: "idle", + }); + } + } + const openedWithResume = (yield* Ref.get(openedNativeThreads)).has(nativeThreadId); + // openedNativeThreads is per session instance and is lost when the + // provider session is idle-released. A prior turn on this native id + // requires resume; sessionId would fail with "already in use". + const hasPersistedProviderTurn = + turnInput.nativeThreadHasTurns ?? turnInput.providerTurnOrdinal > 1; + const shouldResume = + resumeSessionAt !== undefined || openedWithResume || hasPersistedProviderTurn; + const queryOptions = makeClaudeQueryOptions({ + modelSelection: turnInput.modelSelection, + modelCatalog, + nativeThreadId, + title: turnInput.appThread.title, + resume: shouldResume, + ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), + cwd: turnInput.runtimePolicy.cwd, + attachmentsDir, + settings: adapterOptions.settings, + // Delegated threads have no one typing, so they never ask for suggestions. + promptSuggestions: + !isDelegatedAppThread(turnInput.appThread) && (yield* promptSuggestionsEnabled), + ...(prep.launchArgSettings === undefined + ? {} + : { sdkSettings: claudeSdkLaunchArgSettings(prep.launchArgSettings) }), + ...(prep.toolSearchOverride === undefined + ? {} + : { toolSearchOverride: prep.toolSearchOverride }), + environment: claudeQueryEnvironment( + adapterOptions.environment, + turnInput.appThread, + turnInput.appThread.lineage.relationshipToParent === "subagent" + ? prep.subagentPromptCacheTtl + : undefined, + shouldResume, + ), + tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, + ...(mcpOverrides.allowedTools === undefined + ? {} + : { allowedTools: mcpOverrides.allowedTools }), + ...(mcpOverrides.mcpServers === undefined + ? {} + : { mcpServers: mcpOverrides.mcpServers }), + deviceToolsAvailable: mcpSession?.capabilities?.has("device") === true, + permissionMode: queryPolicy.permissionMode, + ...(queryPolicy.allowDangerouslySkipPermissions === undefined + ? {} + : { + allowDangerouslySkipPermissions: + queryPolicy.allowDangerouslySkipPermissions, + }), + canUseTool, + onUserDialog, + onElicitation, + supportedDialogKinds: ["resume_return"], + ...(prep.appendSystemPromptFileText === undefined + ? {} + : { appendSystemPromptFileText: prep.appendSystemPromptFileText }), + }); + const openQueryRuntime = queryRunner.open({ + threadId: turnInput.threadId, + providerSessionId: input.providerSessionId, + options: queryOptions, + }); + const querySession = yield* ( + adapterOptions.installationGate === undefined + ? openQueryRuntime + : adapterOptions.installationGate.execute( + adapterOptions.instanceId, + openQueryRuntime, + ) + ).pipe( + Effect.onError(() => + closedExistingNativeThreadId === nativeThreadId + ? Effect.gen(function* () { + yield* clearWakeStateForNativeThread(nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "idle", + }); + }) + : Effect.void, + ), + ); + // Marked only after a successful open: a failed create must not + // leave the runtime believing the native session exists, or the + // retry would resume a session that was never created. + yield* Ref.update(openedNativeThreads, (current) => { + if (current.has(nativeThreadId)) { + return current; + } + const updated = new Set(current); + updated.add(nativeThreadId); + return updated; + }); + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "active", + }); + const closed = yield* Deferred.make(); + const context: ClaudeLiveQueryContext = { + modelLedger: new Map(), + taskOutputRoot: claudeTaskOutputRoot( + adapterOptions.environment, + path, + prep.resolvedTurnCwd, + hostPlatform, + ), + queryPolicy, + nativeThreadId, + settingsCwd: turnInput.runtimePolicy.cwd, + launchArgSettings: prep.launchArgSettings, + query: querySession, + queryPolicyKey, + selection: compiledSelection, + closed, + lastTurn: null, + promptEchoMode: "unknown", + openedPermissionMode: queryOptions.permissionMode, + receivedInit: false, + capabilities: new Set(), + claudeCodeVersion: undefined, + permissionMode: queryOptions.permissionMode, + stopping: false, + handedOffSubagents: new Set(), + subagentsFromEarlierProcesses: new Set( + [...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter( + (subagent) => subagent.task.status === "running", + ), + ), + }; + // Install the replacement and record inherited subagents in one step. + yield* Ref.update(queryContext, (replaced) => { + if (replaced !== null) { + replaced.handedOffSubagents = new Set( + [...context.subagentsFromEarlierProcesses].filter( + (subagent) => !replaced.subagentsFromEarlierProcesses.has(subagent), + ), + ); + } + return context; + }); + yield* querySession.messages.pipe( + Stream.runForEach((message) => + sdkMessageLock.withPermits(1)( + Effect.gen(function* () { + if ( + message.type === "system" && + message.subtype === "init" && + Reflect.get(message, "parent_tool_use_id") == null + ) { + yield* adoptNativeSessionId(context, message.session_id); + } + if ( + message.type === "system" && + message.subtype === "init" && + !context.receivedInit + ) { + context.openedPermissionMode = + message.permissionMode ?? context.openedPermissionMode; + context.receivedInit = true; + context.capabilities = new Set(message.capabilities ?? []); + context.claudeCodeVersion = message.claude_code_version; + } + if ( + message.type === "system" && + (message.subtype === "init" || message.subtype === "status") && + message.permissionMode !== undefined + ) { + context.permissionMode = message.permissionMode; + } + if (message.type === "result") { + const ledger = claudeModelLedger(message.modelUsage); + const growth = claudeLedgerDelta(context.modelLedger, ledger); + if (growth !== undefined) resultLedgerUsage.set(message, growth); + // A ledger that went backwards cannot anchor the next turn's growth. + context.modelLedger = + growth === undefined && context.modelLedger !== undefined + ? undefined + : ledger; + for (const [model, window] of claudeReportedContextWindows( + message.modelUsage, + )) { + if (reportedContextWindows.get(model) === window) continue; + reportedContextWindows.set(model, window); + resultsReportingNewContextWindow.add(message); + } + } + yield* handleSdkMessage({ query: querySession, message }); + }), + ), + ), + Effect.exit, + Effect.flatMap( + Effect.fnUntraced(function* (exit: ClaudeQueryStreamExit) { + const heldContext = yield* Ref.get(activeTurn); + if ( + (yield* Ref.get(queryContext))?.query === querySession && + heldContext !== null && + heldContext.heldRootFrames.length > 0 + ) { + yield* releaseHeldRootFrames(heldContext); + } + const ownsLiveQuery = yield* Ref.modify(queryContext, (current) => + current?.query === querySession ? [true, null] : [false, current], + ); + if (ownsLiveQuery) { + yield* finalizeActiveTurnAfterQueryExit( + context, + exit._tag === "Failure" ? exit.cause : undefined, + ); + yield* endSubagentCallsOfClosedQuery( + context.nativeThreadId, + context.stopping ? "interrupted" : "failed", + ); + } else { + yield* endSubagentsOfReplacedQuery(context); + } + }), + ), + Effect.ensuring(Deferred.succeed(closed, undefined)), + Effect.forkIn(sessionScope), + ); + return { _tag: "Opened", live: context } as const; + }), + ); + if (decision._tag === "Prepare") { + // Read before closing the live process so a bad path leaves it running. + const cwd = decision.turnInput.runtimePolicy.cwd; + const [appendSystemPromptFileText, { launchArgSettings, toolSearchOverride }] = + yield* Effect.all( + [ + readClaudeAppendSystemPromptFile({ + settings: adapterOptions.settings, + cwd, + fileSystem, + path, + }), + Effect.gen(function* () { + const launchArgSettings = yield* readClaudeLaunchArgSettings({ + settings: adapterOptions.settings, + cwd, + fileSystem, + path, + }); + const toolSearchOverride = yield* resolveToolSearchOverride( + cwd, + launchArgSettings, + ); + return { launchArgSettings, toolSearchOverride }; + }), + ], + { concurrency: "unbounded" }, + ); + replacementPrep = { + resolvedTurnCwd: yield* fileSystem + .realPath(cwd ?? ".") + .pipe(Effect.orElseSucceed(() => cwd)), + launchArgSettings, + toolSearchOverride, + appendSystemPromptFileText, + subagentPromptCacheTtl: + decision.turnInput.appThread.lineage.relationshipToParent === "subagent" + ? yield* resolveSubagentPromptCacheTtl(cwd) + : undefined, + }; + continue; + } + if (decision._tag !== "Reuse") return decision.live; + const existing = decision.live; + if (existing.selection.queryIdentity !== compiledSelection.queryIdentity) { + yield* applySelectionLive(existing, compiledSelection); + } // A denied ExitPlanMode leaves Claude in plan mode. Restore only from plan: // Auto/default may reflect a policy restriction that forbids restoring bypass. if (existing.permissionMode === "plan" && existing.openedPermissionMode !== "plan") { @@ -12196,277 +12653,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } return existing; } - - // Replacing this thread's process kills its background work, even with a new native identity. - if ( - existing !== null && - (existing.nativeThreadId === nativeThreadId || - existing.lastTurn?.input.threadId === turnInput.threadId) && - !existing.stopping && - (yield* liveProcessRunsBackgroundWork(existing)) - ) { - return yield* new ClaudeBackgroundWorkBlocksQueryReplacementError(); - } - - // Read before closing the live process so a bad path leaves it running. - const [appendSystemPromptFileText, { launchArgSettings, toolSearchOverride }] = - yield* Effect.all( - [ - readClaudeAppendSystemPromptFile({ - settings: adapterOptions.settings, - cwd: turnInput.runtimePolicy.cwd, - fileSystem, - path, - }), - Effect.gen(function* () { - const launchArgSettings = yield* readClaudeLaunchArgSettings({ - settings: adapterOptions.settings, - cwd: turnInput.runtimePolicy.cwd, - fileSystem, - path, - }); - const toolSearchOverride = yield* resolveToolSearchOverride( - turnInput.runtimePolicy.cwd, - launchArgSettings, - ); - return { launchArgSettings, toolSearchOverride }; - }), - ], - { concurrency: "unbounded" }, - ); - const closedExistingNativeThreadId = existing !== null ? existing.nativeThreadId : null; - clearedNativeSessions.delete(nativeThreadId); - if (existing !== null) { - yield* existing.query.close.pipe(Effect.ignore); - if (existing.nativeThreadId !== nativeThreadId) { - yield* clearWakeStateForNativeThread(existing.nativeThreadId); - yield* resetBackgroundTaskStateForNativeThreadProcess(existing.nativeThreadId, { - status: "idle", - }); - } - } - - const openedWithResume = (yield* Ref.get(openedNativeThreads)).has(nativeThreadId); - // Idle release loses openedNativeThreads; persisted turns still require resume to avoid "already in use". - const hasPersistedProviderTurn = - turnInput.nativeThreadHasTurns ?? turnInput.providerTurnOrdinal > 1; - const shouldResume = - resumeSessionAt !== undefined || openedWithResume || hasPersistedProviderTurn; - const queryOptions = makeClaudeQueryOptions({ - modelSelection: turnInput.modelSelection, - modelCatalog, - nativeThreadId, - title: turnInput.appThread.title, - resume: shouldResume, - ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), - cwd: turnInput.runtimePolicy.cwd, - attachmentsDir, - settings: adapterOptions.settings, - // Delegated threads have no one typing, so they never ask for suggestions. - promptSuggestions: - !isDelegatedAppThread(turnInput.appThread) && (yield* promptSuggestionsEnabled), - ...(launchArgSettings === undefined - ? {} - : { sdkSettings: claudeSdkLaunchArgSettings(launchArgSettings) }), - environment: claudeQueryEnvironment( - adapterOptions.environment, - turnInput.appThread, - turnInput.appThread.lineage.relationshipToParent === "subagent" - ? yield* resolveSubagentPromptCacheTtl(turnInput.runtimePolicy.cwd) - : undefined, - shouldResume, - ), - tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, - ...(mcpOverrides.allowedTools === undefined - ? {} - : { allowedTools: mcpOverrides.allowedTools }), - ...(mcpOverrides.mcpServers === undefined - ? {} - : { mcpServers: mcpOverrides.mcpServers }), - deviceToolsAvailable: mcpSession?.capabilities?.has("device") === true, - permissionMode: queryPolicy.permissionMode, - ...(queryPolicy.allowDangerouslySkipPermissions === undefined - ? {} - : { - allowDangerouslySkipPermissions: queryPolicy.allowDangerouslySkipPermissions, - }), - canUseTool, - onUserDialog, - onElicitation, - supportedDialogKinds: ["resume_return"], - ...(appendSystemPromptFileText === undefined ? {} : { appendSystemPromptFileText }), - ...(toolSearchOverride === undefined ? {} : { toolSearchOverride }), - }); - const resolvedTurnCwd = yield* fileSystem - .realPath(turnInput.runtimePolicy.cwd ?? ".") - .pipe(Effect.orElseSucceed(() => turnInput.runtimePolicy.cwd)); - const openQueryRuntime = queryRunner.open({ - threadId: turnInput.threadId, - providerSessionId: input.providerSessionId, - options: queryOptions, - }); - // The SDK spawns the CLI per turn, so only the open waits for a provider update. - const querySession = yield* ( - adapterOptions.installationGate === undefined - ? openQueryRuntime - : adapterOptions.installationGate.execute(adapterOptions.instanceId, openQueryRuntime) - ).pipe( - Effect.onError(() => - closedExistingNativeThreadId === nativeThreadId - ? Effect.gen(function* () { - yield* clearWakeStateForNativeThread(nativeThreadId); - yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { - status: "idle", - }); - }) - : Effect.void, - ), - ); - // Mark only successful opens; retries must not resume sessions that failed to create. - yield* Ref.update(openedNativeThreads, (current) => { - if (current.has(nativeThreadId)) { - return current; - } - const updated = new Set(current); - updated.add(nativeThreadId); - return updated; - }); - yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { - status: "active", - }); - const closed = yield* Deferred.make(); - const context: ClaudeLiveQueryContext = { - queryPolicy, - nativeThreadId, - settingsCwd: turnInput.runtimePolicy.cwd, - launchArgSettings, - query: querySession, - queryPolicyKey, - selection: compiledSelection, - closed, - lastTurn: null, - promptEchoMode: "unknown", - openedPermissionMode: queryOptions.permissionMode, - receivedInit: false, - capabilities: new Set(), - claudeCodeVersion: undefined, - permissionMode: queryOptions.permissionMode, - stopping: false, - handedOffSubagents: new Set(), - subagentsFromEarlierProcesses: new Set( - [...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter( - (subagent) => subagent.task.status === "running", - ), - ), - modelLedger: new Map(), - taskOutputRoot: claudeTaskOutputRoot( - adapterOptions.environment, - path, - resolvedTurnCwd, - hostPlatform, - ), - }; - // One step, so the replaced process cannot exit between the install and the handoff. - yield* Ref.update(queryContext, (replaced) => { - if (replaced !== null) { - replaced.handedOffSubagents = new Set( - [...context.subagentsFromEarlierProcesses].filter( - (subagent) => !replaced.subagentsFromEarlierProcesses.has(subagent), - ), - ); - } - return context; - }); - yield* querySession.messages.pipe( - Stream.runForEach((message) => - sdkMessageLock.withPermits(1)( - Effect.suspend(() => { - if ( - message.type === "system" && - message.subtype === "init" && - !context.receivedInit - ) { - context.openedPermissionMode = - message.permissionMode ?? context.openedPermissionMode; - context.receivedInit = true; - context.capabilities = new Set(message.capabilities ?? []); - context.claudeCodeVersion = message.claude_code_version; - } - if ( - message.type === "system" && - message.subtype === "init" && - message.session_id.length > 0 && - message.session_id !== context.nativeThreadId - ) { - clearedNativeSessions.set(context.nativeThreadId, message.session_id); - } - if ( - message.type === "system" && - (message.subtype === "init" || message.subtype === "status") && - message.permissionMode !== undefined - ) { - context.permissionMode = message.permissionMode; - } - if (message.type === "result") { - const ledger = claudeModelLedger(message.modelUsage); - const growth = claudeLedgerDelta(context.modelLedger, ledger); - if (growth !== undefined) resultLedgerUsage.set(message, growth); - // A ledger that went backwards cannot anchor the next turn's growth. - context.modelLedger = - growth === undefined && context.modelLedger !== undefined - ? undefined - : ledger; - for (const [model, window] of claudeReportedContextWindows( - message.modelUsage, - )) { - if (reportedContextWindows.get(model) === window) continue; - reportedContextWindows.set(model, window); - resultsReportingNewContextWindow.add(message); - } - } - return handleSdkMessage({ query: querySession, message }); - }), - ), - ), - Effect.exit, - Effect.flatMap( - Effect.fnUntraced(function* (exit: ClaudeQueryStreamExit) { - const heldContext = yield* Ref.get(activeTurn); - if ( - (yield* Ref.get(queryContext))?.query === querySession && - heldContext !== null && - heldContext.heldRootFrames.length > 0 - ) { - yield* releaseHeldRootFrames(heldContext); - } - const ownsLiveQuery = yield* Ref.modify(queryContext, (current) => - current?.query === querySession ? [true, null] : [false, current], - ); - if (ownsLiveQuery) { - yield* finalizeActiveTurnAfterQueryExit( - context, - exit._tag === "Failure" ? exit.cause : undefined, - ); - yield* endSubagentCallsOfClosedQuery( - context.nativeThreadId, - context.stopping ? "interrupted" : "failed", - ); - } else { - yield* endSubagentsOfReplacedQuery(context); - } - }), - ), - Effect.ensuring(Deferred.succeed(closed, undefined)), - Effect.forkIn(sessionScope), - ); - return context; }); const startTurn = Effect.fn("ClaudeAdapterV2.startTurn")( - function* (turnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { + function* (requestedTurnInput: ProviderAdapter.ProviderAdapterV2TurnInput) { modelCatalog = yield* loadModelCatalog; + const turnInput = followAdoptedNativeSession(requestedTurnInput); const startedAt = yield* DateTime.now; - const nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); + let nativeThreadId = yield* getNativeThreadId(turnInput.providerThread); const nativeTurnId = `turn:${turnInput.attemptId}`; // Use fresh uuids: native sessions can outlive this database, and Claude skips previously recorded prompts. const promptUuid = isClaudeProviderContinuationTurn(turnInput) @@ -12571,7 +12765,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(Path.Path, path), ); - const querySession = yield* openQuery(turnInput, nativeThreadId, queryPolicy); + const querySession = yield* openQuery(turnInput, queryPolicy); const autoCompactWindow = yield* resolveAutoCompactWindow( querySession.settingsCwd, turnInput.modelSelection, @@ -12636,6 +12830,22 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( executedModel: undefined, }; const attachTurn = Effect.gen(function* () { + if (querySession.nativeThreadId !== context.nativeThreadId) { + context.nativeThreadId = querySession.nativeThreadId; + context.input = { + ...turnInput, + providerThread: { + ...turnInput.providerThread, + nativeThreadRef: { + driver: CLAUDE_PROVIDER, + nativeId: querySession.nativeThreadId, + strength: "strong", + }, + nativeConversationHeadRef: null, + }, + }; + } + nativeThreadId = context.nativeThreadId; // Set before claiming: a query that exits after the claim finalizes this turn; one that exits before it fails the start. yield* Ref.set(activeTurn, context); const attached = yield* Ref.modify(queryContext, (current) => { @@ -12652,6 +12862,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( detail: CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL, }); } + for (const plan of supersededTodoPlans.splice(0)) { + yield* emitProviderEvent({ type: "plan.updated", driver: CLAUDE_PROVIDER, plan }); + } if ( !isClaudeProviderContinuationTurn(turnInput) && permissionModeForClaudeRuntimePolicy(turnInput.runtimePolicy) === @@ -12708,8 +12921,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); }); if (userMessage !== null) { - yield* attachTurn; - yield* startUserTurnForWakeReports(nativeThreadId); + yield* sdkMessageLock.withPermits(1)( + attachTurn.pipe( + Effect.andThen(() => startUserTurnForWakeReports(context.nativeThreadId)), + ), + ); yield* Effect.uninterruptibleMask((restore) => restore( querySession.query.offer(userMessage, () => { @@ -13182,7 +13398,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if ( existing === null || (existing.nativeThreadId !== nativeThreadId && - clearedNativeSessions.get(existing.nativeThreadId) !== nativeThreadId) + resolveAdoptedNativeId(existing.nativeThreadId) !== nativeThreadId) ) { return; } @@ -13343,7 +13559,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), resumeThread: Effect.fn("ClaudeAdapterV2.resumeThread")( function* (threadInput: { readonly providerThread: OrchestrationV2ProviderThread }) { - const nativeThreadId = threadInput.providerThread.nativeThreadRef?.nativeId; + // A projection read before an idle `/clear` adoption still names the pre-clear session. + const providerThread = followAdoptedProviderThread(threadInput.providerThread); + const nativeThreadId = providerThread.nativeThreadRef?.nativeId; if ( nativeThreadId !== undefined && nativeThreadId !== null && @@ -13370,7 +13588,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const updatedAt = yield* DateTime.now; return { - ...threadInput.providerThread, + ...providerThread, providerSessionId: input.providerSessionId, status: "idle" as const, updatedAt, @@ -13694,7 +13912,23 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const nativeThreadId = yield* getNativeThreadId(rollbackInput.providerThread); - yield* closeLiveQueryForNativeThread(nativeThreadId); + // Rolling back into a session `/clear` left makes it current again. The process + // still running the adopted session must stop, and forgetting that session keeps + // its exit from republishing it over the rollback. + const adoptedNativeThreadId = resolveAdoptedNativeId(nativeThreadId); + adoptedNativeSessionIds.delete(nativeThreadId); + if (adoptedNativeThreadId !== nativeThreadId) { + yield* Ref.update(lastProviderThreadByNativeThread, (entries) => { + const next = new Map(entries); + next.delete(adoptedNativeThreadId); + return next; + }); + yield* closeLiveQueryForNativeThread(adoptedNativeThreadId); + } + const liveNativeThreadId = (yield* Ref.get(queryContext))?.nativeThreadId; + if (liveNativeThreadId !== undefined) { + yield* closeLiveQueryForNativeThread(liveNativeThreadId); + } const now = yield* DateTime.now; const resumeSessionAt = yield* resolveClaudeRollbackResumeSessionAt(rollbackInput); diff --git a/apps/server/src/orchestration-v2/AttemptNativeSession.ts b/apps/server/src/orchestration-v2/AttemptNativeSession.ts new file mode 100644 index 0000000000000..1fae2eb69c84d --- /dev/null +++ b/apps/server/src/orchestration-v2/AttemptNativeSession.ts @@ -0,0 +1,19 @@ +import type { OrchestrationV2ProviderThread } from "@t3tools/contracts"; + +// Historical attempts can belong to a session from before Claude's /clear. +export function withAttemptNativeSession( + providerThread: OrchestrationV2ProviderThread, + nativeThreadId: string | undefined, +): OrchestrationV2ProviderThread { + if ( + providerThread.nativeThreadRef === null || + nativeThreadId === undefined || + providerThread.nativeThreadRef.nativeId === nativeThreadId + ) + return providerThread; + return { + ...providerThread, + nativeThreadRef: { ...providerThread.nativeThreadRef, nativeId: nativeThreadId }, + nativeConversationHeadRef: null, + }; +} diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index 224fafb02c92d..17233dccf3816 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -1,3 +1,4 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it, vi } from "@effect/vitest"; import { @@ -12,12 +13,14 @@ import { RunAttemptId, RunId, CheckpointScopeId, + ClaudeSettings, ProjectId, ProviderInstanceId, ProviderSessionId, ProviderThreadId, ThreadId, } from "@t3tools/contracts"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; import * as DateTime from "effect/DateTime"; import * as SqlClient from "effect/sql/SqlClient"; @@ -27,6 +30,9 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Option from "effect/Option"; import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerConfig from "../config.ts"; @@ -34,6 +40,7 @@ import * as ServerSettings from "../serverSettings.ts"; import * as GitWorkflow from "../git/GitWorkflowService.ts"; import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts"; import { ProviderWorkspaceMissingError } from "../provider/Errors.ts"; +import * as ClaudeAdapterV2 from "./Adapters/ClaudeAdapterV2.ts"; import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts"; import * as CheckpointService from "./CheckpointService.ts"; import * as CheckpointRollbackService from "./CheckpointRollbackService.ts"; @@ -46,6 +53,7 @@ import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAda import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as RuntimePolicy from "./RuntimePolicy.ts"; +const DEFAULT_CLAUDE_SETTINGS = Schema.decodeSync(ClaudeSettings)({}); const unrelatedProject = Option.some({ workspaceRoot: "/nonexistent/t3-rollback-project", } as never); @@ -874,6 +882,284 @@ it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMe }), ); + // regression: #24153 a rollback across Claude's /clear paired the adopted session with a + // pre-clear message cursor, so the next resume asked the new session for a missing message. + it.effect.each([1, 2] as const)("keeps Claude session at turn %s", (targetOrdinal) => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("rollback-clear"); + const providerThreadId = ProviderThreadId.make("rollback-clear-provider"); + const providerSessionId = ProviderSessionId.make("rollback-clear-session"); + const providerInstanceId = ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID; + const driver = ClaudeAdapterV2.CLAUDE_PROVIDER; + const modelSelection = { instanceId: providerInstanceId, model: "claude-sonnet-4-6" }; + const cwd = "/nonexistent/t3-rollback-clear"; + const scopeId = CheckpointScopeId.make("rollback-clear-scope"); + const runId = (ordinal: number) => RunId.make(`rollback-clear-run-${ordinal}`); + const nodeId = (ordinal: number) => NodeId.make(`rollback-clear-node-${ordinal}`); + const checkpointId = (ordinal: number) => + CheckpointId.make(`rollback-clear-checkpoint-${ordinal}`); + let sequence = 0; + const apply = (event: Record) => + store.apply({ + ...event, + id: EventId.make(`rollback-clear-event-${sequence++}`), + threadId, + occurredAt: now, + } as OrchestrationV2DomainEvent); + yield* apply({ + type: "thread.created", + payload: { + id: threadId, + createdBy: "user", + creationSource: "web", + projectId: ProjectId.make("rollback-clear-project"), + title: "Rollback across /clear", + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + activeProviderThreadId: providerThreadId, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + }); + // Run 2 started in the original session, and its `/clear` adopted a new one. + yield* apply({ + type: "provider-thread.updated", + payload: { + id: providerThreadId, + driver, + providerInstanceId, + providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-after-clear", strength: "strong" }, + nativeConversationHeadRef: { driver, nativeId: "current-head", strength: "weak" }, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }); + yield* apply({ + type: "checkpoint-scope.created", + payload: { + id: scopeId, + threadId, + runId: runId(1), + nodeId: nodeId(1), + parentScopeId: null, + providerThreadId, + kind: "root_run", + ordinalWithinParent: 0, + advancesAppRunCount: true, + cwd, + createdAt: now, + }, + }); + for (const ordinal of [1, 2]) { + const attemptId = RunAttemptId.make(`rollback-clear-attempt-${ordinal}`); + const turnId = ProviderTurnId.make(`rollback-clear-turn-${ordinal}`); + yield* apply({ + type: "run.created", + payload: { + id: runId(ordinal), + threadId, + ordinal, + providerInstanceId, + modelSelection, + providerThreadId, + userMessageId: MessageId.make(`rollback-clear-message-${ordinal}`), + rootNodeId: nodeId(ordinal), + activeAttemptId: attemptId, + status: "completed", + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: checkpointId(ordinal), + contextHandoffId: null, + }, + }); + yield* apply({ + type: "node.updated", + payload: { + id: nodeId(ordinal), + threadId, + runId: runId(ordinal), + parentNodeId: null, + rootNodeId: nodeId(ordinal), + kind: "root_turn", + status: "completed", + countsForRun: true, + providerThreadId, + providerTurnId: turnId, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: scopeId, + startedAt: now, + completedAt: now, + }, + }); + yield* apply({ + type: "run-attempt.created", + runId: runId(ordinal), + payload: { + id: attemptId, + nativeThreadId: "native-before-clear", + runId: runId(ordinal), + attemptOrdinal: 1, + rootNodeId: nodeId(ordinal), + providerInstanceId, + providerThreadId, + providerTurnId: turnId, + reason: "initial", + status: "completed", + startedAt: now, + completedAt: now, + }, + }); + yield* apply({ + type: "provider-turn.updated", + payload: { + id: turnId, + providerThreadId, + nodeId: nodeId(ordinal), + runAttemptId: attemptId, + nativeTurnRef: { driver, nativeId: `assistant-${ordinal}`, strength: "weak" }, + ordinal, + status: "completed", + startedAt: now, + completedAt: now, + }, + }); + } + for (const ordinal of [0, 1, 2]) { + yield* apply({ + type: "checkpoint.captured", + payload: { + id: checkpointId(ordinal), + threadId, + scopeId, + runId: ordinal === 0 ? null : runId(ordinal), + nodeId: nodeId(Math.max(1, ordinal)), + parentCheckpointId: null, + ordinalWithinScope: ordinal, + appRunOrdinal: ordinal === 0 ? null : ordinal, + ref: CheckpointRef.make(`refs/t3/rollback-clear/${ordinal}`), + status: "ready", + files: [], + capturedAt: now, + }, + }); + } + + const fileSystem = yield* FileSystem.FileSystem; + const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: providerInstanceId, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir: yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-rollback-clear-", + }), + fileSystem, + path: yield* Path.Path, + random: yield* SecureRandom.SecureRandom, + idAllocator: yield* IdAllocator.IdAllocatorV2, + queryRunner: { + allocateSessionId: Effect.die("unused allocateSessionId"), + open: () => Effect.die("unused open"), + forkSession: () => Effect.die("unused forkSession"), + subagentLaunchToolUseId: () => Effect.succeed(null), + subagentModel: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy: { runtimeMode: "full-access", interactionMode: "default", cwd: null }, + }); + yield* runtime.events.pipe(Stream.runDrain, Effect.forkScoped); + const committed: OrchestrationV2DomainEvent[] = []; + yield* Effect.gen(function* () { + const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2; + yield* service.execute({ + threadId, + providerThreadId, + scopeId, + checkpointId: checkpointId(targetOrdinal), + restoreFiles: false, + }); + }).pipe( + Effect.provide( + layerCheckpointRollbackService.pipe( + Layer.provide( + Layer.mergeAll( + ThreadCommandExecutor.layer, + IdAllocator.layer, + Layer.mock(CheckpointService.CheckpointServiceV2)({ + restore: () => Effect.void, + deleteStaleRefs: () => Effect.void, + }), + Layer.mock(EventSink.EventSinkV2)({ + write: ({ events }) => + Effect.sync(() => { + committed.push(...events); + return []; + }), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + open: () => Effect.succeed(runtime as never), + }), + Layer.mock(RuntimePolicy.RuntimePolicyV2)({ + resolve: () => Effect.succeed({} as never), + }), + ), + ), + ), + ), + ); + + const providerThread = committed.find( + (event) => event.type === "provider-thread.updated", + )?.payload; + assert.deepEqual( + { + session: providerThread?.nativeThreadRef?.nativeId, + resumeAt: providerThread?.nativeConversationHeadRef?.nativeId, + }, + targetOrdinal === 1 + ? { session: "native-before-clear", resumeAt: "assistant-1" } + : { session: "native-after-clear", resumeAt: "current-head" }, + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect( "rewinds stopped runs repeatedly without counting discarded provider turns again", () => diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 8924a3eb9a78b..8cfda1c6f3430 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -29,6 +29,7 @@ import { isCheckpointRestoreIsolated, SHARED_WORKSPACE_RESTORE_MESSAGE, } from "./CheckpointRestoreSafety.ts"; +import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { CheckpointRestoreError, CheckpointServiceV2 } from "./CheckpointService.ts"; import { EventSinkV2 } from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; @@ -336,7 +337,10 @@ export const layer: Layer.Layer< runsToRollback.length === 0 ? { providerThread } : yield* session.rollbackThread({ - providerThread, + providerThread: withAttemptNativeSession( + providerThread, + targetAttempt?.nativeThreadId, + ), target: rollbackTarget, providerThreadTurns, }); diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index edae3077912ec..ba7d27495b6f4 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -159,10 +159,17 @@ export interface EventSinkV2Shape { EventSinkV2Error >; /** Commits only if the full provider row equals expected; some writers do not advance updatedAt. */ - readonly writeIfProviderThreadUnchanged: (input: { - readonly expected: OrchestrationV2ProviderThread; - readonly events: ReadonlyArray; - }) => Effect.Effect< + readonly writeIfProviderThreadUnchanged: ( + input: { + readonly events: ReadonlyArray; + } & ( + | { readonly expected: OrchestrationV2ProviderThread } + | { + readonly expected: null; + readonly providerThreadId: ProviderThreadId; + } + ), + ) => Effect.Effect< { readonly committed: boolean; readonly storedEvents: ReadonlyArray; @@ -648,7 +655,7 @@ const layerBase: Layer.Layer< // Appends the events only if the stored payload still matches, in one transaction. const writeIfStoredPayloadMatches = ( readStoredPayload: Effect.Effect, E>, - expectedJson: string, + expectedJson: string | undefined, events: ReadonlyArray, ) => commitThenPublish( @@ -675,16 +682,20 @@ const layerBase: Layer.Layer< const writeIfProviderThreadUnchangedEffect = Effect.fn( "orchestrationV2.EventSink.writeIfProviderThreadUnchanged", )(function* (input: Parameters[0]) { + const providerThreadId = input.expected === null ? input.providerThreadId : input.expected.id; yield* Effect.annotateCurrentSpan({ "orchestration_v2.event_count": input.events.length, - "orchestration_v2.provider_thread_id": input.expected.id, + "orchestration_v2.provider_thread_id": providerThreadId, }); - const expectedJson = yield* ProjectionStore.encodeProviderThreadPayload(input.expected); + const expectedJson = + input.expected === null + ? undefined + : yield* ProjectionStore.encodeProviderThreadPayload(input.expected); return yield* writeIfStoredPayloadMatches( sql<{ readonly payload_json: string }>` SELECT payload_json FROM orchestration_v2_projection_provider_threads - WHERE provider_thread_id = ${input.expected.id} + WHERE provider_thread_id = ${providerThreadId} LIMIT 1 `, expectedJson, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 3bdbbffc0373c..f985e7d868148 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -275,21 +275,15 @@ export interface ProjectionCheckpointCaptureContext { readonly sideChat: boolean; } -const RollbackTargetAttempts = Schema.Array( - OrchestrationV2RunAttemptJsonSchema.mapFields(({ id, providerTurnId }) => ({ - id, - providerTurnId, - })), -); -const decodeRollbackTargetAttempts = Schema.decodeUnknownEffect(RollbackTargetAttempts); - export interface ProjectionCheckpointRollbackContext { readonly thread: OrchestrationV2AppThread; readonly providerThread: OrchestrationV2ProviderThread | undefined; readonly providerSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined; readonly checkpoint: OrchestrationV2Checkpoint | undefined; readonly scope: OrchestrationV2CheckpointScope | undefined; - readonly targetAttempt: (typeof RollbackTargetAttempts.Type)[number] | undefined; + readonly targetAttempt: + | Pick + | undefined; readonly providerTurns: ReadonlyArray; readonly rolledBackAttemptIds: ReadonlyArray; readonly runsToRollback: ReadonlyArray; @@ -6019,7 +6013,7 @@ export const layer: Layer.Layer = const targetOrdinal = checkpoint.appRunOrdinal ?? 0; const [attemptRows, runRows, sessionRows, rolledBackAttempts, staleRows] = yield* Effect.all([ - sql`SELECT attempt.attempt_id AS id, attempt.provider_turn_id AS "providerTurnId" + sql`SELECT attempt.payload_json FROM orchestration_v2_projection_run_attempts AS attempt JOIN orchestration_v2_projection_runs AS run ON attempt.attempt_id = json_extract(run.payload_json, '$.activeAttemptId') @@ -6045,7 +6039,10 @@ export const layer: Layer.Layer = AND app_run_ordinal > ${targetOrdinal} ORDER BY ordinal_within_scope ASC`, ]); - const targetAttempt = (yield* decodeRollbackTargetAttempts(attemptRows))[0]; + const targetAttempt = (yield* decodeRows( + decodeRunAttemptPayload, + threadId, + )(attemptRows))[0]; const runsToRollback = yield* decodeRows(decodeRunPayload, threadId)(runRows); const [nodeRows, turnRows] = yield* Effect.all([ sql`SELECT payload_json FROM orchestration_v2_projection_nodes diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index ebe5fba1efb0e..f569e2bef34cc 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -591,7 +591,7 @@ layer("ProviderEventIngestorV2", (it) => { }), ); - it.effect("rolls back runs a provider rewound off its active branch", () => + it.effect.each([false, true])("rewinds runs (skip snapshot: %s)", (skipSnapshot) => Effect.gen(function* () { const now = yield* DateTime.now; const eventSink = yield* EventSink.EventSinkV2; @@ -776,7 +776,63 @@ layer("ProviderEventIngestorV2", (it) => { nativeTurnId: null, }); - yield* ingestor.ingestNormalized({ + const reportedProviderThread: OrchestrationV2ProviderThread = { + id: rewoundThreadId, + driver: CODEX_DRIVER, + providerInstanceId: modelSelection.instanceId, + providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "rewound-thread", + strength: "strong", + }, + nativeConversationHeadRef: { + driver: CODEX_DRIVER, + nativeId: "kept-turn-reply", + strength: "strong", + }, + status: "idle", + firstRunOrdinal: 2, + lastRunOrdinal: 5, + handoffIds: [], + forkedFrom: null, + pendingBackgroundTasks: [], + contextUsage: null, + nativeMetadata: null, + goal: null, + createdAt: now, + updatedAt: now, + }; + const currentProviderThread: OrchestrationV2ProviderThread = { + ...reportedProviderThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "current-native-session", + strength: "strong", + }, + nativeConversationHeadRef: { + driver: CODEX_DRIVER, + nativeId: "current-head", + strength: "strong", + }, + lastRunOrdinal: 6, + }; + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: currentProviderThread, + }, + ], + }); + + const storedEvents = yield* ingestor.ingestNormalized({ + skipProviderThreadSnapshot: skipSnapshot, providerSessionId, providerInstanceId: modelSelection.instanceId, threadId, @@ -784,31 +840,7 @@ layer("ProviderEventIngestorV2", (it) => { event: { type: "provider_thread.updated", driver: CODEX_DRIVER, - providerThread: { - id: rewoundThreadId, - driver: CODEX_DRIVER, - providerInstanceId: modelSelection.instanceId, - providerSessionId, - appThreadId: threadId, - ownerNodeId: null, - nativeThreadRef: { - driver: CODEX_DRIVER, - nativeId: "rewound-thread", - strength: "strong", - }, - nativeConversationHeadRef: { - driver: CODEX_DRIVER, - nativeId: "kept-turn-reply", - strength: "strong", - }, - status: "idle", - firstRunOrdinal: 2, - lastRunOrdinal: 5, - handoffIds: [], - forkedFrom: null, - createdAt: now, - updatedAt: now, - }, + providerThread: reportedProviderThread, retainedNativeTurnIds: ["turn-started-outside-t3", "kept-turn"], }, }); @@ -830,9 +862,13 @@ layer("ProviderEventIngestorV2", (it) => { ), ["ready", "ready", "stale"], ); + assert.deepEqual( + projection.providerThreads.find((thread) => thread.id === rewoundThreadId), + skipSnapshot ? currentProviderThread : reportedProviderThread, + ); assert.equal( - projection.providerThreads.find((thread) => thread.id === rewoundThreadId)?.lastRunOrdinal, - 5, + storedEvents.some((stored) => stored.event.type === "provider-thread.updated"), + !skipSnapshot, ); }), ); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index fddffcd6ee19d..dbd1d7b3a734b 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -192,6 +192,7 @@ function runsOffNativeBranch(input: { } export interface ProviderEventIngestInput { + readonly skipProviderThreadSnapshot?: boolean; readonly providerSessionId: ProviderSessionId; readonly providerInstanceId: ProviderInstanceId; readonly commandId?: CommandId; @@ -503,15 +504,20 @@ export const layer: Layer.Layer< ]; case "provider_thread.updated": { const threadId = input.event.providerThread.appThreadId ?? input.threadId; - const updated = yield* makeDomainEvent(input, { - type: "provider-thread.updated", - threadId, - payload: input.event.providerThread, - }); + const updates = + input.skipProviderThreadSnapshot === true + ? [] + : [ + yield* makeDomainEvent(input, { + type: "provider-thread.updated", + threadId, + payload: input.event.providerThread, + }), + ]; return input.event.retainedNativeTurnIds === undefined - ? [updated] + ? updates : [ - updated, + ...updates, ...(yield* rollBackRewoundRuns(input, { threadId, providerThreadId: input.event.providerThread.id, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 5740faf8f8757..991c0062fb757 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -18,6 +18,7 @@ import { ProviderDriverKind, ProviderInstanceId, type ProviderSessionId, + ProviderThreadId, RunId, ThreadId, TurnItemId, @@ -4478,6 +4479,674 @@ it.effect( }), ); +it.effect("ProviderSessionManagerV2 persists a provider thread update without a run", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-idle-thread-update", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-idle-thread-update", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const afterSequence = yield* eventSink.latestSequence({ threadId }); + const persistedFiber = yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter((stored) => stored.event.type === "provider-thread.updated"), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterEvents); + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "cleared-session", + strength: "strong", + }, + }, + }); + yield* Fiber.join(persistedFiber); + + const projection = yield* projectionStore.getThreadProjection(threadId); + assert.deepEqual( + projection.providerThreads.map((candidate) => candidate.nativeThreadRef?.nativeId), + ["cleared-session"], + ); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps the event pump alive when a runless thread update fails", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const beforeUnchangedWrite = yield* Ref.make>(Effect.void); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-runless-update-failure", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-runless-update-failure", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const afterOpen = yield* eventSink.latestSequence({ threadId }); + const persistedFiber = yield* eventSink.stream({ threadId, afterSequence: afterOpen }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-thread.updated" && + stored.event.payload.nativeThreadRef?.nativeId === "next-session", + ), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterEvents); + // Fail the first unread-snapshot write, then let later writes through. + yield* Ref.set( + beforeUnchangedWrite, + Effect.die(new Error("simulated unread snapshot write failure")), + ); + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "cleared-session", + strength: "strong", + }, + }, + }); + // The pump must keep processing after the failed fallback write. + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "next-session", + strength: "strong", + }, + }, + }); + yield* Fiber.join(persistedFiber); + + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0]?.nativeThreadRef + ?.nativeId, + "next-session", + ); + }); + + yield* effect.pipe( + Effect.provide(layerTest({ state, idleTimeoutMs: 1000, beforeUnchangedWrite })), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 persists a provider thread update queued behind a closing run", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-thread-update-behind-terminal", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-thread-update-behind-terminal", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + // The run consumer is still subscribed while it finalizes the terminal. + const runSubscription = yield* runtime.subscribeEvents!; + const afterSequence = yield* eventSink.latestSequence({ threadId }); + const sessionUpdates = yield* Queue.unbounded(); + yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter((stored) => stored.event.type === "provider-session.updated"), + Stream.runForEach(() => Queue.offer(sessionUpdates, undefined)), + Effect.forkScoped, + ); + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterEvents); + yield* Queue.offer(adapterEvents!, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "turn-before-clear", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + // Claude's root init for the cleared session lands before the run closes. + yield* Queue.offer(adapterEvents!, { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: CODEX_DRIVER, + nativeId: "cleared-session", + strength: "strong", + }, + }, + }); + // The event pump handles events in order, so once this sentinel is + // persisted the thread update has already been routed. + yield* Queue.offer(adapterEvents!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + yield* Queue.take(sessionUpdates); + + const consumed = yield* runSubscription.events.pipe(Stream.take(1), Stream.runCollect); + assert.deepEqual( + Array.from(consumed).map((event) => event.type), + ["turn.terminal"], + ); + yield* runSubscription.close; + yield* Queue.offer(adapterEvents!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + yield* Queue.take(sessionUpdates); + + const projection = yield* projectionStore.getThreadProjection(threadId); + assert.deepEqual( + projection.providerThreads.map((candidate) => candidate.nativeThreadRef?.nativeId), + ["cleared-session"], + ); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 }))); + }), +); + +it.effect.each([ + "late subscriber", + "last recipient", + "runtime failure", + "newer received update", + "unrelated reader", + "coalesced updates", + "repeated committed snapshot", + "rejected write", + "transient release failure", + "persistent release failure", + "idle release retry", + "stopped pump release failure", + "released rewind", + "replacement owner", + "concurrent owner switch", + "replacement insert", + "same-owner write", +] as const)("ProviderSessionManagerV2 preserves unread thread updates with %s", (scenario) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const beforeUnchangedWrite = yield* Ref.make>(Effect.void); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: `unread-update-${scenario}`, + projectId: yield* idAllocator.allocate.project({ fixtureName: "unread-updates" }), + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const first = yield* runtime.subscribeEvents!; + const secondBefore = + scenario === "last recipient" || scenario === "unrelated reader" + ? yield* runtime.subscribeEvents! + : undefined; + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + const sessionUpdates = yield* Queue.unbounded(); + if ( + scenario === "replacement owner" || + scenario === "concurrent owner switch" || + scenario === "same-owner write" + ) { + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: providerThread, + }, + ], + }); + } + const afterSequence = yield* eventSink.latestSequence({ threadId }); + yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter((stored) => stored.event.type === "provider-session.updated"), + Stream.runForEach(() => Queue.offer(sessionUpdates, undefined)), + Effect.forkScoped, + ); + const update = (nativeId: string) => ({ + type: "provider_thread.updated" as const, + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + nativeThreadRef: { driver: CODEX_DRIVER, nativeId, strength: "strong" as const }, + }, + }); + const routed = Effect.gen(function* () { + yield* Queue.offer(adapterEvents, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + yield* Queue.take(sessionUpdates); + }); + const persist = + (event: ProviderAdapter.ProviderAdapterV2Event) => (snapshotCurrent: boolean) => + !snapshotCurrent || event.type !== "provider_thread.updated" + ? Effect.succeed([]) + : eventSink.write({ + events: [ + { + id: yieldEventId, + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: event.providerThread, + }, + ], + }); + const yieldEventId = yield* idAllocator.allocate.event({ threadId }); + if (scenario === "runtime failure") { + yield* Queue.offer(adapterEvents, update("old-session")); + yield* Queue.offer(adapterEvents, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "release-terminal", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + } + if (scenario === "released rewind") + yield* Queue.offer(adapterEvents, { + ...update("rewound-session"), + retainedNativeTurnIds: [], + }); + yield* Queue.offer(adapterEvents, update("cleared-session")); + if (scenario === "coalesced updates") + yield* Queue.offer(adapterEvents, update("newer-session")); + yield* routed; + if (scenario === "idle release retry" || scenario === "stopped pump release failure") { + if (scenario === "stopped pump release failure") { + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Start a turn before the pump fails.", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + } + const failedWrites = yield* Queue.unbounded(); + let attempts = 0; + const failWrite: Effect.Effect = Effect.suspend(() => { + attempts++; + return Ref.set(beforeUnchangedWrite, failWrite).pipe( + Effect.andThen(Queue.offer(failedWrites, undefined)), + Effect.andThen(Effect.die("simulated pending snapshot failure")), + ); + }); + yield* Ref.set(beforeUnchangedWrite, failWrite); + if (scenario === "idle release retry") yield* TestClock.adjust("60 seconds"); + else yield* Queue.end(adapterEvents); + for (let attempt = 0; attempt < 3; attempt++) yield* Queue.take(failedWrites); + assert.equal(attempts, 3); + assert.isTrue(yield* manager.isLive(providerSessionId)); + if (scenario === "stopped pump release failure") { + const openError = yield* manager + .open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal(openError._tag, "ProviderSessionOpenError"); + const lookupError = yield* manager.get(providerSessionId).pipe(Effect.flip); + assert.equal(lookupError._tag, "ProviderSessionLookupError"); + } + yield* Ref.set(beforeUnchangedWrite, Effect.void); + yield* TestClock.adjust("60 seconds"); + assert.isFalse(yield* manager.isLive(providerSessionId)); + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0]?.nativeThreadRef + ?.nativeId, + "cleared-session", + ); + return; + } + if (scenario === "runtime failure" || scenario === "released rewind") { + const released = yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-session.updated" && + stored.event.payload.status === "error", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Queue.end(adapterEvents); + yield* Fiber.join(released); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + yield* first.events.pipe( + Stream.mapEffect((event) => + first.persistEvent!(event, (snapshotCurrent) => { + assert.isFalse( + event.type === "provider_thread.updated" && + event.retainedNativeTurnIds !== undefined, + "A committed rewind must not run again after release", + ); + return persist(event)(snapshotCurrent); + }), + ), + Stream.take(2), + Stream.runDrain, + ); + } else if ( + scenario === "transient release failure" || + scenario === "persistent release failure" + ) { + if (scenario === "persistent release failure") { + yield* Queue.offer(adapterEvents, { + ...update("later-session"), + providerThread: { + ...update("later-session").providerThread, + id: ProviderThreadId.make("later-unread-provider-thread"), + }, + }); + yield* routed; + } + let attempts = 0; + const failWrite: Effect.Effect = Effect.suspend(() => { + attempts++; + return ( + scenario === "persistent release failure" && attempts < 3 + ? Ref.set(beforeUnchangedWrite, failWrite) + : Effect.void + ).pipe(Effect.andThen(Effect.die("simulated release save failure"))); + }); + yield* Ref.set(beforeUnchangedWrite, failWrite); + const release = manager.release({ providerSessionId, reason: "manual_shutdown" }); + if (scenario === "persistent release failure") { + const error = yield* release.pipe(Effect.flip); + assert.equal(error._tag, "ProviderSessionReleaseError"); + } else yield* release; + if (scenario === "persistent release failure") { + assert.equal(attempts, 3); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.deepEqual( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads.map( + (thread) => thread.nativeThreadRef?.nativeId, + ), + ["later-session"], + ); + yield* Ref.set(beforeUnchangedWrite, Effect.void); + yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + } else assert.equal(attempts, 1); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + if (scenario === "persistent release failure") { + assert.deepEqual( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads + .map((thread) => thread.nativeThreadRef?.nativeId) + .sort(), + ["cleared-session", "later-session"], + ); + return; + } + } else if (scenario === "repeated committed snapshot") { + yield* first.events.pipe( + Stream.mapEffect((event) => first.persistEvent!(event, persist(event))), + Stream.take(1), + Stream.runDrain, + ); + yield* Queue.offer(adapterEvents, update("cleared-session")); + yield* routed; + } else if (scenario === "newer received update") { + const second = yield* runtime.subscribeEvents!; + const newer = update("newer-session"); + yield* Queue.offer(adapterEvents, newer); + yield* routed; + yield* second.events.pipe( + Stream.mapEffect((event) => second.persistEvent!(event, persist(event))), + Stream.take(1), + Stream.runDrain, + ); + yield* first.close; + } else if (scenario === "unrelated reader") { + const discarded = yield* Deferred.make(); + const reader = yield* secondBefore!.events.pipe( + Stream.tap((event) => + secondBefore!.discardEvent!(event).pipe( + Effect.andThen( + event.type === "provider_thread.updated" + ? Deferred.succeed(discarded, undefined) + : Effect.void, + ), + ), + ), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Deferred.await(discarded); + yield* first.close; + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0]?.nativeThreadRef + ?.nativeId, + "cleared-session", + ); + yield* Fiber.interrupt(reader); + } else if ( + scenario === "replacement owner" || + scenario === "concurrent owner switch" || + scenario === "replacement insert" || + scenario === "same-owner write" + ) { + const replacement = { + ...update("newer-session").providerThread, + providerSessionId: + scenario === "same-owner write" + ? providerSessionId + : yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }), + }; + const switchOwner = eventSink + .write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: replacement, + }, + ], + }) + .pipe(Effect.asVoid, Effect.orDie); + if (scenario !== "replacement owner") yield* Ref.set(beforeUnchangedWrite, switchOwner); + else yield* switchOwner; + yield* first.close; + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerThreads[0] + ?.providerSessionId, + replacement.providerSessionId, + ); + } else if (scenario === "rejected write") { + yield* first.events.pipe( + Stream.mapEffect((event) => first.persistEvent!(event, () => Effect.succeed([]))), + Stream.take(1), + Stream.runDrain, + ); + } else { + const second = secondBefore ?? (yield* runtime.subscribeEvents!); + yield* first.close; + if (scenario === "last recipient") { + assert.isEmpty((yield* projectionStore.getThreadProjection(threadId)).providerThreads); + } + yield* second.close; + } + const eventStore = yield* EventStore.EventStoreV2; + const persistedUpdates = yield* eventStore + .read({ threadId, afterSequence, eventType: "provider-thread.updated" }) + .pipe(Stream.runCollect); + assert.lengthOf(persistedUpdates, scenario === "same-owner write" ? 2 : 1); + const projection = yield* projectionStore.getThreadProjection(threadId); + assert.deepEqual( + projection.providerThreads.map((thread) => thread.nativeThreadRef?.nativeId), + [ + scenario === "newer received update" || + scenario === "coalesced updates" || + scenario === "replacement owner" || + scenario === "concurrent owner switch" || + scenario === "replacement insert" + ? "newer-session" + : "cleared-session", + ], + ); + }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, beforeUnchangedWrite }))); + }), +); + it.effect.each([ { name: "default load key", normalizeThreadLoadModelSelection: undefined }, { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index eb56df9d049ac..eb59d729a00c0 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -8,7 +8,7 @@ import { OrchestrationV2DomainEvent, OrchestrationV2ProviderSession, type OrchestrationV2PromptSuggestion, - type OrchestrationV2ProviderThread, + OrchestrationV2ProviderThread, type OrchestrationV2Run, OrchestrationV2RuntimeRequest, type OrchestrationV2Subagent, @@ -62,6 +62,8 @@ import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +const providerThreadsEqual = Schema.toEquivalence(OrchestrationV2ProviderThread); + const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; @@ -300,6 +302,13 @@ interface LiveSessionEntry { readonly eventSubscribers: Ref.Ref< ReadonlyMap> >; + readonly eventPumpAlive: Ref.Ref; + readonly latestThreadUpdates: Map; + readonly threadUpdateDeliveries: WeakMap< + ProviderAdapter.ProviderAdapterV2Event, + ProviderThreadUpdateDelivery + >; + readonly pendingRewinds: Map; readonly requestEventPermit: Semaphore.Semaphore; readonly scope: Scope.Closeable; readonly idleGeneration: number; @@ -315,8 +324,21 @@ interface IdleThreadUnload { readonly fiber: Fiber.Fiber | null; } +interface ProviderThreadUpdateDelivery { + readonly event: Extract< + ProviderAdapter.ProviderAdapterV2Event, + { readonly type: "provider_thread.updated" } + >; + readonly recipients: Set; + received: boolean; +} + type ProviderSessionEventSignal = - | { readonly type: "event"; readonly event: ProviderAdapter.ProviderAdapterV2Event } + | { + readonly type: "event"; + readonly event: ProviderAdapter.ProviderAdapterV2Event; + readonly threadUpdate?: ProviderThreadUpdateDelivery; + } | { readonly type: "failure"; readonly cause: Cause.Cause; @@ -636,19 +658,36 @@ export const layerWithOptions = ( Effect.Effect >(); - const publishToSubscribers = ( - subscribers: Ref.Ref< - ReadonlyMap> - >, - signal: ProviderSessionEventSignal, - ) => - Ref.get(subscribers).pipe( - Effect.flatMap((current) => - Effect.forEach(current.values(), (queue) => Queue.offer(queue, signal), { - discard: true, - }), - ), - ); + const publishToSubscribers = Effect.fn("ProviderSessionManager.publishToSubscribers")( + function* (entry: LiveSessionEntry, signal: ProviderSessionEventSignal) { + const current = yield* Ref.get(entry.eventSubscribers); + const threadUpdate = + signal.type === "event" && signal.event.type === "provider_thread.updated" + ? { event: signal.event, recipients: new Set(), received: false } + : undefined; + if (threadUpdate !== undefined) { + entry.latestThreadUpdates.set(threadUpdate.event.providerThread.id, threadUpdate); + entry.threadUpdateDeliveries.set(threadUpdate.event, threadUpdate); + if (threadUpdate.event.retainedNativeTurnIds !== undefined) { + entry.pendingRewinds.set(threadUpdate.event, threadUpdate); + } + } + let delivered = false; + for (const [id, queue] of current) { + const accepted = yield* Queue.offer( + queue, + threadUpdate === undefined + ? signal + : { type: "event", event: threadUpdate.event, threadUpdate }, + ); + if (accepted) { + delivered = true; + threadUpdate?.recipients.add(id); + } + } + return delivered; + }, + ); const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* ( entry: LiveSessionEntry, @@ -1143,14 +1182,30 @@ export const layerWithOptions = ( readonly onlyIfIdleGeneration?: number; readonly keepIfBusyOtherThan?: OwnBusyTurns; readonly keepIf?: Effect.Effect; - }): Effect.fn.Return, DateTime.Utc]> { + }): Effect.fn.Return, DateTime.Utc, boolean]> { const key = sessionKey(input.providerSessionId); const candidate = (yield* Ref.get(sessions)).get(key); if (candidate === undefined) { - return [Option.none(), yield* DateTime.now] as const; + return [Option.none(), yield* DateTime.now, false] as const; } - const remove = (keepRequested: boolean) => - Effect.zip( + const removed = yield* Effect.gen(function* () { + const latest = (yield* Ref.get(sessions)).get(key); + if (latest !== candidate) { + return [latest === undefined ? "gone" : "changed", yield* DateTime.now] as const; + } + if ( + input.onlyIfIdleGeneration !== undefined && + (latest.busyTurns.size > 0 || latest.idleGeneration !== input.onlyIfIdleGeneration) + ) { + return ["kept", yield* DateTime.now] as const; + } + const keepRequested = yield* input.keepIf ?? Effect.succeed(false); + // Unread updates still belong to this runtime. Save them before removal makes + // subscription cleanup unable to find it, including on an event-stream failure. + if (!keepRequested && !(yield* persistPendingThreadUpdates(candidate, true))) { + return ["persist-failed", yield* DateTime.now] as const; + } + return yield* Effect.zip( Ref.modify(sessions, (current) => { const existing = current.get(key); if (existing !== candidate) { @@ -1187,14 +1242,13 @@ export const layerWithOptions = ( }), DateTime.now, ); - const [outcome, releasedAt] = yield* (input.keepIf ?? Effect.succeed(false)).pipe( - Effect.flatMap(remove), - candidate.requestEventPermit.withPermits(1), - ); + }).pipe(candidate.requestEventPermit.withPermits(1)); + const [outcome, releasedAt] = removed; if (outcome === "changed") return yield* removeLiveEntry(input); return [ outcome === "removed" ? Option.some(candidate) : Option.none(), releasedAt, + outcome === "persist-failed", ] as const; }, ); @@ -1248,7 +1302,19 @@ export const layerWithOptions = ( readonly eventStreamCause?: Cause.Cause; }) => Effect.acquireUseRelease( - removeLiveEntry(input), + removeLiveEntry(input).pipe( + Effect.flatMap(([entry, releasedAt, persistFailed]) => + persistFailed + ? Effect.fail( + new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: input.reason, + cause: "Pending provider-thread updates could not be saved before release.", + }), + ) + : Effect.succeed([entry, releasedAt] as const), + ), + ), ([entry, releasedAt]) => Option.match(entry, { onNone: () => Effect.void, @@ -1446,7 +1512,10 @@ export const layerWithOptions = ( Effect.logWarning("orchestration-v2.driver-session.idle-release-failed", { providerSessionId: input.providerSessionId, cause, - }), + }).pipe( + Effect.andThen(Effect.sleep(Duration.millis(idleTimeoutMs))), + Effect.andThen(Effect.suspend(() => releaseIfStillIdle(input))), + ), ), ); }, @@ -1719,6 +1788,8 @@ export const layerWithOptions = ( providerSessionId, Effect.gen(function* () { const key = sessionKey(providerSessionId); + const live = (yield* Ref.get(sessions)).get(key); + if (live === undefined || !(yield* Ref.get(live.eventPumpAlive))) return false; const marked = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); if (entry === undefined || entry.runtime !== runtime) { @@ -2144,6 +2215,133 @@ export const layerWithOptions = ( ), ); + // Callers hold `entry.requestEventPermit`. + const ingestRunlessEventHeld = Effect.fn("ProviderSessionManager.ingestRunlessEventHeld")( + function* ( + entry: LiveSessionEntry, + threadId: ThreadId, + event: ProviderAdapter.ProviderAdapterV2Event, + skipProviderThreadSnapshot = false, + ) { + const input = { + providerSessionId: entry.runtime.providerSessionId, + providerInstanceId: entry.runtime.instanceId, + threadId, + event, + ...(skipProviderThreadSnapshot ? { skipProviderThreadSnapshot: true } : {}), + }; + for (let attempt = 0; attempt < 3; attempt++) { + const current = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + if (current?.runtime !== entry.runtime) return true; + if ( + ((event.type === "provider_thread.updated" && event.sessionScoped === true) || + (event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.sessionScoped === true)) && + !current.attachedThreadIds.has(threadId) + ) + return true; + if (event.type !== "provider_thread.updated") { + yield* providerEventIngestor.ingestNormalized(input); + return true; + } + const projection = yield* projectionStore.getThreadRecords(threadId, [ + "providerThreads", + ]); + const owner = projection.providerThreads.find( + (thread) => thread.id === event.providerThread.id, + ); + // A pooled old runtime can outlive a selection restart. Its unread snapshot + // must not reclaim a thread that has moved to the replacement session. + if (owner !== undefined && owner.providerSessionId !== entry.runtime.providerSessionId) + return true; + // A provider may repeat an already committed snapshot after the run + // closes. Keep rewind effects, but do not store that snapshot a second time. + const events = yield* providerEventIngestor.normalize({ + ...input, + skipProviderThreadSnapshot: + skipProviderThreadSnapshot || + (owner !== undefined && + providerThreadsEqual(owner, { + ...event.providerThread, + pendingBackgroundTasks: event.providerThread.pendingBackgroundTasks ?? [], + contextUsage: event.providerThread.contextUsage ?? null, + nativeMetadata: event.providerThread.nativeMetadata ?? null, + goal: event.providerThread.goal ?? null, + })), + }); + if (events.length === 0) return true; + const written = yield* eventSink.writeIfProviderThreadUnchanged({ + ...(owner === undefined + ? { expected: null, providerThreadId: event.providerThread.id } + : { expected: owner }), + events, + }); + if (written.committed) return true; + } + return false; + }, + (effect, entry) => + effect.pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: entry.runtime.driver, + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + ), + ), + ); + + const ingestRunlessEvent = ( + entry: LiveSessionEntry, + threadId: ThreadId, + event: ProviderAdapter.ProviderAdapterV2Event, + ) => + ingestRunlessEventHeld(entry, threadId, event).pipe( + entry.requestEventPermit.withPermits(1), + ); + + // Keep only the newest snapshot per provider thread. A subscriber joining later + // is not a recipient of an earlier signal, and must not prevent its persistence. + // Callers hold requestEventPermit, including the release cutoff. + const persistPendingThreadUpdates = Effect.fn( + "ProviderSessionManager.persistPendingThreadUpdates", + )(function* (entry: LiveSessionEntry, releasing = false) { + const pending = new Set([ + ...entry.pendingRewinds.values(), + ...entry.latestThreadUpdates.values(), + ]); + let saved = true; + for (const update of pending) { + if (update.received || (!releasing && update.recipients.size > 0)) continue; + const threadId = update.event.providerThread.appThreadId; + if (threadId === null) continue; + const latest = entry.latestThreadUpdates.get(update.event.providerThread.id); + for (let attempt = 0; attempt < (releasing ? 3 : 1); attempt++) { + update.received = yield* ingestRunlessEventHeld( + entry, + threadId, + update.event, + latest !== update, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.unread-thread-update-failed", { + cause, + }).pipe(Effect.as(false)), + ), + ); + if (update.received) break; + } + if (update.received) entry.pendingRewinds.delete(update.event); + else saved = false; + } + return saved; + }); + const makeEventSubscription = Effect.fn("ProviderSessionManager.makeEventSubscription")( function* ( subscribers: Ref.Ref< @@ -2157,18 +2355,53 @@ export const layerWithOptions = ( updated.set(subscriberId, queue); return updated; }); - const close = Ref.modify(subscribers, (current) => { - if (!current.has(subscriberId)) { - return [false, current] as const; - } - const updated = new Map(current); - updated.delete(subscriberId); - return [true, updated] as const; - }).pipe( - Effect.flatMap((removed) => - removed - ? Queue.clear(queue).pipe(Effect.andThen(Queue.end(queue)), Effect.asVoid) - : Effect.void, + const removeAndDrain = (entry: LiveSessionEntry | undefined) => + Ref.modify(subscribers, (current) => { + if (!current.has(subscriberId)) { + return [false, current] as const; + } + const updated = new Map(current); + updated.delete(subscriberId); + return [true, updated] as const; + }).pipe( + Effect.flatMap((removed) => + removed + ? // Ending first makes every later offer report it was refused, so a signal is + // either drained here or seen as undelivered by its publisher. + Queue.end(queue).pipe( + Effect.andThen(Queue.clear(queue)), + Effect.flatMap((unread) => + Effect.gen(function* () { + for (const signal of unread) { + if (signal.type === "event") + signal.threadUpdate?.recipients.delete(subscriberId); + } + if (entry !== undefined) { + for (const update of new Set([ + ...entry.latestThreadUpdates.values(), + ...entry.pendingRewinds.values(), + ])) { + update.recipients.delete(subscriberId); + } + yield* persistPendingThreadUpdates(entry); + } + }), + ), + ) + : Effect.void, + ), + ); + // The pump persists a refused update under the same permit. Holding it from + // the end of the queue through the drained writes keeps an older drained + // update from committing over a newer refused one. + const close = Ref.get(sessions).pipe( + Effect.map((current) => + [...current.values()].find((candidate) => candidate.eventSubscribers === subscribers), + ), + Effect.flatMap((entry) => + entry === undefined + ? removeAndDrain(undefined) + : removeAndDrain(entry).pipe(entry.requestEventPermit.withPermits(1)), ), // Detached so the consumer's run failure is not ordered behind release cleanup. Effect.andThen( @@ -2179,7 +2412,8 @@ export const layerWithOptions = ( }), ), ); - const events = Stream.fromQueue(queue).pipe( + // One signal per pull, so what a reader never pulled is still queued at close. + const events = Stream.fromEffectRepeat(Queue.take(queue)).pipe( Stream.mapEffect((signal) => signal.type === "event" ? Effect.succeed(signal.event) @@ -2187,7 +2421,64 @@ export const layerWithOptions = ( ), Stream.ensuring(close), ); - return { events, close } satisfies ProviderAdapter.ProviderAdapterV2EventSubscription; + const entry = [...(yield* Ref.get(sessions)).values()].find( + (candidate) => candidate.eventSubscribers === subscribers, + ); + const persistEvent: NonNullable< + ProviderAdapter.ProviderAdapterV2EventSubscription["persistEvent"] + > = (event, persist) => + event.type !== "provider_thread.updated" || entry === undefined + ? persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))) + : Effect.gen(function* () { + const current = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + const latest = entry.latestThreadUpdates.get(event.providerThread.id); + const delivery = entry.threadUpdateDeliveries.get(event); + if (delivery?.received === true) { + // The release path may persist the update before this subscriber reads it. + // Report the committed snapshot so finalization cannot emit the old session. + return { + storedEvents: [], + ...(current?.runtime === entry.runtime && latest?.received === true + ? { persistedProviderThread: latest.event.providerThread } + : {}), + }; + } + const snapshotCurrent = + current?.runtime === entry.runtime && + (latest?.event === event || latest?.received !== true); + const stored = yield* persist(snapshotCurrent); + const persistedSnapshot = stored.some( + (record) => record.event.type === "provider-thread.updated", + ); + if ( + delivery !== undefined && + (persistedSnapshot || + (event.retainedNativeTurnIds !== undefined && stored.length > 0)) + ) { + delivery.received = true; + } + if (delivery?.received === true) entry.pendingRewinds.delete(event); + return { + storedEvents: stored, + ...(persistedSnapshot ? { persistedProviderThread: event.providerThread } : {}), + }; + }).pipe(entry.requestEventPermit.withPermits(1)); + const discardEvent = (event: ProviderAdapter.ProviderAdapterV2Event) => + entry === undefined || event.type !== "provider_thread.updated" + ? Effect.void + : Effect.gen(function* () { + const delivery = entry.threadUpdateDeliveries.get(event); + delivery?.recipients.delete(subscriberId); + yield* persistPendingThreadUpdates(entry); + }).pipe(entry.requestEventPermit.withPermits(1)); + return { + events, + close, + persistEvent, + discardEvent, + } satisfies ProviderAdapter.ProviderAdapterV2EventSubscription; }, ); @@ -2728,40 +3019,44 @@ export const layerWithOptions = ( // Requests and recovered background commands can arrive before a run subscribes. const threadId = sessionScopedArtifactThreadId(event); if (threadId !== undefined) { - yield* Effect.gen(function* () { - const current = (yield* Ref.get(sessions)).get( - sessionKey(entry.runtime.providerSessionId), - ); - if ( - current?.runtime !== entry.runtime || - ((event.type === "provider_thread.updated" || - (event.type === "turn_item.updated" && - event.turnItem.type === "command_execution")) && - !current.attachedThreadIds.has(threadId)) - ) - return; - yield* providerEventIngestor - .ingestNormalized({ - providerSessionId: entry.runtime.providerSessionId, - providerInstanceId: entry.runtime.instanceId, - threadId, - event, - }) - .pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterEventStreamError({ - driver: entry.runtime.driver, - providerSessionId: entry.runtime.providerSessionId, - cause, - }), - ), - ); - }).pipe(entry.requestEventPermit.withPermits(1)); + yield* ingestRunlessEvent(entry, threadId, event); yield* releaseLiveOutput; return; } - yield* publishToSubscribers(entry.eventSubscribers, { type: "event", event }); + yield* Effect.gen(function* () { + const delivered = yield* publishToSubscribers(entry, { + type: "event", + event, + }); + // A provider thread can also change while idle (Claude adopts + // a new session after `/clear`). With no run reading it, that + // update would be dropped and a later resume would reopen the + // old native session. + if ( + !delivered && + event.type === "provider_thread.updated" && + event.providerThread.appThreadId !== null + ) { + // Keep the update pending when persistence fails. Letting the + // failure escape would end the event pump and tear down the + // session for an unread snapshot that a later retry can save. + const received = yield* ingestRunlessEventHeld( + entry, + event.providerThread.appThreadId, + event, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "orchestration-v2.driver-session.unread-thread-update-failed", + { providerSessionId: entry.runtime.providerSessionId, cause }, + ).pipe(Effect.as(false)), + ), + ); + const update = entry.threadUpdateDeliveries.get(event); + if (update !== undefined) update.received = received; + if (received) entry.pendingRewinds.delete(event); + } + }).pipe(entry.requestEventPermit.withPermits(1)); yield* releaseLiveOutput; }), ), @@ -2770,6 +3065,15 @@ export const layerWithOptions = ( Effect.exit, Effect.flatMap((exit) => Effect.gen(function* () { + yield* Ref.set(entry.eventPumpAlive, false); + // No more terminal events can clear turns owned by this pump. + yield* Ref.update(sessions, (current) => { + const key = sessionKey(entry.runtime.providerSessionId); + const live = current.get(key); + return live?.runtime === entry.runtime && live.busyTurns.size > 0 + ? new Map(current).set(key, { ...live, busyTurns: new Set() }) + : current; + }); if (shutdownSignal.received) return; const current = (yield* Ref.get(sessions)).get( sessionKey(entry.runtime.providerSessionId), @@ -2807,7 +3111,10 @@ export const layerWithOptions = ( reason: "runtime_error", detail: makeProviderFailure({ cause }).message, eventStreamCause: cause, - }).pipe(logReleaseFailure(entry.runtime.providerSessionId)); + }).pipe( + Effect.tapError(() => scheduleIdleRelease(entry.runtime.providerSessionId)), + logReleaseFailure(entry.runtime.providerSessionId), + ); }), ), Effect.forkIn(layerScope), @@ -2865,6 +3172,14 @@ export const layerWithOptions = ( if (live !== undefined) yield* releaseForLinkedEnvironments(live); const existing = (yield* Ref.get(sessions)).get(key); if (existing !== undefined) { + if (!(yield* Ref.get(existing.eventPumpAlive))) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: + "The provider event stream stopped; pending session updates must be saved before reopening.", + }); + } const attached = existing.attachedThreadIds.has(input.threadId); if (!attached && !existing.supportsMultipleProviderThreads) { return yield* new ProviderSessionOpenError({ @@ -2981,6 +3296,10 @@ export const layerWithOptions = ( runtime, exposedRuntime, eventSubscribers, + eventPumpAlive: yield* Ref.make(true), + latestThreadUpdates: new Map(), + threadUpdateDeliveries: new WeakMap(), + pendingRewinds: new Map(), requestEventPermit: yield* Semaphore.make(1), scope: sessionScope, idleGeneration: 0, @@ -3036,6 +3355,14 @@ export const layerWithOptions = ( if (entry === undefined) { return Option.none(); } + if (!(yield* Ref.get(entry.eventPumpAlive))) { + return yield* new ProviderSessionOpenError({ + instanceId: entry.runtime.instanceId, + providerSessionId, + cause: + "The provider event stream stopped; pending session updates must be saved before reopening.", + }); + } yield* touchActivity(providerSessionId); return Option.some(entry.exposedRuntime); }, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 7fe02946f2c80..8c7d5b2ee81b1 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -26,6 +26,7 @@ import * as Predicate from "effect/Predicate"; import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; +import { withAttemptNativeSession } from "./AttemptNativeSession.ts"; import { CLAUDE_PROVIDER, CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL, @@ -1041,12 +1042,19 @@ export const layer: Layer.Layer< const sourceRun = sourceProjection.runs.find( (candidate) => candidate.id === nativeForkTransfer.sourcePoint.runId, ); - const sourceProviderThread = sourceProjection.providerThreads.find( - (candidate) => candidate.id === sourceRun?.providerThreadId, - ); const sourceAttempt = sourceProjection.attempts.find( (candidate) => candidate.id === sourceRun?.activeAttemptId, ); + const currentSourceProviderThread = sourceProjection.providerThreads.find( + (candidate) => candidate.id === sourceRun?.providerThreadId, + ); + const sourceProviderThread = + currentSourceProviderThread === undefined + ? undefined + : withAttemptNativeSession( + currentSourceProviderThread, + sourceAttempt?.nativeThreadId, + ); const sourceProviderTurn = latestProviderTurnForAttempt(sourceProjection.providerTurns, sourceAttempt?.id) ?? sourceProjection.providerTurns.find( @@ -1714,6 +1722,17 @@ export const layer: Layer.Layer< !noteContinuation ? session : makeDeliverySession(session, startWithHandoffs); + // An adopted native id has already been written by the provider even before its first app turn. + const runningNativeThreadId = runningProviderThread.nativeThreadRef?.nativeId; + const providerAdoptedNativeThread = + sameNativeThread && + runningNativeThreadId != null && + acceptedAttempts.some((source) => source.nativeThreadId !== undefined) && + !projection.attempts.some( + (source) => + source.providerThreadId === providerThread.id && + source.nativeThreadId === runningNativeThreadId, + ); return runExecution.startRootRun({ commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, @@ -1742,6 +1761,7 @@ export const layer: Layer.Layer< ) + 1, nativeThreadHasTurns: nativeInputRunIds.size > 0 || + providerAdoptedNativeThread || (legacyInputRunIds.size > 0 && sameNativeThread && !acceptedAttempts.some((source) => source.nativeThreadId !== undefined)), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 0e8898cf0623b..fbb4e0a237afc 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2681,6 +2681,23 @@ it.effect( }), ); +it.effect.each([undefined, "native-session-before-clear"])( + "records the reported native session when finalizing an attempt initialized with %s", + (nativeThreadId) => + Effect.gen(function* () { + const { finalEvents } = yield* captureRootRunTermination({ + key: "finalize-reported-native-session", + shouldFinalizeRun: () => Effect.succeed(true), + ...(nativeThreadId === undefined ? {} : { nativeThreadId }), + reportedNativeThreadId: "native-session-after-clear", + }); + const attempt = finalEvents.find((event) => event.type === "run-attempt.updated"); + assert.isDefined(attempt); + assert.equal(attempt.payload.nativeThreadId, "native-session-after-clear"); + assert.equal(attempt.payload.status, "interrupted"); + }), +); + it.effect("settles native subagents when a workspace handoff completes the root run", () => Effect.gen(function* () { const ids = backgroundScenarioIds("subagent-workspace-handoff"); @@ -3513,6 +3530,77 @@ it.effect("keeps completed runs completed when pull request refresh fails", () = }), ); +it.effect("finalizes with a provider snapshot already persisted by a skipped event write", () => + Effect.gen(function* () { + const { finalEvents } = yield* captureRootRunTermination({ + key: "skipped-provider-thread-persist", + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.make( + { + type: "provider_thread.updated", + driver, + providerThread: { + id: ids.providerThreadId, + driver, + nativeThreadRef: { driver, nativeId: "adopted-session", strength: "strong" }, + } as unknown as OrchestrationV2ProviderThread, + } as ProviderAdapter.ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + ), + persistEvent: (event, persist) => + event.type === "provider_thread.updated" + ? Effect.succeed({ + storedEvents: [], + persistedProviderThread: event.providerThread, + }) + : persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))), + }); + assert.equal( + finalEvents.find((event) => event.type === "provider-thread.updated")?.payload.nativeThreadRef + ?.nativeId, + "adopted-session", + ); + }), +); + +it.effect("does not adopt a stale snapshot that only stored rewind events", () => + Effect.gen(function* () { + const rewind = { + sequence: 1, + commandId: null, + event: { type: "turn-item.updated", payload: { id: "rewind" } }, + } as unknown as ProviderAdapter.ProviderAdapterV2PersistEventResult["storedEvents"][number]; + const { finalEvents } = yield* captureRootRunTermination({ + key: "stale-provider-thread-persist", + shouldFinalizeRun: () => Effect.succeed(true), + reportedNativeThreadId: "original-session", + events: (ids) => + Stream.make( + { + type: "provider_thread.updated", + driver, + providerThread: { + id: ids.providerThreadId, + driver, + nativeThreadRef: { driver, nativeId: "stale-session", strength: "strong" }, + } as unknown as OrchestrationV2ProviderThread, + } as ProviderAdapter.ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + ), + persistEvent: (event, persist) => + event.type === "provider_thread.updated" + ? Effect.succeed({ storedEvents: [rewind] }) + : persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))), + }); + assert.equal( + finalEvents.find((event) => event.type === "provider-thread.updated")?.payload.nativeThreadRef + ?.nativeId, + "original-session", + ); + }), +); + function captureRootRunTermination(input: { readonly key: string; readonly shouldFinalizeRun: () => Effect.Effect; @@ -3522,6 +3610,7 @@ function captureRootRunTermination(input: { readonly loadRunInterruptReason?: () => Effect.Effect; readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"]; readonly seedOpenSubagent?: boolean; + readonly persistEvent?: ProviderAdapter.ProviderAdapterV2EventSubscription["persistEvent"]; readonly events?: ( ids: BackgroundScenarioIds, ) => Stream.Stream< @@ -3532,6 +3621,8 @@ function captureRootRunTermination(input: { readonly interruptTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["interruptTurn"]; readonly messageText?: string; readonly initialProviderTurnId?: ProviderTurnId | null; + readonly nativeThreadId?: string; + readonly reportedNativeThreadId?: string; readonly onStartFailure?: ( error: RunExecutionService.RunExecutionServiceV2Error, ) => Effect.Effect; @@ -3675,6 +3766,7 @@ function captureRootRunTermination(input: { close: Deferred.succeed(ingestionDone, undefined).pipe( Effect.andThen(input.onSubscriptionClose ?? Effect.void), ), + ...(input.persistEvent === undefined ? {} : { persistEvent: input.persistEvent }), }), startTurn: input.startTurn ?? (() => Effect.void), interruptTurn: input.interruptTurn ?? (() => Effect.void), @@ -3696,9 +3788,15 @@ function captureRootRunTermination(input: { providerThread: { id: ids.providerThreadId, driver, + nativeThreadRef: + input.reportedNativeThreadId === undefined + ? null + : { driver, nativeId: input.reportedNativeThreadId, strength: "strong" }, } as OrchestrationV2ProviderThread, attempt: { id: ids.attemptId, + providerThreadId: ids.providerThreadId, + ...(input.nativeThreadId === undefined ? {} : { nativeThreadId: input.nativeThreadId }), providerTurnId: input.initialProviderTurnId === undefined ? ids.rootProviderTurnId @@ -4176,9 +4274,16 @@ function runBackgroundItemScenario( `subagent:${event.subagent.status}`, ]); } - // Only the count of stored events matters to the run. + // Snapshot identity distinguishes a saved session from unrelated rewind writes. return event.type === "provider_thread.updated" - ? [{} as OrchestrationV2StoredEvent] + ? [ + { + event: { + type: "provider-thread.updated", + payload: event.providerThread, + }, + } as OrchestrationV2StoredEvent, + ] : []; }); // A write outside a batch waits for queued snapshots, as the event sink's does. diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 10f10a98de0a4..c77d12bc9fbe3 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -672,8 +672,15 @@ export const layer: Layer.Layer< }; }) { const completedAt = input.completedAt ?? (yield* DateTime.now); + // Claude's `/clear` can move a run to a new native session after its attempt was + // written; rollback and fork must pair this run's cursor with that session. + const reportedNativeThreadId = + input.providerThread.id === input.attempt.providerThreadId + ? input.providerThread.nativeThreadRef?.nativeId + : undefined; const finalizedAttempt: OrchestrationV2RunAttempt = { ...input.attempt, + ...(reportedNativeThreadId == null ? {} : { nativeThreadId: reportedNativeThreadId }), status: input.terminal.status, completedAt, }; @@ -1088,7 +1095,7 @@ export const layer: Layer.Layer< attemptId: input.attempt.id, providerThreadId: input.providerThread.id, }; - const eventSubscription = + const eventSubscription: ProviderAdapter.ProviderAdapterV2EventSubscription = input.session.subscribeEvents === undefined ? { events: input.session.events, close: Effect.void } : yield* input.session.subscribeEvents; @@ -1502,50 +1509,71 @@ export const layer: Layer.Layer< if (event.type === "provider_turn.updated" && !nativeChildTurn) { firstRootProviderTurnId ??= event.providerTurn.id; } - const storedEvents = yield* providerEventIngestor.ingestNormalized({ - analyticsContext: nativeChildTurn - ? { - ...analyticsContext, - origin: NATIVE_SUBAGENT_TURN_ORIGIN, - segment: "native_child", - } - : { - ...analyticsContext, - segment: - event.type === "provider_turn.updated" && - event.providerTurn.id !== firstRootProviderTurnId - ? "continuation" - : input.attempt.reason, - }, - providerSessionId: input.providerSessionId, - providerInstanceId: input.run.providerInstanceId, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - event: deliveredEvent, - ...(isRootProviderThreadUpdate - ? rootTerminalAlreadySeen + const persist = (snapshotCurrent: boolean) => + providerEventIngestor.ingestNormalized({ + ...(!snapshotCurrent ? { skipProviderThreadSnapshot: true } : {}), + analyticsContext: nativeChildTurn ? { - writeIfProviderThreadOwner: { - providerThreadId: input.providerThread.id, - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedLastRunOrdinal: input.run.ordinal, - }, + ...analyticsContext, + origin: NATIVE_SUBAGENT_TURN_ORIGIN, + segment: "native_child", } : { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } - : {}), - }); - if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) { + ...analyticsContext, + segment: + event.type === "provider_turn.updated" && + event.providerTurn.id !== firstRootProviderTurnId + ? "continuation" + : input.attempt.reason, + }, + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: deliveredEvent, + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } + : {}), + }); + const persisted: ProviderAdapter.ProviderAdapterV2PersistEventResult = + yield* eventSubscription.persistEvent === undefined + ? persist(true).pipe(Effect.map((storedEvents) => ({ storedEvents }))) + : eventSubscription.persistEvent(event, persist); + if ( + isRootProviderThreadUpdate && + rootTerminalAlreadySeen && + persisted.storedEvents.length === 0 && + persisted.persistedProviderThread === undefined + ) { yield* Ref.set(providerThreadOwnerLost, true); } - return storedEvents.length; + return { + persistedProviderThread: persisted.persistedProviderThread, + persistedProviderThreadSnapshot: + persisted.persistedProviderThread !== undefined || + (event.type === "provider_thread.updated" && + persisted.storedEvents.some( + (record) => + record.event.type === "provider-thread.updated" && + record.event.payload.id === event.providerThread.id, + )), + }; }); // The timer only wakes ingestion. The flush runs as an ingestion step outside any provider // batch, so it stays ordered with routing and snapshots, and a write has committed when @@ -1609,7 +1637,11 @@ export const layer: Layer.Layer< event: ProviderAdapter.ProviderAdapterV2Event; accepted: boolean; }) { - if (!accepted) return; + if (!accepted) { + if (eventSubscription.discardEvent !== undefined) + yield* eventSubscription.discardEvent(event); + return; + } if ( event.type === "provider_turn.updated" && event.providerTurn.runAttemptId === input.attempt.id && @@ -1617,7 +1649,8 @@ export const layer: Layer.Layer< ) { yield* Ref.set(latestRootProviderTurn, event.providerTurn); } - let storedEventCount = 0; + let persistedProviderThread: OrchestrationV2ProviderThread | undefined; + let persistedProviderThreadSnapshot = false; const deliveredEvent = assistantStreaming.filter( event, DateTime.toEpochMillis(yield* DateTime.now), @@ -1642,7 +1675,7 @@ export const layer: Layer.Layer< shouldStartProviderTurn().pipe(Effect.orElseSucceed(() => true)), ]), ); - storedEventCount = stored; + ({ persistedProviderThread, persistedProviderThreadSnapshot } = stored); if (!owned) { yield* Ref.get(latestProviderThread).pipe( Effect.flatMap((providerThread) => @@ -1663,7 +1696,10 @@ export const layer: Layer.Layer< ); } } else if (deliveredEvent) { - storedEventCount = yield* ingestDelivered(event, deliveredEvent); + ({ persistedProviderThread, persistedProviderThreadSnapshot } = yield* ingestDelivered( + event, + deliveredEvent, + )); } else if ( event.type === "turn_item.updated" && (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") && @@ -1678,8 +1714,13 @@ export const layer: Layer.Layer< reservedHeldTurnItemIds.add(event.turnItem.id); } if (event.type === "provider_thread.updated") { - if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { - yield* Ref.set(latestProviderThread, event.providerThread); + if ( + event.providerThread.id === input.providerThread.id && + persistedProviderThreadSnapshot + ) { + // Only a snapshot that actually persisted may replace the session. + // Rewind events stored alongside a skipped snapshot are not evidence. + yield* Ref.set(latestProviderThread, persistedProviderThread ?? event.providerThread); rootSnapshotPendingWork = event.pendingBackgroundWork === true; } } diff --git a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts index 745980da07988..b1d26d9ce3e07 100644 --- a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts +++ b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts @@ -1,5 +1,8 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { + ClaudeSettings, CheckpointId, CheckpointRef, CheckpointScopeId, @@ -17,22 +20,31 @@ import { ThreadId, TurnItemId, } from "@t3tools/contracts"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; import * as Statement from "effect/sql/Statement"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as ClaudeAdapterV2 from "./Adapters/ClaudeAdapterV2.ts"; import { ClaudeProviderCapabilitiesV2 } from "./Adapters/ClaudeAdapterV2.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +const DEFAULT_CLAUDE_SETTINGS = Schema.decodeSync(ClaudeSettings)({}); const forkCases = ( [ ["codex", "interrupted"], @@ -649,6 +661,205 @@ it.effect.each([ }, ); +// regression: #24153 a fork across Claude's /clear asked the adopted session for a message +// that only exists in the session the source turn ran in. +it.effect("forks a Claude run from the session it ran in after /clear adopted another", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const forkCalls: Array = []; + const claude = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir: yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-fork-clear-" }), + fileSystem, + path: yield* Path.Path, + random: yield* SecureRandom.SecureRandom, + idAllocator: yield* IdAllocator.IdAllocatorV2, + queryRunner: { + allocateSessionId: Effect.succeed("unused-native-session"), + open: () => Effect.die("unused open"), + forkSession: (input) => + Effect.sync(() => forkCalls.push(input)).pipe( + Effect.andThen(Effect.die("fork inspected")), + ), + subagentLaunchToolUseId: () => Effect.succeed(null), + subagentModel: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + const driver = claude.driver; + const instanceId = claude.instanceId; + const modelSelection = { instanceId, model: "claude-sonnet-4-6" }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const now = yield* DateTime.now; + const sourceThreadId = ThreadId.make("fork-clear-source"); + const providerThreadId = ProviderThreadId.make("fork-clear-provider-thread"); + const sourceRunId = RunId.make("fork-clear-source-run"); + const attemptId = RunAttemptId.make("fork-clear-source-attempt"); + const providerTurnId = ProviderTurnId.make("fork-clear-source-turn"); + const rootNodeId = NodeId.make("fork-clear-source-node"); + + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("fork-clear-create-source"), + threadId: sourceThreadId, + projectId: ProjectId.make("fork-clear-project"), + title: "Fork across /clear", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* eventSink.write({ + events: [ + // A later run's `/clear` adopted a new native session for this provider thread. + { + id: EventId.make("fork-clear-provider-thread"), + type: "provider-thread.updated", + threadId: sourceThreadId, + occurredAt: now, + payload: { + id: providerThreadId, + driver, + providerInstanceId: instanceId, + providerSessionId: null, + appThreadId: sourceThreadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-after-clear", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("fork-clear-source-run"), + type: "run.created", + threadId: sourceThreadId, + runId: sourceRunId, + occurredAt: now, + payload: { + id: sourceRunId, + threadId: sourceThreadId, + ordinal: 1, + providerInstanceId: instanceId, + modelSelection, + providerThreadId, + userMessageId: MessageId.make("fork-clear-source-message"), + rootNodeId, + activeAttemptId: attemptId, + status: "completed", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }, + }, + { + id: EventId.make("fork-clear-source-attempt"), + type: "run-attempt.created", + threadId: sourceThreadId, + runId: sourceRunId, + occurredAt: now, + payload: { + id: attemptId, + nativeThreadId: "native-before-clear", + runId: sourceRunId, + attemptOrdinal: 1, + rootNodeId, + providerInstanceId: instanceId, + providerThreadId, + providerTurnId, + reason: "initial", + status: "completed", + startedAt: now, + completedAt: now, + }, + }, + { + id: EventId.make("fork-clear-source-turn"), + type: "provider-turn.updated", + threadId: sourceThreadId, + occurredAt: now, + payload: { + id: providerTurnId, + providerThreadId, + nodeId: rootNodeId, + runAttemptId: attemptId, + nativeTurnRef: { driver, nativeId: "assistant-before-clear", strength: "weak" }, + ordinal: 1, + status: "completed", + startedAt: now, + completedAt: now, + }, + }, + ], + }); + const targetThreadId = ThreadId.make("fork-clear-target"); + yield* orchestrator.dispatch({ + type: "thread.fork", + commandId: CommandId.make("fork-clear-fork"), + sourceThreadId, + targetThreadId, + sourcePoint: { type: "run", runId: sourceRunId }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("fork-clear-message"), + threadId: targetThreadId, + messageId: MessageId.make("fork-clear-message"), + text: "Continue from here", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "fork-across-clear" }, + ProviderAdapterRegistry.layerFromAdapters([claude]), + ), + ), + ); + + assert.deepEqual( + forkCalls.map((call) => ({ + sessionId: call.sessionId, + upToMessageId: call.options.upToMessageId, + })), + [{ sessionId: "native-before-clear", upToMessageId: "assistant-before-clear" }], + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), +); + const writeParentRun = (input: { readonly threadId: ThreadId; readonly runId: RunId; diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 965e5bd44f2dc..03349ddf6f904 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -108,6 +108,7 @@ function makeTestAdapter(input: { readonly nativeThreadGeneration?: Ref.Ref; readonly failResume?: boolean; readonly failResumeOnce?: Ref.Ref; + readonly clearNativeThreadOnRunOrdinal?: number; readonly initialContextUsage?: OrchestrationV2ProviderThread["contextUsage"]; readonly getModelContextWindow?: (selection: ModelSelection) => number | undefined; readonly canReuseContextUsage?: ProviderAdapter.ProviderAdapterV2SessionRuntime["canReuseContextUsage"]; @@ -439,6 +440,23 @@ function makeTestAdapter(input: { }, }); } + // Claude's `/clear` starts a new native session inside the turn. + if (turnInput.runOrdinal === input.clearNativeThreadOnRunOrdinal) { + yield* PubSub.publish(events, { + type: "provider_thread.updated", + driver: input.driver, + providerThread: { + ...turnInput.providerThread, + nativeThreadRef: { + driver: input.driver, + nativeId: `${turnInput.providerThread.nativeThreadRef?.nativeId}:cleared`, + strength: "strong", + }, + nativeConversationHeadRef: null, + updatedAt: eventTime, + }, + }); + } for (const event of providerEvents) { yield* PubSub.publish(events, event); } @@ -1506,6 +1524,107 @@ describe("orchestration v2 provider switching", () => { }), ); + it.live("resumes the native session /clear selected after the provider session restarts", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("clear-native-session-restart"); + const capturedTurns = yield* Ref.make>([]); + const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([ + makeTestAdapter({ + instanceId: CLAUDE_MODEL_SELECTION.instanceId, + driver: CLAUDE_DRIVER, + capabilities: ClaudeProviderCapabilitiesV2, + modelSelection: CLAUDE_MODEL_SELECTION, + responseByRunOrdinal: {}, + capturedTurns, + clearNativeThreadOnRunOrdinal: 1, + }), + ]); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const runToCompletion = Effect.fn("runToCompletion")(function* ( + ordinal: number, + text: string, + ) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`clear-restart:${ordinal}`), + threadId, + messageId: MessageId.make(`clear-restart:${ordinal}`), + createdBy: "user", + creationSource: "web", + text, + attachments: [], + modelSelection: CLAUDE_MODEL_SELECTION, + dispatchMode: { type: "start_immediately" }, + }); + yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && + event.payload.ordinal === ordinal && + (event.payload.status === "completed" || event.payload.status === "failed"), + ), + Stream.runHead, + ); + yield* worker.drain(); + }); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("clear-restart:create"), + threadId, + projectId, + createdBy: "user", + creationSource: "web", + title: "Clear then restart", + modelSelection: CLAUDE_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + yield* runToCompletion(1, "/clear"); + const providerThread = (yield* orchestrator.getThreadProjection( + threadId, + )).providerThreads.at(0); + if (providerThread === undefined || providerThread.providerSessionId === null) { + return yield* Effect.die("Expected a provider session after /clear."); + } + yield* orchestrator.dispatch({ + type: "provider-session.detach", + commandId: CommandId.make("clear-restart:detach"), + threadId, + providerSessionId: providerThread.providerSessionId, + }); + yield* worker.drain(); + yield* runToCompletion(2, "Continue after the restart"); + + const turns = yield* Ref.get(capturedTurns); + assert.equal(turns.length, 2); + assert.equal(turns[1]?.nativeThreadId, `${turns[0]?.nativeThreadId}:cleared`); + // Claude writes the cleared session's transcript during /clear, so + // opening it with a fresh sessionId fails with "already in use". + assert.isTrue(turns[1]?.nativeThreadHasTurns); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { + name: "clear-native-session-restart", + runtimePolicyOverride: { + cwd, + approvalPolicy: "never", + sandboxPolicy: { type: "readOnly" }, + }, + }, + layerRegistry, + ), + ), + ); + }), + ), + ); + it.live.each(["an older handoff", "a missed request"] as const)( "keeps the latest turn when a resume fallback also delivers %s", (older) => diff --git a/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts index d90886835a234..7f6b364a3ce69 100644 --- a/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ThreadFork.integration.test.ts @@ -44,6 +44,7 @@ const CLAUDE_MODEL_SELECTION = { const PRIOR_TURN_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_prior_turn/codex_transcript.ndjson`; const CLAUDE_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native/claude_transcript.ndjson`; const CLAUDE_PRIOR_TURN_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_prior_turn/claude_transcript.ndjson`; +const CLAUDE_MID_TURN_CLEAR_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson`; const CLAUDE_FORK_LOCAL_ROLLBACK_TRANSCRIPT_PATH = `${import.meta.dirname}/fixtures/thread_fork_native_fork_local_rollback/claude_transcript.ndjson`; const CODEX_READ_ONLY_NEVER_POLICY = { approvalPolicy: "never", @@ -661,6 +662,156 @@ describe("orchestration V2 thread fork", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + // regression: #24153 a run whose `/clear` adopted a new native session mid-turn kept the + // session it started in, so forking it asked that session for a message only the new one has. + it.effect("forks a Claude run from the session it adopted mid-turn", () => + Effect.gen(function* () { + const rawTranscript = yield* readTranscript(CLAUDE_MID_TURN_CLEAR_TRANSCRIPT_PATH); + const transcript = yield* ClaudeOrchestratorReplayHarness.decodeTranscript(rawTranscript); + const cwd = yield* Effect.acquireRelease(makeCheckpointWorkspace, (directory) => + Effect.service(FileSystem.FileSystem).pipe( + Effect.flatMap((fs) => fs.remove(directory, { recursive: true, force: true })), + Effect.orDie, + ), + ); + + const materialized = yield* Effect.gen(function* () { + const ids = yield* IdAllocator.IdAllocatorV2; + const fixtureName = "thread-fork-native-mid-turn-clear"; + const projectId = yield* ids.allocate.project({ fixtureName }); + const sourceThreadId = yield* ids.allocate.thread({ + fixtureName: `${fixtureName}-source`, + projectId, + }); + const targetThreadId = ThreadId.make(`${fixtureName}-target`); + const sourceMessage = (commandName: string, text: string) => + Effect.map( + ids.allocate.command({ fixtureName, commandName }), + (commandId) => + ({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId, + threadId: sourceThreadId, + messageId: MessageId.make(`message-${fixtureName}-${commandName}`), + text, + attachments: [], + modelSelection: CLAUDE_MODEL_SELECTION, + dispatchMode: { type: "start_immediately" }, + }) satisfies OrchestrationV2Command, + ); + const commands = [ + { + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: yield* ids.allocate.command({ + fixtureName, + commandName: "thread-create-source", + }), + threadId: sourceThreadId, + projectId, + title: "Source thread", + modelSelection: CLAUDE_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }, + yield* sourceMessage("source-message-alpha", THREAD_FORK_NATIVE_PRIOR_TURN_ALPHA_PROMPT), + // The provider reports a new native session inside this run, as Claude does for `/clear`. + yield* sourceMessage("source-message-beta", THREAD_FORK_NATIVE_PRIOR_TURN_BETA_PROMPT), + { + type: "thread.fork", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`command-${fixtureName}`), + sourceThreadId, + targetThreadId, + sourcePoint: { + type: "run", + runId: ids.derive.run({ threadId: sourceThreadId, ordinal: 2 }), + }, + title: "Forked from the cleared run", + }, + { + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: yield* ids.allocate.command({ + fixtureName, + commandName: "target-message-repeat", + }), + threadId: targetThreadId, + messageId: MessageId.make(`message-${fixtureName}-repeat`), + text: THREAD_FORK_NATIVE_PRIOR_TURN_REPEAT_PROMPT, + attachments: [], + modelSelection: CLAUDE_MODEL_SELECTION, + dispatchMode: { type: "start_immediately" }, + }, + ] satisfies ReadonlyArray; + + return { sourceThreadId, targetThreadId, commands }; + }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime); + + // The transcript expects `session.fork` against the adopted session, so forking the + // session the run started in fails the replay. + const result = yield* runOrchestratorV2ProviderReplayScenario( + { + name: "thread_fork_native_mid_turn_clear/claude", + transcript, + commands: materialized.commands, + steps: [ + { type: "dispatch", command: materialized.commands[0]!, await: true }, + { type: "advance_clock", duration: "1 millis" }, + { type: "dispatch", command: materialized.commands[1]!, await: true }, + { type: "await_thread_idle", threadId: materialized.sourceThreadId }, + { type: "dispatch", command: materialized.commands[2]!, await: true }, + { type: "await_thread_idle", threadId: materialized.sourceThreadId }, + { type: "dispatch", command: materialized.commands[3]!, await: true }, + { type: "dispatch", command: materialized.commands[4]!, await: true }, + { type: "await_thread_idle", threadId: materialized.targetThreadId }, + ], + projectionThreadIds: [materialized.sourceThreadId, materialized.targetThreadId], + runtimePolicyOverride: { cwd }, + }, + ClaudeOrchestratorReplayHarness, + ).pipe(provideDeterministicTestRuntime); + + const sourceProjection = result.projections.get(materialized.sourceThreadId); + const targetProjection = result.projections.get(materialized.targetThreadId); + assert.isDefined(sourceProjection); + assert.isDefined(targetProjection); + assert.equal( + sourceProjection.providerThreads[0]?.nativeThreadRef?.nativeId, + metadataString(transcript, "clearedNativeSessionId"), + ); + assert.equal( + targetProjection.providerThreads[0]?.nativeThreadRef?.nativeId, + metadataString(transcript, "forkedNativeSessionId"), + ); + assert.equal(targetProjection.contextTransfers[0]?.resolution?.strategy, "native_fork"); + const targetAssistantText = targetProjection.turnItems + .filter((item) => item.type === "assistant_message") + .map((item) => item.text) + .join("\n"); + assert.include(targetAssistantText, "**Assistant:** fork boundary beta"); + assert.notInclude( + targetAssistantText, + "fork boundary alpha", + "the cleared native session must not recover pre-clear conversation context", + ); + const visibleText = targetProjection.visibleTurnItems + .map((row) => row.item) + .filter((item) => item.type === "user_message" || item.type === "assistant_message") + .map((item) => item.text) + .join("\n"); + assert.include(visibleText, "fork boundary alpha"); + assert.include(visibleText, "fork boundary beta"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("rolls back a Claude native fork to an earlier fork-local turn", () => Effect.gen(function* () { const rawTranscript = yield* readTranscript(CLAUDE_FORK_LOCAL_ROLLBACK_TRANSCRIPT_PATH); diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson new file mode 100644 index 0000000000000..722e406f914aa --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_mid_turn_clear/claude_transcript.ndjson @@ -0,0 +1,26 @@ +{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"thread_fork_native_mid_turn_clear","metadata":{"prompts":["For this fork-boundary fixture, respond with exactly: fork boundary alpha","For this fork-boundary fixture, respond with exactly: fork boundary beta","Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."],"model":"claude-sonnet-4-6","nativeSessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","queryMode":"fork_session_prior_turn","tools":"claude_code","permissionMode":"bypassPermissions","generatedBy":"synthetic adaptation of thread_fork_native_prior_turn/claude_transcript.ndjson","sourceAssistantMessageUuids":["be0e3276-8c27-44bf-8ed0-0b4424e9f902","c517b74b-9537-4260-afcb-186c4056c0a4"],"forkUpToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4","forkedNativeSessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","clearedNativeSessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} +{"type":"expect_outbound","label":"query.open:source","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"sessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","settings":{"showThinkingSummaries":true}}}} +{"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"For this fork-boundary fixture, respond with exactly: fork boundary alpha"},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"25b085e7-a6e9-4f30-9708-b189c961d06f","hook_name":"SessionStart:startup","hook_event":"SessionStart","uuid":"42c40147-86c8-4294-bdb4-f1815f48808e","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"25b085e7-a6e9-4f30-9708-b189c961d06f","hook_name":"SessionStart:startup","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"9804deb1-9abf-4e58-9d04-6e163dc7b0af","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_mid_turn_clear","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"b9412e24-eefc-4838-9ede-bb6e00951d15","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_014hdmSJdwbpJ5NkFgqbhfEe","type":"message","role":"assistant","content":[{"type":"text","text":"fork boundary alpha"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":2342,"cache_read_input_tokens":9455,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2342},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","uuid":"be0e3276-8c27-44bf-8ed0-0b4424e9f902"}} +{"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"0c649726-b7a2-4e89-a62a-a9ae534905c7","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":1174,"duration_api_ms":897,"num_turns":1,"result":"fork boundary alpha","stop_reason":"end_turn","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","total_cost_usd":0.011718000000000001,"usage":{"input_tokens":3,"cache_creation_input_tokens":2342,"cache_read_input_tokens":9455,"output_tokens":6,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2342,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":6,"cache_read_input_tokens":9455,"cache_creation_input_tokens":2342,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2342},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":6,"cacheReadInputTokens":9455,"cacheCreationInputTokens":2342,"webSearchRequests":0,"costUSD":0.011718000000000001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"d384185c-9b38-44a4-8917-53d7a8765a5a"}} +{"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"For this fork-boundary fixture, respond with exactly: fork boundary beta"},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_mid_turn_clear","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"5fd1f879-e742-4911-95e3-3dfb0141dc52","session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_0181K4CdzyVanM6Px26CQP7L","type":"message","role":"assistant","content":[{"type":"text","text":"fork boundary beta"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea","uuid":"c517b74b-9537-4260-afcb-186c4056c0a4"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2545,"duration_api_ms":3001,"num_turns":1,"result":"fork boundary beta","stop_reason":"end_turn","session_id":"c1ea7ed0-5e55-4100-8000-00000000c1ea","total_cost_usd":0.01544235,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"output_tokens":6,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":23,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":6,"cache_read_input_tokens":11797,"cache_creation_input_tokens":23,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":6,"outputTokens":12,"cacheReadInputTokens":21252,"cacheCreationInputTokens":2365,"webSearchRequests":0,"costUSD":0.01544235,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"e4f26af4-3a60-45f3-8d5c-c0f8b968c400"}} +{"type":"runtime_exit","status":"success"} +{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"c1ea7ed0-5e55-4100-8000-00000000c1ea","options":{"upToMessageId":"c517b74b-9537-4260-afcb-186c4056c0a4"}}} +{"type":"emit_inbound","label":"session.forked","frame":{"type":"session.forked","sessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} +{"type":"expect_outbound","label":"query.open:fork","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"resume":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","settings":{"showThinkingSummaries":true}}}} +{"type":"expect_outbound","label":"prompt.offer:3","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"6ce9327c-4534-484c-96e5-b47fa1a3143f","hook_name":"SessionStart:resume","hook_event":"SessionStart","uuid":"d4280b57-8334-4021-971d-93bba0e38b4c","session_id":"1dc8dab2-9e1b-4475-bd46-f40e2ebca07d"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"6ce9327c-4534-484c-96e5-b47fa1a3143f","hook_name":"SessionStart:resume","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"ac1dee1e-63c5-4db8-81b7-e1f68ecadf01","session_id":"1dc8dab2-9e1b-4475-bd46-f40e2ebca07d"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.111","cwd":"/tmp/claude-replay-thread_fork_native_mid_turn_clear","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"65e2ff1d-ded8-42d2-a491-70fed78b2368","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01NP2pmrWJHY1AZ8HWqgsCQS","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"The user wants me to repeat the user-visible conversation so far verbatim, including only user and assistant messages.","signature":"ErQCClsIDRgCKkAHbcjlRRK3rHsyqzZn5jI5+wWIl32x7k4E5feJpl1bBZVM6FJVJDBBV5ZgPlHGZMMALoCcDsULDBqzwNbRQ8o0MhFjbGF1ZGUtc29ubmV0LTQtNjgAEgzxdU+J4dE9HPjPJBwaDFLfMOU1oTIY4zJlVSIwkqZMdRU0W/uaPIztqHfQSORVwpEKdR6RAOJb0t5E+IOBu+tFmopmwfCbQjON3NoEKoYBs8E4pkuYLNz2Q+Qp7XeZOdCvvXUTYVIBJ90r2tl9NtEyDgCKce2VtEShQxnPZH9OhcyuEuKrL8c+Fsc8rTj6vYelMB2UJfj9KI1bREqSwoBf1oD7uwiuOz+S40+1Tx7LprJUcmw+JzkTO73YmVr8EHYGCHbQlLSeUHm7NgypEkTcOAaETiIYAQ=="}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"output_tokens":7,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","uuid":"3a81c9d0-22dd-4bbd-8f27-103c75e2878c"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01NP2pmrWJHY1AZ8HWqgsCQS","type":"message","role":"assistant","content":[{"type":"text","text":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary beta\n\n**Assistant:** fork boundary beta\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"output_tokens":7,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","uuid":"90f4c83b-8fff-454a-a95d-aa0cf87ff967"}} +{"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"74ba363a-dc66-4335-b18c-b335f75d4c9b","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2557,"duration_api_ms":2194,"num_turns":1,"result":"**User:** For this fork-boundary fixture, respond with exactly: fork boundary beta\n\n**Assistant:** fork boundary beta\n\n**User:** Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content.","stop_reason":"end_turn","session_id":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd","total_cost_usd":0.0051756,"usage":{"input_tokens":3,"cache_creation_input_tokens":38,"cache_read_input_tokens":11797,"output_tokens":99,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":38,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":99,"cache_read_input_tokens":11797,"cache_creation_input_tokens":38,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":38},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":99,"cacheReadInputTokens":11797,"cacheCreationInputTokens":38,"webSearchRequests":0,"costUSD":0.0051756,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"6fe96da3-b212-41e9-be7b-12d5aee4f191"}} +{"type":"runtime_exit","status":"success"} diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 77e13b2ce0216..c0aa83924ca7d 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -14,6 +14,7 @@ import { OrchestrationV2ProviderFailure, OrchestrationV2ProviderRetry, OrchestrationV2ProviderThread, + OrchestrationV2StoredEvent, OrchestrationV2ProviderTurn, OrchestrationV2RuntimeRequest, OrchestrationV2Subagent, @@ -611,9 +612,25 @@ export interface ProviderAdapterV2ForkThreadInput { readonly runtimePolicy?: ProviderAdapterV2RuntimePolicy; } +export interface ProviderAdapterV2PersistEventResult { + readonly storedEvents: ReadonlyArray; + // Set when the snapshot is already persisted, so a skipped write still lets + // consumers (finalization) use the committed provider thread. + readonly persistedProviderThread?: OrchestrationV2ProviderThread; +} + export interface ProviderAdapterV2EventSubscription { readonly events: Stream.Stream; readonly close: Effect.Effect; + // Managed sessions serialize snapshot persistence with delivery and release. Rewind + // side effects may still apply when the accompanying thread snapshot is stale. + readonly discardEvent?: (event: ProviderAdapterV2Event) => Effect.Effect; + readonly persistEvent?: ( + event: ProviderAdapterV2Event, + persist: ( + snapshotCurrent: boolean, + ) => Effect.Effect, E, R>, + ) => Effect.Effect; } export interface ProviderAdapterV2HistoricalContext {