diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 1cb40e1fab2ed..c519c1931bc55 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -909,7 +909,11 @@ describe("AcpAdapterV2", () => { flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, - makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, protocolEvents }), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + protocolEvents, + }), }, selfInvocation, }); @@ -2324,6 +2328,99 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(layerTest), Effect.scoped), ); + it.live("creates a fresh ACP identity when recovering an exhausted target", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const path = yield* Path.Path; + const selfInvocation = yield* resolveSelfInvocation(); + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.unbounded(); + const instanceId = ProviderInstanceId.make("acp-test-exhausted-target"); + const adapter = yield* makeAcpAdapterV2({ + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + protocolEvents, + wrapRuntime: (runtime, ordinal) => ({ + ...runtime, + start: () => + runtime.start().pipe( + Effect.map((started) => ({ + ...started, + sessionId: `mock-recovery-session-${ordinal}`, + })), + ), + }), + }), + }, + selfInvocation, + }); + const threadId = ThreadId.make("thread-acp-exhausted-target"); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-exhausted-target"), + modelSelection, + runtimePolicy, + }); + const original = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread: original, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 1, + }), + ); + yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runDrain, + ); + const replacement = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + existingProviderThread: { ...original, nativeThreadRef: null }, + forceNewNativeThread: true, + }); + assert.notEqual(replacement.nativeThreadRef?.nativeId, original.nativeThreadRef?.nativeId); + assert.equal(replacement.id, original.id); + assert.equal(replacement.nativeMetadata?.itemIdentityVersion, 2); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread: replacement, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 2, + }), + ); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ); + assert.isTrue( + events.some((event) => event.type === "turn.terminal" && event.status === "completed"), + ); + const methods = yield* pollProtocolMethods(protocolEvents); + assert.equal(methods.filter((method) => method === "session/new").length, 2); + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it.live("preserves new-session fallback when an eager ACP session load is stale", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -5158,6 +5255,19 @@ describe("AcpAdapterV2", () => { yield* hasPendingBackgroundWork, "carryover live subagent must pin hasPendingBackgroundWork after root settle", ); + const refusedReplacement = yield* runtime + .ensureThread({ + threadId, + modelSelection, + runtimePolicy, + existingProviderThread: { ...providerThread, nativeThreadRef: null }, + forceNewNativeThread: true, + }) + .pipe(Effect.flip); + assert.equal(refusedReplacement._tag, "ProviderAdapterEnsureThreadError"); + const preserved = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); + assert.equal(preserved.nativeThreadRef?.nativeId, providerThread.nativeThreadRef?.nativeId); + assert.isTrue(yield* hasPendingBackgroundWork); assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); subagentPhase = "complete"; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 7fece9817ce01..f015adda88ddf 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7652,7 +7652,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect("stops background work after the turn settled", () => + it.effect("preserves settled background work until an explicit stop", () => Effect.scoped( Effect.gen(function* () { let closes = 0; @@ -7681,6 +7681,29 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(settledThread?.pendingBackgroundTasks?.[0]?.taskId, WAKE_TASK_ID); assert.isTrue(yield* harness.hasPendingBackgroundWork); + const replacementAttempt = yield* harness.runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: { + ...harness.providerThread, + nativeThreadRef: { + ...harness.providerThread.nativeThreadRef!, + nativeId: "replacement-session", + }, + }, + now, + attemptId: RunAttemptId.make("attempt-claude-background-replacement"), + text: "Continue in a fresh session.", + attachments: [], + providerTurnOrdinal: 2, + }), + ) + .pipe(Effect.result); + assert.equal(replacementAttempt._tag, "Failure"); + assert.equal(closes, 0); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + yield* harness.runtime.interruptTurn({ providerThread: settledThread ?? harness.providerThread, providerTurnId: harness.terminalEvents()[0]!.providerTurnId, @@ -12108,6 +12131,25 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* changes.until(() => terminals().length === 1); const settledTurn = terminals()[0]!; + assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(providerThread)); + assert.isTrue( + yield* runtime.hasPendingBackgroundWorkForThread!(providerThread, { + includeSubagents: true, + }), + ); + assert.isFalse( + yield* runtime.hasPendingBackgroundWorkForThread!( + { + ...providerThread, + nativeThreadRef: { + ...providerThread.nativeThreadRef!, + nativeId: "sibling-native-session", + }, + }, + { includeSubagents: true }, + ), + ); + const alternateModel = CLAUDE_TEST_RESTART_MODEL_SELECTION; const switchTurn = (attempt: string) => runtime.startTurn( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index baf9a0b0a690b..6366c91b5c599 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -9697,10 +9697,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return existing; } - // Selection changes kill CLI background work; refuse until it finishes or Stop closes the process. + // Replacing this thread's process kills its background work, even with a new native identity. if ( existing !== null && - existing.nativeThreadId === nativeThreadId && + (existing.nativeThreadId === nativeThreadId || + existing.lastTurn?.input.threadId === turnInput.threadId) && !existing.stopping && (yield* liveProcessRunsBackgroundWork(existing)) ) { @@ -10565,11 +10566,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), hasPendingBackgroundWorkForThread: Effect.fn( "ClaudeAdapterV2.hasPendingBackgroundWorkForThread", - )(function* (providerThread) { + )(function* (providerThread, options) { const nativeThreadId = providerThread.nativeThreadRef?.nativeId; if (nativeThreadId === undefined || nativeThreadId === null) { return false; } + if (options?.includeSubagents) { + const live = yield* Ref.get(queryContext); + if ( + live?.nativeThreadId === nativeThreadId && + !live.stopping && + (yield* liveProcessRunsBackgroundWork(live)) + ) + return true; + } return ( rosterForNativeThread( yield* Ref.get(pendingBackgroundTasksByNativeThread), diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 97fdb7c85ca71..70949b02af187 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -4,6 +4,7 @@ import { CommandId, latestProviderTurnForAttempt, type OrchestrationV2DomainEvent, + type OrchestrationV2ContextHandoff, type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderThread, type OrchestrationV2Run, @@ -49,6 +50,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import { + ContextHandoffBackgroundWorkError, knownProviderFailureMessage, makeProviderFailure, } from "@t3tools/provider-core/server/failure"; @@ -695,6 +697,279 @@ export const layer: Layer.Layer< }); let effectiveHandoffs = handoffs; let resumeFellBack = false; + const userText = projectComposerContextForProvider({ + text: message.text, + records: message.context?.records ?? [], + }); + // Codex and Claude parse /goal from the prompt, so it goes verbatim and history waits a turn. + const goalDriver = session.driver === "codex" || session.driver === "claudeAgent"; + const goalCommand = goalDriver && parseGoalCommand(message) !== null; + // The provider turn marks cancellation work delivered; restart continuations use their own text or resume natively. + const noteContinuation = isRestartNoteContinuation( + run, + projection.runs, + projection.providerTurns, + projection.attempts, + ); + const restartCancelledWork = pendingRestartCancelledBackgroundWork({ + runs: projection.runs, + providerTurns: projection.providerTurns, + compactionMessageIds: new Set( + projection.messages + .filter( + (candidate) => + isCompactionCommandMessage(candidate) || + (goalDriver && parseGoalCommand(candidate) !== null), + ) + .map((candidate) => candidate.id), + ), + run, + attempts: projection.attempts, + }); + const restartNote = + restartCancelledWork.length === 0 + ? "" + : restartCancelledBackgroundWorkNote(restartCancelledWork); + const tokenCap = yield* handoffTokenCapConfig.pipe( + Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), + ); + const handoffContextForThread = (loadedProviderThread: OrchestrationV2ProviderThread) => { + const measuredContext = latestNativeContextUsage(projection, providerThread); + const previousSelection = + measuredContext?.modelSelection ?? + projection.runs.findLast( + (source) => + source.ordinal < run.ordinal && + source.startedAt !== null && + source.providerThreadId === providerThread.id, + )?.modelSelection; + const sameSelection = + previousSelection === undefined || + modelSelectionsEqual(previousSelection, run.modelSelection); + const sameNativeThread = + loadedProviderThread.nativeThreadRef?.nativeId === + providerThread.nativeThreadRef?.nativeId; + const threadUsage = loadedProviderThread.contextUsage ?? providerThread.contextUsage; + const previousUsage = measuredContext + ? { ...threadUsage, ...measuredContext.usage } + : threadUsage; + const reuseTelemetry = + sameSelection || + (previousSelection !== undefined && + session.canReuseContextUsage?.(previousSelection, run.modelSelection) === true); + const knownModelWindow = session.getModelContextWindow?.( + run.modelSelection, + resolvedRuntimePolicy.cwd, + ); + const handoffUsage = contextUsageForHandoff({ + sameNativeThread, + sameSelection, + reuseTelemetry, + previousUsage, + knownModelWindow, + }); + const settledHandoffs = projection.contextHandoffs.filter( + (handoff) => + handoff.toProviderThreadId === providerThread.id && + handoff.delivery?.nativeThreadId === loadedProviderThread.nativeThreadRef?.nativeId && + handoff.delivery?.status !== "pending", + ); + const deliveredItemIds = new Set( + settledHandoffs.flatMap((handoff) => handoff.delivery?.itemIds ?? []), + ); + const coveredItemIds = new Set([ + ...deliveredItemIds, + ...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []), + ]); + const deliveredAttemptIds = new Set( + projection.providerTurns.map((turn) => turn.runAttemptId), + ); + const acceptedAttempts = projection.attempts.filter( + (source) => + source.providerThreadId === providerThread.id && deliveredAttemptIds.has(source.id), + ); + const nativeInputRunIds = new Set( + acceptedAttempts + .filter( + (source) => + source.nativeThreadId !== undefined && + source.nativeThreadId === loadedProviderThread.nativeThreadRef?.nativeId, + ) + .map((source) => source.runId), + ); + const legacyInputRunIds = new Set( + acceptedAttempts + .filter((source) => source.nativeThreadId === undefined) + .map((source) => source.runId), + ); + const legacyRecoveredRunIds = new Set( + projection.runs + .filter( + (source) => + source.providerThreadId === providerThread.id && + settledHandoffs.some( + (handoff) => + handoff.strategy === "full_thread_summary" && + handoff.fromProviderThreadIds.includes(providerThread.id) && + source.ordinal >= handoff.coveredRunOrdinals.from && + source.ordinal <= handoff.coveredRunOrdinals.to, + ), + ) + .map((source) => source.id), + ); + // Row identity survives recovery; only accepted native input and delivered history occupy the new session. + const nativeContextEstimate = Effect.gen(function* () { + return sameNativeThread + ? (yield* projectionStore.getTurnStartHistory(input.threadId)).reduce((sum, item) => { + const acceptedNatively = + item.providerThreadId === providerThread.id && + item.runId !== null && + (nativeInputRunIds.has(item.runId) || + (legacyInputRunIds.has(item.runId) && + !coveredItemIds.has(item.id) && + !legacyRecoveredRunIds.has(item.runId))); + if (item.runId === run.id || (!acceptedNatively && !deliveredItemIds.has(item.id))) + return sum; + const historical = historicalMessage(item); + const nativeAttachments = + item.type === "user_message" && acceptedNatively + ? attachmentTokenAllowance(item.attachments) + : 0; + return ( + sum + + (historical === null ? 0 : Buffer.byteLength(historical.text)) + + nativeAttachments + ); + }, 0) + : 0; + }); + const modelContextWindow = + knownModelWindow ?? + (handoffUsage !== null || reuseTelemetry ? previousUsage?.maxTokens : undefined); + const budgetProviderThread = { + ...loadedProviderThread, + contextUsage: handoffUsage, + }; + // The resume fallback summary already covers every earlier run in order. A + // retry copy after it would rank older missed requests as the newest history. + const missedRuns = projection.runs.filter( + (source) => + !resumeFellBack && + source.ordinal < run.ordinal && + source.providerThreadId === providerThread.id && + (source.status === "failed" || source.status === "interrupted") && + !deliveredAttemptIds.has(source.activeAttemptId), + ); + const missedRunIds = new Set(missedRuns.map((source) => source.id)); + return { + sameNativeThread, + handoffUsage, + deliveredItemIds, + coveredItemIds, + acceptedAttempts, + nativeInputRunIds, + legacyInputRunIds, + missedRuns, + missedRunIds, + budget: Effect.gen(function* () { + return handoffBudget({ + tokenCap, + modelContextWindow, + userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, + attachments: message.attachments, + providerThread: budgetProviderThread, + nativeContextEstimate: + budgetProviderThread.contextUsage?.usedTokens === undefined + ? yield* nativeContextEstimate + : 0, + }); + }), + }; + }; + const prepareRetryHandoff = Effect.fn("ProviderTurnStartService.prepareRetryHandoff")( + function* (context: ReturnType) { + const { missedRunIds, coveredItemIds, missedRuns } = context; + const missedItems = + missedRunIds.size === 0 + ? [] + : (yield* projectionStore.getTurnStartHistory(input.threadId, [ + ...missedRunIds, + ])).filter( + (item) => + item.runId !== null && + missedRunIds.has(item.runId) && + !coveredItemIds.has(item.id) && + historicalMessage(item) !== null, + ); + const retryHandoff = + missedItems.length === 0 + ? [] + : [ + yield* contextHandoffService.prepareProviderHandoff({ + threadId: projection.thread.id, + targetRunId: run.id, + transferId: null, + fromProviderThreadIds: [providerThread.id], + toProviderThreadId: providerThread.id, + fromProviderInstanceId: run.providerInstanceId, + toProviderInstanceId: run.providerInstanceId, + coveredRunOrdinals: { + from: missedRuns[0]!.ordinal, + to: missedRuns.at(-1)!.ordinal, + }, + strategy: "delta_since_target_last_seen", + items: missedItems, + runs: projection.runs, + createdAt: yield* DateTime.now, + }), + ]; + return retryHandoff; + }, + ); + const prepareRecoveryHandoff = Effect.fn("ProviderTurnStartService.prepareRecoveryHandoff")( + function* () { + const transferId = yield* idAllocator.allocate.contextTransfer({ + sourceThreadId: projection.thread.id, + targetThreadId: projection.thread.id, + type: "provider_resume_fallback", + }); + const createdAt = yield* DateTime.now; + const handoff = yield* contextHandoffService.prepareProviderHandoff({ + threadId: projection.thread.id, + targetRunId: run.id, + transferId, + fromProviderThreadIds: [providerThread.id], + toProviderThreadId: providerThread.id, + fromProviderInstanceId: providerThread.providerInstanceId, + toProviderInstanceId: run.providerInstanceId, + coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, + strategy: "full_thread_summary", + runs: projection.runs, + items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter( + (item) => + item.runId === null || + projection.runs.some( + (source) => source.id === item.runId && source.ordinal < run.ordinal, + ), + ), + createdAt, + }); + return { handoff, transferId, createdAt }; + }, + ); + const recoveryHandoff = yield* Effect.cached(prepareRecoveryHandoff()); + const withRecoveryHandoff = (handoff: OrchestrationV2ContextHandoff) => { + // Delivery ranks later handoffs as newer history. Pending handoffs from + // earlier runs predate this summary; handoffs for this run (provider-switch + // delta, merge-back) are newer and must stay last. + return [ + ...effectiveHandoffs.filter((candidate) => candidate.targetRunId !== run.id), + handoff, + ...effectiveHandoffs.filter((candidate) => candidate.targetRunId === run.id), + ]; + }; + let resumedRetryHandoff: ReadonlyArray | undefined; + let resumedHandoffContext: ReturnType | undefined; const loadedProviderThread = yield* Effect.gen(function* () { if (nativeForkTransfer !== undefined) { const sourceProjection = yield* projectionStore.getThreadRecords( @@ -772,15 +1047,71 @@ export const layer: Layer.Layer< }), ); if (resumed._tag === "Success") { - return resumed.success; + const context = handoffContextForThread(resumed.success); + context.budget = yield* Effect.cached(context.budget); + resumedHandoffContext = context; + resumedRetryHandoff = yield* prepareRetryHandoff(context); + const fits = yield* deliverContextHandoffs({ + handoffs: [...handoffs, ...resumedRetryHandoff], + providerThread: { ...resumed.success, id: providerThread.id }, + budget: context.budget, + deferInline: isCompactionCommandMessage(message) || goalCommand, + alreadyDeliveredItemIds: context.deliveredItemIds, + persist: () => Effect.void, + }).pipe( + Effect.as(true), + Effect.catchTags({ ContextHandoffBudgetError: () => Effect.succeed(false) }), + ); + if (fits) return resumed.success; + const freshThread = { ...resumed.success, nativeThreadRef: null }; + const freshContext = handoffContextForThread(freshThread); + const { handoff } = yield* recoveryHandoff; + const fitsFresh = yield* deliverContextHandoffs({ + handoffs: withRecoveryHandoff(handoff), + providerThread: freshThread, + budget: freshContext.budget, + alreadyDeliveredItemIds: freshContext.deliveredItemIds, + persist: () => Effect.void, + }).pipe( + Effect.as(true), + Effect.catchTags({ ContextHandoffBudgetError: () => Effect.succeed(false) }), + ); + if (!fitsFresh) return resumed.success; + const hasPendingWork = yield* ( + session.hasPendingBackgroundWorkForThread?.(resumed.success, { + includeSubagents: true, + }) ?? + session.hasPendingBackgroundWork ?? + Effect.succeed(false) + ); + if (hasPendingWork) { + return yield* loadFromProvider( + Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: session.driver, + threadId: projection.thread.id, + providerThreadId: providerThread.id, + runId, + cause: new ContextHandoffBackgroundWorkError(), + }), + ), + ); + } } + resumedHandoffContext = undefined; + resumedRetryHandoff = []; + if (!(yield* isCurrentAttemptInStatus("starting"))) return undefined; - yield* Effect.logWarning("Provider resume failed; attempting a fresh native session", { + yield* Effect.logWarning("Recovering provider context in a fresh native session", { driver: session.driver, providerThreadId: providerThread.id, runId, - reason: uncertainDelivery ? "uncertain_history_delivery" : "resume_failed", - errorTag: resumed.failure._tag, + reason: uncertainDelivery + ? "uncertain_history_delivery" + : resumed._tag === "Success" + ? "handoff_budget_exhausted" + : "resume_failed", + errorTag: resumed._tag === "Failure" ? resumed.failure._tag : "ContextHandoffBudgetError", }); const replacement = yield* loadFromProvider( session.ensureThread({ @@ -791,43 +1122,12 @@ export const layer: Layer.Layer< providerSessionId, // Clear the native ref to force a fresh session while preserving row identity. existingProviderThread: { ...providerThread, nativeThreadRef: null }, + forceNewNativeThread: true, }), ); if (replacement === undefined) return undefined; - const transferId = yield* idAllocator.allocate.contextTransfer({ - sourceThreadId: projection.thread.id, - targetThreadId: projection.thread.id, - type: "provider_resume_fallback", - }); - const createdAt = yield* DateTime.now; - const handoff = yield* contextHandoffService.prepareProviderHandoff({ - threadId: projection.thread.id, - targetRunId: run.id, - transferId, - fromProviderThreadIds: [providerThread.id], - toProviderThreadId: providerThread.id, - fromProviderInstanceId: providerThread.providerInstanceId, - toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, - strategy: "full_thread_summary", - runs: projection.runs, - items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter( - (item) => - item.runId === null || - projection.runs.some( - (source) => source.id === item.runId && source.ordinal < run.ordinal, - ), - ), - createdAt, - }); - // Delivery ranks later handoffs as newer history. Pending handoffs from - // earlier runs predate this summary; handoffs for this run (provider-switch - // delta, merge-back) are newer and must stay last. - effectiveHandoffs = [ - ...effectiveHandoffs.filter((candidate) => candidate.targetRunId !== run.id), - handoff, - ...effectiveHandoffs.filter((candidate) => candidate.targetRunId === run.id), - ]; + const { handoff, transferId, createdAt } = yield* recoveryHandoff; + effectiveHandoffs = withRecoveryHandoff(handoff); resumeFellBack = true; yield* eventSink.write({ events: [ @@ -875,39 +1175,17 @@ export const layer: Layer.Layer< return; } const now = yield* DateTime.now; - const measuredContext = latestNativeContextUsage(projection, providerThread); - const previousSelection = - measuredContext?.modelSelection ?? - projection.runs.findLast( - (source) => - source.ordinal < run.ordinal && - source.startedAt !== null && - source.providerThreadId === providerThread.id, - )?.modelSelection; - const sameSelection = - previousSelection === undefined || - modelSelectionsEqual(previousSelection, run.modelSelection); - const sameNativeThread = - loadedProviderThread.nativeThreadRef?.nativeId === providerThread.nativeThreadRef?.nativeId; - const threadUsage = loadedProviderThread.contextUsage ?? providerThread.contextUsage; - const previousUsage = measuredContext - ? { ...threadUsage, ...measuredContext.usage } - : threadUsage; - const reuseTelemetry = - sameSelection || - (previousSelection !== undefined && - session.canReuseContextUsage?.(previousSelection, run.modelSelection) === true); - const knownModelWindow = session.getModelContextWindow?.( - run.modelSelection, - resolvedRuntimePolicy.cwd, - ); - const handoffUsage = contextUsageForHandoff({ + const handoffContext = resumedHandoffContext ?? handoffContextForThread(loadedProviderThread); + const { sameNativeThread, - sameSelection, - reuseTelemetry, - previousUsage, - knownModelWindow, - }); + handoffUsage, + deliveredItemIds, + acceptedAttempts, + nativeInputRunIds, + legacyInputRunIds, + budget, + } = handoffContext; + const retryHandoff = resumedRetryHandoff ?? (yield* prepareRetryHandoff(handoffContext)); const runningProviderThread: OrchestrationV2ProviderThread = { ...loadedProviderThread, contextUsage: handoffUsage, @@ -1035,191 +1313,13 @@ export const layer: Layer.Layer< const routableSubagents = projection.subagents.filter((subagent) => RunExecutionService.canRouteRelatedSubagent(subagent.status), ); - const userText = projectComposerContextForProvider({ - text: message.text, - records: message.context?.records ?? [], - }); - // Codex and Claude parse /goal from the prompt, so it goes verbatim and history waits a turn. - const goalDriver = session.driver === "codex" || session.driver === "claudeAgent"; - const goalCommand = goalDriver && parseGoalCommand(message) !== null; - // The provider turn marks cancellation work delivered; restart continuations use their own text or resume natively. - const noteContinuation = isRestartNoteContinuation( - run, - projection.runs, - projection.providerTurns, - projection.attempts, - ); - const restartCancelledWork = pendingRestartCancelledBackgroundWork({ - runs: projection.runs, - providerTurns: projection.providerTurns, - compactionMessageIds: new Set( - projection.messages - .filter( - (candidate) => - isCompactionCommandMessage(candidate) || - (goalDriver && parseGoalCommand(candidate) !== null), - ) - .map((candidate) => candidate.id), - ), - run, - attempts: projection.attempts, - }); - const restartNote = - restartCancelledWork.length === 0 - ? "" - : restartCancelledBackgroundWorkNote(restartCancelledWork); - const tokenCap = yield* handoffTokenCapConfig.pipe( - Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), - ); - const settledHandoffs = projection.contextHandoffs.filter( - (handoff) => - handoff.toProviderThreadId === providerThread.id && - handoff.delivery?.nativeThreadId === runningProviderThread.nativeThreadRef?.nativeId && - handoff.delivery?.status !== "pending", - ); - const deliveredItemIds = new Set( - settledHandoffs.flatMap((handoff) => handoff.delivery?.itemIds ?? []), - ); - const coveredItemIds = new Set([ - ...deliveredItemIds, - ...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []), - ]); - const deliveredAttemptIds = new Set( - projection.providerTurns.map((turn) => turn.runAttemptId), - ); - const acceptedAttempts = projection.attempts.filter( - (source) => - source.providerThreadId === providerThread.id && deliveredAttemptIds.has(source.id), - ); - const nativeInputRunIds = new Set( - acceptedAttempts - .filter( - (source) => - source.nativeThreadId !== undefined && - source.nativeThreadId === runningProviderThread.nativeThreadRef?.nativeId, - ) - .map((source) => source.runId), - ); - const legacyInputRunIds = new Set( - acceptedAttempts - .filter((source) => source.nativeThreadId === undefined) - .map((source) => source.runId), - ); - const legacyRecoveredRunIds = new Set( - projection.runs - .filter( - (source) => - source.providerThreadId === providerThread.id && - settledHandoffs.some( - (handoff) => - handoff.strategy === "full_thread_summary" && - handoff.fromProviderThreadIds.includes(providerThread.id) && - source.ordinal >= handoff.coveredRunOrdinals.from && - source.ordinal <= handoff.coveredRunOrdinals.to, - ), - ) - .map((source) => source.id), - ); - // Legacy attempts lack native ids; exclude recovered attachments absent from the replacement thread. - const nativeContextEstimate = Effect.gen(function* () { - return sameNativeThread - ? (yield* projectionStore.getTurnStartHistory(input.threadId)).reduce((sum, item) => { - if ( - item.runId === run.id || - (item.runId !== null && - missedRunIds.has(item.runId) && - !deliveredItemIds.has(item.id)) || - (item.providerThreadId !== providerThread.id && !deliveredItemIds.has(item.id)) - ) - return sum; - const historical = historicalMessage(item); - const nativeAttachments = - item.type === "user_message" && - item.providerThreadId === providerThread.id && - item.runId !== null && - (nativeInputRunIds.has(item.runId) || - (legacyInputRunIds.has(item.runId) && - !coveredItemIds.has(item.id) && - !legacyRecoveredRunIds.has(item.runId))) - ? attachmentTokenAllowance(item.attachments) - : 0; - return ( - sum + - (historical === null ? 0 : Buffer.byteLength(historical.text)) + - nativeAttachments - ); - }, 0) - : 0; - }); - const modelContextWindow = - knownModelWindow ?? - (handoffUsage !== null || reuseTelemetry ? previousUsage?.maxTokens : undefined); - const budgetProviderThread = { - ...runningProviderThread, - contextUsage: handoffUsage, - }; - // The resume fallback summary already covers every earlier run in order. A - // retry copy after it would rank older missed requests as the newest history. - const missedRuns = projection.runs.filter( - (source) => - !resumeFellBack && - source.ordinal < run.ordinal && - source.providerThreadId === providerThread.id && - (source.status === "failed" || source.status === "interrupted") && - !deliveredAttemptIds.has(source.activeAttemptId), - ); - const missedRunIds = new Set(missedRuns.map((source) => source.id)); - const missedItems = - missedRunIds.size === 0 - ? [] - : (yield* projectionStore.getTurnStartHistory(input.threadId, [...missedRunIds])).filter( - (item) => - item.runId !== null && - missedRunIds.has(item.runId) && - !coveredItemIds.has(item.id) && - historicalMessage(item) !== null, - ); const startWithHandoffs = Effect.fn("ProviderTurnStartService.startWithHandoffs")( function* (turnInput: Parameters[0], compact = false) { - const retryHandoff = - missedItems.length === 0 - ? [] - : [ - yield* contextHandoffService.prepareProviderHandoff({ - threadId: projection.thread.id, - targetRunId: run.id, - transferId: null, - fromProviderThreadIds: [providerThread.id], - toProviderThreadId: providerThread.id, - fromProviderInstanceId: run.providerInstanceId, - toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { - from: missedRuns[0]!.ordinal, - to: missedRuns.at(-1)!.ordinal, - }, - strategy: "delta_since_target_last_seen", - items: missedItems, - runs: projection.runs, - createdAt: yield* DateTime.now, - }), - ]; const delivery = yield* deliverContextHandoffs({ handoffs: [...effectiveHandoffs, ...retryHandoff], deferInline: compact || goalCommand, providerThread: runningProviderThread, - budget: Effect.gen(function* () { - return handoffBudget({ - tokenCap, - modelContextWindow, - userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, - attachments: message.attachments, - providerThread: budgetProviderThread, - nativeContextEstimate: - budgetProviderThread.contextUsage?.usedTokens === undefined - ? yield* nativeContextEstimate - : 0, - }); - }), + budget, alreadyDeliveredItemIds: deliveredItemIds, ...(session.injectHistory === undefined ? {} @@ -1302,7 +1402,7 @@ export const layer: Layer.Layer< ); const deliverySession = effectiveHandoffs.length === 0 && - missedItems.length === 0 && + retryHandoff.length === 0 && restartNote === "" && !noteContinuation ? session diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 452509bb97b0d..bfca75a27aa9c 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -114,6 +114,8 @@ function makeTestAdapter(input: { readonly failedRunOrdinals?: ReadonlySet; readonly interruptedRunOrdinals?: ReadonlySet; readonly holdRunOrdinal?: number; + readonly pendingBackgroundWork?: Ref.Ref; + readonly subagentOnlyBackgroundWork?: boolean; readonly holdFirstTurn?: Deferred.Deferred; readonly releaseFirstTurn?: Deferred.Deferred; }): ProviderAdapter.ProviderAdapterV2["Service"] { @@ -145,6 +147,14 @@ function makeTestAdapter(input: { providerSessionId: sessionInput.providerSessionId, providerSession, events: Stream.fromPubSub(events), + ...(input.pendingBackgroundWork === undefined + ? {} + : { + hasPendingBackgroundWorkForThread: (_thread, options) => + input.subagentOnlyBackgroundWork && options?.includeSubagents !== true + ? Effect.succeed(false) + : Ref.get(input.pendingBackgroundWork!), + }), ...(input.getModelContextWindow === undefined ? {} : { getModelContextWindow: input.getModelContextWindow }), @@ -412,6 +422,13 @@ describe("orchestration v2 provider switching", () => { "screenshot-fallback", "screenshot-reported-capacity-native", "screenshot-option-change-retry-native", + "screenshot-exhausted-target-native", + "screenshot-exhausted-target-fallback", + "screenshot-exhausted-target-estimate-native", + "screenshot-exhausted-target-oversized-native", + "screenshot-exhausted-target-tight-native", + "screenshot-exhausted-target-background-native", + "screenshot-exhausted-target-subagent-native", "prior-images-reasoning-change-replacement-small-native", "imported-prior-images-fallback", "prior-images-telemetry-native", @@ -431,11 +448,16 @@ describe("orchestration v2 provider switching", () => { const failStartOnce = yield* Ref.make(false); const failResumeOnce = yield* Ref.make(false); const generation = yield* Ref.make(0); + const pendingBackgroundWork = yield* Ref.make(false); const priorImages = scenario.includes("prior-images"); + const exhaustedTarget = scenario.includes("exhausted-target"); const replaceNative = scenario.includes("replacement"); const capacityScenario = modelScenario || scenario.includes("reported-capacity"); const returning = - scenario === "large-current-input" || scenario.includes("-change-") || priorImages; + scenario === "large-current-input" || + scenario.includes("-change-") || + priorImages || + exhaustedTarget; const targetSelection: ModelSelection = !modelScenario ? CLAUDE_MODEL_SELECTION : { @@ -460,10 +482,19 @@ describe("orchestration v2 provider switching", () => { driver: CLAUDE_DRIVER, capabilities: ClaudeProviderCapabilitiesV2, modelSelection: CLAUDE_MODEL_SELECTION, - responseByRunOrdinal: {}, + responseByRunOrdinal: scenario.includes("target-estimate") + ? { 2: "x".repeat(10_000) } + : {}, capturedTurns, ...(scenario.includes("retry") || scenario.includes("unsent") ? { failStartOnce } : {}), ...(replaceNative ? { failResumeOnce, nativeThreadGeneration: generation } : {}), + ...(exhaustedTarget ? { nativeThreadGeneration: generation } : {}), + ...(scenario.includes("target-background") || scenario.includes("target-subagent") + ? { + pendingBackgroundWork, + subagentOnlyBackgroundWork: scenario.includes("target-subagent"), + } + : {}), ...(scenario.includes("reported-capacity") ? { initialContextUsage: { usedTokens: 999_999, maxTokens: 1_000_000 } } : {}), @@ -491,8 +522,11 @@ describe("orchestration v2 provider switching", () => { ? 1_000_000 : 32_000, } - : scenario === "large-current-input" || priorImages - ? { getModelContextWindow: () => 32_000 } + : scenario === "large-current-input" || priorImages || exhaustedTarget + ? { + getModelContextWindow: () => + scenario.includes("target-tight") ? 25_121 : 32_000, + } : {}), ...(scenario.endsWith("-native") || scenario === "large-current-input" ? { injectedHistory } @@ -539,7 +573,9 @@ describe("orchestration v2 provider switching", () => { })) : turnUsageScenario && ordinal >= targetOrdinal ? [screenshot, { ...screenshot, id: "current-2" }] - : scenario.startsWith("screenshot") && ordinal >= targetOrdinal + : scenario.startsWith("screenshot") && + ordinal >= targetOrdinal && + (!exhaustedTarget || ordinal === targetOrdinal) ? screenshots : priorImages && ordinal === (scenario.startsWith("imported") ? 1 : 2) ? [screenshot] @@ -595,19 +631,49 @@ describe("orchestration v2 provider switching", () => { yield* Effect.addFinalizer(() => Effect.sync(() => spy?.mockRestore())); const current = scenario.startsWith("compact") ? "/compact" - : capacityScenario && !turnUsageScenario - ? "x".repeat(70_000) - : scenario === "large-current-input" - ? "x".repeat(9_000) - : "Continue work"; + : scenario.includes("target-oversized") + ? "x".repeat(40_000) + : capacityScenario && !turnUsageScenario + ? "x".repeat(70_000) + : scenario === "large-current-input" + ? "x".repeat(9_000) + : "Continue work"; if (returning) { if (scenario.includes("unsent")) yield* Ref.set(failStartOnce, true); yield* dispatch( 2, - turnUsageScenario ? "x".repeat(27_460) : "Establish target", + turnUsageScenario + ? "x".repeat(27_460) + : scenario.includes("target-estimate") + ? "x".repeat(12_000) + : "Establish target", CLAUDE_MODEL_SELECTION, ); yield* wait(2); + if (exhaustedTarget) { + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + } + if (exhaustedTarget && !scenario.includes("target-estimate")) { + const target = (yield* orchestrator.getThreadProjection( + threadId, + )).providerThreads.find( + (thread) => thread.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId, + )!; + yield* eventSink.write({ + events: [ + { + id: EventId.make("exhausted-target-usage"), + type: "provider-thread.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...target, contextUsage: { usedTokens: 30_000, maxTokens: 32_000 } }, + }, + ], + }); + } if (modelScenario && !turnUsageScenario) { const target = (yield* orchestrator.getThreadProjection( threadId, @@ -688,6 +754,8 @@ describe("orchestration v2 provider switching", () => { yield* Ref.set(failResumeOnce, true); } if (scenario.includes("retry")) yield* Ref.set(failStartOnce, true); + if (scenario.includes("target-background") || scenario.includes("target-subagent")) + yield* Ref.set(pendingBackgroundWork, true); yield* dispatch(targetOrdinal, current, targetSelection); yield* wait(targetOrdinal); if (scenario.includes("retry")) { @@ -773,7 +841,73 @@ describe("orchestration v2 provider switching", () => { yield* wait(targetOrdinal + 3); } const projection = yield* orchestrator.getThreadProjection(threadId); + if ( + scenario.includes("target-oversized") || + scenario.includes("target-tight") || + scenario.includes("target-background") || + scenario.includes("target-subagent") + ) { + assert.equal(projection.runs.at(-1)?.status, "failed"); + assert.equal(yield* Ref.get(generation), 1); + assert.equal(projection.messages.at(-1)!.text, current); + assert.deepEqual(projection.messages.at(-1)!.attachments, screenshots); + if (scenario.includes("target-background") || scenario.includes("target-subagent")) { + assert.isTrue(yield* Ref.get(pendingBackgroundWork)); + const failure = projection.turnItems.findLast((item) => item.type === "error"); + assert.equal( + failure?.type === "error" ? failure.failure.message : null, + "The target provider still has background work running. Wait for it to finish or stop it before retrying the handoff.", + ); + } + return; + } assert.equal(projection.runs.at(-1)?.status, "completed"); + if (exhaustedTarget) { + assert.equal(yield* Ref.get(generation), 2); + const turns = (yield* Ref.get(capturedTurns)).filter( + (turn) => turn.driver === CLAUDE_DRIVER, + ); + assert.notEqual(turns[0]!.nativeThreadId, turns[1]!.nativeThreadId); + const latestRun = projection.runs.at(-1)!; + assert.equal( + projection.attempts.find((attempt) => attempt.id === latestRun.activeAttemptId)! + .nativeThreadId, + turns[1]!.nativeThreadId, + ); + const context = scenario.endsWith("-native") + ? yield* encodeJson(yield* Ref.get(injectedHistory)) + : turns[1]!.text; + assert.include(context, "Original request with constraints"); + assert.include(context, "Original partial work"); + assert.include(context, "New source constraint"); + assert.include(turns[1]!.text, current); + assert.deepEqual(turns[1]!.attachments, screenshots); + const historyBeforeFollowup = yield* Ref.get(injectedHistory); + yield* dispatch(5, "Continue after automatic recovery", CLAUDE_MODEL_SELECTION); + yield* wait(5); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + assert.equal(yield* Ref.get(generation), 2); + assert.equal( + (yield* Ref.get(capturedTurns)).at(-1)!.text, + "Continue after automatic recovery", + ); + assert.deepEqual(yield* Ref.get(injectedHistory), historyBeforeFollowup); + if (scenario.includes("target-estimate")) { + yield* dispatch(6, "Another source turn", CODEX_MODEL_SELECTION); + yield* wait(6); + yield* dispatch(7, "Return after recovery", CLAUDE_MODEL_SELECTION); + yield* wait(7); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + assert.equal(yield* Ref.get(generation), 2); + } + return; + } if (priorImages) { const handoff = projection.contextHandoffs.at(-1)!; const context = scenario.endsWith("-native") @@ -894,7 +1028,12 @@ describe("orchestration v2 provider switching", () => { }), ), ); - it.live.each(["turn-start", "injection", "large-missed-request"] as const)( + it.live.each([ + "turn-start", + "injection", + "large-missed-request", + "exhausted-target-retry", + ] as const)( "recovers %s failure without duplicating history in the same native thread", (failure) => Effect.scoped( @@ -992,6 +1131,22 @@ describe("orchestration v2 provider switching", () => { failure !== "injection" ? "injected" : "pending", ); const historyBeforeRetry = (yield* Ref.get(injectedHistory)).length; + if (failure === "exhausted-target-retry") { + const target = failed.providerThreads.find( + (thread) => thread.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId, + )!; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("exhausted-retry-usage"), + type: "provider-thread.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...target, contextUsage: { usedTokens: 30_000, maxTokens: 32_000 } }, + }, + ], + }); + } const retryText = failure === "large-missed-request" ? "y".repeat(10_000) : "Retry target request"; yield* dispatch(3, retryText, CLAUDE_MODEL_SELECTION); @@ -999,7 +1154,7 @@ describe("orchestration v2 provider switching", () => { const retried = yield* orchestrator.getThreadProjection(threadId); const lastTurn = (yield* Ref.get(capturedTurns)).at(-1)!; assert.equal(lastTurn.text, retryText); - if (failure !== "injection") { + if (failure !== "injection" && failure !== "exhausted-target-retry") { const delta = yield* encodeJson( (yield* Ref.get(injectedHistory)).slice(historyBeforeRetry), ); diff --git a/docs/user/portable-handoffs.md b/docs/user/portable-handoffs.md index e2d84458ed936..04b2a8ce14987 100644 --- a/docs/user/portable-handoffs.md +++ b/docs/user/portable-handoffs.md @@ -20,9 +20,13 @@ agent-written summary. ## Context limits A handoff must leave room for existing provider context, your request and attachments, instructions, -tools, and subsequent work. If even its retrieval references cannot fit, T3 Code reports an error -instead of shortening your request. Compact the target conversation or select a larger-context -model before trying again. +tools, and subsequent work. If the target conversation has no room for the handoff, T3 Code starts +a fresh provider conversation with a bounded selection of saved history and sends your request +automatically. You do not need to compact manually. If the request and attachments leave too little +room even in a fresh conversation, shorten the request or select a larger-context model. T3 Code +never shortens your request to make it fit. + +Recovery leaves running background work intact. Let it finish or stop it before retrying the handoff. Server operators can set `T3CODE_CONTEXT_HANDOFF_TOKEN_CAP` to change the initial history allowance (default 16,000; clamped to 1,024–64,000). This is an upper bound, not a provider context-window diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index b3ce13af0a705..b47799cca6f6b 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -6511,6 +6511,46 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( return replacementExit.value.started; }); + const commitReplacementSession = Effect.fn("AcpAdapterV2.commitReplacementSession")( + function* ( + threadId: ThreadId | null, + candidate: AcpSessionRuntime.AcpSessionRuntimeStartResult, + ) { + rememberTerminalEnvironment(candidate.sessionId, yield* readMcpContext(threadId)); + yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(activeSessionId, candidate.sessionId); + yield* setActiveSessionSetup(candidate); + yield* Ref.set(activeSelection, null); + yield* Ref.set(activeInteractionMode, null); + yield* Ref.set(promptInstructionStates, new Map()); + yield* Ref.set(compactionMarkers, new Map()); + yield* Ref.set(providerTurns, new Map()); + yield* Ref.set(snapshot, { + order: [], + messages: new Map(), + loadingRole: null, + loadingMessageId: null, + loadingIndex: 0, + }); + yield* continuationPermit.withPermit( + Effect.gen(function* () { + yield* Ref.update(continuationGeneration, (value) => value + 1); + yield* Ref.set(stoppedRunQuarantine, false); + yield* Ref.set(wakeBuffer, []); + yield* Ref.set(wakeReports, noWakeReports); + yield* Ref.set(continuationRequested, false); + yield* Ref.set(runningBackgroundTaskIds, new Set()); + yield* Ref.set(endedBackgroundTaskIds, new Set()); + yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); + yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); + yield* Ref.set(carryoverSubagents, null); + yield* Ref.set(suppressPostSettleMonitorPrompt, false); + yield* Ref.set(lastTurnRoute, null); + }), + ); + }, + ); + const initialStart = yield* Effect.result( startAcpRuntime(input.threadId, input.initialNativeThreadId), ); @@ -7745,28 +7785,61 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( : {}), ensureThread: Effect.fn("AcpAdapterV2.ensureThread")( function* (threadInput: ProviderAdapter.ProviderAdapterV2EnsureThreadInput) { - const now = yield* DateTime.now; - const sessionId = yield* Ref.get(activeSessionId); - if (sessionId === null) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: "ACP runtime did not produce a session id", - }); - } - const providerThread = makeProviderThread({ - driver, - providerInstanceId: options.instanceId, - idAllocator, - appThreadId: threadInput.threadId, - providerSessionId: input.providerSessionId, - nativeThreadId: sessionId, - ...(itemIdentityVersion === undefined ? {} : { itemIdentityVersion }), - now, - }); - yield* Ref.update(providerThreadByNativeSessionId, (current) => - new Map(current).set(sessionId, providerThread), + return yield* runtimeTransitionPermit.withPermit( + Effect.gen(function* () { + if (threadInput.forceNewNativeThread) { + const currentTurn = yield* Ref.get(activeTurn); + const pendingWork = yield* ( + sessionRuntime.hasPendingBackgroundWork ?? Effect.succeed(false) + ); + if (currentTurn !== null || pendingWork) + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: + "Cannot replace ACP session while a turn or background work is active", + }); + yield* awaitRuntimeTeardown(); + const previousItemIdentityVersion = itemIdentityVersion; + itemIdentityVersion = 2; + prepareTerminalEnvironment(yield* readMcpContext(threadInput.threadId)); + yield* startReplacementAcpRuntime(threadInput.threadId, (candidate) => + commitReplacementSession(threadInput.threadId, candidate), + ).pipe( + Effect.tapError(() => + Effect.sync(() => { + itemIdentityVersion = previousItemIdentityVersion; + }), + ), + ); + } + const now = yield* DateTime.now; + const sessionId = yield* Ref.get(activeSessionId); + if (sessionId === null) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: "ACP runtime did not produce a session id", + }); + } + const created = makeProviderThread({ + driver, + providerInstanceId: options.instanceId, + idAllocator, + appThreadId: threadInput.threadId, + providerSessionId: input.providerSessionId, + nativeThreadId: sessionId, + ...(itemIdentityVersion === undefined ? {} : { itemIdentityVersion }), + now, + }); + const providerThread = + threadInput.forceNewNativeThread && threadInput.existingProviderThread + ? { ...created, id: threadInput.existingProviderThread.id } + : created; + yield* Ref.update(providerThreadByNativeSessionId, (current) => + new Map(current).set(sessionId, providerThread), + ); + return providerThread; + }), ); - return providerThread; }, (effect, threadInput) => effect.pipe( @@ -8309,43 +8382,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const replacement = yield* startReplacementAcpRuntime( rollbackInput.providerThread.appThreadId, (candidate) => - Effect.gen(function* () { - rememberTerminalEnvironment( - candidate.sessionId, - yield* readMcpContext(rollbackInput.providerThread.appThreadId), - ); - yield* Ref.set(runtimeRestartRequired, false); - yield* Ref.set(activeSessionId, candidate.sessionId); - yield* setActiveSessionSetup(candidate); - yield* Ref.set(activeSelection, null); - yield* Ref.set(activeInteractionMode, null); - yield* Ref.set(promptInstructionStates, new Map()); - yield* Ref.set(compactionMarkers, new Map()); - yield* Ref.set(providerTurns, new Map()); - yield* Ref.set(snapshot, { - order: [], - messages: new Map(), - loadingRole: null, - loadingMessageId: null, - loadingIndex: 0, - }); - yield* continuationPermit.withPermit( - Effect.gen(function* () { - yield* Ref.update(continuationGeneration, (value) => value + 1); - yield* Ref.set(stoppedRunQuarantine, false); - yield* Ref.set(wakeBuffer, []); - yield* Ref.set(wakeReports, noWakeReports); - yield* Ref.set(continuationRequested, false); - yield* Ref.set(runningBackgroundTaskIds, new Set()); - yield* Ref.set(endedBackgroundTaskIds, new Set()); - yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); - yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); - yield* Ref.set(carryoverSubagents, null); - yield* Ref.set(suppressPostSettleMonitorPrompt, false); - yield* Ref.set(lastTurnRoute, null); - }), - ); - }), + commitReplacementSession(rollbackInput.providerThread.appThreadId, candidate), ).pipe(Effect.retry({ times: 1 })); const now = yield* DateTime.now; return { diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 7c77b7bb0ac6f..3396117c31b09 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -411,6 +411,7 @@ export interface ProviderAdapterV2EnsureThreadInput { readonly runtimePolicy: ProviderAdapterV2RuntimePolicy; readonly providerSessionId?: ProviderSessionId; readonly existingProviderThread?: OrchestrationV2ProviderThread; + readonly forceNewNativeThread?: boolean; } export interface ProviderAdapterV2TurnInput { @@ -549,9 +550,10 @@ export interface ProviderAdapterV2SessionRuntime { /** Manager runtimes broadcast independent subscriptions; adapters may expose only a single-consumer stream. */ readonly subscribeEvents?: Effect.Effect; readonly hasPendingBackgroundWork?: Effect.Effect; - /** Overrides the session-wide probe so sibling threads cannot keep this root subscription open. */ + /** Root-subscription probe; replacement checks opt in to subagents owned by the live process. */ readonly hasPendingBackgroundWorkForThread?: ( providerThread: OrchestrationV2ProviderThread, + options?: { readonly includeSubagents?: boolean }, ) => Effect.Effect; /** Set only if all session background tasks end with its process; otherwise threads must keep Waiting. */ readonly backgroundTasksEndWithProcess?: true; diff --git a/packages/provider-core/src/server/failure.test.ts b/packages/provider-core/src/server/failure.test.ts index 16f0f871ccc26..8fb2e153aa109 100644 --- a/packages/provider-core/src/server/failure.test.ts +++ b/packages/provider-core/src/server/failure.test.ts @@ -16,6 +16,7 @@ import { makeProviderFailureTurnItem, MAX_PROVIDER_FAILURE_MESSAGE_LENGTH, ContextHandoffBudgetError, + ContextHandoffBackgroundWorkError, } from "./failure.ts"; import * as IdAllocator from "./IdAllocator.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -69,17 +70,16 @@ it("does not expose arbitrary cause messages and preserves explicit messages", ( }); it("preserves actionable handoff errors wrapped by turn startup", () => { - const cause = new ProviderAdapter.ProviderAdapterTurnStartError({ - driver: ProviderDriverKind.make("codex"), - threadId: ThreadId.make("thread:handoff-error"), - providerThreadId: ProviderThreadId.make("provider-thread:handoff-error"), - runId: RunId.make("run:handoff-error"), - cause: new ContextHandoffBudgetError(), - }); - assert.equal( - makeProviderFailure({ cause: Cause.fail(cause) }).message, - new ContextHandoffBudgetError().message, - ); + for (const error of [new ContextHandoffBudgetError(), new ContextHandoffBackgroundWorkError()]) { + const cause = new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: ProviderDriverKind.make("codex"), + threadId: ThreadId.make("thread:handoff-error"), + providerThreadId: ProviderThreadId.make("provider-thread:handoff-error"), + runId: RunId.make("run:handoff-error"), + cause: error, + }); + assert.equal(makeProviderFailure({ cause: Cause.fail(cause) }).message, error.message); + } }); it("keeps unrecognized reasons and provider text out of category messages", () => { diff --git a/packages/provider-core/src/server/failure.ts b/packages/provider-core/src/server/failure.ts index 48331a10eac26..4e24fcc945c7b 100644 --- a/packages/provider-core/src/server/failure.ts +++ b/packages/provider-core/src/server/failure.ts @@ -22,7 +22,16 @@ export class ContextHandoffBudgetError extends Schema.TaggedError()( + "ContextHandoffBackgroundWorkError", + {}, +) { + override get message() { + return "The target provider still has background work running. Wait for it to finish or stop it before retrying the handoff."; } } @@ -87,6 +96,11 @@ function curatedCauseMessages(cause: unknown): { switch (tag) { case "ContextHandoffBudgetError": return { known: new ContextHandoffBudgetError().message, category: categoryMessage() }; + case "ContextHandoffBackgroundWorkError": + return { + known: new ContextHandoffBackgroundWorkError().message, + category: categoryMessage(), + }; case "AcpModelSelectionError": case "ClaudeBackgroundWorkBlocksQueryReplacementError": case "ClaudeAppendSystemPromptFileError":