diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index d71b0f7b5e2b8..3a14f9a702aa0 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -1329,6 +1329,10 @@ describe("AcpAdapterV2", () => { ); it.live("fails an end_turn the outcome check rejects and restarts before the next turn", () => Effect.gen(function* () { + const checking = yield* Deferred.make(); + const releaseCheck = yield* Deferred.make(); + const cancelled = yield* Deferred.make(); + const idAllocator = yield* IdAllocator.IdAllocatorV2; const instanceId = ProviderInstanceId.make("acp-test-prompt-outcome"); const threadId = ThreadId.make("thread-acp-prompt-outcome"); const quota = { @@ -1338,6 +1342,12 @@ describe("AcpAdapterV2", () => { retryable: true, resetAt: "2030-01-01T00:00:00.000Z", } as const; + const poisoned = { + class: "provider_error", + message: "Saved history is unusable.", + code: "INVALID_THINKING_SIGNATURE", + retryable: true, + } as const; // The runtime ordinal of each prompt, and the session each check watched. const promptRuntimes: Array = []; const promptTexts: Array = []; @@ -1351,14 +1361,27 @@ describe("AcpAdapterV2", () => { watchPromptOutcome: (sessionId) => Effect.sync(() => { watchedSessions.push(sessionId); - const failed = watchedSessions.length === 1; - return () => Effect.succeed(failed ? quota : undefined); + const outcome = + watchedSessions.length === 1 + ? { failure: quota, threadDisposition: "reusable" as const } + : watchedSessions.length >= 3 + ? { failure: poisoned, threadDisposition: "broken" as const } + : undefined; + const interrupted = watchedSessions.length === 4; + return () => + interrupted + ? Deferred.succeed(checking, undefined).pipe( + Effect.andThen(Deferred.await(releaseCheck)), + Effect.as(outcome), + ) + : Effect.succeed(outcome); }), makeRuntime: makeMockRuntime({ childProcessSpawner: yield* ChildProcessSpawner.ChildProcessSpawner, mockAgentPath: yield* bundledMockAgentPath, wrapRuntime: (runtime, runtimeOrdinal) => ({ ...runtime, + cancel: Deferred.succeed(cancelled, undefined).pipe(Effect.asVoid), prompt: (payload) => Effect.sync(() => { promptRuntimes.push(runtimeOrdinal); @@ -1402,19 +1425,20 @@ describe("AcpAdapterV2", () => { ordinal, }), ); - const events = yield* runtime.events.pipe( - Stream.takeUntil((event) => event.type === "turn.terminal"), - Stream.runCollect, + return Array.from( + yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ), ); - return events.at(-1); }); - const failed = yield* runTurn(1); + const failed = (yield* runTurn(1)).at(-1); if (failed?.type !== "turn.terminal" || failed.status !== "failed") { return assert.fail("expected a failed terminal"); } assert.deepEqual(failed.failure, quota); - const completed = yield* runTurn(2); + const completed = (yield* runTurn(2)).at(-1); assert.equal(completed?.type === "turn.terminal" ? completed.status : null, "completed"); assert.deepEqual(promptRuntimes, [1, 2], "the next turn runs on a fresh runtime"); // The fresh runtime has no record of what reached the agent, so it sends T3's notes again. @@ -1422,6 +1446,61 @@ describe("AcpAdapterV2", () => { assert.include(promptTexts[1] ?? "", "T3 Code interaction mode: Default"); assert.equal(watchedSessions.length, 2); assert.equal(watchedSessions[0], watchedSessions[1]); + + // Poisoned native history marks the thread broken; its persisted error row is never resumed. + const poisonedEvents = yield* runTurn(3); + const broken = poisonedEvents.at(-1); + if (broken?.type !== "turn.terminal" || broken.status !== "failed") { + return assert.fail("expected a failed terminal"); + } + assert.deepEqual(broken.failure, poisoned); + assert.equal(broken.threadDisposition, "broken"); + const brokenThread = poisonedEvents.findLast( + (event) => event.type === "provider_thread.updated", + ); + assert.equal( + brokenThread?.type === "provider_thread.updated" + ? brokenThread.providerThread.status + : null, + "error", + ); + const resumed = yield* Effect.result( + runtime.resumeThread({ + threadId, + providerThread: { ...providerThread, status: "error" }, + modelSelection, + runtimePolicy, + }), + ); + assert.equal( + resumed._tag === "Failure" ? resumed.failure._tag : null, + "ProviderAdapterResumeThreadError", + ); + + const turn = yield* Effect.forkChild(runTurn(4)); + yield* Deferred.await(checking); + const stop = yield* Effect.forkChild( + runtime.interruptTurn({ + providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: acpScopedNativeId(instanceId, `${watchedSessions[3]}:turn:4`), + }), + }), + ); + yield* Deferred.await(cancelled); + yield* Deferred.succeed(releaseCheck, undefined); + const interruptedEvents = yield* Fiber.join(turn); + yield* Fiber.join(stop); + const interrupted = interruptedEvents.at(-1); + assert.equal( + interrupted?.type === "turn.terminal" ? interrupted.status : null, + "interrupted", + ); + assert.equal( + interrupted?.type === "turn.terminal" ? interrupted.threadDisposition : null, + "broken", + ); }).pipe(Effect.provide(layerTest)), ); it.live("retains returned usage when Stop finalizes before the prompt callback", () => diff --git a/apps/server/src/orchestration-v2/ContextHandoffService.ts b/apps/server/src/orchestration-v2/ContextHandoffService.ts index ca6b5ff0fcdf1..faeeb4c06f120 100644 --- a/apps/server/src/orchestration-v2/ContextHandoffService.ts +++ b/apps/server/src/orchestration-v2/ContextHandoffService.ts @@ -26,6 +26,7 @@ import { import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline"; import type { ProjectionStoreV2Shape } from "./ProjectionStore.ts"; +import { cancelledBeforeStart } from "./RestartBackgroundNote.ts"; export class ContextHandoffPrepareError extends Schema.TaggedError()( "ContextHandoffPrepareError", @@ -116,6 +117,42 @@ function compactText(text: string, maxLength = 240): string { return `${compacted.slice(0, last >= 0xd800 && last <= 0xdbff ? end - 1 : end)}...`; } +/** A fresh-session schedule run starts a side conversation; the thread's own one continues past it. */ +export function isScheduledFreshRun( + run: Pick, + turnItems: OrchestrationV2ThreadProjection["turnItems"], +): boolean { + // Turn items, unlike messages, are not narrowed to the runs since the latest compaction. + return ( + run.freshContext === true && + turnItems.some( + (item) => + item.type === "user_message" && + item.messageId === run.userMessageId && + item.scheduledTaskId !== undefined, + ) + ); +} + +/** Ordinal of the latest started fresh-context run. Runs before it never reach a provider again. */ +export function contextBoundaryOrdinal( + projection: Pick, +): number { + let boundary = 0; + for (const run of projection.runs) { + if ( + run.freshContext === true && + run.status !== "queued" && + !cancelledBeforeStart(run) && + run.ordinal > boundary && + !isScheduledFreshRun(run, projection.turnItems) + ) { + boundary = run.ordinal; + } + } + return boundary; +} + export function visibleDeltaRunOrdinals( projection: { readonly runs: ReadonlyArray> }, items: ReadonlyArray>, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 5257330e2cb5d..b0450ea7112b1 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -120,6 +120,8 @@ import { import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts"; import { ContextHandoffServiceV2, + contextBoundaryOrdinal, + isScheduledFreshRun, readForkHistoryItems, readInheritedHandoffItems, visibleConversationItems, @@ -1364,42 +1366,6 @@ function appendContextHandoffId( return handoffId === null ? handoffIds : Array.from(new Set([...handoffIds, handoffId])); } -/** A fresh-session schedule run starts a side conversation; the thread's own one continues past it. */ -function isScheduledFreshRun( - run: Pick, - turnItems: OrchestrationV2ThreadProjection["turnItems"], -): boolean { - // Turn items, unlike messages, are not narrowed to the runs since the latest compaction. - return ( - run.freshContext === true && - turnItems.some( - (item) => - item.type === "user_message" && - item.messageId === run.userMessageId && - item.scheduledTaskId !== undefined, - ) - ); -} - -/** Ordinal of the latest started fresh-context run. Runs before it never reach a provider again. */ -function contextBoundaryOrdinal( - projection: Pick, -): number { - let boundary = 0; - for (const run of projection.runs) { - if ( - run.freshContext === true && - run.status !== "queued" && - !cancelledBeforeStart(run) && - run.ordinal > boundary && - !isScheduledFreshRun(run, projection.turnItems) - ) { - boundary = run.ordinal; - } - } - return boundary; -} - /** The provider thread a message continues: the active one, unless a fresh-session schedule run started it. */ function conversationProviderThread( projection: Pick< @@ -2745,6 +2711,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const deliveryProviderThread = canResumeAcrossInstances ? { ...queuedProviderThread, + status: activeProviderThread.status, nativeThreadRef: activeProviderThread.nativeThreadRef, nativeConversationHeadRef: activeProviderThread.nativeConversationHeadRef, nativeMetadata: activeProviderThread.nativeMetadata, @@ -2936,7 +2903,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const providerThread: OrchestrationV2ProviderThread = { ...deliveryProviderThread, providerSessionId, - status: "not_loaded", + status: deliveryProviderThread.status === "error" ? "error" : "not_loaded", firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? ordinal, lastRunOrdinal: ordinal, handoffIds: appendContextHandoffId( @@ -8721,7 +8688,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : null; const providerThread: OrchestrationV2ProviderThread = { ...ensuredProviderThread, - status: "active", + status: ensuredProviderThread.status === "error" ? "error" : "active", firstRunOrdinal: ensuredProviderThread.firstRunOrdinal ?? ordinal, lastRunOrdinal: ordinal, handoffIds: [ diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 7b0759ae99b0b..0fb859763f609 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -2915,11 +2915,20 @@ effectIt.effect("does not mistake a failed state read for a superseded run", () }), ); -effectIt.effect( - "signs out the existing native provider before opening the newly selected provider", - () => +effectIt.effect.each(["not_loaded", "error"] as const)( + "signs out the existing native provider before opening the newly selected provider with status %s", + (providerThreadStatus) => Effect.gen(function* () { const harness = makeLocalCommandHarness({ text: "/logout", previousNativeSession: true }); + const initial = harness.projection(); + harness.setProjection({ + ...initial, + providerThreads: initial.providerThreads.map((thread) => + thread.id === "new-provider-thread" + ? { ...thread, status: providerThreadStatus } + : thread, + ), + }); yield* harness.start; yield* harness.start; @@ -2945,6 +2954,9 @@ effectIt.effect( ]); expect(projection.providerTurns).toEqual([]); expect(projection.checkpoints).toEqual([]); + expect( + projection.providerThreads.find((thread) => thread.id === "new-provider-thread")?.status, + ).toBe(providerThreadStatus); }), ); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 6587e742c7cd0..fa5409adfa21c 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -624,7 +624,12 @@ export const layer: Layer.Layer< }, providerThreadUpdate: { ...providerThread, - status: providerThread.nativeThreadRef === null ? "not_loaded" : "idle", + status: + providerThread.status === "error" + ? "error" + : providerThread.nativeThreadRef === null + ? "not_loaded" + : "idle", updatedAt: now, }, }); @@ -780,6 +785,12 @@ export const layer: Layer.Layer< run, checkpointScope, }); + const brokenNativeThread = providerThread.status === "error"; + // Opened without the broken native id, a session that was not live starts fresh. + const opensFreshSession = + brokenNativeThread && + providerThread.nativeThreadRef !== null && + !(yield* providerSessions.isLive(providerSessionId)); const sessionResult = yield* Effect.result( providerSessions.open({ threadId: projection.thread.id, @@ -789,7 +800,7 @@ export const layer: Layer.Layer< ...(existingSessionProjection === undefined ? {} : { resumeFromSession: existingSessionProjection }), - ...(providerThread.nativeThreadRef?.nativeId == null + ...(providerThread.nativeThreadRef?.nativeId == null || brokenNativeThread ? {} : { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }), ...(providerThread.nativeMetadata?.itemIdentityVersion === undefined @@ -1071,6 +1082,13 @@ export const layer: Layer.Layer< type: "provider_resume_fallback", }); const createdAt = yield* DateTime.now; + const history = yield* projectionStore.getTurnStartHistory(input.threadId); + // Turn-start context carries only this run's turn items; the boundary needs the thread's. + const boundary = ContextHandoffService.contextBoundaryOrdinal({ + runs: projection.runs, + turnItems: history, + }); + const from = Math.max(1, boundary); const handoff = yield* contextHandoffService.prepareProviderHandoff({ threadId: projection.thread.id, targetRunId: run.id, @@ -1079,15 +1097,18 @@ export const layer: Layer.Layer< toProviderThreadId: providerThread.id, fromProviderInstanceId: providerThread.providerInstanceId, toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, + coveredRunOrdinals: { from, to: Math.max(from, run.ordinal - 1) }, strategy: "full_thread_summary", runs: projection.runs, - items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter( - (item) => - item.runId === null || - projection.runs.some( - (source) => source.id === item.runId && source.ordinal < run.ordinal, - ), + items: history.filter((item) => + item.runId === null + ? boundary === 0 + : projection.runs.some( + (source) => + source.id === item.runId && + source.ordinal >= from && + source.ordinal < run.ordinal, + ), ), createdAt, }); @@ -1417,7 +1438,10 @@ export const layer: Layer.Layer< providerSessionId, // Clear the native ref to force a fresh session while preserving row identity. existingProviderThread: { ...providerThread, nativeThreadRef: null }, - forceNewNativeThread: true, + // A fresh session that never loaded the broken history needs no replacement. + ...(opensFreshSession && resumed._tag === "Failure" + ? {} + : { forceNewNativeThread: true }), }), ); if (replacement === undefined) return undefined; diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index edc12dcd98840..911aec5a48780 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -58,6 +58,7 @@ import * as EffectOutbox from "../EffectOutbox.ts"; import * as ProjectionReplay from "./ProjectionReplay.ts"; import * as ProjectionStore from "../ProjectionStore.ts"; import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; +import * as ProviderSessionManager from "../ProviderSessionManager.ts"; import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import { CLAUDE_MODEL_SELECTION, @@ -116,6 +117,7 @@ function makeTestAdapter(input: { >; readonly failure?: ReturnType; readonly failedRunOrdinals?: ReadonlySet; + readonly brokenRunOrdinals?: ReadonlySet; readonly interruptedRunOrdinals?: ReadonlySet; readonly cancelledRunOrdinals?: ReadonlySet; readonly holdRunOrdinal?: number; @@ -131,6 +133,8 @@ function makeTestAdapter(input: { readonly backgroundTasksEndWithProcess?: true; readonly backgroundWork?: Ref.Ref; readonly closedSessions?: Ref.Ref>; + /** Like ACP: opening resumes the initial native id, and forcing a new thread restarts the runtime. */ + readonly runtimeStarts?: Ref.Ref>; }): ProviderAdapter.ProviderAdapterV2["Service"] { return { instanceId: input.instanceId, @@ -140,6 +144,12 @@ function makeTestAdapter(input: { openSession: (sessionInput) => Effect.gen(function* () { const events = yield* PubSub.unbounded(); + if (input.runtimeStarts !== undefined) { + yield* Ref.update(input.runtimeStarts, (starts) => [ + ...starts, + sessionInput.initialNativeThreadId ?? "new", + ]); + } if (input.closedSessions !== undefined) { const closedSessions = input.closedSessions; yield* Effect.addFinalizer(() => @@ -193,6 +203,9 @@ function makeTestAdapter(input: { }), ensureThread: (threadInput) => Effect.gen(function* () { + if (threadInput.forceNewNativeThread && input.runtimeStarts !== undefined) { + yield* Ref.update(input.runtimeStarts, (starts) => [...starts, "replacement"]); + } const createdAt = yield* DateTime.now; const generation = input.nativeThreadGeneration === undefined @@ -225,6 +238,7 @@ function makeTestAdapter(input: { resumeThread: ({ providerThread }) => Effect.gen(function* () { if ( + (input.brokenRunOrdinals !== undefined && providerThread.status === "error") || input.failResume || (input.failResumeOnce !== undefined && (yield* Ref.getAndSet(input.failResumeOnce, false))) @@ -423,7 +437,9 @@ function makeTestAdapter(input: { }), } : { status: terminalStatus, failure: null }), - threadDisposition: "reusable", + threadDisposition: input.brokenRunOrdinals?.has(turnInput.runOrdinal) + ? "broken" + : "reusable", }, ]; const reportedUsage = input.tokenUsageByRunOrdinal?.[turnInput.runOrdinal]; @@ -439,6 +455,17 @@ function makeTestAdapter(input: { }, }); } + if (input.brokenRunOrdinals?.has(turnInput.runOrdinal)) { + yield* PubSub.publish(events, { + type: "provider_thread.updated", + driver: input.driver, + providerThread: { + ...turnInput.providerThread, + status: "error", + updatedAt: eventTime, + }, + }); + } for (const event of providerEvents) { yield* PubSub.publish(events, event); } @@ -1742,7 +1769,8 @@ describe("orchestration v2 provider switching", () => { it.live.each( ( [ - ["failed", false, false, false], + ["failed", false, false, false, false], + ["failed", false, false, false, true], ["failed", true, true, true], ["interrupted", false, true, false], ["interrupted", true, false, true], @@ -1750,13 +1778,13 @@ describe("orchestration v2 provider switching", () => { ["cancelled", true, true, false], ] as const ).map( - ([status, queued, returning, native]) => + ([status, queued, returning, native, broken]) => [ - `${status} context ${queued ? "through the queue" : "immediately"} to ${returning ? "a returning" : "a new"} provider via ${native ? "native history" : "text"}`, - { status, queued, returning, native }, + `${status}${broken ? " broken" : ""} context ${queued ? "through the queue" : "immediately"} to ${returning ? "a returning" : "a new"} provider via ${native ? "native history" : "text"}`, + { status, queued, returning, native, broken }, ] as const, ), - )("hands off %s", ([, { status, queued, returning, native }]) => + )("hands off %s", ([, { status, queued, returning, native, broken }]) => Effect.gen(function* () { const cwd = yield* checkpointWorkspace(`handoff-${status}-${queued}-${returning}`); const capturedTurns = yield* Ref.make>([]); @@ -1766,6 +1794,7 @@ describe("orchestration v2 provider switching", () => { const sourceOrdinal = returning ? 2 : 1; const sourceSelection = returning ? CLAUDE_MODEL_SELECTION : CODEX_MODEL_SELECTION; const targetSelection = returning ? CODEX_MODEL_SELECTION : CLAUDE_MODEL_SELECTION; + const generation = yield* Ref.make(0); const originalPrompt = "Keep the release marker violet and preserve the existing API."; const partialResponse = "I checked the API and found the release configuration."; const layerRegistry = ProviderAdapterRegistry.layerFromAdapters( @@ -1788,6 +1817,12 @@ describe("orchestration v2 provider switching", () => { ...(modelSelection.instanceId === sourceSelection.instanceId ? { failedRunOrdinals: new Set(status === "failed" ? [sourceOrdinal] : []), + ...(broken + ? { + brokenRunOrdinals: new Set([sourceOrdinal]), + nativeThreadGeneration: generation, + } + : {}), interruptedRunOrdinals: new Set(status === "interrupted" ? [sourceOrdinal] : []), cancelledRunOrdinals: new Set(status === "cancelled" ? [sourceOrdinal] : []), holdRunOrdinal: sourceOrdinal, @@ -1901,8 +1936,17 @@ describe("orchestration v2 provider switching", () => { yield* dispatch("back", "Finish the remaining work", sourceSelection); yield* waitForRun(sourceOrdinal + 2, "completed"); const back = (yield* Ref.get(capturedTurns)).at(-1)!; - assert.notInclude(back.text, originalPrompt); - assert.notInclude(back.text, partialResponse); + if (broken) { + const source = (yield* Ref.get(capturedTurns)).find( + (turn) => turn.modelSelection.instanceId === sourceSelection.instanceId, + )!; + assert.notEqual(back.nativeThreadId, source.nativeThreadId); + assert.include(back.text, originalPrompt); + assert.include(back.text, partialResponse); + } else { + assert.notInclude(back.text, originalPrompt); + assert.notInclude(back.text, partialResponse); + } }).pipe( Effect.provide( ProviderReplayHarness.layerWithRegistry( @@ -3288,15 +3332,23 @@ describe("orchestration v2 provider switching", () => { }), ); - it.live.each(["immediately", "behind a running turn"] as const)( + it.live.each([ + "immediately", + "behind a running turn", + "and recovers it after it breaks", + "and recovers it after it breaks and the server restarts", + ] as const)( "starts a fresh provider session %s and never hands earlier runs to a later provider", (variant) => Effect.gen(function* () { - const queued = variant !== "immediately"; - const key = queued ? "queued" : "immediate"; + const queued = variant === "behind a running turn"; + const restarted = variant === "and recovers it after it breaks and the server restarts"; + const broken = restarted || variant === "and recovers it after it breaks"; + const key = queued ? "queued" : restarted ? "restarted" : broken ? "broken" : "immediate"; const cwd = yield* checkpointWorkspace(`fresh-context-${key}`); const capturedTurns = yield* Ref.make>([]); const closedSessions = yield* Ref.make>([]); + const runtimeStarts = yield* Ref.make>([]); // A queued start has nobody to refuse, so a busy session must stay up. const backgroundWork = yield* Ref.make(queued); const started = yield* Deferred.make(); @@ -3310,11 +3362,18 @@ describe("orchestration v2 provider switching", () => { responseByRunOrdinal: {}, capturedTurns, closedSessions, + runtimeStarts, backgroundWork, nativeThreadGeneration: yield* Ref.make(0), ...(queued ? { holdFirstTurn: started, releaseFirstTurn: release, holdRunOrdinal: 2 } : {}), + ...(broken + ? { + failedRunOrdinals: new Set([3]), + brokenRunOrdinals: new Set([3]), + } + : {}), }), makeTestAdapter({ instanceId: CODEX_MODEL_SELECTION.instanceId, @@ -3408,6 +3467,22 @@ describe("orchestration v2 provider switching", () => { // Claude hosts one native thread per session, so an idle old one is closed. assert.deepEqual(yield* Ref.get(closedSessions), queued ? [] : [firstSession.id]); yield* Ref.set(backgroundWork, false); + let startsBeforeRecovery = 0; + if (broken) { + if (restarted) { + const brokenProjection = yield* orchestrator.getThreadProjection(freshThreadId); + const brokenThread = brokenProjection.providerThreads.find( + (thread) => thread.id === brokenProjection.thread.activeProviderThreadId, + )!; + assert.equal(brokenThread.status, "error"); + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + yield* sessions.close(brokenThread.providerSessionId!); + } + startsBeforeRecovery = (yield* Ref.get(runtimeStarts)).length; + yield* send("epsilon", CLAUDE_MODEL_SELECTION); + yield* worker.drain(); + yield* waitForIdle(freshThreadId); + } yield* send("delta", CODEX_MODEL_SELECTION); yield* worker.drain(); yield* waitForIdle(freshThreadId); @@ -3436,6 +3511,27 @@ describe("orchestration v2 provider switching", () => { if (queued) { assert.equal(turnFor("gamma").providerThreadId, turnFor("beta").providerThreadId); } + if (broken) { + const recovered = turnFor("epsilon"); + assert.equal(recovered.providerThreadId, turnFor("beta").providerThreadId); + assert.notEqual(recovered.nativeThreadId, turnFor("beta").nativeThreadId); + assert.include(recovered.text, "Prompt beta"); + assert.notInclude(recovered.text, "Prompt alpha"); + assert.notInclude(recovered.text, "Prompt zero"); + // The broken native id is never reopened; a reopened session starts fresh only once. + assert.deepEqual( + (yield* Ref.get(runtimeStarts)).slice(startsBeforeRecovery), + restarted ? ["new"] : ["replacement"], + ); + const recoveredRun = projection.runs.find( + (run) => run.userMessageId === `message:fresh-context:${key}:epsilon`, + )!; + assert.equal( + projection.contextHandoffs.find((handoff) => handoff.targetRunId === recoveredRun.id)! + .coveredRunOrdinals.from, + 3, + ); + } const codexTurn = turnFor("delta"); assert.notEqual(codexTurn.providerThreadId, turnFor("zero").providerThreadId); assert.include(codexTurn.text, "Prompt beta"); @@ -3569,6 +3665,8 @@ describe("orchestration v2 provider switching", () => { responseByRunOrdinal: {}, capturedTurns, nativeThreadGeneration: yield* Ref.make(0), + failedRunOrdinals: new Set([5]), + brokenRunOrdinals: new Set([5]), }), ]); const threadId = ThreadId.make("thread:scheduled-fresh-user-chat"); @@ -3616,6 +3714,7 @@ describe("orchestration v2 provider switching", () => { ["compaction", false], ["scheduled-after-compaction", true], ["user-third", false], + ["user-fourth", false], ] as const) { if (name === "compaction") { // The user's run compacts, so later dispatches no longer read the first scheduled run's message. @@ -3666,6 +3765,21 @@ describe("orchestration v2 provider switching", () => { // The scheduled run's fresh session must not replace the user's conversation. assert.equal(turnFor("user-second")!.nativeThreadId, turnFor("user-first")!.nativeThreadId); assert.equal(turnFor("user-third")!.nativeThreadId, turnFor("user-first")!.nativeThreadId); + // The user's session broke; recovery reads the whole thread, so scheduled runs do not cut it. + const recovered = turnFor("user-fourth")!; + assert.equal(recovered.providerThreadId, turnFor("user-first")!.providerThreadId); + assert.notEqual(recovered.nativeThreadId, turnFor("user-first")!.nativeThreadId); + assert.include(recovered.text, "user-first"); + assert.include(recovered.text, "user-third"); + const projection = yield* orchestrator.getThreadProjection(threadId); + const recoveredRun = projection.runs.find( + (run) => run.userMessageId === "message:scheduled-fresh-user-chat:user-fourth", + )!; + assert.equal( + projection.contextHandoffs.find((handoff) => handoff.targetRunId === recoveredRun.id)! + .coveredRunOrdinals.from, + 1, + ); }).pipe( Effect.provide( ProviderReplayHarness.layerWithRegistry( @@ -3802,6 +3916,7 @@ describe("orchestration v2 provider switching", () => { it.live.each([ ["automatic", "Claude"], ["queue.resume", "compatible Codex"], + ["queue.resume", "broken compatible Codex"], ] as const)( "delivers a queued PR update on the current provider through %s (%s)", ([delivery, target]) => @@ -3810,7 +3925,9 @@ describe("orchestration v2 provider switching", () => { const capturedTurns = yield* Ref.make>([]); const started = yield* Deferred.make(); const release = yield* Deferred.make(); - const compatible = target === "compatible Codex"; + const compatible = target !== "Claude"; + const broken = target === "broken compatible Codex"; + const generation = yield* Ref.make(0); const selection = { ...(compatible ? { ...CODEX_MODEL_SELECTION, instanceId: ProviderInstanceId.make("codex-alternate") } @@ -3822,7 +3939,7 @@ describe("orchestration v2 provider switching", () => { ...CodexProviderCapabilitiesV2, context: { ...CodexProviderCapabilitiesV2.context, - canConsumeHandoffSummaries: false, + canConsumeHandoffSummaries: broken, }, } : ClaudeProviderCapabilitiesV2; @@ -3837,6 +3954,9 @@ describe("orchestration v2 provider switching", () => { holdFirstTurn: started, releaseFirstTurn: release, failedRunOrdinals: new Set([1]), + ...(broken + ? { brokenRunOrdinals: new Set([1]), nativeThreadGeneration: generation } + : {}), failure: makeProviderFailure({ message: "Usage limit reached", class: "usage_limit", @@ -3850,6 +3970,9 @@ describe("orchestration v2 provider switching", () => { modelSelection: selection, responseByRunOrdinal: {}, capturedTurns, + ...(broken + ? { brokenRunOrdinals: new Set(), nativeThreadGeneration: generation } + : {}), }), ]; const layerRegistry = compatible @@ -3987,7 +4110,12 @@ describe("orchestration v2 provider switching", () => { assert.equal(executed.providerThreadId, turns[1]!.providerThreadId); assert.equal(executed.nativeThreadId, turns[1]!.nativeThreadId); } - if (compatible) { + if (broken) { + assert.notEqual(executed.nativeThreadId, turns[0]!.nativeThreadId); + assert.isFalse(executed.nativeThreadHasTurns); + assert.include(executed.text, "Original work"); + } + if (compatible && !broken) { assert.equal(executed.nativeThreadId, turns[0]!.nativeThreadId); assert.isEmpty(after.contextHandoffs); } diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index 52e96caa08aea..eb0695948b30c 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -6,6 +6,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import type * as EffectAcpSchema from "effect-acp/compat"; +import type { OrchestrationV2ProviderFailure } from "@t3tools/contracts"; import { watchAntigravityPromptOutcome } from "./AntigravityTurnOutcome.ts"; @@ -32,10 +33,19 @@ const RATE_LIMIT_ERROR = errorPayload( const INVALID_ARGUMENT_ERROR = errorPayload( "Error 400, Message: Request contains an invalid argument., Status: INVALID_ARGUMENT, Details: []", ); +// Vertex Claude rejects a resumed history whose saved thinking block lost its signature. +const MISSING_SIGNATURE_ERROR = errorPayload( + 'Error 400, Message: {"type":"error","error":{"type":"invalid_request_error","message":"messages.1.content.0.thinking.signature: Field required"},"request_id":"req_vrtx_test"}, Status: INVALID_ARGUMENT, Details: []', +); +const INVALID_SIGNATURE_ERROR = errorPayload( + 'Error 400, Message: {"type":"error","error":{"type":"invalid_request_error","message":"messages.3.content.0: Invalid `signature` in `thinking` block"}}, Status: INVALID_ARGUMENT, Details: []', +); // The model's reply quoting the agent's quota notice is an ordinary planner response. const QUOTED_NOTICE = Buffer.from( "Usage Limit Reached\n\nYou have reached your current quota for this period. Your limit will reset in 1 hour.", ); +const reusable = (failure: OrchestrationV2ProviderFailure) => + ({ failure, threadDisposition: "reusable" }) as const; describe("watchAntigravityPromptOutcome", () => { it.effect("fails only turns whose own steps end in an upstream error", () => @@ -83,13 +93,13 @@ describe("watchAntigravityPromptOutcome", () => { [USER_INPUT, input], [ERROR_MESSAGE, QUOTA_ERROR], ]), - { + reusable({ class: "usage_limit", message: "You have reached your Antigravity quota for this period.", code: "RESOURCE_EXHAUSTED", retryable: true, resetAt: "1970-01-01T01:00:00.000Z", - }, + }), ); // The earlier error step is outside this turn. assert.isUndefined( @@ -109,26 +119,35 @@ describe("watchAntigravityPromptOutcome", () => { [USER_INPUT, input], [ERROR_MESSAGE, RATE_LIMIT_ERROR], ]), - { + reusable({ class: "usage_limit", message: "Resource has been exhausted (e.g. check quota).", code: "RESOURCE_EXHAUSTED", retryable: true, resetAt: "1970-01-01T00:00:42.000Z", - }, + }), ); assert.deepEqual( yield* turn([ [USER_INPUT, input], [ERROR_MESSAGE, INVALID_ARGUMENT_ERROR], ]), - { + reusable({ class: "provider_error", message: "Request contains an invalid argument.", code: "INVALID_ARGUMENT", retryable: false, - }, + }), ); + // A saved thinking block without a valid signature fails every later prompt. + for (const error of [MISSING_SIGNATURE_ERROR, INVALID_SIGNATURE_ERROR]) { + const outcome = yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, error], + ]); + assert.equal(outcome?.failure.code, "INVALID_THINKING_SIGNATURE"); + assert.equal(outcome?.threadDisposition, "broken"); + } // A completed tool keeps its usage, and the turn still fails. assert.equal( (yield* turn( @@ -142,7 +161,7 @@ describe("watchAntigravityPromptOutcome", () => { stopReason: "end_turn", usage: { inputTokens: 40, outputTokens: 5, totalTokens: 45 }, }, - ))?.class, + ))?.failure.class, "usage_limit", ); assert.isUndefined( @@ -152,12 +171,15 @@ describe("watchAntigravityPromptOutcome", () => { ]), ); // The agent maps the SDK's QUOTA_EXHAUSTED stop reason to `refusal`. - assert.deepEqual(yield* turn([], { stopReason: "refusal" }), { - class: "usage_limit", - message: "You have reached your Antigravity quota for this period.", - code: "QUOTA_EXHAUSTED", - retryable: true, - }); + assert.deepEqual( + yield* turn([], { stopReason: "refusal" }), + reusable({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "QUOTA_EXHAUSTED", + retryable: true, + }), + ); }).pipe(Effect.provide(NodeServices.layer)), ); @@ -193,19 +215,22 @@ describe("watchAntigravityPromptOutcome", () => { const linked = yield* watchAntigravityPromptOutcome(profileDirectory, own); append(peer, ERROR_MESSAGE, QUOTA_ERROR); assert.isUndefined(yield* linked({ stopReason: "end_turn" })); - assert.equal((yield* linked({ stopReason: "refusal" }))?.class, "usage_limit"); + assert.equal((yield* linked({ stopReason: "refusal" }))?.failure.class, "usage_limit"); // An unreadable log gives no evidence, but the agent's typed refusal still fails the turn. yield* fs.remove(path.join(conversations, `${own}.db`)); yield* fs.writeFileString(path.join(conversations, `${own}.db`), "not a database"); const corrupt = yield* watchAntigravityPromptOutcome(profileDirectory, own); assert.isUndefined(yield* corrupt({ stopReason: "end_turn" })); - assert.deepEqual(yield* corrupt({ stopReason: "refusal" }), { - class: "usage_limit", - message: "You have reached your Antigravity quota for this period.", - code: "QUOTA_EXHAUSTED", - retryable: true, - }); + assert.deepEqual( + yield* corrupt({ stopReason: "refusal" }), + reusable({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "QUOTA_EXHAUSTED", + retryable: true, + }), + ); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 41e25f2e417e1..2cbe4d62b3a4b 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -20,6 +20,10 @@ const MAX_ERROR_PAYLOAD_BYTES = 16_384; // The Go harness's rendering of a Google API error, as the agent's api_errors.py reads it. const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s; const QUOTA_EXHAUSTED = /\breason:QUOTA_EXHAUSTED\b/; +// Claude rejects a saved thinking block without a valid signature. It sits in the native +// history, so every later prompt to that session fails the same way. +const INVALID_THINKING_SIGNATURE = + /\bmessages\.\d+\.content\.\d+(?:\.thinking\.signature: Field required|: Invalid `signature` in `thinking` block)/; const RESET_TIMESTAMP = /\bquotaResetTimeStamp:([^\s\]]+)/; const RESET_DELAY = /\bquotaResetDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/; // google.rpc.RetryInfo, which plain rate limits carry instead of a quota reset. @@ -106,7 +110,12 @@ function turnFailure( stopReason: EffectAcpSchema.StopReason, step: LastStep | undefined, now: DateTime.Utc, -): OrchestrationV2ProviderFailure | undefined { +): + | { + readonly failure: OrchestrationV2ProviderFailure; + readonly threadDisposition: "reusable" | "broken"; + } + | undefined { const errorText = step?.type === ERROR_STEP_TYPE ? new TextDecoder().decode(step.payload ?? new Uint8Array()) @@ -115,30 +124,50 @@ function turnFailure( if (errorText === undefined && stopReason !== "refusal") return undefined; const text = errorText ?? ""; const apiError = API_ERROR.exec(text); + const reusable = (failure: OrchestrationV2ProviderFailure) => + ({ failure, threadDisposition: "reusable" }) as const; if (stopReason === "refusal" || QUOTA_EXHAUSTED.test(text)) { - return makeProviderFailure({ - class: "usage_limit", - message: "You have reached your Antigravity quota for this period.", - code: apiError?.[3] ?? "QUOTA_EXHAUSTED", - retryable: true, - resetAt: resetAtFrom(text, now) ?? null, - }); + return reusable( + makeProviderFailure({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: apiError?.[3] ?? "QUOTA_EXHAUSTED", + retryable: true, + resetAt: resetAtFrom(text, now) ?? null, + }), + ); + } + if (INVALID_THINKING_SIGNATURE.test(text)) { + return { + failure: makeProviderFailure({ + class: "provider_error", + message: + "Antigravity's saved conversation can no longer be sent to the model. The next message starts a fresh Antigravity session with this thread's context.", + code: "INVALID_THINKING_SIGNATURE", + retryable: true, + }), + threadDisposition: "broken", + }; } if (apiError === null) { - return makeProviderFailure({ - class: "provider_error", - message: "Antigravity stopped this turn because of an error.", - }); + return reusable( + makeProviderFailure({ + class: "provider_error", + message: "Antigravity stopped this turn because of an error.", + }), + ); } const [, httpStatus, message, status] = apiError; const rateLimited = httpStatus === "429" || status === "RESOURCE_EXHAUSTED"; - return makeProviderFailure({ - class: rateLimited ? "usage_limit" : "provider_error", - message: message!.trim(), - code: status ?? null, - retryable: rateLimited || Number(httpStatus) >= 500, - resetAt: resetAtFrom(text, now) ?? null, - }); + return reusable( + makeProviderFailure({ + class: rateLimited ? "usage_limit" : "provider_error", + message: message!.trim(), + code: status ?? null, + retryable: rateLimited || Number(httpStatus) >= 500, + resetAt: resetAtFrom(text, now) ?? null, + }), + ); } const conversationSegments = ["antigravity-acp", "conversations"]; diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index b2a28c68c6c08..0c6687dc872d1 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -339,6 +339,11 @@ export function acpPerTurnPromptUsage( }; } +interface AcpPromptOutcomeFailure { + readonly failure: OrchestrationV2ProviderFailure; + readonly threadDisposition: "reusable" | "broken"; +} + export interface AcpAdapterV2Flavor { readonly buildAttachmentPrompt?: (input: { readonly text: string; @@ -353,14 +358,13 @@ export interface AcpAdapterV2Flavor { /** * Runs before `session/prompt`. The returned check sees a prompt the agent ended * without an error or cancel, and reports a failure its stop reason hid. A reported - * failure restarts the runtime before the next turn. + * failure restarts the runtime before the next turn. A `broken` thread's native + * history is unusable, so the next turn falls back to a fresh session. */ readonly watchPromptOutcome?: ( sessionId: string, ) => Effect.Effect< - ( - result: EffectAcpSchema.PromptResponse, - ) => Effect.Effect + (result: EffectAcpSchema.PromptResponse) => Effect.Effect >; readonly requiresSignIn?: (cause: unknown) => boolean; /** @@ -8008,10 +8012,14 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( context: ActiveAcpTurn, status: "completed" | "interrupted" | "failed" | "cancelled", failure?: OrchestrationV2ProviderFailure, - options?: { readonly childTurnsContinue?: boolean }, + options?: { + readonly childTurnsContinue?: boolean; + readonly threadDisposition?: "reusable" | "broken"; + }, ): Effect.fn.Return { if (context.finalized) return; const settledStatus = context.interrupted ? "interrupted" : status; + const threadDisposition = options?.threadDisposition ?? "reusable"; context.finalizedStatus = settledStatus; context.finalized = true; const watchdog = context.inactivityWatchdog; @@ -8110,7 +8118,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const updatedProviderThread: OrchestrationV2ProviderThread = { ...context.input.providerThread, providerSessionId: input.providerSessionId, - status: "active", + status: threadDisposition === "broken" ? "error" : "active", lastRunOrdinal: context.input.runOrdinal, firstRunOrdinal: context.input.providerThread.firstRunOrdinal ?? context.input.runOrdinal, @@ -8154,7 +8162,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( retry: context.providerRetry.retry, retryStartedAt: context.providerRetry.startedAt, }), - threadDisposition: "reusable", + threadDisposition, } : { type: "turn.terminal", @@ -8164,7 +8172,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( runOrdinal: context.input.runOrdinal, status: settledStatus, failure: null, - threadDisposition: "reusable", + threadDisposition, }, ); const subagentsRequiringCarryover = [...context.subagents.values()].filter( @@ -8898,7 +8906,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* Ref.set(runtimeRestartRequired, true); } const failure = - hiddenFailure ?? + hiddenFailure?.failure ?? (context.interrupted ? undefined : stopReasonFailure(result.stopReason, result.error)); @@ -8924,6 +8932,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( // background tools and subagents keep reporting to this run. yield* finalizeTurn(context, "failed", failure, { childTurnsContinue: hasSurvivingBackgroundWork(context), + ...(hiddenFailure === undefined + ? {} + : { threadDisposition: hiddenFailure.threadDisposition }), }); return; } @@ -9197,6 +9208,13 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( readonly modelSelection?: ModelSelection; readonly runtimePolicy?: ProviderAdapter.ProviderAdapterV2RuntimePolicy; }) { + if (threadInput.providerThread.status === "error") { + // A broken turn left this native history unusable; the caller recovers in a fresh session. + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: "The native session history is unusable.", + }); + } return yield* runtimeTransitionPermit.withPermit( Effect.gen(function* () { yield* awaitRuntimeTeardown();