diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 8e231195de8fd..2f791df42ac61 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -5819,6 +5819,57 @@ describe("CodexAdapterV2 post-settle continuation", () => { notification: false, expectedClass: "usage_limit", }, + { + name: "bad-request", + code: "other", + message: '{"detail":"Bad Request"}', + notification: true, + expectedClass: "provider_error", + expectedDisposition: "broken", + }, + { + name: "bad-request-completion-only", + code: "other", + message: ' { "detail": "Bad Request" } ', + notification: false, + expectedClass: "provider_error", + expectedDisposition: "broken", + }, + { + name: "bad-request-prose", + code: "other", + message: "Tool returned Bad Request; try a different argument.", + notification: false, + expectedClass: "provider_error", + }, + { + name: "bad-request-usage-limit", + code: "usageLimitExceeded", + message: '{"detail":"Bad Request"}', + notification: false, + expectedClass: "usage_limit", + }, + { + name: "bad-request-workspace-limit", + code: "other", + message: '{"detail":"Bad Request"}', + details: "subscription_sharing_workplace_individual_usage_limit_exceeded", + notification: true, + expectedClass: "usage_limit", + expectedCode: "subscription_sharing_workplace_individual_usage_limit_exceeded", + expectedMessage: "workspace admin", + }, + { + name: "bad-request-sharing-retry", + code: "other", + message: '{"detail":"Bad Request"}', + details: "subscription_sharing_usage_limit_exceeded", + notification: true, + willRetry: true, + expectedClass: "usage_limit", + expectedCode: "subscription_sharing_usage_limit_exceeded", + expectedMessage: "Usage settings", + }, { name: "exhausted-500", code: "responseTooManyFailedAttempts", @@ -6038,6 +6089,30 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }, + ...("expectedDisposition" in scenario && scenario.expectedDisposition === "broken" + ? [ + { + type: "emit_inbound" as const, + label: "thread/goal/updated after failure", + frame: { + method: "thread/goal/updated", + params: { + threadId: nativeThreadId, + goal: { + threadId: nativeThreadId, + objective: "Continue the task", + status: "paused", + tokenBudget: null, + tokensUsed: 0, + timeUsedSeconds: 0, + createdAt: 1782622440, + updatedAt: 1782622450, + }, + }, + }, + }, + ] + : []), ...(readsAccount ? [readRequest] : []), ...(scenario.name === "newer-reset-during-read" ? [newerSnapshot] : []), ...(readsReset @@ -6140,7 +6215,36 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(terminal?.status, "failed"); if (terminal?.status !== "failed") return; assert.equal(terminal.failure.class, scenario.expectedClass); - assert.equal(terminal.threadDisposition, "reusable"); + assert.equal( + terminal.threadDisposition, + "expectedDisposition" in scenario ? scenario.expectedDisposition : "reusable", + ); + if (terminal.threadDisposition === "broken") { + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.goal?.objective === "Continue the task", + ), + ); + const goalUpdate = harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_thread.updated" } + > => + event.type === "provider_thread.updated" && + event.providerThread.goal?.objective === "Continue the task", + ); + assert.equal(goalUpdate?.providerThread.status, "error"); + const error = yield* Effect.flip( + harness.runtime.resumeThread({ + providerThread: goalUpdate!.providerThread, + }), + ); + assert.instanceOf(error, ProviderAdapter.ProviderAdapterResumeThreadError); + } if (scenario.name === "deferred-reset" || scenario.name === "first-request-reset") assert.equal(terminal.failure.resetAt, resetAt); if (scenario.name === "newer-reset-during-read") @@ -6188,6 +6292,188 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("ignores goal updates from a native conversation replaced after rejection", () => + Effect.scoped( + Effect.gen(function* () { + const original = "rejected-native"; + const replacement = "replacement-native"; + const preamble = (nativeThreadId: string) => + codexReplayPreamble({ + nativeThreadId, + nativeTurnId: `${nativeThreadId}-turn`, + prompt: "work", + }); + const completed = ( + nativeThreadId: string, + failed = false, + ): CodexReplay.CodexAppServerReplayEntry => ({ + type: "emit_inbound", + label: `completed ${nativeThreadId}`, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: { + ...makeCodexReplayTurn({ + id: `${nativeThreadId}-turn`, + status: failed ? "failed" : "completed", + }), + ...(failed + ? { error: { message: '{"detail":"Bad Request"}', codexErrorInfo: "other" } } + : {}), + }, + }, + }, + }); + const goal = (nativeThreadId: string, status: "active" | "paused" = "paused") => ({ + threadId: nativeThreadId, + objective: `Goal for ${nativeThreadId}`, + status, + tokenBudget: null, + tokensUsed: 0, + timeUsedSeconds: 0, + createdAt: 1782622440, + updatedAt: 1782622450, + }); + const goalUpdate = ( + nativeThreadId: string, + status: "active" | "paused" = "paused", + ): CodexReplay.CodexAppServerReplayEntry => ({ + type: "emit_inbound", + label: `goal update ${nativeThreadId}`, + frame: { + method: "thread/goal/updated", + params: { + threadId: nativeThreadId, + goal: goal(nativeThreadId, status), + }, + }, + }); + const transcript = makeCodexReplayTranscript({ + scenario: "replaced-native-goal-update", + entries: [ + ...preamble(original), + completed(original, true), + ...preamble(replacement) + .slice(3, 4) + .map((entry) => withReplayRequestId(entry, 4)), + goalUpdate(original, "active"), + ...preamble(replacement) + .slice(4, 5) + .map((entry) => withReplayRequestId(entry, 4)), + ...preamble(replacement) + .slice(-3) + .map((entry, index) => (index < 2 ? withReplayRequestId(entry, 5) : entry)), + goalUpdate(original, "active"), + { + type: "emit_inbound", + label: "orphan goal turn", + frame: { + method: "turn/started", + params: { + threadId: original, + turn: makeCodexReplayTurn({ id: "orphan-goal-turn", status: "inProgress" }), + }, + }, + }, + { + type: "expect_outbound", + label: "pause retired goal", + frame: { + id: 6, + method: "thread/goal/set", + params: { threadId: original, status: "paused" }, + }, + }, + { + type: "emit_inbound", + label: "retired goal paused", + frame: { id: 6, result: { goal: goal(original) } }, + }, + { + type: "expect_outbound", + label: "interrupt orphan goal turn", + frame: { + id: 7, + method: "turn/interrupt", + params: { threadId: original, turnId: "orphan-goal-turn" }, + }, + }, + { + type: "emit_inbound", + label: "orphan goal interrupted", + frame: { id: 7, result: {} }, + }, + goalUpdate(replacement), + completed(replacement), + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("rejected-attempt"), + text: "work", + }), + ); + yield* harness.awaitTerminal(1); + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.threadDisposition, "broken"); + const resumeError = yield* Effect.flip( + harness.runtime.resumeThread({ + providerThread: { ...harness.providerThread, status: "error" }, + }), + ); + assert.instanceOf(resumeError, ProviderAdapter.ProviderAdapterResumeThreadError); + const beforeReplacement = harness.events.length; + const created = yield* harness.runtime.ensureThread({ + threadId: harness.threadId, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + }); + const replacementThread = { + ...created, + id: harness.providerThread.id, + status: "active" as const, + }; + yield* harness.runtime.startTurn({ + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: replacementThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("replacement-attempt"), + text: "work", + }), + runOrdinal: 2, + providerTurnOrdinal: 2, + }); + yield* harness.awaitTerminal(2); + const publishedThreads = harness.events + .slice(beforeReplacement) + .flatMap((event) => + event.type === "provider_thread.updated" ? [event.providerThread] : [], + ); + assert.isNotEmpty(publishedThreads); + assert.deepEqual( + publishedThreads.map((thread) => thread.nativeThreadRef?.nativeId), + [replacement], + ); + assert.equal(publishedThreads[0]?.goal?.objective, `Goal for ${replacement}`); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + // regression: #17501 Codex 429 retry exhaustion classifies as a usage limit it.effect.each([ { diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 038266ba93426..149f4d8c816bb 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1127,6 +1127,24 @@ function codexErrorInfoCode(value: unknown): string | null { type CodexErrorInfo = CodexSchema.ServerNotification__CodexErrorInfo; +// This opaque rejection can persist with retained history; only this signature +// opts into fresh-history recovery, without assuming all HTTP 400s corrupt a session. +function isCodexRejectedConversation(message: string, code: string | null): boolean { + if (code !== "other") return false; + try { + const payload: unknown = JSON.parse(message); + return ( + typeof payload === "object" && + payload !== null && + "detail" in payload && + Object.keys(payload).length === 1 && + payload.detail === "Bad Request" + ); + } catch { + return false; + } +} + /** * Retry exhaustion only says why it failed through the last attempt's HTTP * status. Exhaustion on 429 is a usage or rate limit, so it keeps the Limited @@ -1998,7 +2016,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // Serialize native completion and interrupt timeout so only one publishes terminal events. const turnTerminalizationPermit = yield* Semaphore.make(1); const goalsByNativeThread = new Map(); - const rootProviderThreads = new Map(); + // Retired roots keep a null binding so late goal turns are still stopped. + const rootProviderThreads = new Map(); const goalHolds = new Map(); const continuedRuns = new Map>(); const capacityContinuations = new Map< @@ -2084,7 +2103,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const emitGoalUpdate = Effect.fnUntraced(function* (nativeThreadId: string) { const providerThread = rootProviderThreads.get(nativeThreadId); if ( - providerThread === undefined || + providerThread == null || !goalsByNativeThread.has(nativeThreadId) || goalUpdatesHeld.has(nativeThreadId) ) { @@ -5917,7 +5936,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( retry: input.providerRetry.retry, retryStartedAt: input.providerRetry.startedAt, }), - threadDisposition: "reusable", + threadDisposition: isCodexRejectedConversation( + input.failureMessage ?? previousFailure?.nativeMessage ?? "", + failure.code, + ) + ? "broken" + : "reusable", }; } return { @@ -5997,8 +6021,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const nativeThreadId = context.providerThread.nativeThreadRef?.nativeId; const rootProviderThread = nativeThreadId == null ? undefined : rootProviderThreads.get(nativeThreadId); - if (nativeThreadId != null && rootProviderThread !== undefined) { - rootProviderThreads.set(nativeThreadId, { ...rootProviderThread, status: "idle" }); + if (nativeThreadId != null && rootProviderThread != null) { + rootProviderThreads.set(nativeThreadId, { + ...rootProviderThread, + status: event.threadDisposition === "broken" ? "error" : "idle", + }); } // Notification handlers block response routing, so the read and terminal get a fiber. if (awaitsReset) { @@ -7271,8 +7298,23 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( updatedAt: codexTimestamp(response.thread.updatedAt), } satisfies OrchestrationV2ProviderThread; }, - (effect, threadInput) => - effect.pipe( + (effect, threadInput) => { + // Retire before recovery delivers history so late updates cannot restore this thread. + if (threadInput.providerThread.status === "error") { + return Effect.suspend(() => { + const nativeThreadId = threadInput.providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId != null) rootProviderThreads.set(nativeThreadId, null); + return Effect.fail( + new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: CODEX_PROVIDER, + providerSessionId: input.providerSessionId, + providerThreadId: threadInput.providerThread.id, + cause: "Codex conversation is not reusable.", + }), + ); + }); + } + return effect.pipe( Effect.mapError((cause) => isCodexActiveWriterError(cause) ? new ProviderAdapter.ProviderAdapterThreadInUseError({ @@ -7288,7 +7330,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( cause, }), ), - ), + ); + }, ), compactThread: Effect.fn("CodexAdapterV2.compactThread")( function* (turnInput) { diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 617f18f86688a..d971626f26296 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -2172,6 +2172,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio nativeThreadRef: activeProviderThread.nativeThreadRef, nativeConversationHeadRef: activeProviderThread.nativeConversationHeadRef, nativeMetadata: activeProviderThread.nativeMetadata, + status: activeProviderThread.status, } : queuedProviderThread; const targetCapabilities = yield* targetAdapter.getCapabilities().pipe( @@ -2357,7 +2358,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const providerThread: OrchestrationV2ProviderThread = { ...deliveryProviderThread, providerSessionId, - status: "not_loaded", + status: deliveryProviderThread.status === "error" ? "error" : "not_loaded", firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? ordinal, lastRunOrdinal: ordinal, handoffIds: appendContextHandoffId( @@ -5606,7 +5607,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...providerThread, providerInstanceId: input.modelSelection.instanceId, providerSessionId: nextProviderSessionId, - status: "not_loaded", + status: providerThread.status === "error" ? "error" : "not_loaded", updatedAt: now, }; restartSessionTransition = { @@ -7542,7 +7543,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : null; const providerThread: OrchestrationV2ProviderThread = { ...ensuredProviderThread, - status: "active", + status: ensuredProviderThread.status === "error" ? "error" : "active", firstRunOrdinal: ensuredProviderThread.firstRunOrdinal ?? ordinal, lastRunOrdinal: ordinal, handoffIds: [ diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index b4fc69727c1de..08f2ae5e94962 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -439,6 +439,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( readonly startTurn?: Effect.Effect; readonly interruptTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; + readonly rejectErroredResume?: boolean; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; } = {}, @@ -543,7 +544,20 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( Ref.update(state, (current) => ({ ...current, resumeCount: current.resumeCount + 1, - })).pipe(Effect.as(threadInput.providerThread)), + })).pipe( + Effect.andThen( + options.rejectErroredResume && threadInput.providerThread.status === "error" + ? Effect.fail( + new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: CODEX_DRIVER, + providerSessionId: input.providerSessionId, + providerThreadId: threadInput.providerThread.id, + cause: "Codex conversation is not reusable.", + }), + ) + : Effect.succeed(threadInput.providerThread), + ), + ), startTurn: () => options.startTurn ?? Effect.void, steerTurn: () => Effect.void, interruptTurn: () => @@ -601,6 +615,7 @@ function layerTest(input: { readonly startTurn?: Effect.Effect; readonly interruptTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; + readonly rejectErroredResume?: boolean; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; readonly serverSettingsLayer?: ReturnType; @@ -640,6 +655,9 @@ function layerTest(input: { ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), ...(input.interruptTurn === undefined ? {} : { interruptTurn: input.interruptTurn }), ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), + ...(input.rejectErroredResume === undefined + ? {} + : { rejectErroredResume: input.rejectErroredResume }), ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), ...(input.scopeCloseReached === undefined ? {} @@ -4165,6 +4183,15 @@ it.effect( siblingThread: yield* countRawResumes(sibling), }; + const beforeErroredResume = (yield* Ref.get(state)).resumeCount; + yield* failed.runtime.resumeThread({ + providerThread: { ...failed.providerThread, status: "error" }, + modelSelection, + runtimePolicy, + }); + assert.equal((yield* Ref.get(state)).resumeCount - beforeErroredResume, 1); + assert.equal(yield* countRawResumes(sibling), 0); + yield* publish({ type: "provider_thread.updated", driver: CODEX_DRIVER, @@ -4194,6 +4221,68 @@ it.effect( }), ); +it.effect( + "ProviderSessionManagerV2 unloads a rejected errored thread while its shared session stays live", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("rejected-resume"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: "rejected-native-thread", + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + const rejection = yield* runtime + .resumeThread({ + providerThread: { ...providerThread, status: "error" }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal(rejection._tag, "ProviderAdapterResumeThreadError"); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, []); + + yield* TestClock.adjust("1 second"); + assert.isTrue(yield* manager.isLive(providerSessionId)); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "rejected-native-thread", + ]); + yield* TestClock.adjust("1 second"); + assert.equal((yield* Ref.get(state)).unloadedNativeThreadIds.length, 1); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 1000, + rejectErroredResume: true, + hasPendingBackgroundWork: Effect.succeed(true), + }), + ), + ); + }), +); + it.effect( "ProviderSessionManagerV2 keeps an attached native child loaded when its parent detaches", () => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 0fb31e81a0732..3d546dd62b234 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2015,13 +2015,18 @@ export const layerWithOptions = ( isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey }), ), Effect.flatMap((loaded) => - loaded + loaded && input.providerThread.status !== "error" ? Effect.succeed({ loaded: input.providerThread, linkedEnvironments: undefined, }) : loadNatively(runtime.resumeThread(input)), ), + Effect.onError(() => + input.providerThread.status === "error" + ? scheduleThreadUnload(providerSessionId, input.providerThread.id) + : Effect.void, + ), Effect.tap(({ loaded: providerThread, linkedEnvironments }) => markProviderThreadLoaded({ providerSessionId, diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index e9b2636998386..5b4b987ade2be 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -113,6 +113,7 @@ function makeTestAdapter(input: { >; readonly failure?: ReturnType; readonly failedRunOrdinals?: ReadonlySet; + readonly brokenRunOrdinals?: ReadonlySet; readonly interruptedRunOrdinals?: ReadonlySet; readonly holdRunOrdinal?: number; readonly pendingBackgroundWork?: Ref.Ref; @@ -211,6 +212,13 @@ function makeTestAdapter(input: { }), resumeThread: ({ providerThread }) => Effect.gen(function* () { + if (input.brokenRunOrdinals !== undefined && providerThread.status === "error") { + return yield* new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: input.driver, + providerSessionId: sessionInput.providerSessionId, + providerThreadId: providerThread.id, + }); + } if ( input.failResume || (input.failResumeOnce !== undefined && @@ -367,7 +375,9 @@ function makeTestAdapter(input: { }), } : { status: terminalStatus, failure: null }), - threadDisposition: "reusable", + threadDisposition: input.brokenRunOrdinals?.has(turnInput.runOrdinal) + ? "broken" + : "reusable", }, ]; const reportedUsage = input.tokenUsageByRunOrdinal?.[turnInput.runOrdinal]; @@ -1226,6 +1236,185 @@ describe("orchestration v2 provider switching", () => { ), ); + it.live.each([ + "direct", + "queued-cached", + "queued-detached", + "direct-switched", + "queued-switched", + ] as const)("recovers a rejected conversation with a %s follow-up", (followupMode) => + Effect.scoped( + Effect.gen(function* () { + const queued = followupMode.startsWith("queued-"); + const followupSelection = followupMode.endsWith("-switched") + ? { ...CODEX_MODEL_SELECTION, instanceId: ProviderInstanceId.make("codex-alternate") } + : CODEX_MODEL_SELECTION; + const selections = + followupSelection === CODEX_MODEL_SELECTION + ? [CODEX_MODEL_SELECTION] + : [CODEX_MODEL_SELECTION, followupSelection]; + const holdsQueue = queued && followupMode !== "queued-switched"; + const cwd = yield* checkpointWorkspace(`rejected-queue-${followupMode}`); + const capturedTurns = yield* Ref.make>([]); + const injectedHistory = yield* Ref.make>([]); + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + const generation = yield* Ref.make(0); + const originalPrompt = "Keep the logo abstract and use pastel colors."; + const partialResponse = "The current logo has an asymmetric outline."; + const layerRegistry = ProviderAdapterRegistry.layerFromAdapters( + selections.map((selection) => + makeTestAdapter({ + instanceId: selection.instanceId, + driver: CODEX_DRIVER, + capabilities: CodexProviderCapabilitiesV2, + modelSelection: selection, + responseByRunOrdinal: { 1: partialResponse }, + capturedTurns, + injectedHistory, + failedRunOrdinals: new Set([1]), + brokenRunOrdinals: new Set([1]), + holdFirstTurn: started, + releaseFirstTurn: release, + nativeThreadGeneration: generation, + }), + ), + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const waitForRun = (ordinal: number, status: "failed" | "completed") => + orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && + event.threadId === threadId && + event.payload.ordinal === ordinal && + (event.payload.status === "failed" || event.payload.status === "completed"), + ), + Stream.runHead, + Effect.andThen(worker.drain()), + Effect.andThen( + Effect.gen(function* () { + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + projection.runs.find((run) => run.ordinal === ordinal)?.status, + status, + ); + }), + ), + ); + const dispatch = ( + key: string, + text: string, + queued: boolean, + modelSelection = CODEX_MODEL_SELECTION, + ) => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`rejected-queue:${key}`), + threadId, + messageId: MessageId.make(`rejected-queue:${key}`), + createdBy: "user", + creationSource: "web", + text, + attachments: [], + modelSelection, + dispatchMode: { type: queued ? "queue_after_active" : "start_immediately" }, + }); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("rejected-queue:create"), + threadId, + projectId, + createdBy: "user", + creationSource: "web", + title: "Rejected conversation recovery", + modelSelection: CODEX_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + yield* dispatch("first", originalPrompt, false); + yield* Deferred.await(started); + if (queued) yield* dispatch("queued", "Continue the logo work.", true, followupSelection); + const failed = yield* waitForRun(1, "failed").pipe( + Effect.forkScoped({ startImmediately: true }), + ); + const heldQueue = holdsQueue + ? yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && + event.threadId === threadId && + event.payload.ordinal === 2 && + event.payload.queueHeld === true, + ), + Stream.runHead, + Effect.forkScoped({ startImmediately: true }), + ) + : undefined; + // A queued provider switch may finish before the first failure's follow-up processing. + const completed = yield* waitForRun(2, "completed").pipe( + Effect.forkScoped({ startImmediately: true }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(failed); + if (heldQueue !== undefined) yield* Fiber.join(heldQueue); + const held = yield* orchestrator.getThreadProjection(threadId); + if (holdsQueue) { + assert.equal(held.runs.at(-1)?.status, "queued"); + assert.isTrue(held.runs.at(-1)?.queueHeld); + } + if (holdsQueue || !queued) assert.equal(held.providerThreads.at(0)?.status, "error"); + if (followupMode === "queued-detached") { + yield* orchestrator.dispatch({ + type: "provider-session.detach", + commandId: CommandId.make("rejected-queue:detach"), + threadId, + providerSessionId: held.providerThreads[0]!.providerSessionId!, + }); + yield* worker.drain(); + } + if (queued) { + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("rejected-queue:resume"), + threadId, + }); + } else { + yield* dispatch("followup", "Continue the logo work.", false, followupSelection); + } + yield* Fiber.join(completed); + const turns = yield* Ref.get(capturedTurns); + assert.equal(turns.length, 2); + assert.equal(yield* Ref.get(generation), 2); + assert.notEqual(turns[1]?.nativeThreadId, turns[0]?.nativeThreadId); + assert.equal(turns[1]?.text, "Continue the logo work."); + assert.equal(turns[1]?.modelSelection.instanceId, followupSelection.instanceId); + const history = yield* encodeJson(yield* Ref.get(injectedHistory)); + assert.include(history, originalPrompt); + assert.include(history, partialResponse); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { + name: `rejected-queue-${followupMode}`, + runtimePolicyOverride: { + cwd, + approvalPolicy: "never", + sandboxPolicy: { type: "readOnly" }, + }, + }, + layerRegistry, + ), + ), + ); + }), + ), + ); + it.live.each(["native", "legacy"] as const)( "starts a replacement native thread as new after the resume fallback with %s attempts", (attempts) =>