diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 548ff5cb6643b..e4eb6feb4328c 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -833,6 +833,11 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { // The subagent still runs, so the session is not idle and the next Stop reaches it. assert.isTrue(yield* runtime.hasPendingBackgroundWork!); assert.isTrue(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); + assert.equal( + (yield* runtime.unloadThread!({ providerThread: thread }).pipe(Effect.flip))._tag, + "ProviderAdapterProtocolError", + ); + assert.isTrue(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); }).pipe(Effect.scoped), ); @@ -3400,7 +3405,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ); it.effect( - "registers T3's MCP server for the thread alone and removes it when the thread unloads", + "registers T3's MCP server for the thread alone, retains it when unload is declined, and removes it after work stops", () => Effect.gen(function* () { const mcpSessions = yield* McpProviderSessions.McpProviderSessions; @@ -3429,15 +3434,29 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }, }), reply("mcp.add", null), - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.succeeded", { sessionID: SESSION }), + ...backgroundLaunch(CHILD), + out("session.interrupt", { sessionID: CHILD }), + reply("session.interrupt", { interrupted: true }), + out("session.interrupt", { sessionID: SESSION }), + reply("session.interrupt", { interrupted: true }), + event("session.execution.interrupted", { sessionID: SESSION }), out("mcp.remove", { server, "location[directory]": WORK }), reply("mcp.remove", null), ]); - const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); - yield* runtime.startTurn(turnInput(thread)); - assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + const watch = yield* watchBackgroundTurn(runtime); + yield* runtime.startTurn(withLineage(thread)); + yield* watch.attached; + assert.equal( + (yield* runtime.unloadThread!({ providerThread: thread }).pipe(Effect.flip))._tag, + "ProviderAdapterProtocolError", + ); + yield* runtime.interruptTurn({ + providerThread: thread, + providerTurnId: yield* providerTurnId, + requestRuntimeRestart: true, + }); + assert.equal((yield* watch.terminal)?.status, "interrupted"); + assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); yield* runtime.unloadThread!({ providerThread: thread }); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 36fa2ced0d551..a561aa8e72789 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3,6 +3,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { AgentScope } from "@t3tools/shared/AgentScope"; import { + CheckpointId, EnvironmentId, type ModelSelection, type OrchestrationV2AppThread, @@ -15,6 +16,7 @@ import { ProviderDriverKind, ProviderInstanceId, type ProviderSessionId, + ProviderThreadId, ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -323,22 +325,32 @@ function unimplemented(detail: string) { const makeProviderAdapter = Effect.fnUntraced(function* ( state: Ref.Ref, options: { + readonly instanceId?: ProviderInstanceId; readonly failEventStream?: boolean; readonly capabilities?: OrchestrationV2ProviderCapabilities; readonly mcpConfigs?: Ref.Ref< ReadonlyArray >; + readonly resumeMcpConfigs?: Ref.Ref< + ReadonlyArray + >; readonly beforeOpen?: (input: { readonly providerSessionId: ProviderSessionId; readonly initialProviderItemIdentityVersion?: 2; }) => Effect.Effect; readonly hasPendingBackgroundWork?: Effect.Effect; - readonly hasPendingBackgroundWorkForThread?: Effect.Effect; + readonly hasPendingBackgroundWorkForThread?: + | Effect.Effect + | ((providerThread: OrchestrationV2ProviderThread) => Effect.Effect); /** False models a runtime whose background tasks can outlive its process. */ readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly afterResume?: Effect.Effect; + readonly rollbackThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["rollbackThread"]; readonly beforeUnload?: Effect.Effect; + readonly beforeClose?: Effect.Effect; + readonly withoutUnload?: boolean; /** Registers the process's closeCount finalizer before `beforeOpen` runs. */ readonly spawnBeforeOpen?: boolean; /** Completed when a hanging scope close reaches its wedged finalizer. */ @@ -347,10 +359,14 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ) { const mcpSessions = yield* McpProviderSessions.McpProviderSessions; const countClose = Effect.addFinalizer(() => - Ref.update(state, (current) => ({ - ...current, - closeCount: current.closeCount + 1, - })), + (options.beforeClose ?? Effect.void).pipe( + Effect.andThen( + Ref.update(state, (current) => ({ + ...current, + closeCount: current.closeCount + 1, + })), + ), + ), ); // Registered after countClose so it runs first on scope close, wedging the // close before the closeCount finalizer, like a provider process that never @@ -362,7 +378,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ).pipe(Effect.andThen(Effect.never)), ); return ProviderAdapter.ProviderAdapterV2.of({ - instanceId: ProviderInstanceId.make("codex"), + instanceId: options.instanceId ?? ProviderInstanceId.make("codex"), driver: CODEX_DRIVER, getCapabilities: () => Effect.succeed(options.capabilities ?? CodexCapabilities), planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), @@ -390,6 +406,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( now, ...(options.capabilities === undefined ? {} : { capabilities: options.capabilities }), }); + const instanceId = options.instanceId ?? modelSelection.instanceId; yield* Ref.update(state, (current) => { const eventQueues = new Map(current.eventQueues); eventQueues.set(String(input.providerSessionId), events); @@ -405,10 +422,10 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( } return { - instanceId: ProviderInstanceId.make("codex"), + instanceId, driver: CODEX_DRIVER, providerSessionId: input.providerSessionId, - providerSession: session, + providerSession: { ...session, providerInstanceId: instanceId }, events: options.failEventStream ? Stream.fail( new ProviderAdapter.ProviderAdapterEventStreamError({ @@ -427,14 +444,31 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ...(options.hasPendingBackgroundWorkForThread === undefined ? {} : { - hasPendingBackgroundWorkForThread: () => options.hasPendingBackgroundWorkForThread!, + hasPendingBackgroundWorkForThread: (providerThread) => + typeof options.hasPendingBackgroundWorkForThread === "function" + ? options.hasPendingBackgroundWorkForThread(providerThread) + : options.hasPendingBackgroundWorkForThread!, }), ensureThread: () => unimplemented("ensureThread unused in test"), resumeThread: (threadInput) => Ref.update(state, (current) => ({ ...current, resumeCount: current.resumeCount + 1, - })).pipe(Effect.as(threadInput.providerThread)), + })).pipe( + Effect.andThen( + options.resumeMcpConfigs === undefined + ? Effect.void + : mcpSessions + .read(threadInput.providerThread.appThreadId!) + .pipe( + Effect.flatMap((config) => + Ref.update(options.resumeMcpConfigs!, (configs) => [...configs, config]), + ), + ), + ), + Effect.andThen(options.afterResume ?? Effect.void), + Effect.as(threadInput.providerThread), + ), startTurn: () => options.startTurn ?? Effect.void, steerTurn: () => Effect.void, interruptTurn: () => @@ -442,21 +476,30 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ...current, interruptCount: current.interruptCount + 1, })), - unloadThread: ({ providerThread }) => - (options.beforeUnload ?? Effect.void).pipe( - Effect.andThen( - Ref.update(state, (current) => ({ - ...current, - unloadedNativeThreadIds: [ - ...current.unloadedNativeThreadIds, - providerThread.nativeThreadRef?.nativeId ?? "", - ], - })), - ), - ), + ...(options.withoutUnload + ? {} + : { + unloadThread: ({ + providerThread, + }: Parameters< + NonNullable + >[0]) => + (options.beforeUnload ?? Effect.void).pipe( + Effect.andThen( + Ref.update(state, (current) => ({ + ...current, + unloadedNativeThreadIds: [ + ...current.unloadedNativeThreadIds, + providerThread.nativeThreadRef?.nativeId ?? "", + ], + })), + ), + ), + }), respondToRuntimeRequest: () => Effect.void, readThreadSnapshot: () => unimplemented("readThreadSnapshot unused in test"), - rollbackThread: () => unimplemented("rollbackThread unused in test"), + rollbackThread: (input) => + options.rollbackThread?.(input) ?? unimplemented("rollbackThread unused in test"), forkThread: () => unimplemented("forkThread unused in test"), } satisfies ProviderAdapter.ProviderAdapterV2SessionRuntime; }), @@ -464,6 +507,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( }); function layerTest(input: { + readonly additionalAdapters?: ReadonlyArray>; readonly state: Ref.Ref; readonly idleTimeoutMs: number; readonly maxIdlePinMs?: number; @@ -472,6 +516,9 @@ function layerTest(input: { readonly mcpConfigs?: Ref.Ref< ReadonlyArray >; + readonly resumeMcpConfigs?: Ref.Ref< + ReadonlyArray + >; readonly beforeOpen?: (input: { readonly providerSessionId: ProviderSessionId; readonly initialProviderItemIdentityVersion?: 2; @@ -486,12 +533,24 @@ function layerTest(input: { readonly armed: Ref.Ref; readonly paused: Deferred.Deferred; }; + readonly pauseIssue?: { + readonly armed: Ref.Ref; + readonly issued: Deferred.Deferred; + readonly resume: Deferred.Deferred; + }; readonly hasPendingBackgroundWork?: Effect.Effect; - readonly hasPendingBackgroundWorkForThread?: Effect.Effect; + readonly hasPendingBackgroundWorkForThread?: + | Effect.Effect + | ((providerThread: OrchestrationV2ProviderThread) => Effect.Effect); readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly afterResume?: Effect.Effect; + readonly rollbackThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["rollbackThread"]; readonly beforeUnload?: Effect.Effect; + readonly beforeClose?: Effect.Effect; + readonly mcpRegistryLayer?: Layer.Layer; + readonly withoutUnload?: boolean; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; readonly serverSettingsLayer?: ReturnType; @@ -508,35 +567,50 @@ function layerTest(input: { ? layerFailingReleaseEventSink : layerTestEventSink; const layerRegistry = Layer.unwrap( - makeProviderAdapter(input.state, { - failEventStream: input.failEventStream ?? false, - ...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }), - ...(input.mcpConfigs === undefined ? {} : { mcpConfigs: input.mcpConfigs }), - ...(input.beforeOpen === undefined ? {} : { beforeOpen: input.beforeOpen }), - ...(input.hasPendingBackgroundWork === undefined - ? {} - : { hasPendingBackgroundWork: input.hasPendingBackgroundWork }), - ...(input.hasPendingBackgroundWorkForThread === undefined - ? {} - : { hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread }), - ...(input.backgroundTasksEndWithProcess === undefined - ? {} - : { backgroundTasksEndWithProcess: input.backgroundTasksEndWithProcess }), - ...(input.hangSessionScopeClose === undefined - ? {} - : { hangSessionScopeClose: input.hangSessionScopeClose }), - ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), - ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), - ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), - ...(input.scopeCloseReached === undefined - ? {} - : { scopeCloseReached: input.scopeCloseReached }), - }).pipe(Effect.map(ProviderAdapterRegistry.layerSingle)), + Effect.all([ + makeProviderAdapter(input.state, { + failEventStream: input.failEventStream ?? false, + ...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }), + ...(input.mcpConfigs === undefined ? {} : { mcpConfigs: input.mcpConfigs }), + ...(input.resumeMcpConfigs === undefined + ? {} + : { resumeMcpConfigs: input.resumeMcpConfigs }), + ...(input.beforeOpen === undefined ? {} : { beforeOpen: input.beforeOpen }), + ...(input.hasPendingBackgroundWork === undefined + ? {} + : { hasPendingBackgroundWork: input.hasPendingBackgroundWork }), + ...(input.hasPendingBackgroundWorkForThread === undefined + ? {} + : { hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread }), + ...(input.backgroundTasksEndWithProcess === undefined + ? {} + : { backgroundTasksEndWithProcess: input.backgroundTasksEndWithProcess }), + ...(input.hangSessionScopeClose === undefined + ? {} + : { hangSessionScopeClose: input.hangSessionScopeClose }), + ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), + ...(input.afterResume === undefined ? {} : { afterResume: input.afterResume }), + ...(input.rollbackThread === undefined ? {} : { rollbackThread: input.rollbackThread }), + ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), + ...(input.beforeClose === undefined ? {} : { beforeClose: input.beforeClose }), + ...(input.withoutUnload === undefined ? {} : { withoutUnload: input.withoutUnload }), + ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), + ...(input.scopeCloseReached === undefined + ? {} + : { scopeCloseReached: input.scopeCloseReached }), + }), + ...(input.additionalAdapters ?? []), + ]).pipe(Effect.map(ProviderAdapterRegistry.layerFromAdapters)), ); - const layerConfiguredMcpRegistry = - input.pauseResolve === undefined + const layerResolvingMcpRegistry = + input.mcpRegistryLayer ?? + (input.pauseResolve === undefined ? layerTestMcpRegistry - : layerPausingMcpRegistry(input.pauseResolve); + : layerPausingMcpRegistry(input.pauseResolve)); + const layerConfiguredMcpRegistry = + input.pauseIssue === undefined + ? layerResolvingMcpRegistry + : layerPausingMcpIssue(input.pauseIssue).pipe(Layer.provide(layerResolvingMcpRegistry)); const layerProviderEventIngestorTest = ProviderEventIngestor.layer.pipe( Layer.provide( Layer.mergeAll( @@ -614,6 +688,33 @@ const layerPausingMcpRegistry = (pause: { }), ).pipe(Layer.provide(layerTestMcpRegistry)); +const layerPausingMcpIssue = (pause: NonNullable[0]["pauseIssue"]>) => + Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (request) => + delegate + .issue(request) + .pipe( + Effect.tap((credential) => + Ref.getAndSet(pause.armed, false).pipe( + Effect.flatMap((armed) => + armed + ? Deferred.succeed(pause.issued, credential.config).pipe( + Effect.andThen(Deferred.await(pause.resume)), + ) + : Effect.void, + ), + ), + ), + ), + }); + }), + ); + function makeBrowserAccessProject(projectId: ProjectId): Project { return { id: projectId, @@ -1256,6 +1357,140 @@ it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake" }), ); +it.effect( + "ProviderSessionManagerV2 finishes failed-open token cleanup before a queued provider resumes without waiting for scope close", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const openedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const handshake = yield* Deferred.make(); + const revoking = yield* Deferred.make(); + const finishRevoke = yield* Deferred.make(); + const closing = yield* Deferred.make(); + const finishClose = yield* Deferred.make(); + const queued = yield* Deferred.make(); + const targetInstanceId = ProviderInstanceId.make("failed-open-cleanup-target"); + const registryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (id) => + Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(finishRevoke)), + Effect.andThen(delegate.revokeProviderSession(id)), + ), + }); + }), + ).pipe(Layer.provide(layerTestMcpRegistry)); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make("failed-open-cleanup"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const targetId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: targetInstanceId, + }); + const targetSelection = { ...modelSelection, instanceId: targetInstanceId }; + const target = yield* manager.open({ + threadId, + providerSessionId: targetId, + modelSelection: targetSelection, + runtimePolicy, + }); + const providerSessionId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const opening = yield* manager + .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild); + yield* Deferred.await(handshake); + const failedConfig = (yield* Ref.get(openedConfigs))[0]!; + const failedToken = failedConfig.authorizationHeader.replace(/^Bearer\s+/, ""); + const replacement = yield* Deferred.succeed(queued, undefined).pipe( + Effect.andThen( + target.resumeThread({ + providerThread: { + ...makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: targetId, + now, + }), + providerInstanceId: targetInstanceId, + }, + modelSelection: targetSelection, + runtimePolicy, + }), + ), + Effect.forkChild, + ); + yield* Deferred.await(queued); + const interrupting = yield* Fiber.interrupt(opening).pipe(Effect.forkChild); + yield* Deferred.await(revoking); + yield* Effect.yieldNow; + assert.deepEqual(yield* Ref.get(resumedConfigs), []); + assert.deepEqual(yield* configs.read(threadId), failedConfig); + + yield* Deferred.succeed(finishRevoke, undefined); + yield* Deferred.await(closing); + yield* Fiber.join(replacement); + const replacementConfig = (yield* configs.read(threadId))!; + assert.deepEqual(yield* Ref.get(resumedConfigs), [replacementConfig]); + assert.isUndefined(yield* registry.resolve(failedToken)); + assert.equal( + (yield* registry.resolve(replacementConfig.authorizationHeader.replace(/^Bearer\s+/, ""))) + ?.thread.providerInstanceId, + targetInstanceId, + ); + assert.equal((yield* Ref.get(state)).closeCount, 0); + yield* Deferred.succeed(finishClose, undefined); + yield* Fiber.join(interrupting); + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.deepEqual(yield* configs.read(threadId), replacementConfig); + assert.isDefined( + yield* registry.resolve(replacementConfig.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + mcpRegistryLayer: registryLayer, + mcpConfigs: openedConfigs, + spawnBeforeOpen: true, + beforeOpen: () => + Deferred.succeed(handshake, undefined).pipe(Effect.andThen(Effect.never)), + beforeClose: Deferred.succeed(closing, undefined).pipe( + Effect.andThen(Deferred.await(finishClose)), + ), + additionalAdapters: [ + makeProviderAdapter(targetState, { + instanceId: targetInstanceId, + resumeMcpConfigs: resumedConfigs, + }), + ], + }), + ), + ); + }), +); + it.effect("ProviderSessionManagerV2 cleans up an interrupted open whose scope close hangs", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -2142,7 +2377,7 @@ it.effect( ); it.effect( - "ProviderSessionManagerV2 keeps a replacement session's attachment when a stale attach is interrupted", + "ProviderSessionManagerV2 preserves a replacement queued behind an interrupted stale attach", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -2190,18 +2425,16 @@ it.effect( .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(paused); - // ...while the session is replaced and the thread opens the new one. + // A replacement waits behind the stale attach's credential gate. yield* manager.release({ providerSessionId, reason: "runtime_error" }); - const replacement = yield* manager.open({ - threadId, - providerSessionId, - modelSelection, - runtimePolicy, - }); - const config = yield* mcpSessions.read(threadId); - assert.isDefined(config); + const replacing = yield* manager + .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild({ startImmediately: true })); yield* Fiber.interrupt(stale); + const replacement = yield* Fiber.join(replacing); + const config = yield* mcpSessions.read(threadId); + assert.isDefined(config); // The replacement keeps the thread and its credential. assert.equal( @@ -2237,9 +2470,9 @@ it.effect( }), ); -it.effect( - "ProviderSessionManagerV2 revokes a reused credential after a resume stopped while checking it", - () => +it.effect.each(["release-before-stop", "release-after-stop"] as const)( + "ProviderSessionManagerV2 revokes unowned reused credentials after interrupted lookup (%s)", + (order) => Effect.gen(function* () { const state = yield* Ref.make(emptyState); const armed = yield* Ref.make(false); @@ -2289,11 +2522,19 @@ it.effect( yield* Ref.set(armed, true); const stopped = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(paused); + if (order === "release-before-stop") { + yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + assert.isDefined(yield* registry.resolve(token)); + } yield* Fiber.interrupt(stopped); // Nothing holds the credential now, so a terminal release revokes it. - yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + if (order === "release-after-stop") { + assert.isDefined(yield* registry.resolve(token)); + yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + } assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(yield* mcpSessions.read(threadId)); }).pipe( Effect.provide( layerTest({ state, idleTimeoutMs: 60_000, pauseResolve: { armed, paused } }), @@ -2660,101 +2901,2058 @@ it.effect( }), ); -it.effect("ProviderSessionManagerV2 terminal detach revokes the thread's MCP credential", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const mcpConfigs = yield* Ref.make< - ReadonlyArray - >([]); - const effect = Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const idAllocator = yield* IdAllocator.IdAllocatorV2; - const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; - const registry = yield* McpSessionRegistry.McpSessionRegistry; - const mcpSessions = yield* McpProviderSessions.McpProviderSessions; - const now = yield* DateTime.now; - const threadId = ThreadId.make("thread-provider-session-manager-terminal-detach"); - const providerSessionId = yield* idAllocator.allocate.providerSession({ - providerInstanceId: modelSelection.instanceId, - threadId, - }); - - yield* eventSink.write({ - events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], - }); - yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); - const issued = (yield* Ref.get(mcpConfigs)).at(-1); - const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); - assert.isDefined(yield* registry.resolve(token!)); +it.effect.each(["return", "close-second"] as const)( + "ProviderSessionManagerV2 preserves credential ownership after switching providers (%s)", + (next) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const otherState = yield* Ref.make(emptyState); + const resumeMcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const otherInstanceId = ProviderInstanceId.make("other-codex"); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-provider-switch"); + const unrelatedThreadId = ThreadId.make( + "thread-provider-session-manager-provider-switch-peer", + ); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const otherSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherInstanceId, + threadId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), + yield* makeThreadCreatedEvent({ idAllocator, threadId: unrelatedThreadId, now }), + ], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + const original = (yield* Ref.get(resumeMcpConfigs)).at(-1)!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); - // Archive/delete detaches carry revokeMcpCredential: the token must die - // with the thread even though the shared provider process lives on. - yield* manager.detach({ - providerSessionId, - threadId, - detail: "Thread deleted.", - revokeMcpCredential: true, - }); - assert.isUndefined(yield* registry.resolve(token!)); - assert.isUndefined(yield* mcpSessions.read(threadId)); - }); + // Another thread shares the first provider process; refreshing one + // native conversation must not stop that process or unload its peer. + yield* runtime.resumeThread({ + providerThread: makeProviderThread({ + idAllocator, + threadId: unrelatedThreadId, + providerSessionId, + now, + nativeThreadId: "native-peer", + }), + modelSelection, + runtimePolicy, + }); + const peer = (yield* Ref.get(resumeMcpConfigs)).at(-1)!; + const peerToken = peer.authorizationHeader.replace(/^Bearer\s+/, ""); - yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1_000, mcpConfigs }))); - }), -); + yield* manager.open({ + threadId, + providerSessionId: otherSessionId, + modelSelection: { ...modelSelection, instanceId: otherInstanceId }, + runtimePolicy, + }); + assert.isUndefined(yield* registry.resolve(originalToken)); + + if (next === "close-second") { + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const second = (yield* mcpSessions.read(threadId))!; + const secondToken = second.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* manager.close(otherSessionId); + assert.isUndefined(yield* registry.resolve(secondToken)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.isDefined(yield* registry.resolve(peerToken)); + yield* manager.close(providerSessionId); + assert.isUndefined(yield* registry.resolve(secondToken)); + return; + } -it.effect("ProviderSessionManagerV2 releases idle sessions without sweeping all sessions", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const effect = Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const idAllocator = yield* IdAllocator.IdAllocatorV2; - const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; - const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const now = yield* DateTime.now; - const projectId = yield* idAllocator.allocate.project({ - fixtureName: "provider-session-manager-idle", - }); - const threadId = yield* idAllocator.allocate.thread({ - fixtureName: "provider-session-manager-idle", - projectId, - }); - const providerSessionId = yield* idAllocator.allocate.providerSession({ - providerInstanceId: modelSelection.instanceId, - threadId, - }); + const returned = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + assert.strictEqual(returned, runtime); + yield* returned.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).resumeCount, 3); + const refreshed = (yield* Ref.get(resumeMcpConfigs)).at(-1)!; + const refreshedToken = refreshed.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.notEqual(refreshedToken, originalToken); + const scope = yield* registry.resolve(refreshedToken); + assert.equal(scope?.thread.threadId, threadId); + assert.equal(scope?.thread.providerInstanceId, modelSelection.instanceId); + assert.isUndefined(yield* registry.resolve(originalToken)); + assert.isDefined(yield* registry.resolve(peerToken)); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.equal((yield* Ref.get(state)).closeCount, 0); - yield* eventSink.write({ - events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], - }); - yield* manager.open({ - threadId, - providerSessionId, - modelSelection, - runtimePolicy, + // The obsolete provider session cannot revoke its replacement token. + yield* manager.close(otherSessionId); + assert.isDefined(yield* registry.resolve(refreshedToken)); + yield* returned.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).resumeCount, 3); }); - - yield* TestClock.adjust("1 second"); - yield* Effect.yieldNow; - - const liveSession = yield* manager.get(providerSessionId); - const runtimeState = yield* Ref.get(state); - const projection = yield* projectionStore.getThreadProjection(threadId); - - assert.isTrue(Option.isNone(liveSession)); - assert.equal(runtimeState.openCount, 1); - assert.equal(runtimeState.closeCount, 1); - assert.equal(projection.providerSessions.at(-1)?.status, "stopped"); - }); - - yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); - }), + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + resumeMcpConfigs, + additionalAdapters: [makeProviderAdapter(otherState, { instanceId: otherInstanceId })], + }), + ), + ); + }), ); -it.effect("ProviderSessionManagerV2 persists release when session scope close hangs", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const effect = Effect.gen(function* () { +it.effect.each(["open", "refresh"] as const)( + "ProviderSessionManagerV2 revokes an interrupted MCP issuance during %s", + (operation) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const armed = yield* Ref.make(false); + const issued = yield* Deferred.make(); + const resume = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread-provider-session-manager-interrupted-${operation}`); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + let runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime | undefined; + if (operation === "refresh") { + runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + yield* registry.revokeThread(threadId); + } + const countAttachments = eventStore + .read({ threadId, eventType: "provider-session.attached" }) + .pipe( + Stream.runCollect, + Effect.map((events) => events.length), + ); + const originalAttachmentCount = yield* countAttachments; + yield* Ref.set(armed, true); + const refresh = yield* Effect.gen(function* () { + if (runtime === undefined) { + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + } else { + // Cached native opens defer preparation to the native operation. + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + } + }).pipe(Effect.forkScoped); + const credential = yield* Deferred.await(issued); + const token = credential.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(token)); + // Signal Stop before releasing issuance; no timing or polling is needed. + yield* Effect.sync(() => refresh.interruptUnsafe()); + yield* Deferred.succeed(resume, undefined); + const stopped = yield* Fiber.await(refresh); + assert.isTrue(Exit.isFailure(stopped) && Cause.hasInterrupts(stopped.cause)); + assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.equal((yield* Ref.get(state)).closeCount, 0); + + const returned = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + if (runtime !== undefined) assert.strictEqual(returned, runtime); + yield* returned.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal( + yield* countAttachments, + originalAttachmentCount + (operation === "open" ? 1 : 0), + "refresh must preserve the existing attachment; opening must attach once", + ); + const replacement = (yield* mcpSessions.read(threadId))!; + const replacementToken = replacement.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(replacementToken)); + yield* manager.close(providerSessionId); + assert.isUndefined(yield* registry.resolve(replacementToken)); + assert.isUndefined(yield* registry.resolve(token)); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ state, idleTimeoutMs: 60_000, pauseIssue: { armed, issued, resume } }), + ), + ); + }), +); + +it.effect.each(["resume", "start", "rollback"] as const)( + "ProviderSessionManagerV2 retains accepted native MCP ownership when %s is interrupted", + (operation) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const residentConfig = yield* Ref.make< + McpProviderSession.McpProviderSessionConfig | undefined + >(undefined); + const accepted = yield* Deferred.make(); + const settingsLayer = ServerSettings.layerTest({ + enableAgentBrowserAccess: true, + enableAgentDeviceAccess: true, + }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const settings = yield* ServerSettings.ServerSettingsService; + const configs = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`interrupted-native-${operation}`); + const providerSessionId = yield* ids.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = ids.derive.run({ threadId, ordinal: 1 }); + const firstCall = + operation === "resume" + ? runtime.resumeThread(resumeInput) + : operation === "rollback" + ? runtime.rollbackThread({ + providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("interrupted-native-checkpoint"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }) + : runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: ids.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: ids.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* ids.allocate.message({ threadId, ordinal: 1 }), + text: "Continue the retained native thread", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + const pending = yield* firstCall.pipe(Effect.asVoid, Effect.forkChild); + yield* Deferred.await(accepted); + yield* Fiber.interrupt(pending); + + yield* settings.updateSettings({ + enableAgentBrowserAccess: false, + enableAgentDeviceAccess: false, + }); + yield* runtime.resumeThread(resumeInput); + const refreshed = (yield* configs.read(threadId))!; + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.deepEqual(yield* Ref.get(residentConfig), refreshed); + }).pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + resumeMcpConfigs: resumedConfigs, + beforeUnload: Ref.set(residentConfig, undefined), + afterResume: Effect.gen(function* () { + const config = (yield* Ref.get(resumedConfigs)).at(-1)!; + // A loaded Codex conversation retains its first MCP client until unload. + yield* Ref.update(residentConfig, (current) => current ?? config); + if (yield* Deferred.succeed(accepted, undefined)) { + return yield* Effect.never; + } + }), + }), + ), + ), + ); + }), +); + +it.effect.each(["expired", "browser-disabled", "device-disabled"] as const)( + "ProviderSessionManagerV2 reloads an attached native thread when its MCP credential is %s", + (reason) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const settingsLayer = ServerSettings.layerTest({ + enableAgentBrowserAccess: true, + enableAgentDeviceAccess: true, + }); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const settings = yield* ServerSettings.ServerSettingsService; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread-provider-session-manager-mcp-${reason}`); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + const original = (yield* mcpSessions.read(threadId))!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); + const originalScope = yield* registry.resolve(originalToken); + assert.isTrue(originalScope?.capabilities.has("preview")); + assert.isTrue(originalScope?.capabilities.has("device")); + + if (reason === "expired") { + yield* registry.revokeThread(threadId); + } else { + yield* settings.updateSettings( + reason === "browser-disabled" + ? { enableAgentBrowserAccess: false } + : { enableAgentDeviceAccess: false }, + ); + } + if (reason === "expired") { + // Retained continuations can start a turn without another resume. + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Continue with refreshed MCP credentials", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + } else { + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + } + const refreshed = (yield* mcpSessions.read(threadId))!; + const scope = yield* registry.resolve( + refreshed.authorizationHeader.replace(/^Bearer\s+/, ""), + ); + assert.isUndefined(yield* registry.resolve(originalToken)); + assert.equal(scope?.thread.threadId, threadId); + assert.equal(scope?.capabilities.has("preview"), reason !== "browser-disabled"); + assert.equal(scope?.capabilities.has("device"), reason !== "device-disabled"); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ state, idleTimeoutMs: 60_000, serverSettingsLayer: settingsLayer }), + ), + ), + ); + }), +); + +it.effect.each(["idle", "detach", "failed-reattach", "detach-batch"] as const)( + "ProviderSessionManagerV2 retains native credential identity after a failed unload (%s)", + (scenario) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const unloadAttempts = yield* Ref.make(0); + const backgroundWork = yield* Ref.make(false); + const sessionBackgroundWork = yield* Ref.make(true); + const pauseAttachWrite = { + armed: yield* Ref.make(false), + paused: yield* Deferred.make(), + }; + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const beforeUnload = Effect.gen(function* () { + const attempt = yield* Ref.updateAndGet(unloadAttempts, (count) => count + 1); + if (attempt === (scenario === "detach-batch" ? 2 : 1)) { + return yield* Effect.die(new Error("simulated native unload failure")); + } + }); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread-failed-native-unload-${scenario}`); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const otherProviderThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: "other-native-thread", + }); + const events: OrchestrationV2DomainEvent[] = [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), + ]; + for (const thread of [ + providerThread, + ...(scenario === "detach-batch" ? [otherProviderThread] : []), + ]) { + events.push({ + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: thread, + }); + } + yield* eventSink.write({ events }); + const openInput = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(openInput); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + if (scenario === "detach-batch") + yield* runtime.resumeThread({ ...resumeInput, providerThread: otherProviderThread }); + const original = (yield* mcpSessions.read(threadId))!; + const token = original.authorizationHeader.replace(/^Bearer\s+/, ""); + if (scenario === "idle") { + // Wait for the event pump's receipt before advancing the unload timer. + const subscription = yield* runtime.subscribeEvents!; + const received = yield* subscription.events.pipe(Stream.runHead, Effect.forkChild); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + yield* Queue.offer(queue, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "failed-idle-unload-turn", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(received); + yield* subscription.close; + yield* TestClock.adjust("1 second"); + } else { + yield* manager.detach({ providerSessionId, threadId }); + } + assert.equal(yield* Ref.get(unloadAttempts), scenario === "detach-batch" ? 2 : 1); + assert.equal((yield* Ref.get(state)).resumeCount, scenario === "detach-batch" ? 2 : 1); + if (scenario === "failed-reattach") { + // Interrupt after preparation but before attachment persistence. Its + // rollback must preserve the pre-existing unknown native load marker. + yield* Ref.set(pauseAttachWrite.armed, true); + const reattach = yield* runtime.resumeThread(resumeInput).pipe(Effect.forkChild); + yield* Deferred.await(pauseAttachWrite.paused); + yield* Fiber.interrupt(reattach); + } + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + yield* Ref.set(backgroundWork, true); + assert.strictEqual(yield* manager.open(openInput), runtime); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + // Ensure has no explicit native pointer: the retained tracker guards it. + assert.equal( + (yield* runtime + .ensureThread({ + ...openInput, + title: "Retained native client", + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterEnsureThreadError", + ); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.isDefined(yield* registry.resolve(token)); + yield* Ref.set(backgroundWork, false); + yield* Ref.set(sessionBackgroundWork, false); + yield* runtime.resumeThread(resumeInput); + assert.equal((yield* Ref.get(state)).resumeCount, scenario === "detach-batch" ? 3 : 2); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.isUndefined(yield* registry.resolve(token)); + if (scenario === "detach-batch") { + yield* runtime.resumeThread({ ...resumeInput, providerThread: otherProviderThread }); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "native-thread", + "other-native-thread", + ]); + } + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 1000, + beforeUnload, + pauseAttachWrite, + // Keep the shared process alive while testing its per-thread unload. + hasPendingBackgroundWork: Ref.get(sessionBackgroundWork), + // The tracked client unloaded successfully in a partial batch; + // only the other, failed client retains background work. + hasPendingBackgroundWorkForThread: + scenario === "detach-batch" + ? (providerThread) => + providerThread.nativeThreadRef?.nativeId === "other-native-thread" + ? Ref.get(backgroundWork) + : Effect.succeed(false) + : Ref.get(backgroundWork), + serverSettingsLayer: settingsLayer, + }), + ), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects reopening after reported activity during its background probe", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const checkEntered = yield* Deferred.make(); + const checkGate = yield* Deferred.make(); + const armed = yield* Ref.make(false); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-reopen-activity-during-probe"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const input = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(input); + const subscription = yield* runtime.subscribeEvents!; + yield* registry.revokeThread(threadId); + yield* Ref.set(armed, true); + const reopening = yield* manager.open(input).pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(checkEntered); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + yield* Queue.offer(queue, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + // The subscriber receives this only after the manager records activity. + assert.isTrue(Option.isSome(yield* subscription.events.pipe(Stream.runHead))); + yield* Deferred.succeed(checkGate, undefined); + assert.equal((yield* Fiber.join(reopening))._tag, "ProviderSessionOpenError"); + assert.equal((yield* Ref.get(state)).openCount, 1); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.strictEqual( + (yield* manager.get(providerSessionId)).pipe(Option.getOrThrow), + runtime, + ); + yield* subscription.close; + // A later explicit retry can reopen once no activity invalidates its probe. + assert.notStrictEqual(yield* manager.open(input), runtime); + assert.equal((yield* Ref.get(state)).openCount, 2); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: true, + hasPendingBackgroundWork: Effect.gen(function* () { + if (yield* Ref.getAndSet(armed, false)) { + yield* Deferred.succeed(checkEntered, undefined); + yield* Deferred.await(checkGate); + } + return false; + }), + }), + ), + ); + }), +); + +it.effect.each(["process", "native"] as const)( + "ProviderSessionManagerV2 preserves the current peer token while refresh is blocked (%s)", + (client) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const peerState = yield* Ref.make(emptyState); + const backgroundWork = yield* Ref.make(false); + const peerInstanceId = ProviderInstanceId.make("current-mcp-peer"); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-rejected-switch-peer"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const peerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: peerInstanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const input = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(input); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + yield* manager.open({ + ...input, + providerSessionId: peerSessionId, + modelSelection: { ...modelSelection, instanceId: peerInstanceId }, + }); + yield* Ref.set(backgroundWork, true); + const peerCredential = (yield* mcpSessions.read(threadId))!; + const peerToken = peerCredential.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.equal( + (yield* registry.resolve(peerToken))?.thread.providerInstanceId, + peerInstanceId, + ); + if (client === "process") { + assert.equal( + (yield* manager.open(input).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + } else { + assert.strictEqual(yield* manager.open(input), runtime); + assert.deepEqual(yield* mcpSessions.read(threadId), peerCredential); + } + assert.equal( + (yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.equal( + (yield* runtime + .ensureThread({ + threadId, + title: "Blocked resume fallback", + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterEnsureThreadError", + ); + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + assert.equal( + (yield* runtime + .startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Retained continuation must preserve the current peer", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterTurnStartError", + ); + assert.deepEqual(yield* mcpSessions.read(threadId), peerCredential); + assert.equal( + (yield* registry.resolve(peerToken))?.thread.providerInstanceId, + peerInstanceId, + ); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.equal((yield* Ref.get(peerState)).closeCount, 0); + yield* Ref.set(backgroundWork, false); + const restarted = yield* manager.open(input); + if (client === "process") assert.notStrictEqual(restarted, runtime); + else assert.strictEqual(restarted, runtime); + yield* restarted.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).openCount, client === "process" ? 2 : 1); + assert.equal((yield* Ref.get(state)).closeCount, client === "process" ? 1 : 0); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + assert.deepEqual( + (yield* Ref.get(state)).unloadedNativeThreadIds, + client === "native" ? ["native-thread"] : [], + ); + const returned = (yield* mcpSessions.read(threadId))!; + assert.equal( + (yield* registry.resolve(returned.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + modelSelection.instanceId, + ); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: client === "process", + hasPendingBackgroundWork: Ref.get(backgroundWork), + hasPendingBackgroundWorkForThread: Ref.get(backgroundWork), + additionalAdapters: [makeProviderAdapter(peerState, { instanceId: peerInstanceId })], + }), + ), + ); + }), +); + +it.effect.each(["before-first-resume", "after-resume", "active-turn", "background-work"] as const)( + "ProviderSessionManagerV2 reopens process-bound MCP clients after access changes (%s)", + (phase) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const backgroundWork = yield* Ref.make(false); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-process-mcp-reopen"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const openInput = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(openInput); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + if (phase !== "before-first-resume") { + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + } + if (phase === "active-turn") { + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Keep this turn active until its terminal event", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + } + const original = (yield* mcpSessions.read(threadId))!; + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + if (phase === "background-work") yield* Ref.set(backgroundWork, true); + if (phase === "active-turn" || phase === "background-work") { + const subscription = yield* Effect.acquireRelease( + runtime.subscribeEvents!, + (sub) => sub.close, + ); + const terminal = yield* subscription.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + Effect.forkChild, + ); + const reopenError = yield* manager.open(openInput).pipe(Effect.flip); + assert.equal(reopenError._tag, "ProviderSessionOpenError"); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.equal((yield* Ref.get(state)).openCount, 1); + const current = (yield* mcpSessions.read(threadId))!; + assert.isDefined( + yield* registry.resolve(current.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + const resident = yield* manager.get(providerSessionId); + assert.isTrue(Option.isSome(resident)); + if (Option.isSome(resident)) assert.strictEqual(resident.value, runtime); + if (phase === "active-turn") { + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + yield* Queue.offer(queue, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "busy-reopen-turn", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + assert.isTrue(Option.isSome(yield* Fiber.join(terminal))); + } else { + yield* Ref.set(backgroundWork, false); + yield* Fiber.interrupt(terminal); + } + } else { + const continuationError = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(continuationError._tag, "ProviderAdapterResumeThreadError"); + const fallbackError = yield* runtime + .ensureThread({ + threadId, + title: "Resume fallback", + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal(fallbackError._tag, "ProviderAdapterEnsureThreadError"); + } + const replacement = yield* manager.open(openInput); + assert.notStrictEqual(replacement, runtime); + yield* replacement.resumeThread({ providerThread, modelSelection, runtimePolicy }); + const credential = (yield* mcpSessions.read(threadId))!; + assert.isUndefined( + yield* registry.resolve(original.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isFalse( + (yield* registry.resolve( + credential.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + assert.equal((yield* Ref.get(state)).openCount, 2); + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.equal((yield* Ref.get(state)).resumeCount, phase !== "before-first-resume" ? 2 : 1); + const staleError = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(staleError._tag, "ProviderAdapterResumeThreadError"); + assert.equal( + (yield* mcpSessions.read(threadId))?.providerSessionId, + credential.providerSessionId, + ); + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: true, + hasPendingBackgroundWork: Ref.get(backgroundWork), + serverSettingsLayer: settingsLayer, + }), + ), + ), + ); + }), +); + +it.effect.each(["timeout", "background-work", "access-change"] as const)( + "ProviderSessionManagerV2 preserves stale native threads until reload is safe (%s)", + (reason) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const unloading = yield* Deferred.make(); + const hang = yield* Ref.make(reason === "timeout"); + const backgroundWork = yield* Ref.make(reason !== "timeout"); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const beforeUnload = Ref.getAndSet(hang, false).pipe( + Effect.flatMap((first) => + first + ? Deferred.succeed(unloading, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.void, + ), + ); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-unload-timeout"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + const original = (yield* mcpSessions.read(threadId))!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); + if (reason === "access-change") { + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + } else { + yield* registry.revokeThread(threadId); + } + if (reason === "timeout") { + const pending = yield* runtime + .resumeThread(resumeInput) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(unloading); + yield* TestClock.adjust("11 seconds"); + assert.equal((yield* Fiber.join(pending))._tag, "ProviderAdapterResumeThreadError"); + } else { + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, []); + if (reason === "access-change") { + assert.equal( + (yield* runtime + .ensureThread({ + threadId, + title: "Scope-change resume fallback", + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterEnsureThreadError", + ); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.isDefined(yield* registry.resolve(originalToken)); + } + yield* Ref.set(backgroundWork, false); + } + assert.equal((yield* Ref.get(state)).resumeCount, 1); + yield* runtime.resumeThread(resumeInput); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + if (reason === "access-change") { + assert.isUndefined(yield* registry.resolve(originalToken)); + const current = (yield* mcpSessions.read(threadId))!; + assert.isFalse( + (yield* registry.resolve( + current.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + } + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + beforeUnload, + hasPendingBackgroundWorkForThread: Ref.get(backgroundWork), + serverSettingsLayer: settingsLayer, + }), + ), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 guards rotation against the settings snapshot it actually validates", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const flipAfterRead = yield* Ref.make(false); + const backgroundWork = yield* Ref.make(true); + const baseSettings = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: delegate.getSettings.pipe( + Effect.tap(() => + Ref.getAndSet(flipAfterRead, false).pipe( + Effect.flatMap((flip) => + flip + ? delegate.updateSettings({ enableAgentBrowserAccess: false }) + : Effect.void, + ), + ), + ), + ), + }); + }), + ).pipe(Layer.provide(baseSettings)); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-settings-snapshot-rotation"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + const original = (yield* mcpSessions.read(threadId))!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* Ref.set(flipAfterRead, true); + // The read returns the old snapshot and changes the next read before it + // completes. Reuse needs no rotation; the following operation must guard. + yield* runtime.resumeThread(resumeInput); + assert.isFalse((yield* settings.getSettings).enableAgentBrowserAccess); + assert.isDefined(yield* registry.resolve(originalToken)); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.isDefined(yield* registry.resolve(originalToken)); + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + hasPendingBackgroundWorkForThread: Ref.get(backgroundWork), + }), + ), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 cleans MCP issuance when release wins during preparation", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const armed = yield* Ref.make(false); + const issued = yield* Deferred.make(); + const resumeIssue = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-release-preparation"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + yield* registry.revokeThread(threadId); + yield* Ref.set(armed, true); + const pending = yield* runtime.resumeThread(resumeInput).pipe(Effect.flip, Effect.forkChild); + const credential = yield* Deferred.await(issued); + yield* manager.close(providerSessionId); + yield* Deferred.succeed(resumeIssue, undefined); + assert.equal((yield* Fiber.join(pending))._tag, "ProviderAdapterResumeThreadError"); + assert.isUndefined( + yield* registry.resolve(credential.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.equal((yield* Ref.get(state)).resumeCount, 1); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + pauseIssue: { armed, issued, resume: resumeIssue }, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 cleans unused MCP access when scope clearing is interrupted", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const clearing = yield* Deferred.make(); + const armed = yield* Ref.make(true); + const agentScope = { + ...AgentScope.defaultValue(), + clear: () => + Ref.getAndSet(armed, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(clearing, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.void, + ), + ), + }; + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-cancel-before-adapter-open"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const input = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const opening = yield* manager.open(input).pipe(Effect.forkChild); + yield* Deferred.await(clearing); + const credential = (yield* mcpSessions.read(threadId))!; + const token = credential.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(token)); + yield* Fiber.interrupt(opening); + assert.equal((yield* Ref.get(state)).openCount, 0); + assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + yield* manager.open(input); + assert.equal((yield* Ref.get(state)).openCount, 1); + assert.isDefined(yield* mcpSessions.read(threadId)); + }); + yield* effect.pipe( + Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })), + Effect.provideService(AgentScope, agentScope), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 lets an unloaded thread switch while another shared-runtime thread has retained work", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const targetInstanceId = ProviderInstanceId.make("unloaded-thread-target"); + const threadA = ThreadId.make("unloaded-owner-a"); + const threadB = ThreadId.make("retained-owner-b"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadA, now }), + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadB, now }), + ], + }); + const providerSessionId = ids.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + const owner = yield* manager.open({ + threadId: threadA, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId: threadB, + providerSessionId, + modelSelection, + runtimePolicy, + }); + for (const threadId of [threadA, threadB]) { + yield* owner.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + nativeThreadId: `native-${threadId}`, + }), + modelSelection, + runtimePolicy, + }); + } + const originalA = (yield* configs.read(threadA))!; + const originalB = (yield* configs.read(threadB))!; + yield* manager.detach({ threadId: threadA, providerSessionId }); + const targetId = yield* ids.allocate.providerSession({ + threadId: threadA, + providerInstanceId: targetInstanceId, + }); + yield* manager.open({ + threadId: threadA, + providerSessionId: targetId, + modelSelection: { ...modelSelection, instanceId: targetInstanceId }, + runtimePolicy, + }); + const switched = (yield* configs.read(threadA))!; + assert.equal( + (yield* registry.resolve(switched.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + targetInstanceId, + ); + assert.isUndefined( + yield* registry.resolve(originalA.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.deepEqual(yield* configs.read(threadB), originalB); + assert.isDefined( + yield* registry.resolve(originalB.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [`native-${threadA}`]); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: (thread) => + Effect.succeed(thread.appThreadId === threadB), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ); + }), +); + +it.effect.each([false, true])( + "ProviderSessionManagerV2 refreshes and retains rollback native credentials (rollback failure: %s)", + (failRollback) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const pendingA = yield* Ref.make(false); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const rollbackCalls = yield* Ref.make(0); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const targetInstanceId = ProviderInstanceId.make("rollback-rotation-target"); + const threadA = ThreadId.make("rollback-native-a"); + const threadB = ThreadId.make("rollback-native-b"); + const rollbackError = new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CODEX_DRIVER, + detail: "rollback failed after loading native state", + }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadA, now }), + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadB, now }), + ], + }); + const providerSessionId = ids.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + const runtime = yield* manager.open({ + threadId: threadA, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId: threadB, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator: ids, + threadId: threadA, + providerSessionId, + now, + nativeThreadId: "rollback-native-a", + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + yield* runtime.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId: threadB, + providerSessionId, + now, + nativeThreadId: "rollback-native-b", + }), + modelSelection, + runtimePolicy, + }); + const original = (yield* configs.read(threadA))!; + const siblingConfig = (yield* configs.read(threadB))!; + // Successful idle unload keeps the app thread attached to the shared session. + yield* runtime.unloadThread!({ providerThread }); + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + yield* manager.open({ + threadId: threadA, + providerSessionId, + modelSelection, + runtimePolicy, + }); + assert.deepEqual(yield* configs.read(threadA), original); + const rollbackInput: ProviderAdapter.ProviderAdapterV2RollbackThreadInput = { + providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("rollback-credential-checkpoint"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }; + const result = yield* runtime.rollbackThread(rollbackInput).pipe(Effect.result); + if (failRollback) { + assert.equal(result._tag, "Failure"); + if (result._tag === "Failure") assert.strictEqual(result.failure, rollbackError); + } else { + assert.equal(result._tag, "Success"); + } + const refreshed = (yield* configs.read(threadA))!; + assert.deepEqual((yield* Ref.get(resumedConfigs)).at(-1), refreshed); + assert.equal(yield* Ref.get(rollbackCalls), 1); + assert.isUndefined( + yield* registry.resolve(original.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isFalse( + (yield* registry.resolve( + refreshed.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + assert.deepEqual(yield* configs.read(threadB), siblingConfig); + + // A failed rollback still leaves a live client consuming its refreshed token. + yield* Ref.set(pendingA, true); + yield* settings.updateSettings({ enableAgentBrowserAccess: true }); + assert.equal( + (yield* runtime.rollbackThread(rollbackInput).pipe(Effect.flip))._tag, + "ProviderAdapterRollbackThreadError", + ); + assert.deepEqual(yield* configs.read(threadA), refreshed); + assert.equal(yield* Ref.get(rollbackCalls), 1); + yield* Ref.set(pendingA, false); + const targetId = yield* ids.allocate.providerSession({ + threadId: threadA, + providerInstanceId: targetInstanceId, + }); + yield* manager.open({ + threadId: threadA, + providerSessionId: targetId, + modelSelection: { ...modelSelection, instanceId: targetInstanceId }, + runtimePolicy, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "rollback-native-a", + "rollback-native-a", + ]); + assert.equal((yield* Ref.get(state)).resumeCount, 4); + assert.equal((yield* Ref.get(state)).closeCount, 0); + }).pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + resumeMcpConfigs: resumedConfigs, + hasPendingBackgroundWorkForThread: (thread) => + thread.appThreadId === threadA ? Ref.get(pendingA) : Effect.succeed(false), + rollbackThread: (input) => + Effect.gen(function* () { + yield* Ref.update(rollbackCalls, (count) => count + 1); + if (failRollback) return yield* rollbackError; + return { + providerThread: input.providerThread, + providerTurns: [], + messages: [], + runtimeRequests: [], + }; + }), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ), + ); + }), +); + +it.effect.each(["fresh-process", "cached-native"] as const)( + "ProviderSessionManagerV2 protects the current owner's retained work from an idle target (%s)", + (target) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const pending = yield* Ref.make(false); + const targetInstanceId = ProviderInstanceId.make("idle-rotation-target"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make(`current-owner-${target}`); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const ownerId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const targetId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: targetInstanceId, + }); + const targetSelection = { ...modelSelection, instanceId: targetInstanceId }; + const targetInput = { + threadId, + providerSessionId: targetId, + modelSelection: targetSelection, + runtimePolicy, + }; + const targetThread = makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: targetId, + now, + }); + const cached = target === "cached-native" ? yield* manager.open(targetInput) : undefined; + if (cached !== undefined) + yield* cached.resumeThread({ + providerThread: targetThread, + modelSelection: targetSelection, + runtimePolicy, + }); + const owner = yield* manager.open({ + threadId, + providerSessionId: ownerId, + modelSelection, + runtimePolicy, + }); + if (target === "cached-native") { + yield* owner.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: ownerId, + now, + nativeThreadId: "current-owner-native", + }), + modelSelection, + runtimePolicy, + }); + } + const original = (yield* configs.read(threadId))!; + const token = original.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* Ref.set(pending, true); + const error = + cached === undefined + ? yield* manager.open(targetInput).pipe(Effect.flip) + : yield* cached + .resumeThread({ + providerThread: targetThread, + modelSelection: targetSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal( + error._tag, + cached === undefined + ? "ProviderSessionActivityError" + : "ProviderAdapterResumeThreadError", + ); + assert.deepEqual(yield* configs.read(threadId), original); + assert.equal( + (yield* registry.resolve(token))?.thread.providerInstanceId, + modelSelection.instanceId, + ); + assert.equal((yield* Ref.get(state)).closeCount, 0); + yield* Ref.set(pending, false); + const ready = yield* manager.open(targetInput); + yield* ready.resumeThread({ + providerThread: targetThread, + modelSelection: targetSelection, + runtimePolicy, + }); + const current = (yield* configs.read(threadId))!; + assert.equal( + (yield* registry.resolve(current.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + targetInstanceId, + ); + assert.isUndefined(yield* registry.resolve(token)); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: target === "fresh-process", + hasPendingBackgroundWork: Ref.get(pending), + hasPendingBackgroundWorkForThread: Ref.get(pending), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps sibling consumed credentials distinct through rotation", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const pendingB = yield* Ref.make(false); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const threadId = ThreadId.make("native-sibling-credentials"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const providerSessionId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const a = makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + nativeThreadId: "sibling-a", + }); + const b = { + ...makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + nativeThreadId: "sibling-b", + }), + // A fresh fallback can replace the native bound to the same app row. + id: a.id, + }; + yield* runtime.resumeThread({ providerThread: a, modelSelection, runtimePolicy }); + // Orchestration adopts the app row after ensure/fork returns a native row. + yield* runtime.resumeThread({ + providerThread: { ...a, id: ProviderThreadId.make("canonical-app-row") }, + modelSelection, + runtimePolicy, + }); + assert.equal((yield* Ref.get(state)).resumeCount, 1); + yield* runtime.resumeThread({ providerThread: b, modelSelection, runtimePolicy }); + const original = (yield* configs.read(threadId))!; + const token = original.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* Ref.set(pendingB, true); + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + assert.equal( + (yield* runtime + .resumeThread({ providerThread: a, modelSelection, runtimePolicy }) + .pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.deepEqual(yield* configs.read(threadId), original); + assert.isDefined(yield* registry.resolve(token)); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, []); + yield* Ref.set(pendingB, false); + yield* runtime.resumeThread({ providerThread: a, modelSelection, runtimePolicy }); + const rotated = (yield* configs.read(threadId))!; + yield* runtime.resumeThread({ providerThread: b, modelSelection, runtimePolicy }); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "sibling-a", + "sibling-b", + ]); + assert.equal((yield* Ref.get(state)).resumeCount, 4); + assert.deepEqual((yield* Ref.get(resumedConfigs)).slice(-2), [rotated, rotated]); + assert.isUndefined(yield* registry.resolve(token)); + assert.isFalse( + (yield* registry.resolve( + rotated.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + }).pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + resumeMcpConfigs: resumedConfigs, + hasPendingBackgroundWorkForThread: (thread) => + thread.nativeThreadRef?.nativeId === "sibling-b" + ? Ref.get(pendingB) + : Effect.succeed(false), + }), + ), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 holds preparation through eager config consumption", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const otherState = yield* Ref.make(emptyState); + const entered = yield* Deferred.make(); + const consume = yield* Deferred.make(); + const consumed = yield* Ref.make< + ReadonlyArray + >([]); + const otherInstanceId = ProviderInstanceId.make("competing-config-consumer"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make("config-consumption-lock"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const firstId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const otherId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: otherInstanceId, + }); + const first = yield* manager + .open({ threadId, providerSessionId: firstId, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild); + yield* Deferred.await(entered); + const captured = (yield* configs.read(threadId))!; + const other = yield* manager + .open({ + threadId, + providerSessionId: otherId, + modelSelection: { ...modelSelection, instanceId: otherInstanceId }, + runtimePolicy, + }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + yield* Deferred.succeed(consume, undefined); + yield* Fiber.join(first); + // The first process reports retained work after acceptance, so the waiting + // competitor must reject instead of silently replacing its credential. + const rejected = yield* Fiber.await(other); + assert.isTrue(Exit.isFailure(rejected)); + assert.deepEqual(yield* Ref.get(consumed), [captured]); + assert.deepEqual(yield* configs.read(threadId), captured); + assert.equal( + (yield* registry.resolve(captured.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + modelSelection.instanceId, + ); + assert.equal((yield* Ref.get(otherState)).openCount, 0); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: true, + mcpConfigs: consumed, + beforeOpen: () => + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(consume))), + hasPendingBackgroundWork: Effect.succeed(true), + additionalAdapters: [makeProviderAdapter(otherState, { instanceId: otherInstanceId })], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 pumps reported activity during the central rotation probe", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const armed = yield* Ref.make(false); + const probing = yield* Deferred.make(); + const finishProbe = yield* Deferred.make(); + const targetInstanceId = ProviderInstanceId.make("probe-rotation-target"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make("central-probe-event-pump"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const ownerId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const targetId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: targetInstanceId, + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId: ownerId, + modelSelection, + runtimePolicy, + }); + yield* runtime.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: ownerId, + now, + }), + modelSelection, + runtimePolicy, + }); + const original = (yield* configs.read(threadId))!; + const subscription = yield* runtime.subscribeEvents!; + yield* Ref.set(armed, true); + const targetInput = { + threadId, + providerSessionId: targetId, + modelSelection: { ...modelSelection, instanceId: targetInstanceId }, + runtimePolicy, + }; + const opening = yield* manager.open(targetInput).pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(probing); + yield* Queue.offer((yield* Ref.get(state)).eventQueues.get(String(ownerId))!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + assert.isTrue(Option.isSome(yield* subscription.events.pipe(Stream.runHead))); + yield* Deferred.succeed(finishProbe, undefined); + assert.equal((yield* Fiber.join(opening))._tag, "ProviderSessionActivityError"); + assert.deepEqual(yield* configs.read(threadId), original); + assert.equal( + (yield* registry.resolve(original.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + modelSelection.instanceId, + ); + yield* subscription.close; + yield* manager.open(targetInput); + assert.equal((yield* Ref.get(targetState)).openCount, 1); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + hasPendingBackgroundWorkForThread: Effect.gen(function* () { + if (yield* Ref.getAndSet(armed, false)) { + yield* Deferred.succeed(probing, undefined); + yield* Deferred.await(finishProbe); + } + return false; + }), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 terminal detach revokes the thread's MCP credential", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-terminal-detach"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + const issued = (yield* Ref.get(mcpConfigs)).at(-1); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(token!)); + + // Archive/delete detaches carry revokeMcpCredential: the token must die + // with the thread even though the shared provider process lives on. + yield* manager.detach({ + providerSessionId, + threadId, + detail: "Thread deleted.", + revokeMcpCredential: true, + }); + assert.isUndefined(yield* registry.resolve(token!)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1_000, mcpConfigs }))); + }), +); + +it.effect("ProviderSessionManagerV2 releases idle sessions without sweeping all sessions", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-idle", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-idle", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + + const liveSession = yield* manager.get(providerSessionId); + const runtimeState = yield* Ref.get(state); + const projection = yield* projectionStore.getThreadProjection(threadId); + + assert.isTrue(Option.isNone(liveSession)); + assert.equal(runtimeState.openCount, 1); + assert.equal(runtimeState.closeCount, 1); + assert.equal(projection.providerSessions.at(-1)?.status, "stopped"); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); + }), +); + +it.effect("ProviderSessionManagerV2 persists release when session scope close hangs", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const idAllocator = yield* IdAllocator.IdAllocatorV2; const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; @@ -4505,8 +6703,9 @@ it.effect( yield* resumeSecondThread; assert.equal((yield* Ref.get(state)).resumeCount, 4); - // The second thread has no persisted provider thread, so nothing is unloaded. - assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, []); + // Resume tracked the second thread's native client even though its + // provider-thread record has not been persisted yet. + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); yield* manager.detach({ providerSessionId, threadId: firstThreadId }); assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); @@ -4514,7 +6713,10 @@ it.effect( assert.equal((yield* Ref.get(state)).interruptCount, 1); // The runtime stays up for the second thread; the first thread's // native state is unloaded after its turn is interrupted. - assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "native-thread", + "native-thread", + ]); yield* manager.detach({ providerSessionId, threadId: secondThreadId }); yield* TestClock.adjust("1 second"); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index a9c6e2fc9f7fe..17b55239a34b2 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -201,9 +201,12 @@ export class ProviderSessionManagerV2 extends Context.Service< interface LiveSessionEntry { readonly attachedThreadIds: ReadonlySet; - readonly loadedProviderThreadKeyByThread: ReadonlyMap; + /** Native identity survives orchestration adopting an existing app row's id. */ + readonly loadedProviderThreads: ReadonlyMap; /** - * MCP credential session id issued for each attached thread. Revocation on + * Process/open credential and latest prepared config for each app thread. + * Retained for release cleanup even after all its natives unload; native + * clients retain their own consumed ids in loadedProviderThreads. Revocation on * detach/release is scoped to these ids so tearing down a superseded * session cannot revoke a replacement session's credential for the same * thread (the workspace-handoff sequence opens the replacement before the @@ -237,6 +240,14 @@ interface LiveSessionEntry { readonly idleThreadUnloads: ReadonlyMap; } +interface LoadedProviderThread { + readonly threadId: ThreadId; + readonly providerThread: OrchestrationV2ProviderThread; + readonly key: string | undefined; + /** The credential actually consumed by this native client. */ + readonly mcpCredentialId: string | undefined; +} + interface IdleThreadUnload { readonly providerThread: OrchestrationV2ProviderThread; readonly generation: number; @@ -465,67 +476,125 @@ export const layerWithOptions = ( const isMcpCredentialReserved = (threadId: ThreadId, mcpCredentialId: string) => (mcpCredentialReservations.get(mcpReservationKey(threadId, mcpCredentialId)) ?? 0) > 0; const mcpPrepareLock = yield* KeyedLock.make(); + const mcpCredentialMatches = ( + resolved: Effect.Success>, + threadId: ThreadId, + providerInstanceId: ProviderInstanceId, + access: { readonly browser: boolean; readonly device: boolean }, + ) => + resolved !== undefined && + resolved.thread.threadId === threadId && + resolved.thread.providerInstanceId === providerInstanceId && + resolved.capabilities.has("preview") === access.browser && + resolved.capabilities.has("device") === access.device; + + // Read-only: rejecting a process restart or native reload must + // leave the current provider's credential intact. + const mcpAccessNeedsRefresh = (entry: LiveSessionEntry, threadId: ThreadId) => + Effect.gen(function* () { + if (options.configureMcp === false) return false; + const current = yield* mcpSessions.read(threadId); + const consumed = entry.mcpCredentialIdByThread.get(threadId); + if (current === undefined || current.providerSessionId !== consumed) return true; + const access = yield* agentAccessSettings(threadId); + const resolved = yield* mcpSessionRegistry.resolve( + current.authorizationHeader.replace(/^Bearer\s+/, ""), + ); + return !mcpCredentialMatches(resolved, threadId, entry.runtime.instanceId, access); + }); + /** * Resolves (or mints) the thread's MCP credential and returns it with a * reservation held; the caller must drop the reservation exactly once. - * Serialized per thread so two concurrent prepares cannot interleave - * their rotate steps and revoke each other's freshly minted credential. + * The caller holds mcpPrepareLock through adapter acceptance and publication, + * so another provider cannot substitute the config before it is consumed. */ const prepareMcpSession = ( threadId: ThreadId, providerInstanceId: ProviderInstanceId, - ): Effect.Effect => + onPrepared: (prepared: PreparedMcpCredential) => void, + providerSessionId: ProviderSessionId, + checkAccess?: ( + candidateId: string | undefined, + willRotate: boolean, + ) => Effect.Effect, + providerThread?: OrchestrationV2ProviderThread, + ): Effect.Effect => options.configureMcp === false - ? mcpSessions - .clear(threadId) - .pipe(Effect.as({ mcpCredentialId: undefined, issued: false })) - : mcpPrepareLock.withLock( - threadId, - Effect.gen(function* () { - // Reuse a still-valid credential for this thread instead of - // rotating: long-lived provider processes (codex app-server) - // build their MCP client once per conversation and keep using - // the credential it started with, so a thread that detaches and - // re-attaches across a workspace handoff must come back to the - // same token or the process's tool calls fail auth. - const { browser: browserToolsAvailable, device: deviceToolsAvailable } = - yield* agentAccessSettings(threadId); - const capabilities = new Set< - import("../mcp/McpInvocationContext.ts").McpCapability - >(["orchestration", "worktree", "pull-requests"]); - if (browserToolsAvailable) capabilities.add("preview"); - if (deviceToolsAvailable) capabilities.add("device"); - const existing = yield* mcpSessions.read(threadId); - if (existing !== undefined) { - // Reserve before the async resolve so a release cannot - // revoke the credential between validation and reservation. - reserveMcpCredential(threadId, existing.providerSessionId); - const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); - // The caller only learns of the reservation once this returns, - // so a stop while resolving must drop it here. - const resolved = yield* mcpSessionRegistry - .resolve(rawToken) - .pipe( - Effect.onInterrupt(() => - Effect.sync(() => - dropMcpCredentialReservation(threadId, existing.providerSessionId), + ? Effect.gen(function* () { + const prepared = { mcpCredentialId: undefined, issued: false }; + onPrepared(prepared); + if (checkAccess !== undefined) yield* checkAccess(undefined, false); + yield* mcpSessions.clear(threadId); + return prepared; + }) + : Effect.gen(function* () { + // Reuse a still-valid credential for this thread instead of + // rotating: long-lived provider processes (codex app-server) + // build their MCP client once per conversation and keep using + // the credential it started with, so a thread that detaches and + // re-attaches across a workspace handoff must come back to the + // same token or the process's tool calls fail auth. + const { browser: browserToolsAvailable, device: deviceToolsAvailable } = + yield* agentAccessSettings(threadId); + const capabilities = new Set([ + "orchestration", + "worktree", + "pull-requests", + ]); + if (browserToolsAvailable) capabilities.add("preview"); + if (deviceToolsAvailable) capabilities.add("device"); + const existing = yield* mcpSessions.read(threadId); + if (existing !== undefined) { + // Reserve before the async resolve so a release cannot + // revoke the credential between validation and reservation. + reserveMcpCredential(threadId, existing.providerSessionId); + const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); + // The caller only learns of the reservation once this returns, + // so a failed lookup must drop it here, including on Stop. + const resolved = yield* mcpSessionRegistry + .resolve(rawToken) + .pipe( + Effect.onError(() => + Effect.sync(() => + dropMcpCredentialReservation(threadId, existing.providerSessionId), + ).pipe( + Effect.andThen( + clearUnownedMcpSession(threadId, existing.providerSessionId), ), + Effect.uninterruptible, ), - ); - if ( - resolved !== undefined && - resolved.thread.threadId === threadId && - resolved.thread.providerInstanceId === providerInstanceId && - // A flipped browser-access setting must not survive through - // credential reuse: rotate so the new scope reflects it. - resolved.capabilities.has("preview") === browserToolsAvailable && - resolved.capabilities.has("device") === deviceToolsAvailable - ) { - return { mcpCredentialId: existing.providerSessionId, issued: false }; - } - dropMcpCredentialReservation(threadId, existing.providerSessionId); + ), + ); + if ( + mcpCredentialMatches(resolved, threadId, providerInstanceId, { + browser: browserToolsAvailable, + device: deviceToolsAvailable, + }) + ) { + const prepared = { mcpCredentialId: existing.providerSessionId, issued: false }; + // The caller owns this reservation before a guard can yield + // or fail, including when the last live owner just released it. + onPrepared(prepared); + if (providerThread !== undefined) + yield* checkMcpOwners(threadId, providerSessionId, { + nativeThreadKey: providerThreadRuntimeKey(providerThread), + candidateId: existing.providerSessionId, + }); + if (checkAccess !== undefined) + yield* checkAccess(existing.providerSessionId, false); + return prepared; } - yield* mcpSessionRegistry.revokeThread(threadId); + yield* Effect.sync(() => + dropMcpCredentialReservation(threadId, existing.providerSessionId), + ); + } + // Validate work against this same settings/credential snapshot + // before any live credential is revoked or issued. + if (checkAccess !== undefined) yield* checkAccess(existing?.providerSessionId, true); + yield* checkMcpOwners(threadId, providerSessionId, undefined, providerThread); + yield* mcpSessionRegistry.revokeThread(threadId); + return yield* Effect.gen(function* () { const credential = yield* mcpSessionRegistry.issue({ threadId, providerInstanceId, @@ -534,9 +603,16 @@ export const layerWithOptions = ( }); yield* mcpSessions.set(credential.config); reserveMcpCredential(threadId, credential.config.providerSessionId); - return { mcpCredentialId: credential.config.providerSessionId, issued: true }; - }), - ); + const prepared = { + mcpCredentialId: credential.config.providerSessionId, + issued: true, + }; + // Publish the cleanup id before restoring interruption: a + // stop must not leave a minted token outside session records. + onPrepared(prepared); + return prepared; + }).pipe(Effect.uninterruptible); + }); /** * With a credential id, revocation is scoped to that credential and the * config slot is cleared only while it still holds it; a replacement @@ -558,6 +634,145 @@ export const layerWithOptions = ( ), ); + const clearUnownedMcpSession = ( + threadId: ThreadId, + mcpCredentialId: string, + ignoredRuntime?: ProviderAdapter.ProviderAdapterV2SessionRuntime, + ) => + Ref.get(sessions).pipe( + Effect.flatMap((current) => + isMcpCredentialReserved(threadId, mcpCredentialId) || + Array.from(current.values()).some( + (entry) => + entry.runtime !== ignoredRuntime && + ownsMcpCredential(entry, threadId, mcpCredentialId), + ) + ? Effect.void + : clearMcpSession(threadId, mcpCredentialId), + ), + ); + + const ownsMcpCredential = (entry: LiveSessionEntry, threadId: ThreadId, id: string) => + entry.mcpCredentialIdByThread.get(threadId) === id || + Array.from(entry.loadedProviderThreads.values()).some( + (loaded) => loaded.threadId === threadId && loaded.mcpCredentialId === id, + ); + + // A rotation revokes every credential for the app thread. Inspect every + // owner, including detached natives and process opens that have no native + // identity yet. A stale reload uses the same gate for its own client. + // Probes cannot make unreported native activity atomic with preparation. + const checkMcpOwners = ( + threadId: ThreadId, + providerSessionId: ProviderSessionId, + reload?: { readonly nativeThreadKey: string; readonly candidateId: string }, + providerThread?: OrchestrationV2ProviderThread, + ) => + Effect.gen(function* () { + const snapshot = yield* Ref.get(sessions); + const target = snapshot.get(sessionKey(providerSessionId)); + const owners = Array.from(snapshot.values()).flatMap((entry) => { + const records = Array.from(entry.loadedProviderThreads.values()).filter( + (loaded) => + loaded.threadId === threadId && + (reload === undefined || + (entry.runtime.providerSessionId === providerSessionId && + providerThreadRuntimeKey(loaded.providerThread) === reload.nativeThreadKey && + loaded.mcpCredentialId !== reload.candidateId)), + ); + // An explicit retained native is known even before its first resume + // in this process. Probe it without inventing a loaded client record. + if ( + providerThread !== undefined && + entry.runtime.providerSessionId === providerSessionId && + !entry.loadedProviderThreads.has(providerThreadRuntimeKey(providerThread)) && + reload === undefined + ) { + records.push({ + threadId, + providerThread, + key: undefined, + mcpCredentialId: entry.mcpCredentialIdByThread.get(threadId), + }); + } + // Native-capable sessions retain these ids for release cleanup after + // unload. Only process-bound clients own them without a loaded native. + const processOwner = + reload === undefined && + entry.runtime.unloadThread === undefined && + entry.mcpCredentialIdByThread.has(threadId); + return records.length === 0 && !processOwner ? [] : [{ entry, records }]; + }); + if ( + reload === undefined && + Array.from(mcpCredentialReservations.keys()).some((key) => + key.startsWith(`${threadId}\0`), + ) + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "An in-flight provider configuration prevents refreshing MCP access.", + }); + } + for (const { entry, records } of owners) { + const uncertain = + entry.runtime.unloadThread === undefined || + records.length === 0 || + records.some((record) => record.providerThread.nativeThreadRef === null); + if ( + uncertain + ? entry.busyTurns.size > 0 + : records.some((record) => hasBusyTurn(entry, record.providerThread.id)) + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "An active turn prevents refreshing MCP access.", + }); + } + const pending = yield* ( + uncertain || entry.runtime.hasPendingBackgroundWorkForThread === undefined + ? (entry.runtime.hasPendingBackgroundWork ?? Effect.succeed(false)) + : Effect.forEach(records, (record) => + entry.runtime.hasPendingBackgroundWorkForThread!(record.providerThread), + ).pipe(Effect.map((results) => results.some(Boolean))) + ).pipe( + Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.mapError( + (cause) => new ProviderSessionActivityError({ providerSessionId, cause }), + ), + ); + if (pending) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "Pending background work prevents refreshing MCP access.", + }); + } + } + const latest = yield* Ref.get(sessions); + if (latest.get(sessionKey(providerSessionId))?.runtime !== target?.runtime) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "The provider session changed while checking pending work.", + }); + } + for (const { entry } of owners) { + const current = latest.get(sessionKey(entry.runtime.providerSessionId)); + if ( + current?.runtime !== entry.runtime || + current.loadedProviderThreads !== entry.loadedProviderThreads || + current.mcpCredentialIdByThread !== entry.mcpCredentialIdByThread || + current.busyTurns !== entry.busyTurns || + current.idleGeneration !== entry.idleGeneration || + current.lastActivityAtMs !== entry.lastActivityAtMs + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "The provider session changed or became active while checking pending work.", + }); + } + } + }); + const publishToSubscribers = ( subscribers: Ref.Ref< ReadonlyMap> @@ -972,23 +1187,52 @@ export const layerWithOptions = ( const removeLiveEntry = (input: { readonly providerSessionId: ProviderSessionId; readonly onlyIfIdleGeneration?: number; - }): Effect.Effect, DateTime.Utc]> => + readonly requireIdle?: boolean; + }): Effect.Effect< + readonly [Option.Option, DateTime.Utc], + ProviderSessionActivityError + > => Effect.gen(function* () { const key = sessionKey(input.providerSessionId); const candidate = (yield* Ref.get(sessions)).get(key); if (candidate === undefined) { return [Option.none(), yield* DateTime.now] as const; } + const expectedGeneration = + input.onlyIfIdleGeneration ?? + (input.requireIdle ? candidate.idleGeneration : undefined); + if (input.requireIdle) { + const pending = yield* ( + candidate.runtime.hasPendingBackgroundWork ?? Effect.succeed(false) + ).pipe( + Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.mapError( + (cause) => + new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause, + }), + ), + ); + if (pending) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "Pending background work prevents reopening the provider session.", + }); + } + } const removed = yield* Effect.zip( Ref.modify(sessions, (current) => { const existing = current.get(key); if (existing !== candidate) { return [existing === undefined ? "gone" : "changed", current] as const; } + if (input.requireIdle && existing.busyTurns.size > 0) { + return ["busy", current] as const; + } if ( - input.onlyIfIdleGeneration !== undefined && - (existing.busyTurns.size > 0 || - existing.idleGeneration !== input.onlyIfIdleGeneration) + expectedGeneration !== undefined && + (existing.busyTurns.size > 0 || existing.idleGeneration !== expectedGeneration) ) { return ["kept", current] as const; } @@ -999,8 +1243,23 @@ export const layerWithOptions = ( DateTime.now, ).pipe(candidate.requestEventPermit.withPermits(1)); const [outcome, releasedAt] = removed; - // Another entry took this id while the permit was held; release it instead. + if ( + input.requireIdle && + (outcome === "changed" || outcome === "kept" || outcome === "gone") + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "Provider activity changed while checking whether the session can reopen.", + }); + } + // Unconditional releases can follow a replacement; idle checks cannot. if (outcome === "changed") return yield* removeLiveEntry(input); + if (outcome === "busy") { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "The provider session has an active turn and cannot be reopened.", + }); + } return [ outcome === "removed" ? Option.some(candidate) : Option.none(), releasedAt, @@ -1054,6 +1313,7 @@ export const layerWithOptions = ( readonly detachedThreadId?: ThreadId; readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; + readonly requireIdle?: boolean; readonly gracefulSubscribers?: boolean; }) => Effect.acquireUseRelease( @@ -1127,7 +1387,23 @@ export const layerWithOptions = ( Ref.get(sessions).pipe( Effect.flatMap((current) => Effect.forEach( - entry.mcpCredentialIdByThread, + new Map([ + ...Array.from( + entry.mcpCredentialIdByThread, + ([threadId, id]) => + [mcpReservationKey(threadId, id), [threadId, id] as const] as const, + ), + ...Array.from(entry.loadedProviderThreads.values()).flatMap((loaded) => + loaded.mcpCredentialId === undefined + ? [] + : [ + [ + mcpReservationKey(loaded.threadId, loaded.mcpCredentialId), + [loaded.threadId, loaded.mcpCredentialId] as const, + ] as const, + ], + ), + ]).values(), ([threadId, mcpCredentialId]) => { // Id-sensitive: a stale record for the same thread but // a DIFFERENT credential (left behind by an old session @@ -1142,8 +1418,7 @@ export const layerWithOptions = ( Array.from(current.values()).some( (other) => other !== entry && - (other.attachedThreadIds.has(threadId) || - other.mcpCredentialIdByThread.get(threadId) === mcpCredentialId), + ownsMcpCredential(other, threadId, mcpCredentialId), ); return heldElsewhere ? Effect.void @@ -1330,24 +1605,26 @@ export const layerWithOptions = ( }), ); - /** Returns the runtime the thread was attached to, or undefined if it already was. */ + /** Missing/replaced runtimes cannot attach or mint credentials. */ const attachThread = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; + readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; }) => withActivityError( input.providerSessionId, Ref.modify(sessions, (current) => { const entry = current.get(sessionKey(input.providerSessionId)); - if (entry === undefined || entry.attachedThreadIds.has(input.threadId)) { + if (entry === undefined || entry.runtime !== input.runtime) { return [undefined, current] as const; } + if (entry.attachedThreadIds.has(input.threadId)) return [false, current] as const; const updated = new Map(current); updated.set(sessionKey(input.providerSessionId), { ...entry, attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]), }); - return [entry.runtime, updated] as const; + return [true, updated] as const; }), ); @@ -1372,53 +1649,119 @@ export const layerWithOptions = ( } const attachedThreadIds = new Set(entry.attachedThreadIds); attachedThreadIds.delete(input.threadId); - const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread); - loadedProviderThreadKeyByThread.delete(input.threadId); const updated = new Map(current); updated.set(key, { ...entry, attachedThreadIds, - loadedProviderThreadKeyByThread, }); return updated; }); - const isProviderThreadLoaded = (input: { + const providerThreadLoadStatus = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; - readonly providerThreadKey: string; + readonly providerThread: OrchestrationV2ProviderThread; + readonly providerThreadKey: string | undefined; }) => Ref.get(sessions).pipe( - Effect.map( - (current) => - current - .get(sessionKey(input.providerSessionId)) - ?.loadedProviderThreadKeyByThread.get(input.threadId) === input.providerThreadKey, - ), + Effect.map((current) => { + const entry = current.get(sessionKey(input.providerSessionId)); + const loaded = entry?.loadedProviderThreads.get( + providerThreadRuntimeKey(input.providerThread), + ); + const staleMcpCredential = + loaded !== undefined && + loaded.mcpCredentialId !== entry?.mcpCredentialIdByThread.get(input.threadId); + return { + loaded: + input.providerThreadKey !== undefined && + loaded?.key === input.providerThreadKey && + !staleMcpCredential, + staleMcpCredential, + }; + }), ); const markProviderThreadLoaded = (input: { readonly providerSessionId: ProviderSessionId; + readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; readonly threadId: ThreadId; - readonly providerThreadKey: string; + readonly providerThread: OrchestrationV2ProviderThread; + readonly mcpCredentialId: string | undefined; + readonly providerThreadKey: string | undefined; }) => - Ref.update(sessions, (current) => { + Ref.modify(sessions, (current) => { const key = sessionKey(input.providerSessionId); const entry = current.get(key); - if (entry === undefined) { - return current; + if (entry?.runtime !== input.runtime) { + return [false, current] as const; } - const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread); - loadedProviderThreadKeyByThread.set(input.threadId, input.providerThreadKey); + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + loadedProviderThreads.set(providerThreadRuntimeKey(input.providerThread), { + threadId: input.threadId, + providerThread: input.providerThread, + key: input.providerThreadKey, + mcpCredentialId: input.mcpCredentialId, + }); const updated = new Map(current); - updated.set(key, { ...entry, loadedProviderThreadKeyByThread }); - return updated; + updated.set(key, { ...entry, loadedProviderThreads }); + return [true, updated] as const; + }).pipe( + Effect.flatMap((recorded) => + recorded + ? Effect.void + : Effect.fail( + new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: + "The provider session was released or replaced before publishing native MCP access.", + }), + ), + ), + ); + + const unloadNativeThread = ( + runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime, + providerThread: OrchestrationV2ProviderThread, + ) => + Effect.gen(function* () { + const key = sessionKey(runtime.providerSessionId); + const nativeKey = providerThreadRuntimeKey(providerThread); + const unloading = yield* Ref.modify(sessions, (current) => { + const entry = current.get(key); + const loaded = entry?.loadedProviderThreads.get(nativeKey); + if (entry?.runtime !== runtime || loaded === undefined) + return [undefined, current] as const; + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + const unknownLoad = { ...loaded, key: undefined }; + loadedProviderThreads.set(nativeKey, unknownLoad); + return [ + unknownLoad, + new Map(current).set(key, { ...entry, loadedProviderThreads }), + ] as const; + }); + yield* runtime.unloadThread!({ providerThread }); + // Failure or interruption retains the pointer and consumed credential. + yield* Ref.update(sessions, (current) => { + const entry = current.get(key); + if ( + entry?.runtime !== runtime || + entry.loadedProviderThreads.get(nativeKey) !== unloading + ) + return current; + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + loadedProviderThreads.delete(nativeKey); + return new Map(current).set(key, { ...entry, loadedProviderThreads }); + }); }); const ensureThreadAttached = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; + readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; + readonly checkAccess?: Parameters[4]; + readonly providerThread?: OrchestrationV2ProviderThread; }) => Effect.suspend(() => { let attachedTo: ProviderAdapter.ProviderAdapterV2SessionRuntime | undefined; @@ -1437,25 +1780,54 @@ export const layerWithOptions = ( const attached = yield* attachThread(input).pipe( // Recorded with no gap for an interrupt: cleanup undoes only an // attach this call made, never one an earlier open made. - Effect.tap((runtime) => Effect.sync(() => (attachedTo = runtime))), + Effect.tap((attached) => + Effect.sync(() => { + if (attached) attachedTo = input.runtime; + }), + ), Effect.uninterruptible, ); - if (attached !== undefined) { - const prepared = yield* prepareMcpSession(input.threadId, input.providerInstanceId); - preparedForCleanup = prepared; + if (attached === undefined) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "The provider session was released or replaced before attaching.", + }); + } + // A cached runtime may still be attached after another provider + // rotated this thread's token. Validate it on every use, so its + // next resume can reload the native MCP client with a live token. + const prepared = yield* prepareMcpSession( + input.threadId, + input.providerInstanceId, + (prepared) => (preparedForCleanup = prepared), + input.providerSessionId, + input.checkAccess, + input.providerThread, + ); + const recorded = yield* Ref.modify(sessions, (current) => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + if (entry === undefined || entry.runtime !== input.runtime) { + return [false, current] as const; + } + const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); if (prepared.mcpCredentialId !== undefined) { - const mcpCredentialId = prepared.mcpCredentialId; - yield* Ref.update(sessions, (current) => { - const key = sessionKey(input.providerSessionId); - const entry = current.get(key); - if (entry === undefined) return current; - const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); - mcpCredentialIdByThread.set(input.threadId, mcpCredentialId); - const updated = new Map(current); - updated.set(key, { ...entry, mcpCredentialIdByThread }); - return updated; - }); + mcpCredentialIdByThread.set(input.threadId, prepared.mcpCredentialId); } + const updated = new Map(current); + updated.set(key, { ...entry, mcpCredentialIdByThread }); + // Publish ownership and remove the reservation in one synchronous + // update, so a racing release cannot leave this token unowned. + dropReservation(); + return [true, updated] as const; + }); + if (!recorded) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "The provider session was released or replaced while preparing MCP access.", + }); + } + if (attached) { const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId)); if (entry !== undefined) { yield* withActivityError( @@ -1469,47 +1841,32 @@ export const layerWithOptions = ( ); } } + return prepared; }).pipe( // An interrupted attach is undone too, so the next attach writes // the attachment instead of finding the thread already attached. Effect.onError(() => - attachedTo === undefined + (attachedTo === undefined ? Effect.void - : removeThreadAttachment({ ...input, runtime: attachedTo }).pipe( - Effect.andThen( - Effect.suspend(() => { - dropReservation(); - // Revoke only a credential this attach freshly minted: a - // REUSED credential is by definition held by another - // live provider process, and revoking it thread-wide - // would break that process's MCP client mid-conversation. - if (preparedForCleanup?.issued !== true) return Effect.void; - const mcpCredentialId = preparedForCleanup.mcpCredentialId; - const attachedRuntime = attachedTo; - // As in release: a replacement session (or an open - // configuring one) may have taken the credential up. - return Ref.get(sessions).pipe( - Effect.flatMap((current) => - (mcpCredentialId !== undefined && - isMcpCredentialReserved(input.threadId, mcpCredentialId)) || - Array.from(current.values()).some( - (other) => - other.runtime !== attachedRuntime && - (other.attachedThreadIds.has(input.threadId) || - (mcpCredentialId !== undefined && - other.mcpCredentialIdByThread.get(input.threadId) === - mcpCredentialId)), - ) - ? Effect.void - : clearMcpSession(input.threadId, mcpCredentialId), - ), + : removeThreadAttachment({ ...input, runtime: attachedTo }) + ).pipe( + Effect.andThen( + Effect.suspend(() => { + dropReservation(); + const id = preparedForCleanup?.mcpCredentialId; + return id === undefined + ? Effect.void + : clearUnownedMcpSession( + input.threadId, + id, + preparedForCleanup?.issued ? attachedTo : undefined, ); - }), - ), - ), + }), + ), + ), ), ); - return threadAttachment.withLock(threadAttachmentKey(input), attach).pipe( + return attach.pipe( // The entry's own record (written above while the thread is // attached) guards the credential from here on; the reservation // is only needed until then. Ensuring covers defects/interrupts. @@ -1526,7 +1883,7 @@ export const layerWithOptions = ( const idleFiber = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); if (entry === undefined) { - return [null, current] as const; + return [undefined, current] as const; } const updated = new Map(current); updated.set(key, { @@ -1538,6 +1895,12 @@ export const layerWithOptions = ( }); return [entry.idleFiber, updated] as const; }); + if (idleFiber === undefined) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "The provider session was released before registering the turn.", + }); + } yield* cancelIdleFiber(idleFiber); }), ); @@ -1701,6 +2064,7 @@ export const layerWithOptions = ( .hasPendingBackgroundWorkForThread(pending.providerThread) .pipe(Effect.catchCause(() => Effect.succeed(false))); if (hasPendingWork) return "deferred" as const; + const retainedUnload = { ...pending, fiber: null }; const unloading = yield* Ref.modify(sessions, (current) => { const latest = current.get(key); if ( @@ -1709,33 +2073,48 @@ export const layerWithOptions = ( ) { return [false, current] as const; } - const loadedProviderThreadKeyByThread = new Map( - latest.loadedProviderThreadKeyByThread, - ); - loadedProviderThreadKeyByThread.delete(input.threadId); const idleThreadUnloads = new Map(latest.idleThreadUnloads); - idleThreadUnloads.delete(input.threadId); + idleThreadUnloads.set(input.threadId, retainedUnload); const updated = new Map(current); updated.set(key, { ...latest, - loadedProviderThreadKeyByThread, idleThreadUnloads, }); return [true, updated] as const; }); if (!unloading) return "skipped" as const; - yield* unloadThread({ providerThread: pending.providerThread }).pipe( + const unloaded = yield* unloadThread({ providerThread: pending.providerThread }).pipe( Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.as(true), Effect.catchCause((cause) => Effect.logWarning("orchestration-v2.driver-session.idle-unload-failed", { providerSessionId: input.providerSessionId, threadId: input.threadId, providerThreadId: pending.providerThread.id, cause, - }), + }).pipe(Effect.as(false)), ), ); - return "unloaded" as const; + if (unloaded) { + yield* Ref.update(sessions, (current) => { + const latest = current.get(key); + if ( + latest?.runtime !== entry.runtime || + latest.idleThreadUnloads.get(input.threadId) !== retainedUnload + ) { + return current; + } + const idleThreadUnloads = new Map(latest.idleThreadUnloads); + idleThreadUnloads.delete(input.threadId); + const updated = new Map(current); + updated.set(key, { + ...latest, + idleThreadUnloads, + }); + return updated; + }); + } + return unloaded ? ("unloaded" as const) : ("failed" as const); }), ); // Re-check on this fiber after another idle window, outside the @@ -1817,33 +2196,125 @@ export const layerWithOptions = ( modelFamily: normalizeModelMetricLabel(model), }, }); + const withRuntimeThread = ( + threadId: ThreadId, + providerThread: OrchestrationV2ProviderThread | undefined, + use: (prepared: PreparedMcpCredential) => Effect.Effect, + ) => + threadAttachment.withLock( + threadAttachmentKey({ providerSessionId, threadId }), + mcpPrepareLock.withLock( + threadId, + // Adapters must consume the config before returning acceptance. + ensureThreadAttached({ + providerSessionId, + threadId, + providerInstanceId: runtime.instanceId, + runtime, + ...(providerThread === undefined ? {} : { providerThread }), + checkAccess: (candidateId, willRotate) => + Effect.gen(function* () { + const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); + if (entry?.runtime !== runtime) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: + "The provider session was released or replaced before preparing MCP access.", + }); + } + if ( + runtime.unloadThread === undefined && + (willRotate || candidateId !== entry.mcpCredentialIdByThread.get(threadId)) + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "MCP access changed; reopen the provider session before continuing.", + }); + } + }), + }).pipe(Effect.flatMap(use)), + ), + ); + const resumeNativeThread = ( + input: Parameters[0] & { + readonly threadId: ThreadId; + }, + staleMcpCredential: boolean, + mcpCredentialId: string | undefined, + ) => { + // Codex keeps its MCP clients in a loaded native conversation. + // A resume alone retains the revoked token; unload only this + // conversation before resuming it with the refreshed config. + const unload = staleMcpCredential + ? unloadNativeThread(runtime, input.providerThread).pipe( + Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: runtime.driver, + providerSessionId, + providerThreadId: input.providerThread.id, + cause, + }), + ), + ) + : Effect.void; + return unload.pipe( + // Resume may consume the config before its interruptible wait returns. + // Retain that ownership even when acceptance is never published. + Effect.andThen( + markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId: input.threadId, + providerThread: input.providerThread, + mcpCredentialId, + providerThreadKey: undefined, + }), + ), + Effect.andThen(runtime.resumeThread(input)), + ); + }; return { ...runtime, subscribeEvents, + ...(runtime.unloadThread === undefined + ? {} + : { + unloadThread: ({ providerThread }) => unloadNativeThread(runtime, providerThread), + }), events: Stream.unwrap( subscribeEvents.pipe(Effect.map((subscription) => subscription.events)), ), ensureThread: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - Effect.andThen(runtime.ensureThread(input)), - Effect.tap((providerThread) => - markProviderThreadLoaded({ - providerSessionId, - threadId: input.threadId, - providerThreadKey: providerThreadLoadKey({ + withRuntimeThread(input.threadId, undefined, (prepared) => + runtime.ensureThread(input).pipe( + Effect.tap((providerThread) => + markProviderThreadLoaded({ + runtime, providerThread, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, + mcpCredentialId: prepared.mcpCredentialId, + providerSessionId, + threadId: input.threadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + }), }), - }), + ), ), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterEnsureThreadError({ + driver: runtime.driver, + threadId: input.threadId, + cause, + }), + ), + }), ), resumeThread: (input) => { const threadId = input.threadId ?? input.providerThread.appThreadId; @@ -1857,111 +2328,225 @@ export const layerWithOptions = ( : { modelSelection: input.modelSelection }), ...(input.runtimePolicy === undefined ? {} : { runtimePolicy: input.runtimePolicy }), }); - return observeActivity( - providerSessionId, - ensureThreadAttached({ + return withRuntimeThread(threadId, input.providerThread, (prepared) => + holdThreadLoaded({ providerSessionId, threadId, - providerInstanceId: runtime.instanceId, - }), + providerThread: input.providerThread, + }).pipe( + Effect.andThen( + providerThreadLoadStatus({ + providerSessionId, + threadId, + providerThread: input.providerThread, + providerThreadKey, + }), + ), + Effect.flatMap(({ loaded, staleMcpCredential }) => { + if (loaded) return Effect.succeed(input.providerThread); + return resumeNativeThread( + { ...input, threadId }, + staleMcpCredential, + prepared.mcpCredentialId, + ); + }), + Effect.tap((providerThread) => + markProviderThreadLoaded({ + runtime, + providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerSessionId, + threadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + ...(input.modelSelection === undefined + ? {} + : { modelSelection: input.modelSelection }), + ...(input.runtimePolicy === undefined + ? {} + : { runtimePolicy: input.runtimePolicy }), + }), + }), + ), + ), ).pipe( - Effect.andThen( - holdThreadLoaded({ + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: runtime.driver, + providerSessionId, + providerThreadId: input.providerThread.id, + cause, + }), + ), + }), + ); + }, + rollbackThread: (input) => { + const threadId = input.providerThread.appThreadId; + if (threadId === null) return runtime.rollbackThread(input); + return withRuntimeThread(threadId, input.providerThread, (prepared) => + Effect.gen(function* () { + yield* holdThreadLoaded({ providerSessionId, threadId, providerThread: input.providerThread, - }), - ), - Effect.andThen( - isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey }), - ), - Effect.flatMap((loaded) => - loaded ? Effect.succeed(input.providerThread) : runtime.resumeThread(input), - ), - Effect.tap((providerThread) => - markProviderThreadLoaded({ + }); + const cachedKey = (yield* Ref.get(sessions)) + .get(sessionKey(providerSessionId)) + ?.loadedProviderThreads.get(providerThreadRuntimeKey(input.providerThread))?.key; + const { loaded, staleMcpCredential } = yield* providerThreadLoadStatus({ providerSessionId, threadId, - providerThreadKey: providerThreadLoadKey({ - providerThread, - ...(input.modelSelection === undefined - ? {} - : { modelSelection: input.modelSelection }), - ...(input.runtimePolicy === undefined - ? {} - : { runtimePolicy: input.runtimePolicy }), - }), - }), - ), + providerThread: input.providerThread, + providerThreadKey: cachedKey, + }); + const providerThread = loaded + ? input.providerThread + : yield* resumeNativeThread( + { providerThread: input.providerThread, threadId }, + staleMcpCredential, + prepared.mcpCredentialId, + ); + // Rollback has no selection/policy; a default resume establishes + // credential ownership without validating the next turn's load key. + const providerThreadKey = loaded ? cachedKey : undefined; + yield* markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId, + providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerThreadKey, + }); + const snapshot = yield* runtime.rollbackThread({ ...input, providerThread }); + yield* markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId, + providerThread: snapshot.providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerThreadKey: + providerThreadRuntimeKey(snapshot.providerThread) === + providerThreadRuntimeKey(providerThread) + ? providerThreadKey + : undefined, + }); + return snapshot; + }), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver: runtime.driver, + providerThreadId: input.providerThread.id, + cause, + }), + ), + }), ); }, forkThread: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.targetThreadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - Effect.andThen(runtime.forkThread(input)), - Effect.tap((providerThread) => - markProviderThreadLoaded({ - providerSessionId, - threadId: input.targetThreadId, - providerThreadKey: providerThreadLoadKey({ + withRuntimeThread(input.targetThreadId, undefined, (prepared) => + runtime.forkThread(input).pipe( + Effect.tap((providerThread) => + markProviderThreadLoaded({ + runtime, providerThread, - ...(input.modelSelection === undefined - ? {} - : { modelSelection: input.modelSelection }), - ...(input.runtimePolicy === undefined - ? {} - : { runtimePolicy: input.runtimePolicy }), + mcpCredentialId: prepared.mcpCredentialId, + providerSessionId, + threadId: input.targetThreadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + ...(input.modelSelection === undefined + ? {} + : { modelSelection: input.modelSelection }), + ...(input.runtimePolicy === undefined + ? {} + : { runtimePolicy: input.runtimePolicy }), + }), }), - }), + ), ), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterForkThreadError({ + driver: runtime.driver, + providerThreadId: input.sourceProviderThread.id, + cause, + }), + ), + }), ), startTurn: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - // A start that fails or is stopped may never emit turn.terminal, - // so it clears its own turn or the session never goes idle. If - // the adapter emits the terminal anyway, clearing the same turn - // again changes nothing, so another thread's turn on a shared - // session stays busy either way. - Effect.andThen( - holdThreadLoaded({ + withRuntimeThread(input.threadId, input.providerThread, (prepared) => + Effect.gen(function* () { + yield* holdThreadLoaded({ providerSessionId, threadId: input.threadId, providerThread: input.providerThread, - }), - ), - Effect.andThen( - Effect.acquireUseRelease( - observeActivity( - providerSessionId, - markBusy( - providerSessionId, - busyTurnKey(input.providerThread.id, input.runOrdinal), - ), - ), + }); + const { loaded, staleMcpCredential } = yield* providerThreadLoadStatus({ + providerSessionId, + threadId: input.threadId, + providerThread: input.providerThread, + providerThreadKey: providerThreadLoadKey(input), + }); + // Retained continuations can start without the usual resume. + const providerThread = !loaded + ? yield* resumeNativeThread( + { + providerThread: input.providerThread, + threadId: input.threadId, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + }, + staleMcpCredential, + prepared.mcpCredentialId, + ) + : input.providerThread; + yield* markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId: input.threadId, + providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerThreadKey: providerThreadLoadKey({ ...input, providerThread }), + }); + // A failed/stopped start may never emit a terminal event. It + // clears only its own registration, while accepted turns stay busy. + return yield* Effect.acquireUseRelease( + markBusy(providerSessionId, busyTurnKey(providerThread.id, input.runOrdinal)), () => - runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), + runtime + .startTurn({ ...input, providerThread }) + .pipe(turnMetrics("send", input.modelSelection.model)), (_, exit) => Exit.isFailure(exit) ? observeActivity( providerSessionId, - markIdle(providerSessionId, input.providerThread.id, input.runOrdinal), + markIdle(providerSessionId, providerThread.id, input.runOrdinal), ) : Effect.void, - ), - ), + ); + }), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: runtime.driver, + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + cause, + }), + ), + }), ), steerTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( @@ -2153,199 +2738,255 @@ export const layerWithOptions = ( return ProviderSessionManagerV2.of({ shutdown, open: (input) => - sessionOpen.withLock( - input.providerSessionId, - Effect.gen(function* () { - const cwd = input.runtimePolicy.cwd; - if (cwd !== null) { - const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe( - Effect.map((stat) => stat.type === "Directory"), - Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")), - ); - if (!workspaceIsDirectory) { - return yield* new ProviderWorkspaceMissingError({ - threadId: input.threadId, - cwd, - }); - } + Effect.suspend(() => { + let preparedForCleanup: PreparedMcpCredential | undefined; + let openingScope: Scope.Closeable | undefined; + let reservationDropped = false; + const dropReservation = Effect.sync(() => { + const id = preparedForCleanup?.mcpCredentialId; + if (!reservationDropped && id !== undefined) { + reservationDropped = true; + dropMcpCredentialReservation(input.threadId, id); } - const key = sessionKey(input.providerSessionId); - const existing = (yield* Ref.get(sessions)).get(key); - if (existing !== undefined) { - const attached = existing.attachedThreadIds.has(input.threadId); - if (!attached && !existing.supportsMultipleProviderThreads) { - return yield* new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, - providerSessionId: input.providerSessionId, - cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`, - }); - } - // Joining a shared session is a new session for this thread. - if (!attached) yield* agentScope.clear(input.threadId); - yield* ensureThreadAttached({ - providerSessionId: input.providerSessionId, - threadId: input.threadId, - providerInstanceId: existing.runtime.instanceId, - }); - yield* touchActivity(input.providerSessionId); - return existing.exposedRuntime; - } - - const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( - Effect.mapError( - (cause) => - new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, + }); + // Any failure, including a Stop that interrupts a slow handshake, + // stops the provider process this open started. The close is + // time-boxed: this runs under the session's open lock, so a wedged + // adapter finalizer must not hold the interrupter or later opens. + const cleanupOpen = (cause: Cause.Cause): Effect.Effect => + Ref.get(sessions).pipe( + Effect.flatMap((current) => { + const scope = openingScope; + if ( + scope !== undefined && + current.get(sessionKey(input.providerSessionId))?.scope === scope + ) { + // The open lock prevents another open replacing this entry + // while failure cleanup hands it to the normal lifecycle. + return releaseEntry({ providerSessionId: input.providerSessionId, - cause, - }), - ), - ); - const prepared = yield* prepareMcpSession( - input.threadId, - input.modelSelection.instanceId, + reason: Cause.hasInterruptsOnly(cause) ? "manual_shutdown" : "runtime_error", + detail: Cause.hasInterruptsOnly(cause) + ? "The provider session start was interrupted." + : "The provider session did not finish opening.", + }).pipe(logReleaseFailure(input.providerSessionId)); + } + return scope === undefined + ? Effect.void + : closeScopeWithin(scope, { + providerSessionId: input.providerSessionId, + reason: "open_failed", + }); + }), ); - const mcpCredentialId = prepared.mcpCredentialId; - // The reservation from prepare protects the credential (which - // eager adapters bake into the provider process during - // openSession) from racing releases until this session's entry - // is recorded below. Dropped exactly once on every path. - let reservationDropped = mcpCredentialId === undefined; - const dropReservation = Effect.sync(() => { - if (!reservationDropped && mcpCredentialId !== undefined) { - reservationDropped = true; - dropMcpCredentialReservation(input.threadId, mcpCredentialId); + return sessionOpen.withLock( + input.providerSessionId, + Effect.gen(function* () { + const cwd = input.runtimePolicy.cwd; + if (cwd !== null) { + const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe( + Effect.map((stat) => stat.type === "Directory"), + Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")), + ); + if (!workspaceIsDirectory) { + return yield* new ProviderWorkspaceMissingError({ + threadId: input.threadId, + cwd, + }); + } } - }); - const sessionScope = yield* Scope.fork(sessionScopes); - // The previous session's agent scope must not explain this - // session's failures, even when this provider runs without one. - yield* agentScope.clear(input.threadId); - const runtime = yield* adapter - .openSession({ - threadId: input.threadId, - providerSessionId: input.providerSessionId, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, - ...(input.resumeFromSession === undefined - ? {} - : { resumeFromSession: input.resumeFromSession }), - ...(input.initialNativeThreadId === undefined - ? {} - : { initialNativeThreadId: input.initialNativeThreadId }), - ...(input.initialProviderItemIdentityVersion === undefined - ? {} - : { - initialProviderItemIdentityVersion: - input.initialProviderItemIdentityVersion, - }), - }) - .pipe( - Effect.provideService(Scope.Scope, sessionScope), - // Any failure, including a Stop that interrupts a slow - // handshake, stops the provider process this open started. - // The session cleanup runs first, and the close is - // time-boxed: this runs under the session's open lock, so an - // adapter finalizer that never finishes must not hold the - // interrupter, the lock, or later opens. - Effect.onError(() => - dropReservation.pipe( - // Clear only a session this open freshly set up: a reused - // one is held by another live provider process and must - // survive this open's failure. - Effect.andThen( - prepared.issued - ? clearMcpSession(input.threadId, mcpCredentialId) - : Effect.void, + const key = sessionKey(input.providerSessionId); + const existing = (yield* Ref.get(sessions)).get(key); + if (existing !== undefined) { + const attached = existing.attachedThreadIds.has(input.threadId); + if (!attached && !existing.supportsMultipleProviderThreads) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`, + }); + } + if ( + attached && + existing.runtime.unloadThread === undefined && + (yield* mcpAccessNeedsRefresh(existing, input.threadId)) + ) { + // Pi and other process-bound MCP clients can only pick up + // changed access by reopening through the normal lifecycle. + yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "manual_shutdown", + requireIdle: true, + }).pipe( + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), ), - Effect.ensuring( - closeScopeWithin(sessionScope, { - providerSessionId: input.providerSessionId, - reason: "open_failed", - }), + ); + } else { + // Joining a shared session is a new session for this thread. + if (!attached) yield* agentScope.clear(input.threadId); + // Cached native clients prepare access at the operation + // boundary, where pending work can reject a reload safely. + if ( + !attached && + (existing.runtime.unloadThread === undefined || + !Array.from(existing.loadedProviderThreads.values()).some( + (loaded) => loaded.threadId === input.threadId, + )) + ) + yield* threadAttachment.withLock( + threadAttachmentKey(input), + mcpPrepareLock.withLock( + input.threadId, + ensureThreadAttached({ + providerSessionId: input.providerSessionId, + threadId: input.threadId, + providerInstanceId: existing.runtime.instanceId, + runtime: existing.runtime, + }), + ), + ); + yield* touchActivity(input.providerSessionId); + return existing.exposedRuntime; + } + } + + return yield* mcpPrepareLock.withLock( + input.threadId, + Effect.gen(function* () { + const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), ), - ), - ), - Effect.mapError( - (cause) => - new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, + ); + const prepared = yield* prepareMcpSession( + input.threadId, + input.modelSelection.instanceId, + (prepared) => (preparedForCleanup = prepared), + input.providerSessionId, + ); + const mcpCredentialId = prepared.mcpCredentialId; + // The reservation from prepare protects the credential (which + // eager adapters bake into the provider process during + // openSession) from racing releases until this session's entry + // is recorded below. Dropped exactly once on every path. + const sessionScope = yield* Scope.fork(sessionScopes).pipe( + Effect.tap((scope) => Effect.sync(() => (openingScope = scope))), + Effect.uninterruptible, + ); + // The previous session's agent scope must not explain this + // session's failures, even when this provider runs without one. + yield* agentScope.clear(input.threadId); + const runtime = yield* adapter + .openSession({ + threadId: input.threadId, providerSessionId: input.providerSessionId, - cause, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + ...(input.resumeFromSession === undefined + ? {} + : { resumeFromSession: input.resumeFromSession }), + ...(input.initialNativeThreadId === undefined + ? {} + : { initialNativeThreadId: input.initialNativeThreadId }), + ...(input.initialProviderItemIdentityVersion === undefined + ? {} + : { + initialProviderItemIdentityVersion: + input.initialProviderItemIdentityVersion, + }), + }) + .pipe( + Effect.provideService(Scope.Scope, sessionScope), + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), + ), + withMetrics({ + counter: providerSessionsTotal, + attributes: { provider: adapter.driver, operation: "open" }, + }), + ); + const eventSubscribers = yield* Ref.make< + ReadonlyMap> + >(new Map()); + const exposedRuntime = decorateRuntime(runtime, eventSubscribers); + const now = yield* Clock.currentTimeMillis; + const entry: LiveSessionEntry = { + attachedThreadIds: new Set([input.threadId]), + // Process-bound adapters consume MCP access during open, before + // the first native thread is loaded. Keep that credential until + // a successful native reload records its replacement. + loadedProviderThreads: new Map(), + mcpCredentialIdByThread: + mcpCredentialId === undefined + ? new Map() + : new Map([[input.threadId, mcpCredentialId]]), + supportsMultipleProviderThreads: + runtime.providerSession.capabilities.sessions + .supportsMultipleProviderThreadsPerSession, + runtime, + exposedRuntime, + eventSubscribers, + requestEventPermit: yield* Semaphore.make(1), + scope: sessionScope, + idleGeneration: 0, + busyTurns: new Set(), + lastActivityAtMs: now, + idleFiber: null, + pinnedSinceMs: null, + idleThreadUnloads: new Map(), + }; + yield* Ref.update(sessions, (current) => { + const updated = new Map(current); + updated.set(key, entry); + return updated; + }); + // The entry now guards the credential via its recorded id, so + // the pre-open reservation can be dropped. + yield* dropReservation; + yield* withActivityError( + input.providerSessionId, + writeProviderSessionEvents({ + runtime, + threadIds: [input.threadId], + type: "provider-session.attached", + payload: runtime.providerSession, }), + ); + yield* startEventPump(entry); + yield* scheduleIdleRelease(input.providerSessionId); + return exposedRuntime; + }).pipe( + // A waiter must see cleanup finished before preparing its config. + // Scope close stays outside this gate so provider finalizers cannot + // hold up another provider's open. + Effect.onError(() => + Effect.gen(function* () { + yield* dropReservation; + const id = preparedForCleanup?.mcpCredentialId; + if (id !== undefined) yield* clearUnownedMcpSession(input.threadId, id); + }), + ), ), - withMetrics({ - counter: providerSessionsTotal, - attributes: { provider: adapter.driver, operation: "open" }, - }), ); - const eventSubscribers = yield* Ref.make< - ReadonlyMap> - >(new Map()); - const exposedRuntime = decorateRuntime(runtime, eventSubscribers); - const now = yield* Clock.currentTimeMillis; - const entry: LiveSessionEntry = { - attachedThreadIds: new Set([input.threadId]), - loadedProviderThreadKeyByThread: new Map(), - mcpCredentialIdByThread: - mcpCredentialId === undefined - ? new Map() - : new Map([[input.threadId, mcpCredentialId]]), - supportsMultipleProviderThreads: - runtime.providerSession.capabilities.sessions - .supportsMultipleProviderThreadsPerSession, - runtime, - exposedRuntime, - eventSubscribers, - requestEventPermit: yield* Semaphore.make(1), - scope: sessionScope, - idleGeneration: 0, - busyTurns: new Set(), - lastActivityAtMs: now, - idleFiber: null, - pinnedSinceMs: null, - idleThreadUnloads: new Map(), - }; - yield* Ref.update(sessions, (current) => { - const updated = new Map(current); - updated.set(key, entry); - return updated; - }); - // The entry now guards the credential via its recorded id, so - // the pre-open reservation can be dropped. - yield* dropReservation; - yield* withActivityError( - input.providerSessionId, - writeProviderSessionEvents({ - runtime, - threadIds: [input.threadId], - type: "provider-session.attached", - payload: runtime.providerSession, - }), - ).pipe( - // Released on interrupt too: this entry has no event pump or - // idle timer yet, so nothing else would ever release it. - Effect.onError((cause) => - releaseEntry( - Cause.hasInterruptsOnly(cause) - ? { - providerSessionId: input.providerSessionId, - reason: "manual_shutdown", - detail: "The provider session start was interrupted.", - } - : { - providerSessionId: input.providerSessionId, - reason: "runtime_error", - detail: "Failed to persist the provider-session attachment.", - }, - ).pipe(logReleaseFailure(input.providerSessionId)), - ), - ); - yield* startEventPump(entry); - yield* scheduleIdleRelease(input.providerSessionId); - return exposedRuntime; - }), - ), + }).pipe(Effect.onError((cause) => cleanupOpen(cause))), + ); + }), get: (providerSessionId) => Effect.gen(function* () { const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); @@ -2464,10 +3105,11 @@ export const layerWithOptions = ( } const attachedThreadIds = new Set(entry.attachedThreadIds); attachedThreadIds.delete(input.threadId); - const loadedProviderThreadKeyByThread = new Map( - entry.loadedProviderThreadKeyByThread, - ); - loadedProviderThreadKeyByThread.delete(input.threadId); + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + for (const [id, loaded] of loadedProviderThreads) { + if (loaded.threadId === input.threadId) + loadedProviderThreads.set(id, { ...loaded, key: undefined }); + } // For a plain (workspace-change) detach, the credential id stays // recorded: the thread may re-attach and reuse it, and // releaseEntry revokes it when the provider process finally goes @@ -2481,13 +3123,15 @@ export const layerWithOptions = ( return pruned; })() : entry.mcpCredentialIdByThread; - // The detach unloads the thread itself below. + // Keep the native pointer until the detach's unload succeeds. const idleThreadUnloads = new Map(entry.idleThreadUnloads); - idleThreadUnloads.delete(input.threadId); + const pending = idleThreadUnloads.get(input.threadId); + if (pending !== undefined) + idleThreadUnloads.set(input.threadId, { ...pending, fiber: null }); const updatedEntry = { ...entry, attachedThreadIds, - loadedProviderThreadKeyByThread, + loadedProviderThreads, mcpCredentialIdByThread, idleThreadUnloads, }; @@ -2551,8 +3195,52 @@ export const layerWithOptions = ( ) { return; } + const pending = entry.idleThreadUnloads.get(input.threadId); + const providerThreads = new Map( + detachedProviderThreads.map( + (thread) => [providerThreadRuntimeKey(thread), thread] as const, + ), + ); + if (pending !== undefined) + providerThreads.set( + providerThreadRuntimeKey(pending.providerThread), + pending.providerThread, + ); + for (const loaded of entry.loadedProviderThreads.values()) { + if (loaded.threadId === input.threadId) + providerThreads.set( + providerThreadRuntimeKey(loaded.providerThread), + loaded.providerThread, + ); + } + const fallbackCredentialId = + entry.mcpCredentialIdByThread.get(input.threadId) ?? + Array.from(entry.loadedProviderThreads.values()).find( + (loaded) => loaded.threadId === input.threadId, + )?.mcpCredentialId; + yield* Ref.update(sessions, (current) => { + const latest = current.get(key); + if (latest?.runtime !== entry.runtime) return current; + const loadedProviderThreads = new Map(latest.loadedProviderThreads); + for (const providerThread of providerThreads.values()) { + if ( + providerThread.nativeThreadRef !== null && + !loadedProviderThreads.has(providerThreadRuntimeKey(providerThread)) + ) { + loadedProviderThreads.set(providerThreadRuntimeKey(providerThread), { + threadId: input.threadId, + providerThread, + key: undefined, + mcpCredentialId: fallbackCredentialId, + }); + } + } + return new Map(current).set(key, { ...latest, loadedProviderThreads }); + }); yield* Effect.forEach( - detachedProviderThreads.filter((thread) => thread.nativeThreadRef !== null), + [...providerThreads.values()].filter( + (thread) => thread.nativeThreadRef !== null, + ), (providerThread) => unloadThread({ providerThread }).pipe( // Bounded so a wedged provider cannot hold up the @@ -2572,6 +3260,22 @@ export const layerWithOptions = ( ), { concurrency: 1, discard: true }, ); + yield* Ref.update(sessions, (current) => { + const latest = current.get(key); + if ( + latest?.runtime !== entry.runtime || + latest.attachedThreadIds.has(input.threadId) || + latest.idleThreadUnloads.get(input.threadId) !== pending || + (pending !== undefined && + latest.loadedProviderThreads.has( + providerThreadRuntimeKey(pending.providerThread), + )) + ) + return current; + const idleThreadUnloads = new Map(latest.idleThreadUnloads); + idleThreadUnloads.delete(input.threadId); + return new Map(current).set(key, { ...latest, idleThreadUnloads }); + }); }), ); } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts index 525b9b4cff4e7..5c55bafcf78a9 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts @@ -23,9 +23,9 @@ export function assertThreadRollbackAfterRestartOutput( ); const revert = methods.indexOf("thread/revert"); assert.isAbove(revert, 0, "the rollback must reach thread/revert"); - assert.equal( - methods.lastIndexOf("thread/resume", revert), - revert - 2, - "the fresh app-server must resume the thread right before listing turns and reverting", + assert.deepEqual( + methods.slice(revert - 3, revert + 1), + ["thread/resume", "thread/read", "thread/turns/list", "thread/revert"], + "the fresh app-server must resume the thread before reading metadata, listing turns, and reverting", ); } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson index 2e5d54b8ddbce..0a71f53802020 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson @@ -48,14 +48,14 @@ {"type":"expect_outbound","label":"initialize","frame":{"id":1,"method":"initialize","params":{"capabilities":{"experimentalApi":true,"optOutNotificationMethods":["turn/diff/updated"],"extensions":{"io.modelcontextprotocol/ui":{"mimeTypes":["text/html;profile=mcp-app"]}}},"clientInfo":{"name":"T3 Code","title":"T3 Code","version":"0.0.44"}}}} {"type":"emit_inbound","label":"initialize","frame":{"id":1,"result":{"userAgent":"t3code_desktop/0.156.1 (Debian 13.0.0; x86_64) unknown (t3code_desktop; 0.1.0)","codexHome":"/home/replay-user/tmp/bughunt-codex/codex-home","platformFamily":"unix","platformOs":"linux"}}} {"type":"expect_outbound","label":"initialized","frame":{"method":"initialized"}} -{"type":"expect_outbound","label":"thread/read","frame":{"id":2,"method":"thread/read","params":{"includeTurns":false,"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} +{"type":"expect_outbound","label":"thread/resume","frame":{"id":2,"method":"thread/resume","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000},"cwd":"","excludeTurns":true,"model":"gpt-6-luna","threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} {"type":"emit_inbound","label":"configWarning","frame":{"method":"configWarning","params":{"summary":"Codex could not find bubblewrap on PATH. Install bubblewrap with your OS package manager. See the sandbox prerequisites: https://developers.openai.com/codex/concepts/sandboxing#prerequisites. Codex will use the bundled bubblewrap in the meantime.","details":null},"emittedAtMs":1790404935235}} {"type":"emit_inbound","label":"remoteControl/status/changed","frame":{"method":"remoteControl/status/changed","params":{"status":"disabled","serverName":"replay-host","installationId":"00000000-0000-4000-8000-000000000000","environmentId":null},"emittedAtMs":1790404935235}} -{"type":"emit_inbound","label":"thread/read","frame":{"id":2,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":null,"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404934,"recencyAt":1790404931,"status":{"type":"notLoaded"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":null,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]}}}} -{"type":"expect_outbound","label":"thread/resume","frame":{"id":3,"method":"thread/resume","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000},"cwd":"","excludeTurns":true,"model":"gpt-6-luna","threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1790404935482}} {"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","status":{"type":"idle"}},"emittedAtMs":1790404935500}} -{"type":"emit_inbound","label":"thread/resume","frame":{"id":3,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":[{"environmentId":"local","cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"]}],"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404935,"recencyAt":1790404931,"status":{"type":"idle"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]},"model":"gpt-6-luna","modelProvider":"openai","serviceTier":null,"disabledPluginIds":[],"cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"],"instructionSources":["/home/replay-user/t3code/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"workspaceWrite","writableRoots":[],"networkAccess":false,"excludeTmpdirEnvVar":false,"excludeSlashTmp":false},"activePermissionProfile":{"id":":workspace","extends":null},"reasoningEffort":null,"collaborationMode":{"mode":"default","settings":{"model":"gpt-6-luna","reasoning_effort":null,"developer_instructions":null}},"multiAgentMode":"explicitRequestOnly","initialTurnsPage":null,"turnsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":16,\"includeAnchor\":true,\"scope\":{\"kind\":\"turns\"}}","itemsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":22,\"includeAnchor\":true,\"scope\":{\"kind\":\"itemsByCreatedAtOrdinal\"}}"}}} +{"type":"emit_inbound","label":"thread/resume","frame":{"id":2,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":[{"environmentId":"local","cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"]}],"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404935,"recencyAt":1790404931,"status":{"type":"idle"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]},"model":"gpt-6-luna","modelProvider":"openai","serviceTier":null,"disabledPluginIds":[],"cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"],"instructionSources":["/home/replay-user/t3code/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"workspaceWrite","writableRoots":[],"networkAccess":false,"excludeTmpdirEnvVar":false,"excludeSlashTmp":false},"activePermissionProfile":{"id":":workspace","extends":null},"reasoningEffort":null,"collaborationMode":{"mode":"default","settings":{"model":"gpt-6-luna","reasoning_effort":null,"developer_instructions":null}},"multiAgentMode":"explicitRequestOnly","initialTurnsPage":null,"turnsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":16,\"includeAnchor\":true,\"scope\":{\"kind\":\"turns\"}}","itemsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":22,\"includeAnchor\":true,\"scope\":{\"kind\":\"itemsByCreatedAtOrdinal\"}}"}}} +{"type":"expect_outbound","label":"thread/read","frame":{"id":3,"method":"thread/read","params":{"includeTurns":false,"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} +{"type":"emit_inbound","label":"thread/read","frame":{"id":3,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":null,"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404934,"recencyAt":1790404931,"status":{"type":"idle"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":null,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]}}}} {"type":"expect_outbound","label":"thread/turns/list","frame":{"id":4,"method":"thread/turns/list","params":{"cursor":null,"itemsView":"summary","limit":1,"sortDirection":"desc","threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} {"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","turnId":"01a0dc73-2dc4-7911-a448-c66b9bf3318b","tokenUsage":{"total":{"totalTokens":44003,"inputTokens":43985,"cachedInputTokens":19200,"cacheWriteInputTokens":0,"outputTokens":18,"reasoningOutputTokens":0},"last":{"totalTokens":24198,"inputTokens":24189,"cachedInputTokens":19200,"cacheWriteInputTokens":0,"outputTokens":9,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1790404935520}} {"type":"emit_inbound","label":"thread/goal/cleared","frame":{"method":"thread/goal/cleared","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"},"emittedAtMs":1790404935521}} diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 785bf1a06e70c..c68e780b99df2 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -4206,13 +4206,14 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr Effect.gen(function* () { const nativeId = providerThread.nativeThreadRef?.nativeId; const state = nativeId == null ? undefined : threads.get(nativeId); - if ( - nativeId == null || - state === undefined || - state.active !== undefined || - hasBackground(state) - ) { - return; + if (nativeId == null || state === undefined) return; + // Success lets the manager drop native credential ownership; a busy + // native must fail instead of reporting a successful no-op. + if (state.active !== undefined || hasBackground(state)) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: OPENCODE_PROVIDER, + detail: `Cannot unload OpenCode session ${nativeId} while it is still working`, + }); } threads.delete(nativeId); for (const [child, owner] of childOwners) {