From 750c2ca15c3188879a6a673f9200e8d4ba6285c6 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Mon, 12 Oct 2026 22:32:28 -0400 Subject: [PATCH 1/2] fix(server): refresh cached MCP credentials after provider switches Revalidate reused session credentials and reload only native conversations whose MCP token changed. Preserve foreground and background work during refresh, and reopen idle process-bound clients through the existing session lifecycle. Guard every known credential owner across provider sessions and native siblings before rotation. Hold preparation through adapter consumption and acceptance, while keeping event ingestion and replies available. Keep credential ownership exact across cancellation, publication, release, and replacement. Co-Authored-By: GPT-6.1-Sol at high effort Co-Authored-By: GPT-6.1-Sol at high effort, implementation and review subagent for provider credentials Co-Authored-By: GPT-6.1-Sol at xhigh effort, review subagent for credential lifecycle Co-Authored-By: GPT-6-Astra at high effort, design review subagents for credential ownership Co-Authored-By: Claude Opus 5.5 at high effort, design review and cleanup subagents for credential ownership Co-Authored-By: Claude Opus 5.5, review subagent for credential cleanup Upstream-PR: https://github.com/pingdotgg/t3code/pull/18884 --- .../Adapters/OpenCode2AdapterV2.test.ts | 33 +- .../ProviderSessionManager.test.ts | 2395 +++++++++++++++-- .../ProviderSessionManager.ts | 1541 ++++++++--- .../codex_output.ts | 8 +- .../codex_transcript.ndjson | 8 +- .../src/server/v2/adapter.ts | 15 +- 6 files changed, 3392 insertions(+), 608 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 548ff5cb6643b..e4eb6feb4328c 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -833,6 +833,11 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { // The subagent still runs, so the session is not idle and the next Stop reaches it. assert.isTrue(yield* runtime.hasPendingBackgroundWork!); assert.isTrue(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); + assert.equal( + (yield* runtime.unloadThread!({ providerThread: thread }).pipe(Effect.flip))._tag, + "ProviderAdapterProtocolError", + ); + assert.isTrue(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); }).pipe(Effect.scoped), ); @@ -3400,7 +3405,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ); it.effect( - "registers T3's MCP server for the thread alone and removes it when the thread unloads", + "registers T3's MCP server for the thread alone, retains it when unload is declined, and removes it after work stops", () => Effect.gen(function* () { const mcpSessions = yield* McpProviderSessions.McpProviderSessions; @@ -3429,15 +3434,29 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }, }), reply("mcp.add", null), - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.succeeded", { sessionID: SESSION }), + ...backgroundLaunch(CHILD), + out("session.interrupt", { sessionID: CHILD }), + reply("session.interrupt", { interrupted: true }), + out("session.interrupt", { sessionID: SESSION }), + reply("session.interrupt", { interrupted: true }), + event("session.execution.interrupted", { sessionID: SESSION }), out("mcp.remove", { server, "location[directory]": WORK }), reply("mcp.remove", null), ]); - const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); - yield* runtime.startTurn(turnInput(thread)); - assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + const watch = yield* watchBackgroundTurn(runtime); + yield* runtime.startTurn(withLineage(thread)); + yield* watch.attached; + assert.equal( + (yield* runtime.unloadThread!({ providerThread: thread }).pipe(Effect.flip))._tag, + "ProviderAdapterProtocolError", + ); + yield* runtime.interruptTurn({ + providerThread: thread, + providerTurnId: yield* providerTurnId, + requestRuntimeRestart: true, + }); + assert.equal((yield* watch.terminal)?.status, "interrupted"); + assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread)); yield* runtime.unloadThread!({ providerThread: thread }); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 36fa2ced0d551..d3bc1317e1d0f 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3,6 +3,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { AgentScope } from "@t3tools/shared/AgentScope"; import { + CheckpointId, EnvironmentId, type ModelSelection, type OrchestrationV2AppThread, @@ -15,6 +16,7 @@ import { ProviderDriverKind, ProviderInstanceId, type ProviderSessionId, + ProviderThreadId, ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -323,22 +325,31 @@ function unimplemented(detail: string) { const makeProviderAdapter = Effect.fnUntraced(function* ( state: Ref.Ref, options: { + readonly instanceId?: ProviderInstanceId; readonly failEventStream?: boolean; readonly capabilities?: OrchestrationV2ProviderCapabilities; readonly mcpConfigs?: Ref.Ref< ReadonlyArray >; + readonly resumeMcpConfigs?: Ref.Ref< + ReadonlyArray + >; readonly beforeOpen?: (input: { readonly providerSessionId: ProviderSessionId; readonly initialProviderItemIdentityVersion?: 2; }) => Effect.Effect; readonly hasPendingBackgroundWork?: Effect.Effect; - readonly hasPendingBackgroundWorkForThread?: Effect.Effect; + readonly hasPendingBackgroundWorkForThread?: + | Effect.Effect + | ((providerThread: OrchestrationV2ProviderThread) => Effect.Effect); /** False models a runtime whose background tasks can outlive its process. */ readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly rollbackThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["rollbackThread"]; readonly beforeUnload?: Effect.Effect; + readonly beforeClose?: Effect.Effect; + readonly withoutUnload?: boolean; /** Registers the process's closeCount finalizer before `beforeOpen` runs. */ readonly spawnBeforeOpen?: boolean; /** Completed when a hanging scope close reaches its wedged finalizer. */ @@ -347,10 +358,14 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ) { const mcpSessions = yield* McpProviderSessions.McpProviderSessions; const countClose = Effect.addFinalizer(() => - Ref.update(state, (current) => ({ - ...current, - closeCount: current.closeCount + 1, - })), + (options.beforeClose ?? Effect.void).pipe( + Effect.andThen( + Ref.update(state, (current) => ({ + ...current, + closeCount: current.closeCount + 1, + })), + ), + ), ); // Registered after countClose so it runs first on scope close, wedging the // close before the closeCount finalizer, like a provider process that never @@ -362,7 +377,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ).pipe(Effect.andThen(Effect.never)), ); return ProviderAdapter.ProviderAdapterV2.of({ - instanceId: ProviderInstanceId.make("codex"), + instanceId: options.instanceId ?? ProviderInstanceId.make("codex"), driver: CODEX_DRIVER, getCapabilities: () => Effect.succeed(options.capabilities ?? CodexCapabilities), planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), @@ -390,6 +405,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( now, ...(options.capabilities === undefined ? {} : { capabilities: options.capabilities }), }); + const instanceId = options.instanceId ?? modelSelection.instanceId; yield* Ref.update(state, (current) => { const eventQueues = new Map(current.eventQueues); eventQueues.set(String(input.providerSessionId), events); @@ -405,10 +421,10 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( } return { - instanceId: ProviderInstanceId.make("codex"), + instanceId, driver: CODEX_DRIVER, providerSessionId: input.providerSessionId, - providerSession: session, + providerSession: { ...session, providerInstanceId: instanceId }, events: options.failEventStream ? Stream.fail( new ProviderAdapter.ProviderAdapterEventStreamError({ @@ -427,14 +443,30 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ...(options.hasPendingBackgroundWorkForThread === undefined ? {} : { - hasPendingBackgroundWorkForThread: () => options.hasPendingBackgroundWorkForThread!, + hasPendingBackgroundWorkForThread: (providerThread) => + typeof options.hasPendingBackgroundWorkForThread === "function" + ? options.hasPendingBackgroundWorkForThread(providerThread) + : options.hasPendingBackgroundWorkForThread!, }), ensureThread: () => unimplemented("ensureThread unused in test"), resumeThread: (threadInput) => Ref.update(state, (current) => ({ ...current, resumeCount: current.resumeCount + 1, - })).pipe(Effect.as(threadInput.providerThread)), + })).pipe( + Effect.andThen( + options.resumeMcpConfigs === undefined + ? Effect.void + : mcpSessions + .read(threadInput.providerThread.appThreadId!) + .pipe( + Effect.flatMap((config) => + Ref.update(options.resumeMcpConfigs!, (configs) => [...configs, config]), + ), + ), + ), + Effect.as(threadInput.providerThread), + ), startTurn: () => options.startTurn ?? Effect.void, steerTurn: () => Effect.void, interruptTurn: () => @@ -442,21 +474,30 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ...current, interruptCount: current.interruptCount + 1, })), - unloadThread: ({ providerThread }) => - (options.beforeUnload ?? Effect.void).pipe( - Effect.andThen( - Ref.update(state, (current) => ({ - ...current, - unloadedNativeThreadIds: [ - ...current.unloadedNativeThreadIds, - providerThread.nativeThreadRef?.nativeId ?? "", - ], - })), - ), - ), + ...(options.withoutUnload + ? {} + : { + unloadThread: ({ + providerThread, + }: Parameters< + NonNullable + >[0]) => + (options.beforeUnload ?? Effect.void).pipe( + Effect.andThen( + Ref.update(state, (current) => ({ + ...current, + unloadedNativeThreadIds: [ + ...current.unloadedNativeThreadIds, + providerThread.nativeThreadRef?.nativeId ?? "", + ], + })), + ), + ), + }), respondToRuntimeRequest: () => Effect.void, readThreadSnapshot: () => unimplemented("readThreadSnapshot unused in test"), - rollbackThread: () => unimplemented("rollbackThread unused in test"), + rollbackThread: (input) => + options.rollbackThread?.(input) ?? unimplemented("rollbackThread unused in test"), forkThread: () => unimplemented("forkThread unused in test"), } satisfies ProviderAdapter.ProviderAdapterV2SessionRuntime; }), @@ -464,6 +505,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( }); function layerTest(input: { + readonly additionalAdapters?: ReadonlyArray>; readonly state: Ref.Ref; readonly idleTimeoutMs: number; readonly maxIdlePinMs?: number; @@ -472,6 +514,9 @@ function layerTest(input: { readonly mcpConfigs?: Ref.Ref< ReadonlyArray >; + readonly resumeMcpConfigs?: Ref.Ref< + ReadonlyArray + >; readonly beforeOpen?: (input: { readonly providerSessionId: ProviderSessionId; readonly initialProviderItemIdentityVersion?: 2; @@ -486,12 +531,23 @@ function layerTest(input: { readonly armed: Ref.Ref; readonly paused: Deferred.Deferred; }; + readonly pauseIssue?: { + readonly armed: Ref.Ref; + readonly issued: Deferred.Deferred; + readonly resume: Deferred.Deferred; + }; readonly hasPendingBackgroundWork?: Effect.Effect; - readonly hasPendingBackgroundWorkForThread?: Effect.Effect; + readonly hasPendingBackgroundWorkForThread?: + | Effect.Effect + | ((providerThread: OrchestrationV2ProviderThread) => Effect.Effect); readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly rollbackThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["rollbackThread"]; readonly beforeUnload?: Effect.Effect; + readonly beforeClose?: Effect.Effect; + readonly mcpRegistryLayer?: Layer.Layer; + readonly withoutUnload?: boolean; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; readonly serverSettingsLayer?: ReturnType; @@ -508,35 +564,49 @@ function layerTest(input: { ? layerFailingReleaseEventSink : layerTestEventSink; const layerRegistry = Layer.unwrap( - makeProviderAdapter(input.state, { - failEventStream: input.failEventStream ?? false, - ...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }), - ...(input.mcpConfigs === undefined ? {} : { mcpConfigs: input.mcpConfigs }), - ...(input.beforeOpen === undefined ? {} : { beforeOpen: input.beforeOpen }), - ...(input.hasPendingBackgroundWork === undefined - ? {} - : { hasPendingBackgroundWork: input.hasPendingBackgroundWork }), - ...(input.hasPendingBackgroundWorkForThread === undefined - ? {} - : { hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread }), - ...(input.backgroundTasksEndWithProcess === undefined - ? {} - : { backgroundTasksEndWithProcess: input.backgroundTasksEndWithProcess }), - ...(input.hangSessionScopeClose === undefined - ? {} - : { hangSessionScopeClose: input.hangSessionScopeClose }), - ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), - ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), - ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), - ...(input.scopeCloseReached === undefined - ? {} - : { scopeCloseReached: input.scopeCloseReached }), - }).pipe(Effect.map(ProviderAdapterRegistry.layerSingle)), + Effect.all([ + makeProviderAdapter(input.state, { + failEventStream: input.failEventStream ?? false, + ...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }), + ...(input.mcpConfigs === undefined ? {} : { mcpConfigs: input.mcpConfigs }), + ...(input.resumeMcpConfigs === undefined + ? {} + : { resumeMcpConfigs: input.resumeMcpConfigs }), + ...(input.beforeOpen === undefined ? {} : { beforeOpen: input.beforeOpen }), + ...(input.hasPendingBackgroundWork === undefined + ? {} + : { hasPendingBackgroundWork: input.hasPendingBackgroundWork }), + ...(input.hasPendingBackgroundWorkForThread === undefined + ? {} + : { hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread }), + ...(input.backgroundTasksEndWithProcess === undefined + ? {} + : { backgroundTasksEndWithProcess: input.backgroundTasksEndWithProcess }), + ...(input.hangSessionScopeClose === undefined + ? {} + : { hangSessionScopeClose: input.hangSessionScopeClose }), + ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), + ...(input.rollbackThread === undefined ? {} : { rollbackThread: input.rollbackThread }), + ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), + ...(input.beforeClose === undefined ? {} : { beforeClose: input.beforeClose }), + ...(input.withoutUnload === undefined ? {} : { withoutUnload: input.withoutUnload }), + ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), + ...(input.scopeCloseReached === undefined + ? {} + : { scopeCloseReached: input.scopeCloseReached }), + }), + ...(input.additionalAdapters ?? []), + ]).pipe(Effect.map(ProviderAdapterRegistry.layerFromAdapters)), ); - const layerConfiguredMcpRegistry = - input.pauseResolve === undefined + const layerResolvingMcpRegistry = + input.mcpRegistryLayer ?? + (input.pauseResolve === undefined ? layerTestMcpRegistry - : layerPausingMcpRegistry(input.pauseResolve); + : layerPausingMcpRegistry(input.pauseResolve)); + const layerConfiguredMcpRegistry = + input.pauseIssue === undefined + ? layerResolvingMcpRegistry + : layerPausingMcpIssue(input.pauseIssue).pipe(Layer.provide(layerResolvingMcpRegistry)); const layerProviderEventIngestorTest = ProviderEventIngestor.layer.pipe( Layer.provide( Layer.mergeAll( @@ -614,6 +684,33 @@ const layerPausingMcpRegistry = (pause: { }), ).pipe(Layer.provide(layerTestMcpRegistry)); +const layerPausingMcpIssue = (pause: NonNullable[0]["pauseIssue"]>) => + Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (request) => + delegate + .issue(request) + .pipe( + Effect.tap((credential) => + Ref.getAndSet(pause.armed, false).pipe( + Effect.flatMap((armed) => + armed + ? Deferred.succeed(pause.issued, credential.config).pipe( + Effect.andThen(Deferred.await(pause.resume)), + ) + : Effect.void, + ), + ), + ), + ), + }); + }), + ); + function makeBrowserAccessProject(projectId: ProjectId): Project { return { id: projectId, @@ -1256,6 +1353,140 @@ it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake" }), ); +it.effect( + "ProviderSessionManagerV2 finishes failed-open token cleanup before a queued provider resumes without waiting for scope close", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const openedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const handshake = yield* Deferred.make(); + const revoking = yield* Deferred.make(); + const finishRevoke = yield* Deferred.make(); + const closing = yield* Deferred.make(); + const finishClose = yield* Deferred.make(); + const queued = yield* Deferred.make(); + const targetInstanceId = ProviderInstanceId.make("failed-open-cleanup-target"); + const registryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (id) => + Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(finishRevoke)), + Effect.andThen(delegate.revokeProviderSession(id)), + ), + }); + }), + ).pipe(Layer.provide(layerTestMcpRegistry)); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make("failed-open-cleanup"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const targetId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: targetInstanceId, + }); + const targetSelection = { ...modelSelection, instanceId: targetInstanceId }; + const target = yield* manager.open({ + threadId, + providerSessionId: targetId, + modelSelection: targetSelection, + runtimePolicy, + }); + const providerSessionId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const opening = yield* manager + .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild); + yield* Deferred.await(handshake); + const failedConfig = (yield* Ref.get(openedConfigs))[0]!; + const failedToken = failedConfig.authorizationHeader.replace(/^Bearer\s+/, ""); + const replacement = yield* Deferred.succeed(queued, undefined).pipe( + Effect.andThen( + target.resumeThread({ + providerThread: { + ...makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: targetId, + now, + }), + providerInstanceId: targetInstanceId, + }, + modelSelection: targetSelection, + runtimePolicy, + }), + ), + Effect.forkChild, + ); + yield* Deferred.await(queued); + const interrupting = yield* Fiber.interrupt(opening).pipe(Effect.forkChild); + yield* Deferred.await(revoking); + yield* Effect.yieldNow; + assert.deepEqual(yield* Ref.get(resumedConfigs), []); + assert.deepEqual(yield* configs.read(threadId), failedConfig); + + yield* Deferred.succeed(finishRevoke, undefined); + yield* Deferred.await(closing); + yield* Fiber.join(replacement); + const replacementConfig = (yield* configs.read(threadId))!; + assert.deepEqual(yield* Ref.get(resumedConfigs), [replacementConfig]); + assert.isUndefined(yield* registry.resolve(failedToken)); + assert.equal( + (yield* registry.resolve(replacementConfig.authorizationHeader.replace(/^Bearer\s+/, ""))) + ?.thread.providerInstanceId, + targetInstanceId, + ); + assert.equal((yield* Ref.get(state)).closeCount, 0); + yield* Deferred.succeed(finishClose, undefined); + yield* Fiber.join(interrupting); + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.deepEqual(yield* configs.read(threadId), replacementConfig); + assert.isDefined( + yield* registry.resolve(replacementConfig.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + mcpRegistryLayer: registryLayer, + mcpConfigs: openedConfigs, + spawnBeforeOpen: true, + beforeOpen: () => + Deferred.succeed(handshake, undefined).pipe(Effect.andThen(Effect.never)), + beforeClose: Deferred.succeed(closing, undefined).pipe( + Effect.andThen(Deferred.await(finishClose)), + ), + additionalAdapters: [ + makeProviderAdapter(targetState, { + instanceId: targetInstanceId, + resumeMcpConfigs: resumedConfigs, + }), + ], + }), + ), + ); + }), +); + it.effect("ProviderSessionManagerV2 cleans up an interrupted open whose scope close hangs", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -2142,7 +2373,7 @@ it.effect( ); it.effect( - "ProviderSessionManagerV2 keeps a replacement session's attachment when a stale attach is interrupted", + "ProviderSessionManagerV2 preserves a replacement queued behind an interrupted stale attach", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -2190,18 +2421,16 @@ it.effect( .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(paused); - // ...while the session is replaced and the thread opens the new one. + // A replacement waits behind the stale attach's credential gate. yield* manager.release({ providerSessionId, reason: "runtime_error" }); - const replacement = yield* manager.open({ - threadId, - providerSessionId, - modelSelection, - runtimePolicy, - }); - const config = yield* mcpSessions.read(threadId); - assert.isDefined(config); + const replacing = yield* manager + .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild({ startImmediately: true })); yield* Fiber.interrupt(stale); + const replacement = yield* Fiber.join(replacing); + const config = yield* mcpSessions.read(threadId); + assert.isDefined(config); // The replacement keeps the thread and its credential. assert.equal( @@ -2237,9 +2466,9 @@ it.effect( }), ); -it.effect( - "ProviderSessionManagerV2 revokes a reused credential after a resume stopped while checking it", - () => +it.effect.each(["release-before-stop", "release-after-stop"] as const)( + "ProviderSessionManagerV2 revokes unowned reused credentials after interrupted lookup (%s)", + (order) => Effect.gen(function* () { const state = yield* Ref.make(emptyState); const armed = yield* Ref.make(false); @@ -2289,11 +2518,19 @@ it.effect( yield* Ref.set(armed, true); const stopped = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(paused); + if (order === "release-before-stop") { + yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + assert.isDefined(yield* registry.resolve(token)); + } yield* Fiber.interrupt(stopped); // Nothing holds the credential now, so a terminal release revokes it. - yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + if (order === "release-after-stop") { + assert.isDefined(yield* registry.resolve(token)); + yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); + } assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(yield* mcpSessions.read(threadId)); }).pipe( Effect.provide( layerTest({ state, idleTimeoutMs: 60_000, pauseResolve: { armed, paused } }), @@ -2660,101 +2897,1943 @@ it.effect( }), ); -it.effect("ProviderSessionManagerV2 terminal detach revokes the thread's MCP credential", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const mcpConfigs = yield* Ref.make< - ReadonlyArray - >([]); - const effect = Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const idAllocator = yield* IdAllocator.IdAllocatorV2; - const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; - const registry = yield* McpSessionRegistry.McpSessionRegistry; - const mcpSessions = yield* McpProviderSessions.McpProviderSessions; - const now = yield* DateTime.now; - const threadId = ThreadId.make("thread-provider-session-manager-terminal-detach"); - const providerSessionId = yield* idAllocator.allocate.providerSession({ - providerInstanceId: modelSelection.instanceId, - threadId, - }); - - yield* eventSink.write({ - events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], - }); - yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); - const issued = (yield* Ref.get(mcpConfigs)).at(-1); - const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); - assert.isDefined(yield* registry.resolve(token!)); +it.effect.each(["return", "close-second"] as const)( + "ProviderSessionManagerV2 preserves credential ownership after switching providers (%s)", + (next) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const otherState = yield* Ref.make(emptyState); + const resumeMcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const otherInstanceId = ProviderInstanceId.make("other-codex"); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-provider-switch"); + const unrelatedThreadId = ThreadId.make( + "thread-provider-session-manager-provider-switch-peer", + ); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const otherSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherInstanceId, + threadId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), + yield* makeThreadCreatedEvent({ idAllocator, threadId: unrelatedThreadId, now }), + ], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + const original = (yield* Ref.get(resumeMcpConfigs)).at(-1)!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); - // Archive/delete detaches carry revokeMcpCredential: the token must die - // with the thread even though the shared provider process lives on. - yield* manager.detach({ - providerSessionId, - threadId, - detail: "Thread deleted.", - revokeMcpCredential: true, - }); - assert.isUndefined(yield* registry.resolve(token!)); - assert.isUndefined(yield* mcpSessions.read(threadId)); - }); + // Another thread shares the first provider process; refreshing one + // native conversation must not stop that process or unload its peer. + yield* runtime.resumeThread({ + providerThread: makeProviderThread({ + idAllocator, + threadId: unrelatedThreadId, + providerSessionId, + now, + nativeThreadId: "native-peer", + }), + modelSelection, + runtimePolicy, + }); + const peer = (yield* Ref.get(resumeMcpConfigs)).at(-1)!; + const peerToken = peer.authorizationHeader.replace(/^Bearer\s+/, ""); - yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1_000, mcpConfigs }))); - }), -); + yield* manager.open({ + threadId, + providerSessionId: otherSessionId, + modelSelection: { ...modelSelection, instanceId: otherInstanceId }, + runtimePolicy, + }); + assert.isUndefined(yield* registry.resolve(originalToken)); + + if (next === "close-second") { + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const second = (yield* mcpSessions.read(threadId))!; + const secondToken = second.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* manager.close(otherSessionId); + assert.isUndefined(yield* registry.resolve(secondToken)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.isDefined(yield* registry.resolve(peerToken)); + yield* manager.close(providerSessionId); + assert.isUndefined(yield* registry.resolve(secondToken)); + return; + } -it.effect("ProviderSessionManagerV2 releases idle sessions without sweeping all sessions", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const effect = Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const idAllocator = yield* IdAllocator.IdAllocatorV2; - const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; - const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const now = yield* DateTime.now; - const projectId = yield* idAllocator.allocate.project({ - fixtureName: "provider-session-manager-idle", - }); - const threadId = yield* idAllocator.allocate.thread({ - fixtureName: "provider-session-manager-idle", - projectId, - }); - const providerSessionId = yield* idAllocator.allocate.providerSession({ - providerInstanceId: modelSelection.instanceId, - threadId, - }); + const returned = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + assert.strictEqual(returned, runtime); + yield* returned.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).resumeCount, 3); + const refreshed = (yield* Ref.get(resumeMcpConfigs)).at(-1)!; + const refreshedToken = refreshed.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.notEqual(refreshedToken, originalToken); + const scope = yield* registry.resolve(refreshedToken); + assert.equal(scope?.thread.threadId, threadId); + assert.equal(scope?.thread.providerInstanceId, modelSelection.instanceId); + assert.isUndefined(yield* registry.resolve(originalToken)); + assert.isDefined(yield* registry.resolve(peerToken)); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.equal((yield* Ref.get(state)).closeCount, 0); - yield* eventSink.write({ - events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], - }); - yield* manager.open({ - threadId, - providerSessionId, - modelSelection, - runtimePolicy, + // The obsolete provider session cannot revoke its replacement token. + yield* manager.close(otherSessionId); + assert.isDefined(yield* registry.resolve(refreshedToken)); + yield* returned.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).resumeCount, 3); }); - - yield* TestClock.adjust("1 second"); - yield* Effect.yieldNow; - - const liveSession = yield* manager.get(providerSessionId); - const runtimeState = yield* Ref.get(state); - const projection = yield* projectionStore.getThreadProjection(threadId); - - assert.isTrue(Option.isNone(liveSession)); - assert.equal(runtimeState.openCount, 1); - assert.equal(runtimeState.closeCount, 1); - assert.equal(projection.providerSessions.at(-1)?.status, "stopped"); - }); - - yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); - }), + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + resumeMcpConfigs, + additionalAdapters: [makeProviderAdapter(otherState, { instanceId: otherInstanceId })], + }), + ), + ); + }), ); -it.effect("ProviderSessionManagerV2 persists release when session scope close hangs", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const effect = Effect.gen(function* () { +it.effect.each(["open", "refresh"] as const)( + "ProviderSessionManagerV2 revokes an interrupted MCP issuance during %s", + (operation) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const armed = yield* Ref.make(false); + const issued = yield* Deferred.make(); + const resume = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread-provider-session-manager-interrupted-${operation}`); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + let runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime | undefined; + if (operation === "refresh") { + runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + yield* registry.revokeThread(threadId); + } + const countAttachments = eventStore + .read({ threadId, eventType: "provider-session.attached" }) + .pipe( + Stream.runCollect, + Effect.map((events) => events.length), + ); + const originalAttachmentCount = yield* countAttachments; + yield* Ref.set(armed, true); + const refresh = yield* Effect.gen(function* () { + if (runtime === undefined) { + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + } else { + // Cached native opens defer preparation to the native operation. + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + } + }).pipe(Effect.forkScoped); + const credential = yield* Deferred.await(issued); + const token = credential.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(token)); + // Signal Stop before releasing issuance; no timing or polling is needed. + yield* Effect.sync(() => refresh.interruptUnsafe()); + yield* Deferred.succeed(resume, undefined); + const stopped = yield* Fiber.await(refresh); + assert.isTrue(Exit.isFailure(stopped) && Cause.hasInterrupts(stopped.cause)); + assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.equal((yield* Ref.get(state)).closeCount, 0); + + const returned = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + if (runtime !== undefined) assert.strictEqual(returned, runtime); + yield* returned.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal( + yield* countAttachments, + originalAttachmentCount + (operation === "open" ? 1 : 0), + "refresh must preserve the existing attachment; opening must attach once", + ); + const replacement = (yield* mcpSessions.read(threadId))!; + const replacementToken = replacement.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(replacementToken)); + yield* manager.close(providerSessionId); + assert.isUndefined(yield* registry.resolve(replacementToken)); + assert.isUndefined(yield* registry.resolve(token)); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ state, idleTimeoutMs: 60_000, pauseIssue: { armed, issued, resume } }), + ), + ); + }), +); + +it.effect.each(["expired", "browser-disabled", "device-disabled"] as const)( + "ProviderSessionManagerV2 reloads an attached native thread when its MCP credential is %s", + (reason) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const settingsLayer = ServerSettings.layerTest({ + enableAgentBrowserAccess: true, + enableAgentDeviceAccess: true, + }); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const settings = yield* ServerSettings.ServerSettingsService; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread-provider-session-manager-mcp-${reason}`); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + const original = (yield* mcpSessions.read(threadId))!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); + const originalScope = yield* registry.resolve(originalToken); + assert.isTrue(originalScope?.capabilities.has("preview")); + assert.isTrue(originalScope?.capabilities.has("device")); + + if (reason === "expired") { + yield* registry.revokeThread(threadId); + } else { + yield* settings.updateSettings( + reason === "browser-disabled" + ? { enableAgentBrowserAccess: false } + : { enableAgentDeviceAccess: false }, + ); + } + if (reason === "expired") { + // Retained continuations can start a turn without another resume. + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Continue with refreshed MCP credentials", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + } else { + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + } + const refreshed = (yield* mcpSessions.read(threadId))!; + const scope = yield* registry.resolve( + refreshed.authorizationHeader.replace(/^Bearer\s+/, ""), + ); + assert.isUndefined(yield* registry.resolve(originalToken)); + assert.equal(scope?.thread.threadId, threadId); + assert.equal(scope?.capabilities.has("preview"), reason !== "browser-disabled"); + assert.equal(scope?.capabilities.has("device"), reason !== "device-disabled"); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ state, idleTimeoutMs: 60_000, serverSettingsLayer: settingsLayer }), + ), + ), + ); + }), +); + +it.effect.each(["idle", "detach", "failed-reattach", "detach-batch"] as const)( + "ProviderSessionManagerV2 retains native credential identity after a failed unload (%s)", + (scenario) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const unloadAttempts = yield* Ref.make(0); + const backgroundWork = yield* Ref.make(false); + const sessionBackgroundWork = yield* Ref.make(true); + const pauseAttachWrite = { + armed: yield* Ref.make(false), + paused: yield* Deferred.make(), + }; + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const beforeUnload = Effect.gen(function* () { + const attempt = yield* Ref.updateAndGet(unloadAttempts, (count) => count + 1); + if (attempt === (scenario === "detach-batch" ? 2 : 1)) { + return yield* Effect.die(new Error("simulated native unload failure")); + } + }); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread-failed-native-unload-${scenario}`); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const otherProviderThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: "other-native-thread", + }); + const events: OrchestrationV2DomainEvent[] = [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), + ]; + for (const thread of [ + providerThread, + ...(scenario === "detach-batch" ? [otherProviderThread] : []), + ]) { + events.push({ + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: thread, + }); + } + yield* eventSink.write({ events }); + const openInput = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(openInput); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + if (scenario === "detach-batch") + yield* runtime.resumeThread({ ...resumeInput, providerThread: otherProviderThread }); + const original = (yield* mcpSessions.read(threadId))!; + const token = original.authorizationHeader.replace(/^Bearer\s+/, ""); + if (scenario === "idle") { + // Wait for the event pump's receipt before advancing the unload timer. + const subscription = yield* runtime.subscribeEvents!; + const received = yield* subscription.events.pipe(Stream.runHead, Effect.forkChild); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + yield* Queue.offer(queue, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "failed-idle-unload-turn", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(received); + yield* subscription.close; + yield* TestClock.adjust("1 second"); + } else { + yield* manager.detach({ providerSessionId, threadId }); + } + assert.equal(yield* Ref.get(unloadAttempts), scenario === "detach-batch" ? 2 : 1); + assert.equal((yield* Ref.get(state)).resumeCount, scenario === "detach-batch" ? 2 : 1); + if (scenario === "failed-reattach") { + // Interrupt after preparation but before attachment persistence. Its + // rollback must preserve the pre-existing unknown native load marker. + yield* Ref.set(pauseAttachWrite.armed, true); + const reattach = yield* runtime.resumeThread(resumeInput).pipe(Effect.forkChild); + yield* Deferred.await(pauseAttachWrite.paused); + yield* Fiber.interrupt(reattach); + } + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + yield* Ref.set(backgroundWork, true); + assert.strictEqual(yield* manager.open(openInput), runtime); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + // Ensure has no explicit native pointer: the retained tracker guards it. + assert.equal( + (yield* runtime + .ensureThread({ + ...openInput, + title: "Retained native client", + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterEnsureThreadError", + ); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.isDefined(yield* registry.resolve(token)); + yield* Ref.set(backgroundWork, false); + yield* Ref.set(sessionBackgroundWork, false); + yield* runtime.resumeThread(resumeInput); + assert.equal((yield* Ref.get(state)).resumeCount, scenario === "detach-batch" ? 3 : 2); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.isUndefined(yield* registry.resolve(token)); + if (scenario === "detach-batch") { + yield* runtime.resumeThread({ ...resumeInput, providerThread: otherProviderThread }); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "native-thread", + "other-native-thread", + ]); + } + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 1000, + beforeUnload, + pauseAttachWrite, + // Keep the shared process alive while testing its per-thread unload. + hasPendingBackgroundWork: Ref.get(sessionBackgroundWork), + // The tracked client unloaded successfully in a partial batch; + // only the other, failed client retains background work. + hasPendingBackgroundWorkForThread: + scenario === "detach-batch" + ? (providerThread) => + providerThread.nativeThreadRef?.nativeId === "other-native-thread" + ? Ref.get(backgroundWork) + : Effect.succeed(false) + : Ref.get(backgroundWork), + serverSettingsLayer: settingsLayer, + }), + ), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects reopening after reported activity during its background probe", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const checkEntered = yield* Deferred.make(); + const checkGate = yield* Deferred.make(); + const armed = yield* Ref.make(false); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-reopen-activity-during-probe"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const input = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(input); + const subscription = yield* runtime.subscribeEvents!; + yield* registry.revokeThread(threadId); + yield* Ref.set(armed, true); + const reopening = yield* manager.open(input).pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(checkEntered); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + yield* Queue.offer(queue, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + // The subscriber receives this only after the manager records activity. + assert.isTrue(Option.isSome(yield* subscription.events.pipe(Stream.runHead))); + yield* Deferred.succeed(checkGate, undefined); + assert.equal((yield* Fiber.join(reopening))._tag, "ProviderSessionOpenError"); + assert.equal((yield* Ref.get(state)).openCount, 1); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.strictEqual( + (yield* manager.get(providerSessionId)).pipe(Option.getOrThrow), + runtime, + ); + yield* subscription.close; + // A later explicit retry can reopen once no activity invalidates its probe. + assert.notStrictEqual(yield* manager.open(input), runtime); + assert.equal((yield* Ref.get(state)).openCount, 2); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: true, + hasPendingBackgroundWork: Effect.gen(function* () { + if (yield* Ref.getAndSet(armed, false)) { + yield* Deferred.succeed(checkEntered, undefined); + yield* Deferred.await(checkGate); + } + return false; + }), + }), + ), + ); + }), +); + +it.effect.each(["process", "native"] as const)( + "ProviderSessionManagerV2 preserves the current peer token while refresh is blocked (%s)", + (client) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const peerState = yield* Ref.make(emptyState); + const backgroundWork = yield* Ref.make(false); + const peerInstanceId = ProviderInstanceId.make("current-mcp-peer"); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-rejected-switch-peer"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const peerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: peerInstanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const input = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(input); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + yield* manager.open({ + ...input, + providerSessionId: peerSessionId, + modelSelection: { ...modelSelection, instanceId: peerInstanceId }, + }); + yield* Ref.set(backgroundWork, true); + const peerCredential = (yield* mcpSessions.read(threadId))!; + const peerToken = peerCredential.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.equal( + (yield* registry.resolve(peerToken))?.thread.providerInstanceId, + peerInstanceId, + ); + if (client === "process") { + assert.equal( + (yield* manager.open(input).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + } else { + assert.strictEqual(yield* manager.open(input), runtime); + assert.deepEqual(yield* mcpSessions.read(threadId), peerCredential); + } + assert.equal( + (yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.equal( + (yield* runtime + .ensureThread({ + threadId, + title: "Blocked resume fallback", + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterEnsureThreadError", + ); + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + assert.equal( + (yield* runtime + .startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Retained continuation must preserve the current peer", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterTurnStartError", + ); + assert.deepEqual(yield* mcpSessions.read(threadId), peerCredential); + assert.equal( + (yield* registry.resolve(peerToken))?.thread.providerInstanceId, + peerInstanceId, + ); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.equal((yield* Ref.get(peerState)).closeCount, 0); + yield* Ref.set(backgroundWork, false); + const restarted = yield* manager.open(input); + if (client === "process") assert.notStrictEqual(restarted, runtime); + else assert.strictEqual(restarted, runtime); + yield* restarted.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).openCount, client === "process" ? 2 : 1); + assert.equal((yield* Ref.get(state)).closeCount, client === "process" ? 1 : 0); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + assert.deepEqual( + (yield* Ref.get(state)).unloadedNativeThreadIds, + client === "native" ? ["native-thread"] : [], + ); + const returned = (yield* mcpSessions.read(threadId))!; + assert.equal( + (yield* registry.resolve(returned.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + modelSelection.instanceId, + ); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: client === "process", + hasPendingBackgroundWork: Ref.get(backgroundWork), + hasPendingBackgroundWorkForThread: Ref.get(backgroundWork), + additionalAdapters: [makeProviderAdapter(peerState, { instanceId: peerInstanceId })], + }), + ), + ); + }), +); + +it.effect.each(["before-first-resume", "after-resume", "active-turn", "background-work"] as const)( + "ProviderSessionManagerV2 reopens process-bound MCP clients after access changes (%s)", + (phase) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const backgroundWork = yield* Ref.make(false); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-process-mcp-reopen"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const openInput = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const runtime = yield* manager.open(openInput); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + if (phase !== "before-first-resume") { + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + } + if (phase === "active-turn") { + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "Keep this turn active until its terminal event", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + } + const original = (yield* mcpSessions.read(threadId))!; + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + if (phase === "background-work") yield* Ref.set(backgroundWork, true); + if (phase === "active-turn" || phase === "background-work") { + const subscription = yield* Effect.acquireRelease( + runtime.subscribeEvents!, + (sub) => sub.close, + ); + const terminal = yield* subscription.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + Effect.forkChild, + ); + const reopenError = yield* manager.open(openInput).pipe(Effect.flip); + assert.equal(reopenError._tag, "ProviderSessionOpenError"); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.equal((yield* Ref.get(state)).openCount, 1); + const current = (yield* mcpSessions.read(threadId))!; + assert.isDefined( + yield* registry.resolve(current.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + const resident = yield* manager.get(providerSessionId); + assert.isTrue(Option.isSome(resident)); + if (Option.isSome(resident)) assert.strictEqual(resident.value, runtime); + if (phase === "active-turn") { + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + yield* Queue.offer(queue, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "busy-reopen-turn", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + assert.isTrue(Option.isSome(yield* Fiber.join(terminal))); + } else { + yield* Ref.set(backgroundWork, false); + yield* Fiber.interrupt(terminal); + } + } else { + const continuationError = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(continuationError._tag, "ProviderAdapterResumeThreadError"); + const fallbackError = yield* runtime + .ensureThread({ + threadId, + title: "Resume fallback", + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal(fallbackError._tag, "ProviderAdapterEnsureThreadError"); + } + const replacement = yield* manager.open(openInput); + assert.notStrictEqual(replacement, runtime); + yield* replacement.resumeThread({ providerThread, modelSelection, runtimePolicy }); + const credential = (yield* mcpSessions.read(threadId))!; + assert.isUndefined( + yield* registry.resolve(original.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isFalse( + (yield* registry.resolve( + credential.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + assert.equal((yield* Ref.get(state)).openCount, 2); + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.equal((yield* Ref.get(state)).resumeCount, phase !== "before-first-resume" ? 2 : 1); + const staleError = yield* runtime + .resumeThread({ providerThread, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(staleError._tag, "ProviderAdapterResumeThreadError"); + assert.equal( + (yield* mcpSessions.read(threadId))?.providerSessionId, + credential.providerSessionId, + ); + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: true, + hasPendingBackgroundWork: Ref.get(backgroundWork), + serverSettingsLayer: settingsLayer, + }), + ), + ), + ); + }), +); + +it.effect.each(["timeout", "background-work", "access-change"] as const)( + "ProviderSessionManagerV2 preserves stale native threads until reload is safe (%s)", + (reason) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const unloading = yield* Deferred.make(); + const hang = yield* Ref.make(reason === "timeout"); + const backgroundWork = yield* Ref.make(reason !== "timeout"); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const beforeUnload = Ref.getAndSet(hang, false).pipe( + Effect.flatMap((first) => + first + ? Deferred.succeed(unloading, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.void, + ), + ); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-unload-timeout"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + const original = (yield* mcpSessions.read(threadId))!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); + if (reason === "access-change") { + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + } else { + yield* registry.revokeThread(threadId); + } + if (reason === "timeout") { + const pending = yield* runtime + .resumeThread(resumeInput) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(unloading); + yield* TestClock.adjust("11 seconds"); + assert.equal((yield* Fiber.join(pending))._tag, "ProviderAdapterResumeThreadError"); + } else { + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, []); + if (reason === "access-change") { + assert.equal( + (yield* runtime + .ensureThread({ + threadId, + title: "Scope-change resume fallback", + providerSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip))._tag, + "ProviderAdapterEnsureThreadError", + ); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.isDefined(yield* registry.resolve(originalToken)); + } + yield* Ref.set(backgroundWork, false); + } + assert.equal((yield* Ref.get(state)).resumeCount, 1); + yield* runtime.resumeThread(resumeInput); + assert.equal((yield* Ref.get(state)).resumeCount, 2); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + if (reason === "access-change") { + assert.isUndefined(yield* registry.resolve(originalToken)); + const current = (yield* mcpSessions.read(threadId))!; + assert.isFalse( + (yield* registry.resolve( + current.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + } + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + beforeUnload, + hasPendingBackgroundWorkForThread: Ref.get(backgroundWork), + serverSettingsLayer: settingsLayer, + }), + ), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 guards rotation against the settings snapshot it actually validates", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const flipAfterRead = yield* Ref.make(false); + const backgroundWork = yield* Ref.make(true); + const baseSettings = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: delegate.getSettings.pipe( + Effect.tap(() => + Ref.getAndSet(flipAfterRead, false).pipe( + Effect.flatMap((flip) => + flip + ? delegate.updateSettings({ enableAgentBrowserAccess: false }) + : Effect.void, + ), + ), + ), + ), + }); + }), + ).pipe(Layer.provide(baseSettings)); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-settings-snapshot-rotation"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + const original = (yield* mcpSessions.read(threadId))!; + const originalToken = original.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* Ref.set(flipAfterRead, true); + // The read returns the old snapshot and changes the next read before it + // completes. Reuse needs no rotation; the following operation must guard. + yield* runtime.resumeThread(resumeInput); + assert.isFalse((yield* settings.getSettings).enableAgentBrowserAccess); + assert.isDefined(yield* registry.resolve(originalToken)); + assert.deepEqual(yield* mcpSessions.read(threadId), original); + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.isDefined(yield* registry.resolve(originalToken)); + }); + yield* effect.pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + hasPendingBackgroundWorkForThread: Ref.get(backgroundWork), + }), + ), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 cleans MCP issuance when release wins during preparation", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const armed = yield* Ref.make(false); + const issued = yield* Deferred.make(); + const resumeIssue = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-release-preparation"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + yield* runtime.resumeThread(resumeInput); + yield* registry.revokeThread(threadId); + yield* Ref.set(armed, true); + const pending = yield* runtime.resumeThread(resumeInput).pipe(Effect.flip, Effect.forkChild); + const credential = yield* Deferred.await(issued); + yield* manager.close(providerSessionId); + yield* Deferred.succeed(resumeIssue, undefined); + assert.equal((yield* Fiber.join(pending))._tag, "ProviderAdapterResumeThreadError"); + assert.isUndefined( + yield* registry.resolve(credential.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.equal( + (yield* runtime.resumeThread(resumeInput).pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.equal((yield* Ref.get(state)).resumeCount, 1); + }); + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + pauseIssue: { armed, issued, resume: resumeIssue }, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 cleans unused MCP access when scope clearing is interrupted", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const clearing = yield* Deferred.make(); + const armed = yield* Ref.make(true); + const agentScope = { + ...AgentScope.defaultValue(), + clear: () => + Ref.getAndSet(armed, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(clearing, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.void, + ), + ), + }; + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-mcp-cancel-before-adapter-open"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const input = { threadId, providerSessionId, modelSelection, runtimePolicy }; + const opening = yield* manager.open(input).pipe(Effect.forkChild); + yield* Deferred.await(clearing); + const credential = (yield* mcpSessions.read(threadId))!; + const token = credential.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(token)); + yield* Fiber.interrupt(opening); + assert.equal((yield* Ref.get(state)).openCount, 0); + assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + yield* manager.open(input); + assert.equal((yield* Ref.get(state)).openCount, 1); + assert.isDefined(yield* mcpSessions.read(threadId)); + }); + yield* effect.pipe( + Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })), + Effect.provideService(AgentScope, agentScope), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 lets an unloaded thread switch while another shared-runtime thread has retained work", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const targetInstanceId = ProviderInstanceId.make("unloaded-thread-target"); + const threadA = ThreadId.make("unloaded-owner-a"); + const threadB = ThreadId.make("retained-owner-b"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadA, now }), + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadB, now }), + ], + }); + const providerSessionId = ids.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + const owner = yield* manager.open({ + threadId: threadA, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId: threadB, + providerSessionId, + modelSelection, + runtimePolicy, + }); + for (const threadId of [threadA, threadB]) { + yield* owner.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + nativeThreadId: `native-${threadId}`, + }), + modelSelection, + runtimePolicy, + }); + } + const originalA = (yield* configs.read(threadA))!; + const originalB = (yield* configs.read(threadB))!; + yield* manager.detach({ threadId: threadA, providerSessionId }); + const targetId = yield* ids.allocate.providerSession({ + threadId: threadA, + providerInstanceId: targetInstanceId, + }); + yield* manager.open({ + threadId: threadA, + providerSessionId: targetId, + modelSelection: { ...modelSelection, instanceId: targetInstanceId }, + runtimePolicy, + }); + const switched = (yield* configs.read(threadA))!; + assert.equal( + (yield* registry.resolve(switched.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + targetInstanceId, + ); + assert.isUndefined( + yield* registry.resolve(originalA.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.deepEqual(yield* configs.read(threadB), originalB); + assert.isDefined( + yield* registry.resolve(originalB.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [`native-${threadA}`]); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: (thread) => + Effect.succeed(thread.appThreadId === threadB), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ); + }), +); + +it.effect.each([false, true])( + "ProviderSessionManagerV2 refreshes and retains rollback native credentials (rollback failure: %s)", + (failRollback) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const pendingA = yield* Ref.make(false); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const rollbackCalls = yield* Ref.make(0); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + const targetInstanceId = ProviderInstanceId.make("rollback-rotation-target"); + const threadA = ThreadId.make("rollback-native-a"); + const threadB = ThreadId.make("rollback-native-b"); + const rollbackError = new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CODEX_DRIVER, + detail: "rollback failed after loading native state", + }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadA, now }), + yield* makeThreadCreatedEvent({ idAllocator: ids, threadId: threadB, now }), + ], + }); + const providerSessionId = ids.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + const runtime = yield* manager.open({ + threadId: threadA, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId: threadB, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator: ids, + threadId: threadA, + providerSessionId, + now, + nativeThreadId: "rollback-native-a", + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + yield* runtime.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId: threadB, + providerSessionId, + now, + nativeThreadId: "rollback-native-b", + }), + modelSelection, + runtimePolicy, + }); + const original = (yield* configs.read(threadA))!; + const siblingConfig = (yield* configs.read(threadB))!; + // Successful idle unload keeps the app thread attached to the shared session. + yield* runtime.unloadThread!({ providerThread }); + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + yield* manager.open({ + threadId: threadA, + providerSessionId, + modelSelection, + runtimePolicy, + }); + assert.deepEqual(yield* configs.read(threadA), original); + const rollbackInput: ProviderAdapter.ProviderAdapterV2RollbackThreadInput = { + providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("rollback-credential-checkpoint"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }; + const result = yield* runtime.rollbackThread(rollbackInput).pipe(Effect.result); + if (failRollback) { + assert.equal(result._tag, "Failure"); + if (result._tag === "Failure") assert.strictEqual(result.failure, rollbackError); + } else { + assert.equal(result._tag, "Success"); + } + const refreshed = (yield* configs.read(threadA))!; + assert.deepEqual((yield* Ref.get(resumedConfigs)).at(-1), refreshed); + assert.equal(yield* Ref.get(rollbackCalls), 1); + assert.isUndefined( + yield* registry.resolve(original.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isFalse( + (yield* registry.resolve( + refreshed.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + assert.deepEqual(yield* configs.read(threadB), siblingConfig); + + // A failed rollback still leaves a live client consuming its refreshed token. + yield* Ref.set(pendingA, true); + yield* settings.updateSettings({ enableAgentBrowserAccess: true }); + assert.equal( + (yield* runtime.rollbackThread(rollbackInput).pipe(Effect.flip))._tag, + "ProviderAdapterRollbackThreadError", + ); + assert.deepEqual(yield* configs.read(threadA), refreshed); + assert.equal(yield* Ref.get(rollbackCalls), 1); + yield* Ref.set(pendingA, false); + const targetId = yield* ids.allocate.providerSession({ + threadId: threadA, + providerInstanceId: targetInstanceId, + }); + yield* manager.open({ + threadId: threadA, + providerSessionId: targetId, + modelSelection: { ...modelSelection, instanceId: targetInstanceId }, + runtimePolicy, + }); + yield* runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "rollback-native-a", + "rollback-native-a", + ]); + assert.equal((yield* Ref.get(state)).resumeCount, 4); + assert.equal((yield* Ref.get(state)).closeCount, 0); + }).pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + resumeMcpConfigs: resumedConfigs, + hasPendingBackgroundWorkForThread: (thread) => + thread.appThreadId === threadA ? Ref.get(pendingA) : Effect.succeed(false), + rollbackThread: (input) => + Effect.gen(function* () { + yield* Ref.update(rollbackCalls, (count) => count + 1); + if (failRollback) return yield* rollbackError; + return { + providerThread: input.providerThread, + providerTurns: [], + messages: [], + runtimeRequests: [], + }; + }), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ), + ); + }), +); + +it.effect.each(["fresh-process", "cached-native"] as const)( + "ProviderSessionManagerV2 protects the current owner's retained work from an idle target (%s)", + (target) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const pending = yield* Ref.make(false); + const targetInstanceId = ProviderInstanceId.make("idle-rotation-target"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make(`current-owner-${target}`); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const ownerId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const targetId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: targetInstanceId, + }); + const targetSelection = { ...modelSelection, instanceId: targetInstanceId }; + const targetInput = { + threadId, + providerSessionId: targetId, + modelSelection: targetSelection, + runtimePolicy, + }; + const targetThread = makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: targetId, + now, + }); + const cached = target === "cached-native" ? yield* manager.open(targetInput) : undefined; + if (cached !== undefined) + yield* cached.resumeThread({ + providerThread: targetThread, + modelSelection: targetSelection, + runtimePolicy, + }); + const owner = yield* manager.open({ + threadId, + providerSessionId: ownerId, + modelSelection, + runtimePolicy, + }); + if (target === "cached-native") { + yield* owner.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: ownerId, + now, + nativeThreadId: "current-owner-native", + }), + modelSelection, + runtimePolicy, + }); + } + const original = (yield* configs.read(threadId))!; + const token = original.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* Ref.set(pending, true); + const error = + cached === undefined + ? yield* manager.open(targetInput).pipe(Effect.flip) + : yield* cached + .resumeThread({ + providerThread: targetThread, + modelSelection: targetSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal( + error._tag, + cached === undefined + ? "ProviderSessionActivityError" + : "ProviderAdapterResumeThreadError", + ); + assert.deepEqual(yield* configs.read(threadId), original); + assert.equal( + (yield* registry.resolve(token))?.thread.providerInstanceId, + modelSelection.instanceId, + ); + assert.equal((yield* Ref.get(state)).closeCount, 0); + yield* Ref.set(pending, false); + const ready = yield* manager.open(targetInput); + yield* ready.resumeThread({ + providerThread: targetThread, + modelSelection: targetSelection, + runtimePolicy, + }); + const current = (yield* configs.read(threadId))!; + assert.equal( + (yield* registry.resolve(current.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + targetInstanceId, + ); + assert.isUndefined(yield* registry.resolve(token)); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: target === "fresh-process", + hasPendingBackgroundWork: Ref.get(pending), + hasPendingBackgroundWorkForThread: Ref.get(pending), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps sibling consumed credentials distinct through rotation", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const pendingB = yield* Ref.make(false); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const settingsLayer = ServerSettings.layerTest({ enableAgentBrowserAccess: true }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const settings = yield* ServerSettings.ServerSettingsService; + const threadId = ThreadId.make("native-sibling-credentials"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const providerSessionId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const a = makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + nativeThreadId: "sibling-a", + }); + const b = { + ...makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + nativeThreadId: "sibling-b", + }), + // A fresh fallback can replace the native bound to the same app row. + id: a.id, + }; + yield* runtime.resumeThread({ providerThread: a, modelSelection, runtimePolicy }); + // Orchestration adopts the app row after ensure/fork returns a native row. + yield* runtime.resumeThread({ + providerThread: { ...a, id: ProviderThreadId.make("canonical-app-row") }, + modelSelection, + runtimePolicy, + }); + assert.equal((yield* Ref.get(state)).resumeCount, 1); + yield* runtime.resumeThread({ providerThread: b, modelSelection, runtimePolicy }); + const original = (yield* configs.read(threadId))!; + const token = original.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* Ref.set(pendingB, true); + yield* settings.updateSettings({ enableAgentBrowserAccess: false }); + assert.equal( + (yield* runtime + .resumeThread({ providerThread: a, modelSelection, runtimePolicy }) + .pipe(Effect.flip))._tag, + "ProviderAdapterResumeThreadError", + ); + assert.deepEqual(yield* configs.read(threadId), original); + assert.isDefined(yield* registry.resolve(token)); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, []); + yield* Ref.set(pendingB, false); + yield* runtime.resumeThread({ providerThread: a, modelSelection, runtimePolicy }); + const rotated = (yield* configs.read(threadId))!; + yield* runtime.resumeThread({ providerThread: b, modelSelection, runtimePolicy }); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "sibling-a", + "sibling-b", + ]); + assert.equal((yield* Ref.get(state)).resumeCount, 4); + assert.deepEqual((yield* Ref.get(resumedConfigs)).slice(-2), [rotated, rotated]); + assert.isUndefined(yield* registry.resolve(token)); + assert.isFalse( + (yield* registry.resolve( + rotated.authorizationHeader.replace(/^Bearer\s+/, ""), + ))?.capabilities.has("preview"), + ); + }).pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + resumeMcpConfigs: resumedConfigs, + hasPendingBackgroundWorkForThread: (thread) => + thread.nativeThreadRef?.nativeId === "sibling-b" + ? Ref.get(pendingB) + : Effect.succeed(false), + }), + ), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 holds preparation through eager config consumption", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const otherState = yield* Ref.make(emptyState); + const entered = yield* Deferred.make(); + const consume = yield* Deferred.make(); + const consumed = yield* Ref.make< + ReadonlyArray + >([]); + const otherInstanceId = ProviderInstanceId.make("competing-config-consumer"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make("config-consumption-lock"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const firstId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const otherId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: otherInstanceId, + }); + const first = yield* manager + .open({ threadId, providerSessionId: firstId, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild); + yield* Deferred.await(entered); + const captured = (yield* configs.read(threadId))!; + const other = yield* manager + .open({ + threadId, + providerSessionId: otherId, + modelSelection: { ...modelSelection, instanceId: otherInstanceId }, + runtimePolicy, + }) + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + yield* Deferred.succeed(consume, undefined); + yield* Fiber.join(first); + // The first process reports retained work after acceptance, so the waiting + // competitor must reject instead of silently replacing its credential. + const rejected = yield* Fiber.await(other); + assert.isTrue(Exit.isFailure(rejected)); + assert.deepEqual(yield* Ref.get(consumed), [captured]); + assert.deepEqual(yield* configs.read(threadId), captured); + assert.equal( + (yield* registry.resolve(captured.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + modelSelection.instanceId, + ); + assert.equal((yield* Ref.get(otherState)).openCount, 0); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + withoutUnload: true, + mcpConfigs: consumed, + beforeOpen: () => + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(consume))), + hasPendingBackgroundWork: Effect.succeed(true), + additionalAdapters: [makeProviderAdapter(otherState, { instanceId: otherInstanceId })], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 pumps reported activity during the central rotation probe", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const targetState = yield* Ref.make(emptyState); + const armed = yield* Ref.make(false); + const probing = yield* Deferred.make(); + const finishProbe = yield* Deferred.make(); + const targetInstanceId = ProviderInstanceId.make("probe-rotation-target"); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const configs = yield* McpProviderSessions.McpProviderSessions; + const threadId = ThreadId.make("central-probe-event-pump"); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const ownerId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: modelSelection.instanceId, + }); + const targetId = yield* ids.allocate.providerSession({ + threadId, + providerInstanceId: targetInstanceId, + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId: ownerId, + modelSelection, + runtimePolicy, + }); + yield* runtime.resumeThread({ + providerThread: makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId: ownerId, + now, + }), + modelSelection, + runtimePolicy, + }); + const original = (yield* configs.read(threadId))!; + const subscription = yield* runtime.subscribeEvents!; + yield* Ref.set(armed, true); + const targetInput = { + threadId, + providerSessionId: targetId, + modelSelection: { ...modelSelection, instanceId: targetInstanceId }, + runtimePolicy, + }; + const opening = yield* manager.open(targetInput).pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(probing); + yield* Queue.offer((yield* Ref.get(state)).eventQueues.get(String(ownerId))!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + assert.isTrue(Option.isSome(yield* subscription.events.pipe(Stream.runHead))); + yield* Deferred.succeed(finishProbe, undefined); + assert.equal((yield* Fiber.join(opening))._tag, "ProviderSessionActivityError"); + assert.deepEqual(yield* configs.read(threadId), original); + assert.equal( + (yield* registry.resolve(original.authorizationHeader.replace(/^Bearer\s+/, "")))?.thread + .providerInstanceId, + modelSelection.instanceId, + ); + yield* subscription.close; + yield* manager.open(targetInput); + assert.equal((yield* Ref.get(targetState)).openCount, 1); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + hasPendingBackgroundWorkForThread: Effect.gen(function* () { + if (yield* Ref.getAndSet(armed, false)) { + yield* Deferred.succeed(probing, undefined); + yield* Deferred.await(finishProbe); + } + return false; + }), + additionalAdapters: [ + makeProviderAdapter(targetState, { instanceId: targetInstanceId }), + ], + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 terminal detach revokes the thread's MCP credential", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-terminal-detach"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + const issued = (yield* Ref.get(mcpConfigs)).at(-1); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(token!)); + + // Archive/delete detaches carry revokeMcpCredential: the token must die + // with the thread even though the shared provider process lives on. + yield* manager.detach({ + providerSessionId, + threadId, + detail: "Thread deleted.", + revokeMcpCredential: true, + }); + assert.isUndefined(yield* registry.resolve(token!)); + assert.isUndefined(yield* mcpSessions.read(threadId)); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1_000, mcpConfigs }))); + }), +); + +it.effect("ProviderSessionManagerV2 releases idle sessions without sweeping all sessions", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-idle", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-idle", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + + const liveSession = yield* manager.get(providerSessionId); + const runtimeState = yield* Ref.get(state); + const projection = yield* projectionStore.getThreadProjection(threadId); + + assert.isTrue(Option.isNone(liveSession)); + assert.equal(runtimeState.openCount, 1); + assert.equal(runtimeState.closeCount, 1); + assert.equal(projection.providerSessions.at(-1)?.status, "stopped"); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); + }), +); + +it.effect("ProviderSessionManagerV2 persists release when session scope close hangs", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const idAllocator = yield* IdAllocator.IdAllocatorV2; const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; @@ -4505,8 +6584,9 @@ it.effect( yield* resumeSecondThread; assert.equal((yield* Ref.get(state)).resumeCount, 4); - // The second thread has no persisted provider thread, so nothing is unloaded. - assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, []); + // Resume tracked the second thread's native client even though its + // provider-thread record has not been persisted yet. + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); yield* manager.detach({ providerSessionId, threadId: firstThreadId }); assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); @@ -4514,7 +6594,10 @@ it.effect( assert.equal((yield* Ref.get(state)).interruptCount, 1); // The runtime stays up for the second thread; the first thread's // native state is unloaded after its turn is interrupted. - assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, [ + "native-thread", + "native-thread", + ]); yield* manager.detach({ providerSessionId, threadId: secondThreadId }); yield* TestClock.adjust("1 second"); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index a9c6e2fc9f7fe..a23984bf9af8e 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -201,9 +201,12 @@ export class ProviderSessionManagerV2 extends Context.Service< interface LiveSessionEntry { readonly attachedThreadIds: ReadonlySet; - readonly loadedProviderThreadKeyByThread: ReadonlyMap; + /** Native identity survives orchestration adopting an existing app row's id. */ + readonly loadedProviderThreads: ReadonlyMap; /** - * MCP credential session id issued for each attached thread. Revocation on + * Process/open credential and latest prepared config for each app thread. + * Retained for release cleanup even after all its natives unload; native + * clients retain their own consumed ids in loadedProviderThreads. Revocation on * detach/release is scoped to these ids so tearing down a superseded * session cannot revoke a replacement session's credential for the same * thread (the workspace-handoff sequence opens the replacement before the @@ -237,6 +240,14 @@ interface LiveSessionEntry { readonly idleThreadUnloads: ReadonlyMap; } +interface LoadedProviderThread { + readonly threadId: ThreadId; + readonly providerThread: OrchestrationV2ProviderThread; + readonly key: string | undefined; + /** The credential actually consumed by this native client. */ + readonly mcpCredentialId: string | undefined; +} + interface IdleThreadUnload { readonly providerThread: OrchestrationV2ProviderThread; readonly generation: number; @@ -465,67 +476,125 @@ export const layerWithOptions = ( const isMcpCredentialReserved = (threadId: ThreadId, mcpCredentialId: string) => (mcpCredentialReservations.get(mcpReservationKey(threadId, mcpCredentialId)) ?? 0) > 0; const mcpPrepareLock = yield* KeyedLock.make(); + const mcpCredentialMatches = ( + resolved: Effect.Success>, + threadId: ThreadId, + providerInstanceId: ProviderInstanceId, + access: { readonly browser: boolean; readonly device: boolean }, + ) => + resolved !== undefined && + resolved.thread.threadId === threadId && + resolved.thread.providerInstanceId === providerInstanceId && + resolved.capabilities.has("preview") === access.browser && + resolved.capabilities.has("device") === access.device; + + // Read-only: rejecting a process restart or native reload must + // leave the current provider's credential intact. + const mcpAccessNeedsRefresh = (entry: LiveSessionEntry, threadId: ThreadId) => + Effect.gen(function* () { + if (options.configureMcp === false) return false; + const current = yield* mcpSessions.read(threadId); + const consumed = entry.mcpCredentialIdByThread.get(threadId); + if (current === undefined || current.providerSessionId !== consumed) return true; + const access = yield* agentAccessSettings(threadId); + const resolved = yield* mcpSessionRegistry.resolve( + current.authorizationHeader.replace(/^Bearer\s+/, ""), + ); + return !mcpCredentialMatches(resolved, threadId, entry.runtime.instanceId, access); + }); + /** * Resolves (or mints) the thread's MCP credential and returns it with a * reservation held; the caller must drop the reservation exactly once. - * Serialized per thread so two concurrent prepares cannot interleave - * their rotate steps and revoke each other's freshly minted credential. + * The caller holds mcpPrepareLock through adapter acceptance and publication, + * so another provider cannot substitute the config before it is consumed. */ const prepareMcpSession = ( threadId: ThreadId, providerInstanceId: ProviderInstanceId, - ): Effect.Effect => + onPrepared: (prepared: PreparedMcpCredential) => void, + providerSessionId: ProviderSessionId, + checkAccess?: ( + candidateId: string | undefined, + willRotate: boolean, + ) => Effect.Effect, + providerThread?: OrchestrationV2ProviderThread, + ): Effect.Effect => options.configureMcp === false - ? mcpSessions - .clear(threadId) - .pipe(Effect.as({ mcpCredentialId: undefined, issued: false })) - : mcpPrepareLock.withLock( - threadId, - Effect.gen(function* () { - // Reuse a still-valid credential for this thread instead of - // rotating: long-lived provider processes (codex app-server) - // build their MCP client once per conversation and keep using - // the credential it started with, so a thread that detaches and - // re-attaches across a workspace handoff must come back to the - // same token or the process's tool calls fail auth. - const { browser: browserToolsAvailable, device: deviceToolsAvailable } = - yield* agentAccessSettings(threadId); - const capabilities = new Set< - import("../mcp/McpInvocationContext.ts").McpCapability - >(["orchestration", "worktree", "pull-requests"]); - if (browserToolsAvailable) capabilities.add("preview"); - if (deviceToolsAvailable) capabilities.add("device"); - const existing = yield* mcpSessions.read(threadId); - if (existing !== undefined) { - // Reserve before the async resolve so a release cannot - // revoke the credential between validation and reservation. - reserveMcpCredential(threadId, existing.providerSessionId); - const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); - // The caller only learns of the reservation once this returns, - // so a stop while resolving must drop it here. - const resolved = yield* mcpSessionRegistry - .resolve(rawToken) - .pipe( - Effect.onInterrupt(() => - Effect.sync(() => - dropMcpCredentialReservation(threadId, existing.providerSessionId), + ? Effect.gen(function* () { + const prepared = { mcpCredentialId: undefined, issued: false }; + onPrepared(prepared); + if (checkAccess !== undefined) yield* checkAccess(undefined, false); + yield* mcpSessions.clear(threadId); + return prepared; + }) + : Effect.gen(function* () { + // Reuse a still-valid credential for this thread instead of + // rotating: long-lived provider processes (codex app-server) + // build their MCP client once per conversation and keep using + // the credential it started with, so a thread that detaches and + // re-attaches across a workspace handoff must come back to the + // same token or the process's tool calls fail auth. + const { browser: browserToolsAvailable, device: deviceToolsAvailable } = + yield* agentAccessSettings(threadId); + const capabilities = new Set([ + "orchestration", + "worktree", + "pull-requests", + ]); + if (browserToolsAvailable) capabilities.add("preview"); + if (deviceToolsAvailable) capabilities.add("device"); + const existing = yield* mcpSessions.read(threadId); + if (existing !== undefined) { + // Reserve before the async resolve so a release cannot + // revoke the credential between validation and reservation. + reserveMcpCredential(threadId, existing.providerSessionId); + const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); + // The caller only learns of the reservation once this returns, + // so a failed lookup must drop it here, including on Stop. + const resolved = yield* mcpSessionRegistry + .resolve(rawToken) + .pipe( + Effect.onError(() => + Effect.sync(() => + dropMcpCredentialReservation(threadId, existing.providerSessionId), + ).pipe( + Effect.andThen( + clearUnownedMcpSession(threadId, existing.providerSessionId), ), + Effect.uninterruptible, ), - ); - if ( - resolved !== undefined && - resolved.thread.threadId === threadId && - resolved.thread.providerInstanceId === providerInstanceId && - // A flipped browser-access setting must not survive through - // credential reuse: rotate so the new scope reflects it. - resolved.capabilities.has("preview") === browserToolsAvailable && - resolved.capabilities.has("device") === deviceToolsAvailable - ) { - return { mcpCredentialId: existing.providerSessionId, issued: false }; - } - dropMcpCredentialReservation(threadId, existing.providerSessionId); + ), + ); + if ( + mcpCredentialMatches(resolved, threadId, providerInstanceId, { + browser: browserToolsAvailable, + device: deviceToolsAvailable, + }) + ) { + const prepared = { mcpCredentialId: existing.providerSessionId, issued: false }; + // The caller owns this reservation before a guard can yield + // or fail, including when the last live owner just released it. + onPrepared(prepared); + if (providerThread !== undefined) + yield* checkMcpOwners(threadId, providerSessionId, { + nativeThreadKey: providerThreadRuntimeKey(providerThread), + candidateId: existing.providerSessionId, + }); + if (checkAccess !== undefined) + yield* checkAccess(existing.providerSessionId, false); + return prepared; } - yield* mcpSessionRegistry.revokeThread(threadId); + yield* Effect.sync(() => + dropMcpCredentialReservation(threadId, existing.providerSessionId), + ); + } + // Validate work against this same settings/credential snapshot + // before any live credential is revoked or issued. + if (checkAccess !== undefined) yield* checkAccess(existing?.providerSessionId, true); + yield* checkMcpOwners(threadId, providerSessionId, undefined, providerThread); + yield* mcpSessionRegistry.revokeThread(threadId); + return yield* Effect.gen(function* () { const credential = yield* mcpSessionRegistry.issue({ threadId, providerInstanceId, @@ -534,9 +603,16 @@ export const layerWithOptions = ( }); yield* mcpSessions.set(credential.config); reserveMcpCredential(threadId, credential.config.providerSessionId); - return { mcpCredentialId: credential.config.providerSessionId, issued: true }; - }), - ); + const prepared = { + mcpCredentialId: credential.config.providerSessionId, + issued: true, + }; + // Publish the cleanup id before restoring interruption: a + // stop must not leave a minted token outside session records. + onPrepared(prepared); + return prepared; + }).pipe(Effect.uninterruptible); + }); /** * With a credential id, revocation is scoped to that credential and the * config slot is cleared only while it still holds it; a replacement @@ -558,6 +634,145 @@ export const layerWithOptions = ( ), ); + const clearUnownedMcpSession = ( + threadId: ThreadId, + mcpCredentialId: string, + ignoredRuntime?: ProviderAdapter.ProviderAdapterV2SessionRuntime, + ) => + Ref.get(sessions).pipe( + Effect.flatMap((current) => + isMcpCredentialReserved(threadId, mcpCredentialId) || + Array.from(current.values()).some( + (entry) => + entry.runtime !== ignoredRuntime && + ownsMcpCredential(entry, threadId, mcpCredentialId), + ) + ? Effect.void + : clearMcpSession(threadId, mcpCredentialId), + ), + ); + + const ownsMcpCredential = (entry: LiveSessionEntry, threadId: ThreadId, id: string) => + entry.mcpCredentialIdByThread.get(threadId) === id || + Array.from(entry.loadedProviderThreads.values()).some( + (loaded) => loaded.threadId === threadId && loaded.mcpCredentialId === id, + ); + + // A rotation revokes every credential for the app thread. Inspect every + // owner, including detached natives and process opens that have no native + // identity yet. A stale reload uses the same gate for its own client. + // Probes cannot make unreported native activity atomic with preparation. + const checkMcpOwners = ( + threadId: ThreadId, + providerSessionId: ProviderSessionId, + reload?: { readonly nativeThreadKey: string; readonly candidateId: string }, + providerThread?: OrchestrationV2ProviderThread, + ) => + Effect.gen(function* () { + const snapshot = yield* Ref.get(sessions); + const target = snapshot.get(sessionKey(providerSessionId)); + const owners = Array.from(snapshot.values()).flatMap((entry) => { + const records = Array.from(entry.loadedProviderThreads.values()).filter( + (loaded) => + loaded.threadId === threadId && + (reload === undefined || + (entry.runtime.providerSessionId === providerSessionId && + providerThreadRuntimeKey(loaded.providerThread) === reload.nativeThreadKey && + loaded.mcpCredentialId !== reload.candidateId)), + ); + // An explicit retained native is known even before its first resume + // in this process. Probe it without inventing a loaded client record. + if ( + providerThread !== undefined && + entry.runtime.providerSessionId === providerSessionId && + !entry.loadedProviderThreads.has(providerThreadRuntimeKey(providerThread)) && + reload === undefined + ) { + records.push({ + threadId, + providerThread, + key: undefined, + mcpCredentialId: entry.mcpCredentialIdByThread.get(threadId), + }); + } + // Native-capable sessions retain these ids for release cleanup after + // unload. Only process-bound clients own them without a loaded native. + const processOwner = + reload === undefined && + entry.runtime.unloadThread === undefined && + entry.mcpCredentialIdByThread.has(threadId); + return records.length === 0 && !processOwner ? [] : [{ entry, records }]; + }); + if ( + reload === undefined && + Array.from(mcpCredentialReservations.keys()).some((key) => + key.startsWith(`${threadId}\0`), + ) + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "An in-flight provider configuration prevents refreshing MCP access.", + }); + } + for (const { entry, records } of owners) { + const uncertain = + entry.runtime.unloadThread === undefined || + records.length === 0 || + records.some((record) => record.providerThread.nativeThreadRef === null); + if ( + uncertain + ? entry.busyTurns.size > 0 + : records.some((record) => hasBusyTurn(entry, record.providerThread.id)) + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "An active turn prevents refreshing MCP access.", + }); + } + const pending = yield* ( + uncertain || entry.runtime.hasPendingBackgroundWorkForThread === undefined + ? (entry.runtime.hasPendingBackgroundWork ?? Effect.succeed(false)) + : Effect.forEach(records, (record) => + entry.runtime.hasPendingBackgroundWorkForThread!(record.providerThread), + ).pipe(Effect.map((results) => results.some(Boolean))) + ).pipe( + Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.mapError( + (cause) => new ProviderSessionActivityError({ providerSessionId, cause }), + ), + ); + if (pending) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "Pending background work prevents refreshing MCP access.", + }); + } + } + const latest = yield* Ref.get(sessions); + if (latest.get(sessionKey(providerSessionId))?.runtime !== target?.runtime) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "The provider session changed while checking pending work.", + }); + } + for (const { entry } of owners) { + const current = latest.get(sessionKey(entry.runtime.providerSessionId)); + if ( + current?.runtime !== entry.runtime || + current.loadedProviderThreads !== entry.loadedProviderThreads || + current.mcpCredentialIdByThread !== entry.mcpCredentialIdByThread || + current.busyTurns !== entry.busyTurns || + current.idleGeneration !== entry.idleGeneration || + current.lastActivityAtMs !== entry.lastActivityAtMs + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "The provider session changed or became active while checking pending work.", + }); + } + } + }); + const publishToSubscribers = ( subscribers: Ref.Ref< ReadonlyMap> @@ -972,23 +1187,52 @@ export const layerWithOptions = ( const removeLiveEntry = (input: { readonly providerSessionId: ProviderSessionId; readonly onlyIfIdleGeneration?: number; - }): Effect.Effect, DateTime.Utc]> => + readonly requireIdle?: boolean; + }): Effect.Effect< + readonly [Option.Option, DateTime.Utc], + ProviderSessionActivityError + > => Effect.gen(function* () { const key = sessionKey(input.providerSessionId); const candidate = (yield* Ref.get(sessions)).get(key); if (candidate === undefined) { return [Option.none(), yield* DateTime.now] as const; } + const expectedGeneration = + input.onlyIfIdleGeneration ?? + (input.requireIdle ? candidate.idleGeneration : undefined); + if (input.requireIdle) { + const pending = yield* ( + candidate.runtime.hasPendingBackgroundWork ?? Effect.succeed(false) + ).pipe( + Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.mapError( + (cause) => + new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause, + }), + ), + ); + if (pending) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "Pending background work prevents reopening the provider session.", + }); + } + } const removed = yield* Effect.zip( Ref.modify(sessions, (current) => { const existing = current.get(key); if (existing !== candidate) { return [existing === undefined ? "gone" : "changed", current] as const; } + if (input.requireIdle && existing.busyTurns.size > 0) { + return ["busy", current] as const; + } if ( - input.onlyIfIdleGeneration !== undefined && - (existing.busyTurns.size > 0 || - existing.idleGeneration !== input.onlyIfIdleGeneration) + expectedGeneration !== undefined && + (existing.busyTurns.size > 0 || existing.idleGeneration !== expectedGeneration) ) { return ["kept", current] as const; } @@ -999,8 +1243,23 @@ export const layerWithOptions = ( DateTime.now, ).pipe(candidate.requestEventPermit.withPermits(1)); const [outcome, releasedAt] = removed; - // Another entry took this id while the permit was held; release it instead. + if ( + input.requireIdle && + (outcome === "changed" || outcome === "kept" || outcome === "gone") + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "Provider activity changed while checking whether the session can reopen.", + }); + } + // Unconditional releases can follow a replacement; idle checks cannot. if (outcome === "changed") return yield* removeLiveEntry(input); + if (outcome === "busy") { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "The provider session has an active turn and cannot be reopened.", + }); + } return [ outcome === "removed" ? Option.some(candidate) : Option.none(), releasedAt, @@ -1054,6 +1313,7 @@ export const layerWithOptions = ( readonly detachedThreadId?: ThreadId; readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; + readonly requireIdle?: boolean; readonly gracefulSubscribers?: boolean; }) => Effect.acquireUseRelease( @@ -1127,7 +1387,23 @@ export const layerWithOptions = ( Ref.get(sessions).pipe( Effect.flatMap((current) => Effect.forEach( - entry.mcpCredentialIdByThread, + new Map([ + ...Array.from( + entry.mcpCredentialIdByThread, + ([threadId, id]) => + [mcpReservationKey(threadId, id), [threadId, id] as const] as const, + ), + ...Array.from(entry.loadedProviderThreads.values()).flatMap((loaded) => + loaded.mcpCredentialId === undefined + ? [] + : [ + [ + mcpReservationKey(loaded.threadId, loaded.mcpCredentialId), + [loaded.threadId, loaded.mcpCredentialId] as const, + ] as const, + ], + ), + ]).values(), ([threadId, mcpCredentialId]) => { // Id-sensitive: a stale record for the same thread but // a DIFFERENT credential (left behind by an old session @@ -1142,8 +1418,7 @@ export const layerWithOptions = ( Array.from(current.values()).some( (other) => other !== entry && - (other.attachedThreadIds.has(threadId) || - other.mcpCredentialIdByThread.get(threadId) === mcpCredentialId), + ownsMcpCredential(other, threadId, mcpCredentialId), ); return heldElsewhere ? Effect.void @@ -1330,24 +1605,26 @@ export const layerWithOptions = ( }), ); - /** Returns the runtime the thread was attached to, or undefined if it already was. */ + /** Missing/replaced runtimes cannot attach or mint credentials. */ const attachThread = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; + readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; }) => withActivityError( input.providerSessionId, Ref.modify(sessions, (current) => { const entry = current.get(sessionKey(input.providerSessionId)); - if (entry === undefined || entry.attachedThreadIds.has(input.threadId)) { + if (entry === undefined || entry.runtime !== input.runtime) { return [undefined, current] as const; } + if (entry.attachedThreadIds.has(input.threadId)) return [false, current] as const; const updated = new Map(current); updated.set(sessionKey(input.providerSessionId), { ...entry, attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]), }); - return [entry.runtime, updated] as const; + return [true, updated] as const; }), ); @@ -1372,53 +1649,119 @@ export const layerWithOptions = ( } const attachedThreadIds = new Set(entry.attachedThreadIds); attachedThreadIds.delete(input.threadId); - const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread); - loadedProviderThreadKeyByThread.delete(input.threadId); const updated = new Map(current); updated.set(key, { ...entry, attachedThreadIds, - loadedProviderThreadKeyByThread, }); return updated; }); - const isProviderThreadLoaded = (input: { + const providerThreadLoadStatus = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; - readonly providerThreadKey: string; + readonly providerThread: OrchestrationV2ProviderThread; + readonly providerThreadKey: string | undefined; }) => Ref.get(sessions).pipe( - Effect.map( - (current) => - current - .get(sessionKey(input.providerSessionId)) - ?.loadedProviderThreadKeyByThread.get(input.threadId) === input.providerThreadKey, - ), + Effect.map((current) => { + const entry = current.get(sessionKey(input.providerSessionId)); + const loaded = entry?.loadedProviderThreads.get( + providerThreadRuntimeKey(input.providerThread), + ); + const staleMcpCredential = + loaded !== undefined && + loaded.mcpCredentialId !== entry?.mcpCredentialIdByThread.get(input.threadId); + return { + loaded: + input.providerThreadKey !== undefined && + loaded?.key === input.providerThreadKey && + !staleMcpCredential, + staleMcpCredential, + }; + }), ); const markProviderThreadLoaded = (input: { readonly providerSessionId: ProviderSessionId; + readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; readonly threadId: ThreadId; - readonly providerThreadKey: string; + readonly providerThread: OrchestrationV2ProviderThread; + readonly mcpCredentialId: string | undefined; + readonly providerThreadKey: string | undefined; }) => - Ref.update(sessions, (current) => { + Ref.modify(sessions, (current) => { const key = sessionKey(input.providerSessionId); const entry = current.get(key); - if (entry === undefined) { - return current; + if (entry?.runtime !== input.runtime) { + return [false, current] as const; } - const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread); - loadedProviderThreadKeyByThread.set(input.threadId, input.providerThreadKey); + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + loadedProviderThreads.set(providerThreadRuntimeKey(input.providerThread), { + threadId: input.threadId, + providerThread: input.providerThread, + key: input.providerThreadKey, + mcpCredentialId: input.mcpCredentialId, + }); const updated = new Map(current); - updated.set(key, { ...entry, loadedProviderThreadKeyByThread }); - return updated; + updated.set(key, { ...entry, loadedProviderThreads }); + return [true, updated] as const; + }).pipe( + Effect.flatMap((recorded) => + recorded + ? Effect.void + : Effect.fail( + new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: + "The provider session was released or replaced before publishing native MCP access.", + }), + ), + ), + ); + + const unloadNativeThread = ( + runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime, + providerThread: OrchestrationV2ProviderThread, + ) => + Effect.gen(function* () { + const key = sessionKey(runtime.providerSessionId); + const nativeKey = providerThreadRuntimeKey(providerThread); + const unloading = yield* Ref.modify(sessions, (current) => { + const entry = current.get(key); + const loaded = entry?.loadedProviderThreads.get(nativeKey); + if (entry?.runtime !== runtime || loaded === undefined) + return [undefined, current] as const; + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + const unknownLoad = { ...loaded, key: undefined }; + loadedProviderThreads.set(nativeKey, unknownLoad); + return [ + unknownLoad, + new Map(current).set(key, { ...entry, loadedProviderThreads }), + ] as const; + }); + yield* runtime.unloadThread!({ providerThread }); + // Failure or interruption retains the pointer and consumed credential. + yield* Ref.update(sessions, (current) => { + const entry = current.get(key); + if ( + entry?.runtime !== runtime || + entry.loadedProviderThreads.get(nativeKey) !== unloading + ) + return current; + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + loadedProviderThreads.delete(nativeKey); + return new Map(current).set(key, { ...entry, loadedProviderThreads }); + }); }); const ensureThreadAttached = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; + readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; + readonly checkAccess?: Parameters[4]; + readonly providerThread?: OrchestrationV2ProviderThread; }) => Effect.suspend(() => { let attachedTo: ProviderAdapter.ProviderAdapterV2SessionRuntime | undefined; @@ -1437,25 +1780,54 @@ export const layerWithOptions = ( const attached = yield* attachThread(input).pipe( // Recorded with no gap for an interrupt: cleanup undoes only an // attach this call made, never one an earlier open made. - Effect.tap((runtime) => Effect.sync(() => (attachedTo = runtime))), + Effect.tap((attached) => + Effect.sync(() => { + if (attached) attachedTo = input.runtime; + }), + ), Effect.uninterruptible, ); - if (attached !== undefined) { - const prepared = yield* prepareMcpSession(input.threadId, input.providerInstanceId); - preparedForCleanup = prepared; + if (attached === undefined) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "The provider session was released or replaced before attaching.", + }); + } + // A cached runtime may still be attached after another provider + // rotated this thread's token. Validate it on every use, so its + // next resume can reload the native MCP client with a live token. + const prepared = yield* prepareMcpSession( + input.threadId, + input.providerInstanceId, + (prepared) => (preparedForCleanup = prepared), + input.providerSessionId, + input.checkAccess, + input.providerThread, + ); + const recorded = yield* Ref.modify(sessions, (current) => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + if (entry === undefined || entry.runtime !== input.runtime) { + return [false, current] as const; + } + const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); if (prepared.mcpCredentialId !== undefined) { - const mcpCredentialId = prepared.mcpCredentialId; - yield* Ref.update(sessions, (current) => { - const key = sessionKey(input.providerSessionId); - const entry = current.get(key); - if (entry === undefined) return current; - const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); - mcpCredentialIdByThread.set(input.threadId, mcpCredentialId); - const updated = new Map(current); - updated.set(key, { ...entry, mcpCredentialIdByThread }); - return updated; - }); + mcpCredentialIdByThread.set(input.threadId, prepared.mcpCredentialId); } + const updated = new Map(current); + updated.set(key, { ...entry, mcpCredentialIdByThread }); + // Publish ownership and remove the reservation in one synchronous + // update, so a racing release cannot leave this token unowned. + dropReservation(); + return [true, updated] as const; + }); + if (!recorded) { + return yield* new ProviderSessionActivityError({ + providerSessionId: input.providerSessionId, + cause: "The provider session was released or replaced while preparing MCP access.", + }); + } + if (attached) { const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId)); if (entry !== undefined) { yield* withActivityError( @@ -1469,47 +1841,32 @@ export const layerWithOptions = ( ); } } + return prepared; }).pipe( // An interrupted attach is undone too, so the next attach writes // the attachment instead of finding the thread already attached. Effect.onError(() => - attachedTo === undefined + (attachedTo === undefined ? Effect.void - : removeThreadAttachment({ ...input, runtime: attachedTo }).pipe( - Effect.andThen( - Effect.suspend(() => { - dropReservation(); - // Revoke only a credential this attach freshly minted: a - // REUSED credential is by definition held by another - // live provider process, and revoking it thread-wide - // would break that process's MCP client mid-conversation. - if (preparedForCleanup?.issued !== true) return Effect.void; - const mcpCredentialId = preparedForCleanup.mcpCredentialId; - const attachedRuntime = attachedTo; - // As in release: a replacement session (or an open - // configuring one) may have taken the credential up. - return Ref.get(sessions).pipe( - Effect.flatMap((current) => - (mcpCredentialId !== undefined && - isMcpCredentialReserved(input.threadId, mcpCredentialId)) || - Array.from(current.values()).some( - (other) => - other.runtime !== attachedRuntime && - (other.attachedThreadIds.has(input.threadId) || - (mcpCredentialId !== undefined && - other.mcpCredentialIdByThread.get(input.threadId) === - mcpCredentialId)), - ) - ? Effect.void - : clearMcpSession(input.threadId, mcpCredentialId), - ), + : removeThreadAttachment({ ...input, runtime: attachedTo }) + ).pipe( + Effect.andThen( + Effect.suspend(() => { + dropReservation(); + const id = preparedForCleanup?.mcpCredentialId; + return id === undefined + ? Effect.void + : clearUnownedMcpSession( + input.threadId, + id, + preparedForCleanup?.issued ? attachedTo : undefined, ); - }), - ), - ), + }), + ), + ), ), ); - return threadAttachment.withLock(threadAttachmentKey(input), attach).pipe( + return attach.pipe( // The entry's own record (written above while the thread is // attached) guards the credential from here on; the reservation // is only needed until then. Ensuring covers defects/interrupts. @@ -1526,7 +1883,7 @@ export const layerWithOptions = ( const idleFiber = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); if (entry === undefined) { - return [null, current] as const; + return [undefined, current] as const; } const updated = new Map(current); updated.set(key, { @@ -1538,6 +1895,12 @@ export const layerWithOptions = ( }); return [entry.idleFiber, updated] as const; }); + if (idleFiber === undefined) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "The provider session was released before registering the turn.", + }); + } yield* cancelIdleFiber(idleFiber); }), ); @@ -1701,6 +2064,7 @@ export const layerWithOptions = ( .hasPendingBackgroundWorkForThread(pending.providerThread) .pipe(Effect.catchCause(() => Effect.succeed(false))); if (hasPendingWork) return "deferred" as const; + const retainedUnload = { ...pending, fiber: null }; const unloading = yield* Ref.modify(sessions, (current) => { const latest = current.get(key); if ( @@ -1709,33 +2073,48 @@ export const layerWithOptions = ( ) { return [false, current] as const; } - const loadedProviderThreadKeyByThread = new Map( - latest.loadedProviderThreadKeyByThread, - ); - loadedProviderThreadKeyByThread.delete(input.threadId); const idleThreadUnloads = new Map(latest.idleThreadUnloads); - idleThreadUnloads.delete(input.threadId); + idleThreadUnloads.set(input.threadId, retainedUnload); const updated = new Map(current); updated.set(key, { ...latest, - loadedProviderThreadKeyByThread, idleThreadUnloads, }); return [true, updated] as const; }); if (!unloading) return "skipped" as const; - yield* unloadThread({ providerThread: pending.providerThread }).pipe( + const unloaded = yield* unloadThread({ providerThread: pending.providerThread }).pipe( Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.as(true), Effect.catchCause((cause) => Effect.logWarning("orchestration-v2.driver-session.idle-unload-failed", { providerSessionId: input.providerSessionId, threadId: input.threadId, providerThreadId: pending.providerThread.id, cause, - }), + }).pipe(Effect.as(false)), ), ); - return "unloaded" as const; + if (unloaded) { + yield* Ref.update(sessions, (current) => { + const latest = current.get(key); + if ( + latest?.runtime !== entry.runtime || + latest.idleThreadUnloads.get(input.threadId) !== retainedUnload + ) { + return current; + } + const idleThreadUnloads = new Map(latest.idleThreadUnloads); + idleThreadUnloads.delete(input.threadId); + const updated = new Map(current); + updated.set(key, { + ...latest, + idleThreadUnloads, + }); + return updated; + }); + } + return unloaded ? ("unloaded" as const) : ("failed" as const); }), ); // Re-check on this fiber after another idle window, outside the @@ -1817,33 +2196,108 @@ export const layerWithOptions = ( modelFamily: normalizeModelMetricLabel(model), }, }); + const withRuntimeThread = ( + threadId: ThreadId, + providerThread: OrchestrationV2ProviderThread | undefined, + use: (prepared: PreparedMcpCredential) => Effect.Effect, + ) => + threadAttachment.withLock( + threadAttachmentKey({ providerSessionId, threadId }), + mcpPrepareLock.withLock( + threadId, + // Adapters must consume the config before returning acceptance. + ensureThreadAttached({ + providerSessionId, + threadId, + providerInstanceId: runtime.instanceId, + runtime, + ...(providerThread === undefined ? {} : { providerThread }), + checkAccess: (candidateId, willRotate) => + Effect.gen(function* () { + const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); + if (entry?.runtime !== runtime) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: + "The provider session was released or replaced before preparing MCP access.", + }); + } + if ( + runtime.unloadThread === undefined && + (willRotate || candidateId !== entry.mcpCredentialIdByThread.get(threadId)) + ) { + return yield* new ProviderSessionActivityError({ + providerSessionId, + cause: "MCP access changed; reopen the provider session before continuing.", + }); + } + }), + }).pipe(Effect.flatMap(use)), + ), + ); + const resumeNativeThread = ( + input: Parameters[0], + staleMcpCredential: boolean, + ) => { + // Codex keeps its MCP clients in a loaded native conversation. + // A resume alone retains the revoked token; unload only this + // conversation before resuming it with the refreshed config. + const unload = staleMcpCredential + ? unloadNativeThread(runtime, input.providerThread).pipe( + Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS), + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: runtime.driver, + providerSessionId, + providerThreadId: input.providerThread.id, + cause, + }), + ), + ) + : Effect.void; + return unload.pipe(Effect.andThen(runtime.resumeThread(input))); + }; return { ...runtime, subscribeEvents, + ...(runtime.unloadThread === undefined + ? {} + : { + unloadThread: ({ providerThread }) => unloadNativeThread(runtime, providerThread), + }), events: Stream.unwrap( subscribeEvents.pipe(Effect.map((subscription) => subscription.events)), ), ensureThread: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - Effect.andThen(runtime.ensureThread(input)), - Effect.tap((providerThread) => - markProviderThreadLoaded({ - providerSessionId, - threadId: input.threadId, - providerThreadKey: providerThreadLoadKey({ + withRuntimeThread(input.threadId, undefined, (prepared) => + runtime.ensureThread(input).pipe( + Effect.tap((providerThread) => + markProviderThreadLoaded({ + runtime, providerThread, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, + mcpCredentialId: prepared.mcpCredentialId, + providerSessionId, + threadId: input.threadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + }), }), - }), + ), ), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterEnsureThreadError({ + driver: runtime.driver, + threadId: input.threadId, + cause, + }), + ), + }), ), resumeThread: (input) => { const threadId = input.threadId ?? input.providerThread.appThreadId; @@ -1857,111 +2311,219 @@ export const layerWithOptions = ( : { modelSelection: input.modelSelection }), ...(input.runtimePolicy === undefined ? {} : { runtimePolicy: input.runtimePolicy }), }); - return observeActivity( - providerSessionId, - ensureThreadAttached({ + return withRuntimeThread(threadId, input.providerThread, (prepared) => + holdThreadLoaded({ providerSessionId, threadId, - providerInstanceId: runtime.instanceId, - }), + providerThread: input.providerThread, + }).pipe( + Effect.andThen( + providerThreadLoadStatus({ + providerSessionId, + threadId, + providerThread: input.providerThread, + providerThreadKey, + }), + ), + Effect.flatMap(({ loaded, staleMcpCredential }) => { + if (loaded) return Effect.succeed(input.providerThread); + return resumeNativeThread(input, staleMcpCredential); + }), + Effect.tap((providerThread) => + markProviderThreadLoaded({ + runtime, + providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerSessionId, + threadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + ...(input.modelSelection === undefined + ? {} + : { modelSelection: input.modelSelection }), + ...(input.runtimePolicy === undefined + ? {} + : { runtimePolicy: input.runtimePolicy }), + }), + }), + ), + ), ).pipe( - Effect.andThen( - holdThreadLoaded({ + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: runtime.driver, + providerSessionId, + providerThreadId: input.providerThread.id, + cause, + }), + ), + }), + ); + }, + rollbackThread: (input) => { + const threadId = input.providerThread.appThreadId; + if (threadId === null) return runtime.rollbackThread(input); + return withRuntimeThread(threadId, input.providerThread, (prepared) => + Effect.gen(function* () { + yield* holdThreadLoaded({ providerSessionId, threadId, providerThread: input.providerThread, - }), - ), - Effect.andThen( - isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey }), - ), - Effect.flatMap((loaded) => - loaded ? Effect.succeed(input.providerThread) : runtime.resumeThread(input), - ), - Effect.tap((providerThread) => - markProviderThreadLoaded({ + }); + const cachedKey = (yield* Ref.get(sessions)) + .get(sessionKey(providerSessionId)) + ?.loadedProviderThreads.get(providerThreadRuntimeKey(input.providerThread))?.key; + const { loaded, staleMcpCredential } = yield* providerThreadLoadStatus({ providerSessionId, threadId, - providerThreadKey: providerThreadLoadKey({ - providerThread, - ...(input.modelSelection === undefined - ? {} - : { modelSelection: input.modelSelection }), - ...(input.runtimePolicy === undefined - ? {} - : { runtimePolicy: input.runtimePolicy }), - }), - }), - ), + providerThread: input.providerThread, + providerThreadKey: cachedKey, + }); + const providerThread = loaded + ? input.providerThread + : yield* resumeNativeThread( + { providerThread: input.providerThread, threadId }, + staleMcpCredential, + ); + // Rollback has no selection/policy; a default resume establishes + // credential ownership without validating the next turn's load key. + const providerThreadKey = loaded ? cachedKey : undefined; + yield* markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId, + providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerThreadKey, + }); + const snapshot = yield* runtime.rollbackThread({ ...input, providerThread }); + yield* markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId, + providerThread: snapshot.providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerThreadKey: + providerThreadRuntimeKey(snapshot.providerThread) === + providerThreadRuntimeKey(providerThread) + ? providerThreadKey + : undefined, + }); + return snapshot; + }), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver: runtime.driver, + providerThreadId: input.providerThread.id, + cause, + }), + ), + }), ); }, forkThread: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.targetThreadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - Effect.andThen(runtime.forkThread(input)), - Effect.tap((providerThread) => - markProviderThreadLoaded({ - providerSessionId, - threadId: input.targetThreadId, - providerThreadKey: providerThreadLoadKey({ + withRuntimeThread(input.targetThreadId, undefined, (prepared) => + runtime.forkThread(input).pipe( + Effect.tap((providerThread) => + markProviderThreadLoaded({ + runtime, providerThread, - ...(input.modelSelection === undefined - ? {} - : { modelSelection: input.modelSelection }), - ...(input.runtimePolicy === undefined - ? {} - : { runtimePolicy: input.runtimePolicy }), + mcpCredentialId: prepared.mcpCredentialId, + providerSessionId, + threadId: input.targetThreadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + ...(input.modelSelection === undefined + ? {} + : { modelSelection: input.modelSelection }), + ...(input.runtimePolicy === undefined + ? {} + : { runtimePolicy: input.runtimePolicy }), + }), }), - }), + ), ), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterForkThreadError({ + driver: runtime.driver, + providerThreadId: input.sourceProviderThread.id, + cause, + }), + ), + }), ), startTurn: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - // A start that fails or is stopped may never emit turn.terminal, - // so it clears its own turn or the session never goes idle. If - // the adapter emits the terminal anyway, clearing the same turn - // again changes nothing, so another thread's turn on a shared - // session stays busy either way. - Effect.andThen( - holdThreadLoaded({ + withRuntimeThread(input.threadId, input.providerThread, (prepared) => + Effect.gen(function* () { + yield* holdThreadLoaded({ providerSessionId, threadId: input.threadId, providerThread: input.providerThread, - }), - ), - Effect.andThen( - Effect.acquireUseRelease( - observeActivity( - providerSessionId, - markBusy( - providerSessionId, - busyTurnKey(input.providerThread.id, input.runOrdinal), - ), - ), + }); + const { loaded, staleMcpCredential } = yield* providerThreadLoadStatus({ + providerSessionId, + threadId: input.threadId, + providerThread: input.providerThread, + providerThreadKey: providerThreadLoadKey(input), + }); + // Retained continuations can start without the usual resume. + const providerThread = !loaded + ? yield* resumeNativeThread( + { + providerThread: input.providerThread, + threadId: input.threadId, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + }, + staleMcpCredential, + ) + : input.providerThread; + yield* markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId: input.threadId, + providerThread, + mcpCredentialId: prepared.mcpCredentialId, + providerThreadKey: providerThreadLoadKey({ ...input, providerThread }), + }); + // A failed/stopped start may never emit a terminal event. It + // clears only its own registration, while accepted turns stay busy. + return yield* Effect.acquireUseRelease( + markBusy(providerSessionId, busyTurnKey(providerThread.id, input.runOrdinal)), () => - runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), + runtime + .startTurn({ ...input, providerThread }) + .pipe(turnMetrics("send", input.modelSelection.model)), (_, exit) => Exit.isFailure(exit) ? observeActivity( providerSessionId, - markIdle(providerSessionId, input.providerThread.id, input.runOrdinal), + markIdle(providerSessionId, providerThread.id, input.runOrdinal), ) : Effect.void, - ), - ), + ); + }), + ).pipe( + Effect.catchTags({ + ProviderSessionActivityError: (cause) => + Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: runtime.driver, + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + cause, + }), + ), + }), ), steerTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( @@ -2153,199 +2715,255 @@ export const layerWithOptions = ( return ProviderSessionManagerV2.of({ shutdown, open: (input) => - sessionOpen.withLock( - input.providerSessionId, - Effect.gen(function* () { - const cwd = input.runtimePolicy.cwd; - if (cwd !== null) { - const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe( - Effect.map((stat) => stat.type === "Directory"), - Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")), - ); - if (!workspaceIsDirectory) { - return yield* new ProviderWorkspaceMissingError({ - threadId: input.threadId, - cwd, - }); - } + Effect.suspend(() => { + let preparedForCleanup: PreparedMcpCredential | undefined; + let openingScope: Scope.Closeable | undefined; + let reservationDropped = false; + const dropReservation = Effect.sync(() => { + const id = preparedForCleanup?.mcpCredentialId; + if (!reservationDropped && id !== undefined) { + reservationDropped = true; + dropMcpCredentialReservation(input.threadId, id); } - const key = sessionKey(input.providerSessionId); - const existing = (yield* Ref.get(sessions)).get(key); - if (existing !== undefined) { - const attached = existing.attachedThreadIds.has(input.threadId); - if (!attached && !existing.supportsMultipleProviderThreads) { - return yield* new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, - providerSessionId: input.providerSessionId, - cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`, - }); - } - // Joining a shared session is a new session for this thread. - if (!attached) yield* agentScope.clear(input.threadId); - yield* ensureThreadAttached({ - providerSessionId: input.providerSessionId, - threadId: input.threadId, - providerInstanceId: existing.runtime.instanceId, - }); - yield* touchActivity(input.providerSessionId); - return existing.exposedRuntime; - } - - const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( - Effect.mapError( - (cause) => - new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, + }); + // Any failure, including a Stop that interrupts a slow handshake, + // stops the provider process this open started. The close is + // time-boxed: this runs under the session's open lock, so a wedged + // adapter finalizer must not hold the interrupter or later opens. + const cleanupOpen = (cause: Cause.Cause): Effect.Effect => + Ref.get(sessions).pipe( + Effect.flatMap((current) => { + const scope = openingScope; + if ( + scope !== undefined && + current.get(sessionKey(input.providerSessionId))?.scope === scope + ) { + // The open lock prevents another open replacing this entry + // while failure cleanup hands it to the normal lifecycle. + return releaseEntry({ providerSessionId: input.providerSessionId, - cause, - }), - ), - ); - const prepared = yield* prepareMcpSession( - input.threadId, - input.modelSelection.instanceId, + reason: Cause.hasInterruptsOnly(cause) ? "manual_shutdown" : "runtime_error", + detail: Cause.hasInterruptsOnly(cause) + ? "The provider session start was interrupted." + : "The provider session did not finish opening.", + }).pipe(logReleaseFailure(input.providerSessionId)); + } + return scope === undefined + ? Effect.void + : closeScopeWithin(scope, { + providerSessionId: input.providerSessionId, + reason: "open_failed", + }); + }), ); - const mcpCredentialId = prepared.mcpCredentialId; - // The reservation from prepare protects the credential (which - // eager adapters bake into the provider process during - // openSession) from racing releases until this session's entry - // is recorded below. Dropped exactly once on every path. - let reservationDropped = mcpCredentialId === undefined; - const dropReservation = Effect.sync(() => { - if (!reservationDropped && mcpCredentialId !== undefined) { - reservationDropped = true; - dropMcpCredentialReservation(input.threadId, mcpCredentialId); + return sessionOpen.withLock( + input.providerSessionId, + Effect.gen(function* () { + const cwd = input.runtimePolicy.cwd; + if (cwd !== null) { + const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe( + Effect.map((stat) => stat.type === "Directory"), + Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")), + ); + if (!workspaceIsDirectory) { + return yield* new ProviderWorkspaceMissingError({ + threadId: input.threadId, + cwd, + }); + } } - }); - const sessionScope = yield* Scope.fork(sessionScopes); - // The previous session's agent scope must not explain this - // session's failures, even when this provider runs without one. - yield* agentScope.clear(input.threadId); - const runtime = yield* adapter - .openSession({ - threadId: input.threadId, - providerSessionId: input.providerSessionId, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, - ...(input.resumeFromSession === undefined - ? {} - : { resumeFromSession: input.resumeFromSession }), - ...(input.initialNativeThreadId === undefined - ? {} - : { initialNativeThreadId: input.initialNativeThreadId }), - ...(input.initialProviderItemIdentityVersion === undefined - ? {} - : { - initialProviderItemIdentityVersion: - input.initialProviderItemIdentityVersion, - }), - }) - .pipe( - Effect.provideService(Scope.Scope, sessionScope), - // Any failure, including a Stop that interrupts a slow - // handshake, stops the provider process this open started. - // The session cleanup runs first, and the close is - // time-boxed: this runs under the session's open lock, so an - // adapter finalizer that never finishes must not hold the - // interrupter, the lock, or later opens. - Effect.onError(() => - dropReservation.pipe( - // Clear only a session this open freshly set up: a reused - // one is held by another live provider process and must - // survive this open's failure. - Effect.andThen( - prepared.issued - ? clearMcpSession(input.threadId, mcpCredentialId) - : Effect.void, + const key = sessionKey(input.providerSessionId); + const existing = (yield* Ref.get(sessions)).get(key); + if (existing !== undefined) { + const attached = existing.attachedThreadIds.has(input.threadId); + if (!attached && !existing.supportsMultipleProviderThreads) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`, + }); + } + if ( + attached && + existing.runtime.unloadThread === undefined && + (yield* mcpAccessNeedsRefresh(existing, input.threadId)) + ) { + // Pi and other process-bound MCP clients can only pick up + // changed access by reopening through the normal lifecycle. + yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "manual_shutdown", + requireIdle: true, + }).pipe( + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), ), - Effect.ensuring( - closeScopeWithin(sessionScope, { - providerSessionId: input.providerSessionId, - reason: "open_failed", - }), + ); + } else { + // Joining a shared session is a new session for this thread. + if (!attached) yield* agentScope.clear(input.threadId); + // Cached native clients prepare access at the operation + // boundary, where pending work can reject a reload safely. + if ( + !attached && + (existing.runtime.unloadThread === undefined || + !Array.from(existing.loadedProviderThreads.values()).some( + (loaded) => loaded.threadId === input.threadId, + )) + ) + yield* threadAttachment.withLock( + threadAttachmentKey(input), + mcpPrepareLock.withLock( + input.threadId, + ensureThreadAttached({ + providerSessionId: input.providerSessionId, + threadId: input.threadId, + providerInstanceId: existing.runtime.instanceId, + runtime: existing.runtime, + }), + ), + ); + yield* touchActivity(input.providerSessionId); + return existing.exposedRuntime; + } + } + + return yield* mcpPrepareLock.withLock( + input.threadId, + Effect.gen(function* () { + const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), ), - ), - ), - Effect.mapError( - (cause) => - new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, + ); + const prepared = yield* prepareMcpSession( + input.threadId, + input.modelSelection.instanceId, + (prepared) => (preparedForCleanup = prepared), + input.providerSessionId, + ); + const mcpCredentialId = prepared.mcpCredentialId; + // The reservation from prepare protects the credential (which + // eager adapters bake into the provider process during + // openSession) from racing releases until this session's entry + // is recorded below. Dropped exactly once on every path. + const sessionScope = yield* Scope.fork(sessionScopes).pipe( + Effect.tap((scope) => Effect.sync(() => (openingScope = scope))), + Effect.uninterruptible, + ); + // The previous session's agent scope must not explain this + // session's failures, even when this provider runs without one. + yield* agentScope.clear(input.threadId); + const runtime = yield* adapter + .openSession({ + threadId: input.threadId, providerSessionId: input.providerSessionId, - cause, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + ...(input.resumeFromSession === undefined + ? {} + : { resumeFromSession: input.resumeFromSession }), + ...(input.initialNativeThreadId === undefined + ? {} + : { initialNativeThreadId: input.initialNativeThreadId }), + ...(input.initialProviderItemIdentityVersion === undefined + ? {} + : { + initialProviderItemIdentityVersion: + input.initialProviderItemIdentityVersion, + }), + }) + .pipe( + Effect.provideService(Scope.Scope, sessionScope), + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), + ), + withMetrics({ + counter: providerSessionsTotal, + attributes: { provider: adapter.driver, operation: "open" }, + }), + ); + const eventSubscribers = yield* Ref.make< + ReadonlyMap> + >(new Map()); + const exposedRuntime = decorateRuntime(runtime, eventSubscribers); + const now = yield* Clock.currentTimeMillis; + const entry: LiveSessionEntry = { + attachedThreadIds: new Set([input.threadId]), + // Process-bound adapters consume MCP access during open, before + // the first native thread is loaded. Keep that credential until + // a successful native reload records its replacement. + loadedProviderThreads: new Map(), + mcpCredentialIdByThread: + mcpCredentialId === undefined + ? new Map() + : new Map([[input.threadId, mcpCredentialId]]), + supportsMultipleProviderThreads: + runtime.providerSession.capabilities.sessions + .supportsMultipleProviderThreadsPerSession, + runtime, + exposedRuntime, + eventSubscribers, + requestEventPermit: yield* Semaphore.make(1), + scope: sessionScope, + idleGeneration: 0, + busyTurns: new Set(), + lastActivityAtMs: now, + idleFiber: null, + pinnedSinceMs: null, + idleThreadUnloads: new Map(), + }; + yield* Ref.update(sessions, (current) => { + const updated = new Map(current); + updated.set(key, entry); + return updated; + }); + // The entry now guards the credential via its recorded id, so + // the pre-open reservation can be dropped. + yield* dropReservation; + yield* withActivityError( + input.providerSessionId, + writeProviderSessionEvents({ + runtime, + threadIds: [input.threadId], + type: "provider-session.attached", + payload: runtime.providerSession, }), + ); + yield* startEventPump(entry); + yield* scheduleIdleRelease(input.providerSessionId); + return exposedRuntime; + }).pipe( + // A waiter must see cleanup finished before preparing its config. + // Scope close stays outside this gate so provider finalizers cannot + // hold up another provider's open. + Effect.onError(() => + Effect.gen(function* () { + yield* dropReservation; + const id = preparedForCleanup?.mcpCredentialId; + if (id !== undefined) yield* clearUnownedMcpSession(input.threadId, id); + }), + ), ), - withMetrics({ - counter: providerSessionsTotal, - attributes: { provider: adapter.driver, operation: "open" }, - }), ); - const eventSubscribers = yield* Ref.make< - ReadonlyMap> - >(new Map()); - const exposedRuntime = decorateRuntime(runtime, eventSubscribers); - const now = yield* Clock.currentTimeMillis; - const entry: LiveSessionEntry = { - attachedThreadIds: new Set([input.threadId]), - loadedProviderThreadKeyByThread: new Map(), - mcpCredentialIdByThread: - mcpCredentialId === undefined - ? new Map() - : new Map([[input.threadId, mcpCredentialId]]), - supportsMultipleProviderThreads: - runtime.providerSession.capabilities.sessions - .supportsMultipleProviderThreadsPerSession, - runtime, - exposedRuntime, - eventSubscribers, - requestEventPermit: yield* Semaphore.make(1), - scope: sessionScope, - idleGeneration: 0, - busyTurns: new Set(), - lastActivityAtMs: now, - idleFiber: null, - pinnedSinceMs: null, - idleThreadUnloads: new Map(), - }; - yield* Ref.update(sessions, (current) => { - const updated = new Map(current); - updated.set(key, entry); - return updated; - }); - // The entry now guards the credential via its recorded id, so - // the pre-open reservation can be dropped. - yield* dropReservation; - yield* withActivityError( - input.providerSessionId, - writeProviderSessionEvents({ - runtime, - threadIds: [input.threadId], - type: "provider-session.attached", - payload: runtime.providerSession, - }), - ).pipe( - // Released on interrupt too: this entry has no event pump or - // idle timer yet, so nothing else would ever release it. - Effect.onError((cause) => - releaseEntry( - Cause.hasInterruptsOnly(cause) - ? { - providerSessionId: input.providerSessionId, - reason: "manual_shutdown", - detail: "The provider session start was interrupted.", - } - : { - providerSessionId: input.providerSessionId, - reason: "runtime_error", - detail: "Failed to persist the provider-session attachment.", - }, - ).pipe(logReleaseFailure(input.providerSessionId)), - ), - ); - yield* startEventPump(entry); - yield* scheduleIdleRelease(input.providerSessionId); - return exposedRuntime; - }), - ), + }).pipe(Effect.onError((cause) => cleanupOpen(cause))), + ); + }), get: (providerSessionId) => Effect.gen(function* () { const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); @@ -2464,10 +3082,11 @@ export const layerWithOptions = ( } const attachedThreadIds = new Set(entry.attachedThreadIds); attachedThreadIds.delete(input.threadId); - const loadedProviderThreadKeyByThread = new Map( - entry.loadedProviderThreadKeyByThread, - ); - loadedProviderThreadKeyByThread.delete(input.threadId); + const loadedProviderThreads = new Map(entry.loadedProviderThreads); + for (const [id, loaded] of loadedProviderThreads) { + if (loaded.threadId === input.threadId) + loadedProviderThreads.set(id, { ...loaded, key: undefined }); + } // For a plain (workspace-change) detach, the credential id stays // recorded: the thread may re-attach and reuse it, and // releaseEntry revokes it when the provider process finally goes @@ -2481,13 +3100,15 @@ export const layerWithOptions = ( return pruned; })() : entry.mcpCredentialIdByThread; - // The detach unloads the thread itself below. + // Keep the native pointer until the detach's unload succeeds. const idleThreadUnloads = new Map(entry.idleThreadUnloads); - idleThreadUnloads.delete(input.threadId); + const pending = idleThreadUnloads.get(input.threadId); + if (pending !== undefined) + idleThreadUnloads.set(input.threadId, { ...pending, fiber: null }); const updatedEntry = { ...entry, attachedThreadIds, - loadedProviderThreadKeyByThread, + loadedProviderThreads, mcpCredentialIdByThread, idleThreadUnloads, }; @@ -2551,8 +3172,52 @@ export const layerWithOptions = ( ) { return; } + const pending = entry.idleThreadUnloads.get(input.threadId); + const providerThreads = new Map( + detachedProviderThreads.map( + (thread) => [providerThreadRuntimeKey(thread), thread] as const, + ), + ); + if (pending !== undefined) + providerThreads.set( + providerThreadRuntimeKey(pending.providerThread), + pending.providerThread, + ); + for (const loaded of entry.loadedProviderThreads.values()) { + if (loaded.threadId === input.threadId) + providerThreads.set( + providerThreadRuntimeKey(loaded.providerThread), + loaded.providerThread, + ); + } + const fallbackCredentialId = + entry.mcpCredentialIdByThread.get(input.threadId) ?? + Array.from(entry.loadedProviderThreads.values()).find( + (loaded) => loaded.threadId === input.threadId, + )?.mcpCredentialId; + yield* Ref.update(sessions, (current) => { + const latest = current.get(key); + if (latest?.runtime !== entry.runtime) return current; + const loadedProviderThreads = new Map(latest.loadedProviderThreads); + for (const providerThread of providerThreads.values()) { + if ( + providerThread.nativeThreadRef !== null && + !loadedProviderThreads.has(providerThreadRuntimeKey(providerThread)) + ) { + loadedProviderThreads.set(providerThreadRuntimeKey(providerThread), { + threadId: input.threadId, + providerThread, + key: undefined, + mcpCredentialId: fallbackCredentialId, + }); + } + } + return new Map(current).set(key, { ...latest, loadedProviderThreads }); + }); yield* Effect.forEach( - detachedProviderThreads.filter((thread) => thread.nativeThreadRef !== null), + [...providerThreads.values()].filter( + (thread) => thread.nativeThreadRef !== null, + ), (providerThread) => unloadThread({ providerThread }).pipe( // Bounded so a wedged provider cannot hold up the @@ -2572,6 +3237,22 @@ export const layerWithOptions = ( ), { concurrency: 1, discard: true }, ); + yield* Ref.update(sessions, (current) => { + const latest = current.get(key); + if ( + latest?.runtime !== entry.runtime || + latest.attachedThreadIds.has(input.threadId) || + latest.idleThreadUnloads.get(input.threadId) !== pending || + (pending !== undefined && + latest.loadedProviderThreads.has( + providerThreadRuntimeKey(pending.providerThread), + )) + ) + return current; + const idleThreadUnloads = new Map(latest.idleThreadUnloads); + idleThreadUnloads.delete(input.threadId); + return new Map(current).set(key, { ...latest, idleThreadUnloads }); + }); }), ); } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts index 525b9b4cff4e7..5c55bafcf78a9 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_output.ts @@ -23,9 +23,9 @@ export function assertThreadRollbackAfterRestartOutput( ); const revert = methods.indexOf("thread/revert"); assert.isAbove(revert, 0, "the rollback must reach thread/revert"); - assert.equal( - methods.lastIndexOf("thread/resume", revert), - revert - 2, - "the fresh app-server must resume the thread right before listing turns and reverting", + assert.deepEqual( + methods.slice(revert - 3, revert + 1), + ["thread/resume", "thread/read", "thread/turns/list", "thread/revert"], + "the fresh app-server must resume the thread before reading metadata, listing turns, and reverting", ); } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson index 2e5d54b8ddbce..0a71f53802020 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_rollback_after_restart/codex_transcript.ndjson @@ -48,14 +48,14 @@ {"type":"expect_outbound","label":"initialize","frame":{"id":1,"method":"initialize","params":{"capabilities":{"experimentalApi":true,"optOutNotificationMethods":["turn/diff/updated"],"extensions":{"io.modelcontextprotocol/ui":{"mimeTypes":["text/html;profile=mcp-app"]}}},"clientInfo":{"name":"T3 Code","title":"T3 Code","version":"0.0.44"}}}} {"type":"emit_inbound","label":"initialize","frame":{"id":1,"result":{"userAgent":"t3code_desktop/0.156.1 (Debian 13.0.0; x86_64) unknown (t3code_desktop; 0.1.0)","codexHome":"/home/replay-user/tmp/bughunt-codex/codex-home","platformFamily":"unix","platformOs":"linux"}}} {"type":"expect_outbound","label":"initialized","frame":{"method":"initialized"}} -{"type":"expect_outbound","label":"thread/read","frame":{"id":2,"method":"thread/read","params":{"includeTurns":false,"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} +{"type":"expect_outbound","label":"thread/resume","frame":{"id":2,"method":"thread/resume","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000},"cwd":"","excludeTurns":true,"model":"gpt-6-luna","threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} {"type":"emit_inbound","label":"configWarning","frame":{"method":"configWarning","params":{"summary":"Codex could not find bubblewrap on PATH. Install bubblewrap with your OS package manager. See the sandbox prerequisites: https://developers.openai.com/codex/concepts/sandboxing#prerequisites. Codex will use the bundled bubblewrap in the meantime.","details":null},"emittedAtMs":1790404935235}} {"type":"emit_inbound","label":"remoteControl/status/changed","frame":{"method":"remoteControl/status/changed","params":{"status":"disabled","serverName":"replay-host","installationId":"00000000-0000-4000-8000-000000000000","environmentId":null},"emittedAtMs":1790404935235}} -{"type":"emit_inbound","label":"thread/read","frame":{"id":2,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":null,"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404934,"recencyAt":1790404931,"status":{"type":"notLoaded"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":null,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]}}}} -{"type":"expect_outbound","label":"thread/resume","frame":{"id":3,"method":"thread/resume","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000},"cwd":"","excludeTurns":true,"model":"gpt-6-luna","threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1790404935482}} {"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","status":{"type":"idle"}},"emittedAtMs":1790404935500}} -{"type":"emit_inbound","label":"thread/resume","frame":{"id":3,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":[{"environmentId":"local","cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"]}],"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404935,"recencyAt":1790404931,"status":{"type":"idle"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]},"model":"gpt-6-luna","modelProvider":"openai","serviceTier":null,"disabledPluginIds":[],"cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"],"instructionSources":["/home/replay-user/t3code/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"workspaceWrite","writableRoots":[],"networkAccess":false,"excludeTmpdirEnvVar":false,"excludeSlashTmp":false},"activePermissionProfile":{"id":":workspace","extends":null},"reasoningEffort":null,"collaborationMode":{"mode":"default","settings":{"model":"gpt-6-luna","reasoning_effort":null,"developer_instructions":null}},"multiAgentMode":"explicitRequestOnly","initialTurnsPage":null,"turnsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":16,\"includeAnchor\":true,\"scope\":{\"kind\":\"turns\"}}","itemsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":22,\"includeAnchor\":true,\"scope\":{\"kind\":\"itemsByCreatedAtOrdinal\"}}"}}} +{"type":"emit_inbound","label":"thread/resume","frame":{"id":2,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":[{"environmentId":"local","cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"]}],"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404935,"recencyAt":1790404931,"status":{"type":"idle"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]},"model":"gpt-6-luna","modelProvider":"openai","serviceTier":null,"disabledPluginIds":[],"cwd":"/home/replay-user/t3code/apps/server","runtimeWorkspaceRoots":["/home/replay-user/t3code/apps/server"],"instructionSources":["/home/replay-user/t3code/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"workspaceWrite","writableRoots":[],"networkAccess":false,"excludeTmpdirEnvVar":false,"excludeSlashTmp":false},"activePermissionProfile":{"id":":workspace","extends":null},"reasoningEffort":null,"collaborationMode":{"mode":"default","settings":{"model":"gpt-6-luna","reasoning_effort":null,"developer_instructions":null}},"multiAgentMode":"explicitRequestOnly","initialTurnsPage":null,"turnsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":16,\"includeAnchor\":true,\"scope\":{\"kind\":\"turns\"}}","itemsBackwardsCursor":"{\"requestedThreadId\":\"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a\",\"rolloutOrdinal\":22,\"includeAnchor\":true,\"scope\":{\"kind\":\"itemsByCreatedAtOrdinal\"}}"}}} +{"type":"expect_outbound","label":"thread/read","frame":{"id":3,"method":"thread/read","params":{"includeTurns":false,"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} +{"type":"emit_inbound","label":"thread/read","frame":{"id":3,"result":{"thread":{"id":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","environments":null,"extra":null,"sessionId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","forkedFromId":null,"parentThreadId":null,"preview":"Respond with exactly: rollback fixture first turn complete","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790404926,"updatedAt":1790404934,"recencyAt":1790404931,"status":{"type":"idle"},"path":"/home/replay-user/tmp/bughunt-codex/codex-home/sessions/2026/09/25/rollout-2026-09-25T23-42-06-01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a.jsonl","cwd":"/home/replay-user/t3code/apps/server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":null,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":{"sha":"70629cdddbe2c29749e8dce588b1828580b2777b","branch":"fix/codex-rollback-resume","originUrl":"https://github.com/pingdotgg/t3code.git"},"name":null,"daybreakEnabled":null,"turns":[]}}}} {"type":"expect_outbound","label":"thread/turns/list","frame":{"id":4,"method":"thread/turns/list","params":{"cursor":null,"itemsView":"summary","limit":1,"sortDirection":"desc","threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"}}} {"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a","turnId":"01a0dc73-2dc4-7911-a448-c66b9bf3318b","tokenUsage":{"total":{"totalTokens":44003,"inputTokens":43985,"cachedInputTokens":19200,"cacheWriteInputTokens":0,"outputTokens":18,"reasoningOutputTokens":0},"last":{"totalTokens":24198,"inputTokens":24189,"cachedInputTokens":19200,"cacheWriteInputTokens":0,"outputTokens":9,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1790404935520}} {"type":"emit_inbound","label":"thread/goal/cleared","frame":{"method":"thread/goal/cleared","params":{"threadId":"01a0dc73-1bb4-74f1-977f-2fc39b8aaa6a"},"emittedAtMs":1790404935521}} diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 785bf1a06e70c..c68e780b99df2 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -4206,13 +4206,14 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr Effect.gen(function* () { const nativeId = providerThread.nativeThreadRef?.nativeId; const state = nativeId == null ? undefined : threads.get(nativeId); - if ( - nativeId == null || - state === undefined || - state.active !== undefined || - hasBackground(state) - ) { - return; + if (nativeId == null || state === undefined) return; + // Success lets the manager drop native credential ownership; a busy + // native must fail instead of reporting a successful no-op. + if (state.active !== undefined || hasBackground(state)) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: OPENCODE_PROVIDER, + detail: `Cannot unload OpenCode session ${nativeId} while it is still working`, + }); } threads.delete(nativeId); for (const [child, owner] of childOwners) { From ce7cf34c1d3c46db267baa79675f374c7a042602 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Wed, 14 Oct 2026 07:20:42 -0400 Subject: [PATCH 2/2] fix(server): retain MCP ownership when native resume is interrupted Record the prepared credential as an unknown native load after stale unload succeeds and before awaiting resume. Interrupted acceptance can then trigger the required unload on the next credential rotation. Cover resume, direct turn start, and rollback with deterministic cancellation regressions. Co-Authored-By: GPT-6.1-Sol at high effort Co-Authored-By: GPT-6.1-Sol at high effort, implementation subagent for interrupted native resume Co-Authored-By: Claude Opus 5.5 at high effort, cleanup review subagent for interrupted native resume --- .../ProviderSessionManager.test.ts | 119 ++++++++++++++++++ .../ProviderSessionManager.ts | 29 ++++- 2 files changed, 145 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index d3bc1317e1d0f..a561aa8e72789 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -346,6 +346,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly afterResume?: Effect.Effect; readonly rollbackThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["rollbackThread"]; readonly beforeUnload?: Effect.Effect; readonly beforeClose?: Effect.Effect; @@ -465,6 +466,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( ), ), ), + Effect.andThen(options.afterResume ?? Effect.void), Effect.as(threadInput.providerThread), ), startTurn: () => options.startTurn ?? Effect.void, @@ -543,6 +545,7 @@ function layerTest(input: { readonly backgroundTasksEndWithProcess?: boolean; readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; + readonly afterResume?: Effect.Effect; readonly rollbackThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["rollbackThread"]; readonly beforeUnload?: Effect.Effect; readonly beforeClose?: Effect.Effect; @@ -586,6 +589,7 @@ function layerTest(input: { ? {} : { hangSessionScopeClose: input.hangSessionScopeClose }), ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), + ...(input.afterResume === undefined ? {} : { afterResume: input.afterResume }), ...(input.rollbackThread === undefined ? {} : { rollbackThread: input.rollbackThread }), ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), ...(input.beforeClose === undefined ? {} : { beforeClose: input.beforeClose }), @@ -3120,6 +3124,121 @@ it.effect.each(["open", "refresh"] as const)( }), ); +it.effect.each(["resume", "start", "rollback"] as const)( + "ProviderSessionManagerV2 retains accepted native MCP ownership when %s is interrupted", + (operation) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const resumedConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const residentConfig = yield* Ref.make< + McpProviderSession.McpProviderSessionConfig | undefined + >(undefined); + const accepted = yield* Deferred.make(); + const settingsLayer = ServerSettings.layerTest({ + enableAgentBrowserAccess: true, + enableAgentDeviceAccess: true, + }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const settings = yield* ServerSettings.ServerSettingsService; + const configs = yield* McpProviderSessions.McpProviderSessions; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`interrupted-native-${operation}`); + const providerSessionId = yield* ids.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator: ids, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator: ids, + threadId, + providerSessionId, + now, + }); + const resumeInput = { providerThread, modelSelection, runtimePolicy }; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const runId = ids.derive.run({ threadId, ordinal: 1 }); + const firstCall = + operation === "resume" + ? runtime.resumeThread(resumeInput) + : operation === "rollback" + ? runtime.rollbackThread({ + providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("interrupted-native-checkpoint"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }) + : runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: ids.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: ids.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* ids.allocate.message({ threadId, ordinal: 1 }), + text: "Continue the retained native thread", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + const pending = yield* firstCall.pipe(Effect.asVoid, Effect.forkChild); + yield* Deferred.await(accepted); + yield* Fiber.interrupt(pending); + + yield* settings.updateSettings({ + enableAgentBrowserAccess: false, + enableAgentDeviceAccess: false, + }); + yield* runtime.resumeThread(resumeInput); + const refreshed = (yield* configs.read(threadId))!; + assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + assert.deepEqual(yield* Ref.get(residentConfig), refreshed); + }).pipe( + Effect.provide( + Layer.merge( + settingsLayer, + layerTest({ + state, + idleTimeoutMs: 60_000, + serverSettingsLayer: settingsLayer, + resumeMcpConfigs: resumedConfigs, + beforeUnload: Ref.set(residentConfig, undefined), + afterResume: Effect.gen(function* () { + const config = (yield* Ref.get(resumedConfigs)).at(-1)!; + // A loaded Codex conversation retains its first MCP client until unload. + yield* Ref.update(residentConfig, (current) => current ?? config); + if (yield* Deferred.succeed(accepted, undefined)) { + return yield* Effect.never; + } + }), + }), + ), + ), + ); + }), +); + it.effect.each(["expired", "browser-disabled", "device-disabled"] as const)( "ProviderSessionManagerV2 reloads an attached native thread when its MCP credential is %s", (reason) => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index a23984bf9af8e..17b55239a34b2 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2236,8 +2236,11 @@ export const layerWithOptions = ( ), ); const resumeNativeThread = ( - input: Parameters[0], + input: Parameters[0] & { + readonly threadId: ThreadId; + }, staleMcpCredential: boolean, + mcpCredentialId: string | undefined, ) => { // Codex keeps its MCP clients in a loaded native conversation. // A resume alone retains the revoked token; unload only this @@ -2256,7 +2259,21 @@ export const layerWithOptions = ( ), ) : Effect.void; - return unload.pipe(Effect.andThen(runtime.resumeThread(input))); + return unload.pipe( + // Resume may consume the config before its interruptible wait returns. + // Retain that ownership even when acceptance is never published. + Effect.andThen( + markProviderThreadLoaded({ + runtime, + providerSessionId, + threadId: input.threadId, + providerThread: input.providerThread, + mcpCredentialId, + providerThreadKey: undefined, + }), + ), + Effect.andThen(runtime.resumeThread(input)), + ); }; return { ...runtime, @@ -2327,7 +2344,11 @@ export const layerWithOptions = ( ), Effect.flatMap(({ loaded, staleMcpCredential }) => { if (loaded) return Effect.succeed(input.providerThread); - return resumeNativeThread(input, staleMcpCredential); + return resumeNativeThread( + { ...input, threadId }, + staleMcpCredential, + prepared.mcpCredentialId, + ); }), Effect.tap((providerThread) => markProviderThreadLoaded({ @@ -2386,6 +2407,7 @@ export const layerWithOptions = ( : yield* resumeNativeThread( { providerThread: input.providerThread, threadId }, staleMcpCredential, + prepared.mcpCredentialId, ); // Rollback has no selection/policy; a default resume establishes // credential ownership without validating the next turn's load key. @@ -2484,6 +2506,7 @@ export const layerWithOptions = ( runtimePolicy: input.runtimePolicy, }, staleMcpCredential, + prepared.mcpCredentialId, ) : input.providerThread; yield* markProviderThreadLoaded({