From 37010e4016c212f9af3301077d7e5bc54b3f586f Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Sat, 3 Oct 2026 09:26:08 -0400 Subject: [PATCH 01/14] fix(orchestration): resume queued work after planned workspace handoffs --- .../server/src/mcp/WorktreeMcpService.test.ts | 141 +++- apps/server/src/mcp/WorktreeMcpService.ts | 49 +- .../src/orchestration-v2/EffectOutbox.ts | 2 + .../src/orchestration-v2/EffectWorker.ts | 1 + .../src/orchestration-v2/Orchestrator.ts | 43 +- .../ProviderSessionManager.ts | 29 +- .../RunExecutionService.test.ts | 66 +- .../orchestration-v2/RunExecutionService.ts | 81 ++- .../SelectionRestart.integration.test.ts | 624 +++++++++++++++++- docs/internals/overview.md | 7 + docs/user/composer.md | 4 + packages/contracts/src/orchestrationV2.ts | 4 + .../src/server/ProviderAdapter.ts | 11 + 13 files changed, 994 insertions(+), 68 deletions(-) diff --git a/apps/server/src/mcp/WorktreeMcpService.test.ts b/apps/server/src/mcp/WorktreeMcpService.test.ts index 3be8705cc0bb0..0a715a2f8954c 100644 --- a/apps/server/src/mcp/WorktreeMcpService.test.ts +++ b/apps/server/src/mcp/WorktreeMcpService.test.ts @@ -3,7 +3,10 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { CommandId, EnvironmentId, + EventId, + RunId, type OrchestrationV2ThreadProjection, + type OrchestrationV2ServerCommand, type Project, ProjectId, ProviderInstanceId, @@ -11,6 +14,7 @@ import { WorktreeMcpHandoffInput, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -26,6 +30,7 @@ import * as ServerConfig from "../config.ts"; import { OrchestratorDispatchError, OrchestratorProjectionError, + type OrchestratorV2DispatchResult, } from "../orchestration-v2/Orchestrator.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as ProjectService from "../project/ProjectService.ts"; @@ -103,7 +108,7 @@ interface HarnessOptions { readonly threadAttachedOnRecheck?: boolean; readonly threadArchivedOnRecheck?: boolean; readonly threadReadFailsOnRecheck?: boolean; - readonly continuation?: "queued" | "fails" | "dies"; + readonly continuation?: "queued" | "started"; readonly projectMissing?: boolean; readonly projectReadFails?: boolean; readonly existingBranchWorktreePath?: string | null; @@ -118,7 +123,7 @@ interface HarnessOptions { const makeHarness = (options: HarnessOptions = {}) => { const thread = options.thread === undefined ? {} : options.thread; const scope = makeScope(options.capabilities ?? new Set(["preview", "worktree"])); - const dispatch = vi.fn((_: unknown) => + const dispatch = vi.fn((command: OrchestrationV2ServerCommand) => (options.dispatchGate ?? Effect.void).pipe( Effect.andThen( options.dispatchInterrupts @@ -127,7 +132,45 @@ const makeHarness = (options: HarnessOptions = {}) => { ? Effect.die(new Error("dispatch defect")) : options.dispatchFails ? (Effect.fail("simulated dispatch failure") as never) - : Effect.succeed({ sequence: 1, storedEvents: [] }), + : Effect.succeed({ + sequence: 1, + storedEvents: + command.type !== "thread.metadata.update" || + command.worktreeContinuation === undefined + ? [] + : [ + { + sequence: 1, + commandId: command.commandId, + event: { + id: EventId.make("event-worktree-continuation"), + type: "run.created", + threadId, + occurredAt: DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"), + payload: { + id: RunId.make("run-worktree-continuation"), + threadId, + ordinal: 2, + providerInstanceId: ProviderInstanceId.make("provider-test"), + modelSelection: { + instanceId: ProviderInstanceId.make("provider-test"), + model: "test-model", + }, + providerThreadId: null, + userMessageId: command.worktreeContinuation.messageId, + rootNodeId: null, + activeAttemptId: null, + status: options.continuation === "started" ? "starting" : "queued", + requestedAt: DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"), + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + }, + }, + ], + } satisfies OrchestratorV2DispatchResult), ), ), ); @@ -175,22 +218,9 @@ const makeHarness = (options: HarnessOptions = {}) => { ? Effect.succeed(makeProjection(thread)) : Effect.fail(new OrchestratorProjectionError({ threadId: id })); }); - const sendToThread = vi.fn((_: unknown) => { - switch (options.continuation ?? "queued") { - case "fails": - return Effect.fail( - new ThreadManagementService.ThreadManagementThreadArchivedError({ - threadId, - }), - ); - case "dies": - return Effect.die(new Error("send defect")); - default: - return Effect.succeed({ - delivery: "queued", - } as ThreadManagementService.ThreadManagementSendResult); - } - }); + const sendToThread = vi.fn(() => + Effect.die(new Error("Standalone continuation dispatch is unsafe")), + ); const getById = vi.fn((id: ProjectId) => options.projectReadFails ? (Effect.fail("simulated project read failure") as never) @@ -370,19 +400,68 @@ const runStatus = (harness: ReturnType) => const decodeHandoffInput = Schema.decodeUnknownEffect(WorktreeMcpHandoffInput); describe("t3_worktree_handoff", () => { - it.effect("reports a continuation failure without failing the handoff", () => { - const harness = makeHarness({ continuation: "fails" }); + it.effect("queues the continuation prompt as the thread's next message", () => { + const harness = makeHarness(); return Effect.gen(function* () { const result = yield* runHandoff(harness, { - branch: "feature/continue-fails", + branch: "feature/continue", + continuationPrompt: "Keep fixing the login bug in the new worktree.", + }); + + expect(result.continuation).toEqual({ status: "scheduled", delivery: "queued" }); + expect(harness.dispatch).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + type: "thread.metadata.update", + threadId, + worktreePath: result.worktreePath, + expectedWorktreePath: null, + worktreeContinuation: { + messageId: expect.stringContaining("worktree-continuation"), + text: "Keep fixing the login bug in the new worktree.", + }, + }), + ); + expect(harness.sendToThread).not.toHaveBeenCalled(); + // The continuation must be durably queued before anything slower runs. + expect(harness.dispatch.mock.invocationCallOrder[0]).toBeLessThan( + harness.runForThread.mock.invocationCallOrder[0]!, + ); + }); + }); + + it.effect("reports a continuation started when no run was blocking the handoff", () => { + const harness = makeHarness({ continuation: "started" }); + return Effect.gen(function* () { + const result = yield* runHandoff(harness, { + branch: "feature/continue-idle", continuationPrompt: "Keep going.", }); - expect(result.continuation).toMatchObject({ status: "failed" }); - expect(result.worktreePath).toBe("/worktrees/project/feature/continue-fails"); - expect(harness.dispatch).toHaveBeenCalled(); + expect(result.continuation).toEqual({ status: "scheduled", delivery: "started" }); + expect(harness.dispatch).toHaveBeenCalledTimes(1); + expect(harness.sendToThread).not.toHaveBeenCalled(); }); }); + it.effect( + "rolls back the worktree when the atomic bind and continuation dispatch defects", + () => { + const harness = makeHarness({ dispatchDies: true }); + return Effect.gen(function* () { + const exit = yield* Effect.exit( + runHandoff(harness, { + branch: "feature/continue-dies", + continuationPrompt: "Keep going.", + }), + ); + expectTypedFailure(exit, { _tag: "WorktreeMcpFailure", code: "operation_failed" }); + expect(harness.removeWorktree).toHaveBeenCalledTimes(1); + expect(harness.deleteLocalBranch).toHaveBeenCalledTimes(1); + expect(harness.runForThread).not.toHaveBeenCalled(); + expect(harness.sendToThread).not.toHaveBeenCalled(); + }); + }, + ); + it.effect("does not queue a continuation when the thread update fails", () => { const harness = makeHarness({ dispatchFails: true }); return Effect.gen(function* () { @@ -586,6 +665,9 @@ describe("t3_worktree_handoff", () => { const gate = yield* Deferred.make(); const harness = makeHarness({ dispatchGate: Deferred.await(gate) }); + // Interrupt arrives while the metadata dispatch is in flight; the + // binding-plus-continuation section must run to completion anyway so the + // atomic binding and continuation commit survives the pending interruption. const fiber = yield* Effect.forkChild( runHandoff(harness, { branch: "feature/interrupted", @@ -600,7 +682,14 @@ describe("t3_worktree_handoff", () => { yield* Fiber.join(interruption); expect(harness.dispatch).toHaveBeenCalledTimes(1); - expect(harness.sendToThread).toHaveBeenCalledTimes(1); + expect(harness.dispatch).toHaveBeenCalledWith( + expect.objectContaining({ + worktreeContinuation: expect.objectContaining({ text: "Keep going in the worktree." }), + }), + ); + expect(harness.sendToThread).not.toHaveBeenCalled(); + // The setup script must also survive the pending interrupt; otherwise + // the continuation run starts in a worktree that was never set up. expect(harness.runForThread).toHaveBeenCalledTimes(1); }), ); diff --git a/apps/server/src/mcp/WorktreeMcpService.ts b/apps/server/src/mcp/WorktreeMcpService.ts index 267df07f2432f..45185f858443a 100644 --- a/apps/server/src/mcp/WorktreeMcpService.ts +++ b/apps/server/src/mcp/WorktreeMcpService.ts @@ -130,7 +130,6 @@ const make = Effect.gen(function* () { ); return { commandId: CommandId.make(part("command", "worktree-handoff")), - continuationCommandId: CommandId.make(part("command", "worktree-continuation")), continuationMessageId: MessageId.make(part("message", "worktree-continuation")), }; }), @@ -239,6 +238,7 @@ const make = Effect.gen(function* () { ); const worktreePath = worktree.worktree.path; + // Setup errors are reported after the handoff has already committed. const reportFailed = (logMessage: string) => Effect.catchCause((cause: Cause.Cause) => { const detail = errorMessage(Cause.squash(cause)); @@ -275,7 +275,7 @@ const make = Effect.gen(function* () { `Thread '${scope.thread.threadId}' was archived while the worktree was being created; the handoff was rolled back.`, ); } - yield* threadManagement + return yield* threadManagement .dispatch({ type: "thread.metadata.update", commandId: ids.commandId, @@ -283,6 +283,14 @@ const make = Effect.gen(function* () { branch: worktree.worktree.refName, worktreePath, expectedWorktreePath: null, + ...(input.continuationPrompt === undefined + ? {} + : { + worktreeContinuation: { + messageId: ids.continuationMessageId, + text: input.continuationPrompt, + }, + }), }) .pipe( Effect.catchCause((cause) => @@ -303,33 +311,16 @@ const make = Effect.gen(function* () { ), ); - // Queue continuation immediately after binding; detachment terminates the caller, so relaunch needs a durable message. - const queueContinuation: Effect.Effect = - Effect.suspend(() => - input.continuationPrompt === undefined - ? Effect.succeed({ status: "skipped" }) - : threadManagement - .sendToThread({ - projectId: projection.thread.projectId, - commandId: ids.continuationCommandId, - threadId: scope.thread.threadId, - messageId: ids.continuationMessageId, - text: input.continuationPrompt, - attachments: [], - mode: "queue", - createdBy: "agent", - creationSource: "mcp", - }) - .pipe( - Effect.map((sendResult): WorktreeMcpContinuationStatus => ({ - status: "scheduled", - delivery: sendResult.delivery, - })), - reportFailed("worktree handoff continuation failed to queue"), - ), - ); - - const continuation = yield* recheckAndBind.pipe(Effect.andThen(queueContinuation)); + // Binding and continuation events share the detach effect's commit, so + // even an immediately executing detach sees the durable queued message. + const bound = yield* recheckAndBind; + const queued = bound.storedEvents.some( + ({ event }) => event.type === "run.created" && event.payload.status === "queued", + ); + const continuation: WorktreeMcpContinuationStatus = + input.continuationPrompt === undefined + ? { status: "skipped" } + : { status: "scheduled", delivery: queued ? "queued" : "started" }; yield* vcsStatusBroadcaster .refreshStatus(worktreePath) diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 92e1e01ad43f2..d6d7323d5c870 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -38,6 +38,8 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ type: Schema.Literal("provider-session.detach"), providerSessionId: ProviderSessionId, detail: Schema.optional(Schema.String), + reason: Schema.optional(Schema.Literal("workspace_changed")), + /** Set on terminal detaches (thread archive/delete): revoke the thread's MCP credentials. */ revokeMcpCredential: Schema.optional(Schema.Boolean), }), Schema.Struct({ diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index d174db2fb3a29..9c5c4e6b3c3df 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -124,6 +124,7 @@ export const layerExecutor: Layer.Layer< providerSessionId: effect.request.providerSessionId, threadId: effect.threadId, ...(effect.request.detail === undefined ? {} : { detail: effect.request.detail }), + ...(effect.request.reason === undefined ? {} : { reason: effect.request.reason }), ...(effect.request.revokeMcpCredential === undefined ? {} : { revokeMcpCredential: effect.request.revokeMcpCredential }), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 91acb881637c1..96fe5d1c35eba 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -2917,6 +2917,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Thread ${command.threadId} branch changed before the metadata update could be applied.`, }); } + if (command.type === "thread.metadata.update" && command.worktreeContinuation !== undefined) { + if (isProviderNativeSubagentThread(thread)) { + return yield* new OrchestratorSubagentThreadReadOnlyError({ + commandId: command.commandId, + threadId: command.threadId, + }); + } + if ( + command.worktreePath == null || + command.worktreePath === thread.worktreePath || + thread.archivedAt !== null + ) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: + "A worktree continuation requires an active thread and a changed worktree binding.", + }); + } + } if (command.type === "thread.metadata.update" && command.expectedEmpty === true) { const records = yield* projectionStore .getThreadRecords(command.threadId, ["runs"]) @@ -3908,6 +3928,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? "Runtime mode changed." : "Provider or model selection changed.", // Terminal detaches revoke MCP credentials; reattachments need other detach reasons to preserve them. + ...(command.type === "thread.metadata.update" + ? { reason: "workspace_changed" as const } + : {}), ...(command.type === "thread.archive" ? { revokeMcpCredential: true } : {}), }, } satisfies PendingOrchestrationEffectV2; @@ -12461,7 +12484,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "thread.pin.reorder": case "thread.active.reorder": case "thread.mark-unread": - case "thread.metadata.update": case "thread.pull-request.link": case "thread.pull-request.unlink": case "thread.pull-request-link.sync": @@ -12474,6 +12496,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "provider.switch": yield* dispatchThreadMutation(command, events, effects); break; + case "thread.metadata.update": + yield* dispatchThreadMutation(command, events, effects); + if (command.worktreeContinuation !== undefined) { + yield* dispatchMessage( + { + type: "message.dispatch", + commandId: command.commandId, + threadId: command.threadId, + ...command.worktreeContinuation, + attachments: [], + createdBy: "agent", + creationSource: "mcp", + dispatchMode: { type: "queue_after_active" }, + }, + events, + effects, + ); + } + break; case "thread.pull-request-watch.sync": yield* dispatchPullRequestWatchSync(command, events, effects); break; diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index df2ee630df38b..0a021b0d4ee3b 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -178,6 +178,12 @@ export interface ProviderSessionManagerV2Shape { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; readonly detail?: string; + readonly reason?: "workspace_changed"; + /** + * True for terminal detaches (thread archived or deleted): the thread's + * MCP credentials are revoked immediately instead of surviving for a + * potential re-attach. + */ readonly revokeMcpCredential?: boolean; }) => Effect.Effect; } @@ -506,12 +512,20 @@ export const layerWithOptions = ( const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* ( entry: LiveSessionEntry, detail: string, + expectedDetach?: Pick, ) { - const error = new ProviderAdapter.ProviderAdapterEventStreamError({ - driver: entry.runtime.driver, - providerSessionId: entry.runtime.providerSessionId, - cause: detail, - }); + const error = + expectedDetach === undefined + ? new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: entry.runtime.driver, + providerSessionId: entry.runtime.providerSessionId, + cause: detail, + }) + : new ProviderAdapter.ProviderAdapterSessionDetached({ + driver: entry.runtime.driver, + providerSessionId: entry.runtime.providerSessionId, + ...expectedDetach, + }); const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map()); yield* Effect.forEach( subscribers.values(), @@ -972,6 +986,7 @@ export const layerWithOptions = ( readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; readonly gracefulSubscribers?: boolean; + readonly expectedDetach?: Pick; }) => Effect.acquireUseRelease( removeLiveEntry(input), @@ -998,6 +1013,7 @@ export const layerWithOptions = ( yield* failSubscribers( entry, input.detail ?? `Provider session released: ${input.reason}.`, + input.expectedDetach, ); } const closeExit = yield* closeScopeWithin(entry.scope, { @@ -2380,6 +2396,9 @@ export const layerWithOptions = ( reason: "manual_shutdown", ...(input.detail === undefined ? {} : { detail: input.detail }), detachedThreadId: input.threadId, + ...(input.reason === undefined + ? {} + : { expectedDetach: { threadId: input.threadId, reason: input.reason } }), }); return; } diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 394cc1527cb7d..787cef0fa1c63 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2548,6 +2548,62 @@ it.effect("does not refresh pull requests for auxiliary or stale provider termin }), ); +it.effect("finalizes a planned workspace detach before the provider announces its first turn", () => + Effect.gen(function* () { + const key = "workspace-detach-before-turn"; + const { written, observed } = yield* captureRootRunTermination({ + key, + knownProviderTurn: false, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.fail( + new ProviderAdapter.ProviderAdapterSessionDetached({ + driver, + providerSessionId: ProviderSessionId.make(`session:${key}`), + threadId: ids.threadId, + reason: "workspace_changed", + }), + ), + }); + assert.deepEqual(observed, ["run:interrupted", "pull-requests-refreshed"]); + assert.deepEqual( + written.map((item) => item.type), + ["run_interrupt_result"], + ); + const interruption = written.find((item) => item.type === "run_interrupt_result"); + assert.equal(interruption?.message, "Run interrupted because the workspace changed"); + }), +); + +it.effect.each(["thread", "session"] as const)( + "fails rather than treating another %s's planned detach as this run's interruption", + (mismatch) => + Effect.gen(function* () { + const key = `workspace-detach-mismatched-${mismatch}`; + const { written, observed } = yield* captureRootRunTermination({ + key, + knownProviderTurn: false, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.fail( + new ProviderAdapter.ProviderAdapterSessionDetached({ + driver, + providerSessionId: ProviderSessionId.make( + mismatch === "session" ? "session:other" : `session:${key}`, + ), + threadId: mismatch === "thread" ? ThreadId.make("thread:other") : ids.threadId, + reason: "workspace_changed", + }), + ), + }); + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]); + assert.deepEqual( + written.map((item) => item.type), + ["error"], + ); + }), +); + it.effect("refreshes pull requests after a provider stream exits with an error", () => Effect.gen(function* () { const { written, observed } = yield* captureRootRunTermination({ @@ -2978,6 +3034,7 @@ function captureRootRunTermination(input: { readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"]; readonly seedOpenSubagent?: boolean; + readonly knownProviderTurn?: boolean; readonly events?: ( ids: BackgroundScenarioIds, ) => Stream.Stream< @@ -3141,7 +3198,7 @@ function captureRootRunTermination(input: { } as OrchestrationV2Run, rootNode: { id: ids.rootNodeId, - providerTurnId: ids.rootProviderTurnId, + providerTurnId: input.knownProviderTurn === false ? null : ids.rootProviderTurnId, } as OrchestrationV2ExecutionNode, checkpointScope: { id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`), @@ -3152,10 +3209,9 @@ function captureRootRunTermination(input: { } as OrchestrationV2ProviderThread, attempt: { id: ids.attemptId, - providerTurnId: - input.initialProviderTurnId === undefined - ? ids.rootProviderTurnId - : input.initialProviderTurnId, + providerTurnId: input.initialProviderTurnId === undefined + ? input.knownProviderTurn === false ? null : ids.rootProviderTurnId + : input.initialProviderTurnId, } as OrchestrationV2RunAttempt, attemptId: ids.attemptId, providerTurnOrdinal: 1, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 3e3f37a15a936..a70e7d0c88353 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -626,6 +626,7 @@ export const layer: Layer.Layer< // Root turn snapshot written with the run when T3 ends the run itself. readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn; readonly terminal: ProviderTerminalEvent; + readonly interruptionMessage?: string; readonly completedAt?: DateTime.Utc; readonly failureItemPersisted: boolean; readonly refreshAfterTurn: Effect.Effect; @@ -792,6 +793,9 @@ export const layer: Layer.Layer< providerThread: input.providerThread, reason, completedAt, + ...(input.interruptionMessage === undefined + ? {} + : { message: input.interruptionMessage }), }), }, ] @@ -1029,6 +1033,7 @@ export const layer: Layer.Layer< ); const rootTerminalSeen = yield* Ref.make(false); const rootRunFinalized = yield* Ref.make(false); + const interruptionMessage = yield* Ref.make(undefined); const providerThreadOwnerLost = yield* Ref.make(false); const activeChildProviderTurns = yield* Ref.make>(new Set()); const activeChildSubagents = yield* Ref.make>(new Map()); @@ -1044,6 +1049,7 @@ export const layer: Layer.Layer< } const providerThread = yield* Ref.get(latestProviderThread); const openSubagents = yield* Ref.get(openRunOwnedSubagents); + const message = yield* Ref.get(interruptionMessage); yield* writeFinalRunEvents({ run: input.run, rootNode: input.rootNode, @@ -1071,6 +1077,7 @@ export const layer: Layer.Layer< : { loadRunInterruptReason: input.loadRunInterruptReason }), openRunOwnedSubagents: openSubagents, terminal, + ...(message === undefined ? {} : { interruptionMessage: message }), failureItemPersisted: terminal.status === "failed", refreshAfterTurn, }).pipe( @@ -1283,6 +1290,69 @@ export const layer: Layer.Layer< // as a dev server keep this consumer alive afterwards and must not pin preview focus. const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId); const providerEventFiber = yield* eventSubscription.events.pipe( + Stream.catchTag("ProviderAdapterSessionDetached", (detached) => + detached.providerSessionId !== input.providerSessionId || + detached.threadId !== input.run.threadId + ? Stream.fail(detached) + : Stream.unwrap( + Effect.gen(function* () { + if (yield* Ref.get(rootRunFinalized)) return Stream.empty; + yield* Ref.set( + interruptionMessage, + "Run interrupted because the workspace changed", + ); + const routing = yield* Ref.get(eventRouting); + const providerTurn = yield* Ref.get(latestRootProviderTurn); + const completedAt = yield* DateTime.now; + const terminal = { + type: "turn.terminal", + driver: input.providerThread.driver, + providerThreadId: input.providerThread.id, + providerTurnId: + routing.rootProviderTurnId ?? + input.attempt.providerTurnId ?? + idAllocator.derive.providerTurn({ + driver: input.providerThread.driver, + nativeTurnId: `detached:${input.attempt.id}`, + }), + runOrdinal: input.run.ordinal, + status: "interrupted", + failure: null, + threadDisposition: "reusable", + } satisfies ProviderTerminalEvent; + // Before the provider's first turn event, the planned + // detach supplies this run's root identity for routing. + yield* Ref.update(eventRouting, (state) => + state.rootProviderTurnId === null + ? { + ...state, + rootProviderTurnId: terminal.providerTurnId, + ownedProviderTurnIds: new Set([ + ...state.ownedProviderTurnIds, + terminal.providerTurnId, + ]), + } + : state, + ); + return Stream.fromIterable([ + ...(providerTurn?.status === "running" || providerTurn?.status === "pending" + ? [ + { + type: "provider_turn.updated" as const, + driver: input.providerThread.driver, + providerTurn: { + ...providerTurn, + status: "interrupted" as const, + completedAt, + }, + }, + ] + : []), + terminal, + ]); + }), + ), + ), Stream.mapEffect((event) => Effect.gen(function* () { const previous = yield* Ref.get(eventRouting); @@ -1441,6 +1511,14 @@ export const layer: Layer.Layer< yield* Ref.set(latestProviderThread, event.providerThread); } } + if ( + event.type === "provider_turn.updated" && + (event.providerTurn.runAttemptId === input.attempt.id || + event.providerTurn.id === (yield* Ref.get(eventRouting)).rootProviderTurnId) + ) { + yield* Ref.set(latestRootProviderTurn, event.providerTurn); + } + if ( event.type === "turn_item.updated" && event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId @@ -1748,6 +1826,7 @@ export function makeInterruptResultTurnItem(input: { /** The Stop's own reason, from its run_interrupt_request item. */ readonly reason: string | undefined; readonly completedAt: DateTime.Utc; + readonly message?: string; }): OrchestrationV2TurnItem { return { id: input.idAllocator.derive.runSignalTurnItem({ @@ -1771,6 +1850,6 @@ export function makeInterruptResultTurnItem(input: { completedAt: input.completedAt, updatedAt: input.completedAt, type: "run_interrupt_result", - message: input.reason ?? "Stopped by the provider", + message: input.reason ?? input.message ?? "Stopped by the provider", }; } diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 03cdf5e85a642..2b063eeb6b8dd 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -15,8 +15,10 @@ import { ProviderTurnId, type RunId, ThreadId, + TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -81,6 +83,7 @@ function makeRestartAdapter( state: Ref.Ref, sessionCapabilities: OrchestrationV2ProviderCapabilities = pooledCapabilities, providerInstanceId = initialSelection.instanceId, + completeContinuation: Effect.Effect = Effect.void, ): ProviderAdapter.ProviderAdapterV2["Service"] { return { instanceId: providerInstanceId, @@ -225,7 +228,10 @@ function makeRestartAdapter( input, providerTurnId: ProviderTurnId.make(`provider-turn:${input.attemptId}`), } satisfies ActiveTurn; - if (input.modelSelection.model === initialSelection.model) { + if ( + input.modelSelection.model === initialSelection.model && + !input.message.text.endsWith("handoff continuation") + ) { const occurredAt = yield* DateTime.now; yield* Ref.update(state, (current) => ({ ...current, activeTurn: active })); yield* Queue.offer(events, { @@ -249,6 +255,73 @@ function makeRestartAdapter( }); return; } + if (input.message.text.endsWith("handoff continuation")) { + const startedAt = yield* DateTime.now; + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: active.providerTurnId, + providerThreadId: input.providerThread.id, + nodeId: input.rootNodeId, + runAttemptId: input.attemptId, + nativeTurnRef: { + driver, + nativeId: `native:${active.providerTurnId}`, + strength: "strong", + }, + ordinal: input.providerTurnOrdinal, + status: "running", + startedAt, + completedAt: null, + }, + }); + yield* completeContinuation; + const completedAt = yield* DateTime.now; + yield* Queue.offer(events, { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make(`turn-item:${input.attemptId}:result`), + threadId: input.threadId, + runId: input.runId, + nodeId: input.rootNodeId, + providerThreadId: input.providerThread.id, + providerTurnId: active.providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: input.providerTurnOrdinal * 100 + 2, + status: "completed", + title: null, + startedAt, + completedAt, + updatedAt: completedAt, + type: "assistant_message", + messageId: MessageId.make(`message:${input.attemptId}:result`), + text: "Completed inside the new worktree", + attachments: [], + streaming: false, + }, + }); + yield* Queue.offer(events, { + type: "message.updated", + driver, + message: { + id: MessageId.make(`message:${input.attemptId}:result`), + threadId: input.threadId, + runId: input.runId, + nodeId: input.rootNodeId, + role: "assistant", + text: "Completed inside the new worktree", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "provider", + createdAt: completedAt, + updatedAt: completedAt, + }, + }); + } yield* publishTerminal(active, "completed"); }), steerTurn: () => Effect.void, @@ -774,3 +847,552 @@ it.live.each(["active", "idle", "selection-command", "pooled", "separate-home"] }), ), ); + +it.live.each(["before-detach", "after-detach"] as const)( + "starts a workspace handoff continuation %s without holding its queue", + (continuationOrder) => + Effect.scoped( + Effect.gen(function* () { + const name = `workspace-handoff-${continuationOrder}`; + const sourceCwd = yield* checkpointWorkspace(`${name}-source`); + const targetCwd = yield* checkpointWorkspace(`${name}-target`); + const threadId = ThreadId.make(`thread:${name}`); + const state = yield* Ref.make({ + activeTurn: null, + opened: [], + started: [], + closedSessionCount: 0, + failedReplacementOpen: false, + }); + const registry = ProviderAdapterRegistry.makeSingleLayer( + makeRestartAdapter(state, exclusiveCapabilities), + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`${name}:create`), + threadId, + projectId: ProjectId.make(`project:${name}`), + title: name, + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: sourceCwd, + }); + const startSequence = yield* sink.latestSequence(); + const started = yield* sink.stream({ afterSequence: startSequence, threadId }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-turn.updated" && + stored.event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`${name}:first`), + threadId, + messageId: MessageId.make(`${name}:first`), + text: "bind my workspace", + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + yield* worker.drain(); + yield* Fiber.join(started); + const first = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; + const beforeDetach = yield* sink.latestSequence(); + const ended = yield* sink.stream({ afterSequence: beforeDetach, threadId }).pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === first.id && + ["interrupted", "failed", "cancelled", "completed"].includes( + stored.event.payload.status, + ), + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`${name}:bind`), + threadId, + worktreePath: targetCwd, + }); + if (continuationOrder === "after-detach") { + yield* EffectWorker.runDaemon.pipe(Effect.forkScoped); + yield* worker.drain(); + yield* Fiber.join(ended); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs[0]?.status, + "interrupted", + ); + } + const beforeContinuation = yield* sink.latestSequence(); + const continuation = yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`${name}:continue`), + threadId, + messageId: MessageId.make(`${name}:continue`), + text: "handoff continuation", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; + if (continuationOrder === "before-detach") { + yield* EffectWorker.runDaemon.pipe(Effect.forkScoped); + } + if (queued.status === "queued") { + const promoted = yield* sink + .stream({ afterSequence: beforeContinuation, threadId }) + .pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === queued.id && + (stored.event.payload.status === "starting" || + stored.event.payload.queueHeld === true), + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* worker.drain(); + yield* Fiber.join(ended); + yield* Fiber.join(promoted); + const progress = yield* orchestrator.getThreadProjection(threadId); + assert.equal(progress.runs[1]?.queueHeld ?? false, false); + assert.equal(progress.runs[0]?.status, "interrupted"); + } + const completed = yield* sink + .stream({ afterSequence: continuation.sequence, threadId }) + .pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === queued.id && + stored.event.payload.status === "completed", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* worker.drain(); + yield* Fiber.join(completed); + const settled = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + settled.runs.map((run) => run.status), + ["interrupted", "completed"], + ); + assert.equal(settled.providerTurns[0]?.status, "interrupted"); + assert.equal( + settled.turnItems.find((item) => item.type === "run_interrupt_result")?.message, + "Run interrupted because the workspace changed", + ); + assert.isFalse(settled.runs.some((run) => run.queueHeld === true)); + assert.isFalse(settled.turnItems.some((item) => item.type === "error")); + const captured = yield* Ref.get(state); + assert.deepEqual( + captured.started.map((turn) => turn.cwd), + [sourceCwd, targetCwd], + ); + assert.equal(captured.closedSessionCount, 1); + }).pipe( + Effect.provide( + makeOrchestratorV2ReplayLayerWithRegistry({ name }, registry, { + runEffectWorker: false, + }), + ), + ); + }), + ), +); + +it.live("keeps a delegated workspace handoff pending until its atomic continuation completes", () => + Effect.scoped( + Effect.gen(function* () { + const name = "delegated-workspace-handoff"; + const sourceCwd = yield* checkpointWorkspace(`${name}-source`); + const targetCwd = yield* checkpointWorkspace(`${name}-target`); + const parentId = ThreadId.make(`thread:${name}:parent`); + const continuationStarted = yield* Deferred.make(); + const finishContinuation = yield* Deferred.make(); + const state = yield* Ref.make({ + activeTurn: null, + opened: [], + started: [], + closedSessionCount: 0, + failedReplacementOpen: false, + }); + const registry = ProviderAdapterRegistry.makeSingleLayer( + makeRestartAdapter( + state, + exclusiveCapabilities, + providerInstanceId, + Deferred.succeed(continuationStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishContinuation)), + ), + ), + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${name}:parent:create`), + threadId: parentId, + projectId: ProjectId.make(`project:${name}`), + title: name, + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: sourceCwd, + }); + const beforeStart = yield* sink.latestSequence(); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${name}:parent:start`), + threadId: parentId, + messageId: MessageId.make(`${name}:parent:start`), + text: "delegate the work", + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + yield* sink.stream({ afterSequence: beforeStart, threadId: parentId }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-turn.updated" && + stored.event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + ); + const parentRun = (yield* orchestrator.getThreadProjection(parentId)).runs[0]!; + const delegated = yield* orchestrator.dispatch({ + type: "delegated_task.request", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`${name}:delegate`), + parentThreadId: parentId, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId!, + task: "bind my workspace", + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", + }); + const task = (yield* orchestrator.getThreadProjection(parentId)).subagents[0]!; + const childId = task.childThreadId!; + yield* sink.stream({ afterSequence: delegated.sequence, threadId: childId }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-turn.updated" && + stored.event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + ); + const binding = { + type: "thread.metadata.update" as const, + commandId: CommandId.make(`${name}:bind`), + threadId: childId, + worktreePath: targetCwd, + expectedWorktreePath: sourceCwd, + worktreeContinuation: { + messageId: MessageId.make(`${name}:continue`), + text: "handoff continuation", + }, + }; + const bound = yield* orchestrator.dispatch(binding); + assert.lengthOf( + bound.storedEvents.filter((stored) => stored.event.type === "run.created"), + 1, + ); + assert.lengthOf( + bound.storedEvents.filter((stored) => stored.event.type === "message.updated"), + 1, + ); + // Repeating the exact handoff cannot enqueue another continuation. + yield* orchestrator.dispatch(binding); + const unchangedBinding = yield* orchestrator + .dispatch({ + ...binding, + commandId: CommandId.make(`${name}:unchanged-bind`), + expectedWorktreePath: targetCwd, + worktreeContinuation: { + messageId: MessageId.make(`${name}:unchanged-continue`), + text: "unexpected duplicate continuation", + }, + }) + .pipe(Effect.flip); + assert.include(String(unchangedBinding.cause), "changed worktree binding"); + yield* Deferred.await(continuationStarted); + const beforeCompletion = yield* orchestrator.getThreadProjection(parentId); + assert.isNull(beforeCompletion.subagents[0]?.result); + assert.isNull(beforeCompletion.subagents[0]?.completionDelivery ?? null); + assert.isFalse( + beforeCompletion.contextTransfers.some((transfer) => transfer.type === "subagent_result"), + ); + const continuing = yield* orchestrator.getThreadProjection(childId); + assert.deepEqual( + continuing.runs.map((run) => run.status), + ["interrupted", "running"], + ); + assert.isFalse(continuing.runs.some((run) => run.queueHeld)); + const beforeFinish = yield* sink.latestSequence(); + yield* Deferred.succeed(finishContinuation, undefined); + yield* sink.stream({ afterSequence: beforeFinish, threadId: parentId }).pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === task.id && + stored.event.payload.status === "completed", + ), + Stream.take(1), + Stream.runDrain, + ); + const finished = yield* orchestrator.getThreadProjection(parentId); + assert.equal(finished.subagents[0]?.status, "completed"); + assert.equal(finished.subagents[0]?.result, "Completed inside the new worktree"); + assert.lengthOf( + finished.contextTransfers.filter((transfer) => transfer.type === "subagent_result"), + 1, + ); + const childFinished = yield* orchestrator.getThreadProjection(childId); + assert.deepEqual( + childFinished.runs.map((run) => run.status), + ["interrupted", "completed"], + ); + assert.lengthOf( + childFinished.messages.filter((message) => message.text === "handoff continuation"), + 1, + ); + assert.isFalse( + childFinished.messages.some( + (message) => message.text === "unexpected duplicate continuation", + ), + ); + assert.isFalse(childFinished.turnItems.some((item) => item.type === "error")); + const assistantItems = childFinished.visibleTurnItems.flatMap((row) => + row.item.type === "assistant_message" ? [row.item] : [], + ); + assert.lengthOf(assistantItems, 1); + assert.equal(assistantItems[0]?.text, "Completed inside the new worktree"); + const assistantMessage = childFinished.messages.find( + (message) => message.role === "assistant", + ); + assert.equal(assistantItems[0]?.messageId, assistantMessage?.id); + assert.equal(assistantItems[0]?.text, assistantMessage?.text); + + assert.deepEqual( + (yield* Ref.get(state)).started.map((turn) => turn.cwd), + [sourceCwd, sourceCwd, targetCwd], + ); + }).pipe(Effect.provide(makeOrchestratorV2ReplayLayerWithRegistry({ name }, registry))); + }), + ), +); + +it.live.each(["provider-crash", "explicit-stop"] as const)( + "preserves real queued input until explicitly resumed after %s", + (termination) => + Effect.scoped( + Effect.gen(function* () { + const name = `queued-control-${termination}`; + const cwd = yield* checkpointWorkspace(name); + const threadId = ThreadId.make(`thread:${name}`); + const crash = yield* Deferred.make(); + const firstSession = yield* Ref.make(true); + const state = yield* Ref.make({ + activeTurn: null, + opened: [], + started: [], + closedSessionCount: 0, + failedReplacementOpen: false, + }); + const base = makeRestartAdapter(state, exclusiveCapabilities); + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + ...base, + openSession: (input) => + base.openSession(input).pipe( + Effect.flatMap((session) => + Ref.getAndSet(firstSession, false).pipe( + Effect.map((isFirst) => ({ + ...session, + events: + termination === "explicit-stop" || !isFirst + ? session.events + : Stream.merge( + session.events, + Stream.fromEffect( + Deferred.await(crash).pipe( + Effect.andThen( + Effect.fail( + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver, + providerSessionId: input.providerSessionId, + cause: "Simulated provider crash", + }), + ), + ), + ), + ), + ), + })), + ), + ), + ), + }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${name}:create`), + threadId, + projectId: ProjectId.make(`project:${name}`), + title: name, + modelSelection: initialSelection, + createdBy: "user", + creationSource: "web", + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + }); + const beforeStart = yield* sink.latestSequence(); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${name}:first`), + threadId, + messageId: MessageId.make(`${name}:first`), + createdBy: "user", + creationSource: "web", + text: "keep working", + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + yield* sink.stream({ threadId, afterSequence: beforeStart }).pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-turn.updated" && + stored.event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + ); + const queuedCommand = { + type: "message.dispatch" as const, + commandId: CommandId.make(`${name}:queue`), + threadId, + messageId: MessageId.make(`${name}:queue`), + createdBy: "user" as const, + creationSource: "web" as const, + text: "handoff continuation", + attachments: [], + dispatchMode: { type: "queue_after_active" as const }, + }; + const queuedReceipt = yield* orchestrator.dispatch(queuedCommand); + const queuedState = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + queuedState.runs.map((run) => run.status), + ["running", "queued"], + ); + const originalRun = queuedState.runs[0]!; + const queuedRun = queuedState.runs[1]!; + if (termination === "provider-crash") yield* Deferred.succeed(crash, undefined); + else + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make(`${name}:stop`), + threadId, + runId: originalRun.id, + holdQueue: true, + }); + yield* sink.stream({ threadId, afterSequence: queuedReceipt.sequence }).pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === queuedRun.id && + stored.event.payload.queueHeld === true, + ), + Stream.take(1), + Stream.runDrain, + ); + const held = yield* orchestrator.getThreadProjection(threadId); + assert.equal(held.runs[1]?.status, "queued"); + assert.equal( + held.messages.find((message) => message.id === queuedCommand.messageId)?.text, + queuedCommand.text, + ); + assert.lengthOf((yield* Ref.get(state)).started, 1); + const resumed = yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make(`${name}:resume`), + threadId, + }); + yield* sink.stream({ threadId, afterSequence: resumed.sequence }).pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && + stored.event.payload.id === queuedRun.id && + stored.event.payload.status === "completed", + ), + Stream.take(1), + Stream.runDrain, + ); + const completed = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + completed.runs.map((run) => run.status), + [termination === "provider-crash" ? "failed" : "interrupted", "completed"], + ); + assert.isFalse(completed.runs.some((run) => run.queueHeld)); + assert.lengthOf( + completed.messages.filter((message) => message.id === queuedCommand.messageId), + 1, + ); + assert.lengthOf((yield* Ref.get(state)).started, 2); + assert.equal( + completed.messages.find((message) => message.role === "assistant")?.text, + "Completed inside the new worktree", + ); + if (termination === "provider-crash") + assert.isTrue( + completed.turnItems.some( + (item) => item.type === "error" && item.runId === originalRun.id, + ), + ); + else + assert.equal( + completed.turnItems.find((item) => item.type === "run_interrupt_result")?.message, + "Run interrupted by user", + ); + }).pipe( + Effect.provide( + makeOrchestratorV2ReplayLayerWithRegistry( + { name }, + ProviderAdapterRegistry.makeSingleLayer(adapter), + ), + ), + ); + }), + ), +); diff --git a/docs/internals/overview.md b/docs/internals/overview.md index 9e5cf6c9d8a6c..e8d6aacdbddc5 100644 --- a/docs/internals/overview.md +++ b/docs/internals/overview.md @@ -58,6 +58,13 @@ therefore means the intent committed, not that the provider, checkpoint, or othe finished. Keep external I/O out of command decisions and the database transaction. Effects tied to a lost provider process cannot simply replay; recovery retires them before admitting new work. +A worktree handoff with a continuation records the workspace binding, continuation message and +run, and provider detach effect in the same command transaction. The detach can execute immediately +after commit, so recording the continuation afterward would let a delegated child publish its result +too early. A planned exclusive workspace detach interrupts the old turn without a provider failure; +the queued continuation can then start in the new workspace. Unexpected provider failures and +explicit Stop retain queued messages until the user resumes them. + Persisted events must remain decodable on replay. Changing a schema affects old environments at startup as well as live RPC traffic. Compatibility work must account for stored history, not just what the newest client sends. diff --git a/docs/user/composer.md b/docs/user/composer.md index e0d1f541851f0..fad773b937d7f 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -62,6 +62,10 @@ If your **Send shortcut** requires `Cmd+Enter` or `Ctrl+Enter`, that shortcut us your default follow-up behavior; add **Shift** to use the opposite action. This applies only to multiline messages when you choose the multiline send option. +When an agent moves its thread into a worktree with a continuation message, work resumes +automatically in that worktree. An explicit Stop or a provider failure pauses queued messages +until you resume the queue; the messages stay saved. + Use `Cmd+Shift+Enter` on macOS or `Ctrl+Shift+Enter` on Windows and Linux to send the oldest queued message as a steer. This leaves the current draft intact and requires an active turn that supports steering. If that shortcut sends the opposite diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 56d910b752deb..54c20e43cb9c0 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -2987,6 +2987,10 @@ export const OrchestrationV2Command = Schema.Union([ worktreePath: Schema.optional(Schema.NullOr(TrimmedNonEmptyString)), expectedWorktreePath: Schema.optional(Schema.NullOr(TrimmedNonEmptyString)), expectedBranch: Schema.optional(Schema.NullOr(TrimmedNonEmptyString)), + /** Record the next message atomically with a worktree handoff, before session detach. */ + worktreeContinuation: Schema.optional( + Schema.Struct({ messageId: MessageId, text: TrimmedNonEmptyString }), + ), /** Reject unless no message or run has landed on this thread. */ expectedEmpty: Schema.optional(Schema.Boolean), limitRecovery: Schema.optional(Schema.NullOr(OrchestrationV2LimitRecoveryUpdate)), diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 9dc7ec1f62285..ef1cd6dab5a62 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -358,6 +358,16 @@ const ProviderAdapterFailureReason = Schema.Literals([ "attachment_unreadable", ]); export type ProviderAdapterFailureReason = typeof ProviderAdapterFailureReason.Type; +/** The app deliberately closed an exclusive session to move its thread's workspace. */ +export class ProviderAdapterSessionDetached extends Schema.TaggedError()( + "ProviderAdapterSessionDetached", + { + driver: ProviderDriverKind, + providerSessionId: ProviderSessionId, + threadId: ThreadId, + reason: Schema.Literal("workspace_changed"), + }, +) {} export class ProviderAdapterProtocolError extends Schema.TaggedError()( "ProviderAdapterProtocolError", @@ -389,6 +399,7 @@ export const ProviderAdapterV2Error = Schema.Union([ ProviderAdapterInterruptError, ProviderAdapterRuntimeRequestResponseError, ProviderAdapterEventStreamError, + ProviderAdapterSessionDetached, ProviderAdapterProtocolError, ]); export type ProviderAdapterV2Error = typeof ProviderAdapterV2Error.Type; From d5e466921ed7045d211dafd7d302312ce19c307b Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Thu, 8 Oct 2026 15:46:32 -0400 Subject: [PATCH 02/14] fix(test): use current provider replay helpers --- .../SelectionRestart.integration.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 2b063eeb6b8dd..0458baff72c8c 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -864,7 +864,7 @@ it.live.each(["before-detach", "after-detach"] as const)( closedSessionCount: 0, failedReplacementOpen: false, }); - const registry = ProviderAdapterRegistry.makeSingleLayer( + const registry = ProviderAdapterRegistry.layerSingle( makeRestartAdapter(state, exclusiveCapabilities), ); yield* Effect.gen(function* () { @@ -1012,7 +1012,7 @@ it.live.each(["before-detach", "after-detach"] as const)( assert.equal(captured.closedSessionCount, 1); }).pipe( Effect.provide( - makeOrchestratorV2ReplayLayerWithRegistry({ name }, registry, { + ProviderReplayHarness.layerWithRegistry({ name }, registry, { runEffectWorker: false, }), ), @@ -1037,7 +1037,7 @@ it.live("keeps a delegated workspace handoff pending until its atomic continuati closedSessionCount: 0, failedReplacementOpen: false, }); - const registry = ProviderAdapterRegistry.makeSingleLayer( + const registry = ProviderAdapterRegistry.layerSingle( makeRestartAdapter( state, exclusiveCapabilities, @@ -1206,7 +1206,7 @@ it.live("keeps a delegated workspace handoff pending until its atomic continuati (yield* Ref.get(state)).started.map((turn) => turn.cwd), [sourceCwd, sourceCwd, targetCwd], ); - }).pipe(Effect.provide(makeOrchestratorV2ReplayLayerWithRegistry({ name }, registry))); + }).pipe(Effect.provide(ProviderReplayHarness.layerWithRegistry({ name }, registry))); }), ), ); @@ -1387,9 +1387,9 @@ it.live.each(["provider-crash", "explicit-stop"] as const)( ); }).pipe( Effect.provide( - makeOrchestratorV2ReplayLayerWithRegistry( + ProviderReplayHarness.layerWithRegistry( { name }, - ProviderAdapterRegistry.makeSingleLayer(adapter), + ProviderAdapterRegistry.layerSingle(adapter), ), ), ); From 989e7dd0be725df15c3b6abeac49e90fdb38187f Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Fri, 9 Oct 2026 17:32:08 -0400 Subject: [PATCH 03/14] fix(orchestration): use the provider-core event type for handoffs --- apps/server/src/orchestration-v2/RunExecutionService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index a70e7d0c88353..05c855604caa9 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1334,7 +1334,7 @@ export const layer: Layer.Layer< } : state, ); - return Stream.fromIterable([ + return Stream.fromIterable([ ...(providerTurn?.status === "running" || providerTurn?.status === "pending" ? [ { From 28415f049be8996eaeed71c3fe45b78df5867a62 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Thu, 15 Oct 2026 12:59:53 -0400 Subject: [PATCH 04/14] fix(server): interrupt open tool items on worktree handoff --- .../RunExecutionService.test.ts | 96 ++++++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 30 +++++- 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 787cef0fa1c63..8c9a825844d05 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2575,6 +2575,95 @@ it.effect("finalizes a planned workspace detach before the provider announces it }), ); +it.effect.each(["pending", "running"] as const)( + "interrupts %s root tool items before finalizing a planned workspace detach", + (status) => + Effect.gen(function* () { + const key = `workspace-detach-open-tools:${status}`; + const ids = backgroundScenarioIds(key); + const secondItemId = TurnItemId.make(`${ids.itemId}:second`); + const completedItemId = TurnItemId.make(`${ids.itemId}:completed`); + const { ingested, observed } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.concat( + Stream.fromIterable([ + backgroundTurnItemEvent(ids, "dynamic_tool", status, 1), + backgroundTurnItemEvent(ids, "dynamic_tool", status, 2, secondItemId), + backgroundTurnItemEvent(ids, "dynamic_tool", "running", 3, completedItemId), + backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 3, completedItemId), + ]), + Stream.fail( + new ProviderAdapter.ProviderAdapterSessionDetached({ + driver, + providerSessionId: ProviderSessionId.make(`session:${key}`), + threadId: ids.threadId, + reason: "workspace_changed", + }), + ), + ), + }); + const interruptedItems = ingested.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.status === "interrupted" + ? [event.turnItem] + : [], + ); + assert.deepEqual( + interruptedItems.map((item) => item.id), + [ids.itemId, secondItemId], + ); + for (const item of interruptedItems) { + assert.isNotNull(item.completedAt); + assert.deepEqual(item.updatedAt, item.completedAt); + } + assert.equal(ingested.at(-1)?.type, "turn.terminal"); + assert.deepEqual(observed, ["run:interrupted", "pull-requests-refreshed"]); + }), +); + +it.effect("leaves planned-detach subagent items to the run-owned cascade", () => + Effect.gen(function* () { + const key = "workspace-detach-subagent"; + const { ingested, written } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.concat( + Stream.make({ + type: "turn_item.updated" as const, + driver, + turnItem: makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + }), + Stream.fail( + new ProviderAdapter.ProviderAdapterSessionDetached({ + driver, + providerSessionId: ProviderSessionId.make(`session:${key}`), + threadId: ids.threadId, + reason: "workspace_changed", + }), + ), + ), + }); + assert.deepEqual( + ingested.flatMap((event) => + event.type === "turn_item.updated" ? [event.turnItem.status] : [], + ), + ["running"], + ); + assert.deepEqual( + written.filter((item) => item.type === "subagent").map((item) => item.status), + ["interrupted"], + ); + }), +); + it.effect.each(["thread", "session"] as const)( "fails rather than treating another %s's planned detach as this run's interruption", (mismatch) => @@ -3071,6 +3160,7 @@ function captureRootRunTermination(input: { }); const writtenItems = yield* Ref.make>([]); const finalEvents = yield* Ref.make>([]); + const ingested = yield* Ref.make>([]); const observed = yield* Ref.make>([]); const submittedEffects = yield* Ref.make>([]); const committedEffects = yield* Ref.make>([]); @@ -3132,7 +3222,8 @@ function captureRootRunTermination(input: { }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ - ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])), + ingestNormalized: input.ingestNormalized ?? (({ event }) => + Ref.update(ingested, (current) => [...current, event]).pipe(Effect.as([]))), }), ServerSettings.layerTest(), Layer.succeed(RunFinalizationService.RunFinalizationObserver, { @@ -3257,6 +3348,7 @@ function captureRootRunTermination(input: { previewActivityEnded, written: yield* Ref.get(writtenItems), finalEvents: yield* Ref.get(finalEvents), + ingested: yield* Ref.get(ingested), observed: yield* Ref.get(observed), submittedEffects: yield* Ref.get(submittedEffects), committedEffects: yield* Ref.get(committedEffects), @@ -3332,7 +3424,7 @@ function childBackgroundTurnItemEvent( function backgroundTurnItemEvent( ids: BackgroundScenarioIds, type: "command_execution" | "dynamic_tool" | "subagent", - status: "running" | "completed" | "idle", + status: "pending" | "running" | "completed" | "idle", ordinal: number, itemId?: TurnItemId, ): ProviderAdapter.ProviderAdapterV2Event { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 05c855604caa9..c81b5dd54b768 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -991,7 +991,11 @@ export const layer: Layer.Layer< const terminalEvent = yield* Ref.make(null); const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100); const latestProviderThread = yield* Ref.make(input.providerThread); - const latestRootProviderTurn = yield* Ref.make(null); + const latestRootProviderTurn = yield* Ref.make(null); + // A planned detach closes ingestion before the adapter can end its open root items. + const openRootTurnItems = yield* Ref.make< + ReadonlyMap + >(new Map()); const routeIdentity: ProviderEventRouteIdentity = { threadId: input.run.threadId, runId: input.run.id, @@ -1334,7 +1338,19 @@ export const layer: Layer.Layer< } : state, ); + const openItems = yield* Ref.getAndSet(openRootTurnItems, new Map()); return Stream.fromIterable([ + ...[...openItems.values()].map((turnItem) => ({ + type: "turn_item.updated" as const, + driver: input.providerThread.driver, + turnItem: { + ...turnItem, + ...("streaming" in turnItem ? { streaming: false } : {}), + status: "interrupted" as const, + completedAt, + updatedAt: completedAt, + }, + })), ...(providerTurn?.status === "running" || providerTurn?.status === "pending" ? [ { @@ -1526,6 +1542,18 @@ export const layer: Layer.Layer< yield* Ref.update(latestTurnItemOrdinal, (current) => Math.max(current, event.turnItem.ordinal), ); + const turnItem = event.turnItem; + // Subagent items are ended by the run-owned cascade. + if (turnItem.type !== "subagent") { + const open = turnItem.status === "running" || turnItem.status === "pending"; + yield* Ref.update(openRootTurnItems, (items) => { + if (!open && !items.has(turnItem.id)) return items; + const next = new Map(items); + if (open) next.set(turnItem.id, turnItem); + else next.delete(turnItem.id); + return next; + }); + } } if (event.type === "turn.terminal") { yield* Ref.set(terminalEvent, event); From ea9e8bc8338b097b7209517a286ff0193140761f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 08:40:31 +0000 Subject: [PATCH 05/14] fix(orchestration): target planned detach to the moving thread --- apps/server/src/mcp/WorktreeMcpService.ts | 3 - .../src/orchestration-v2/EffectOutbox.ts | 1 - .../ProviderSessionManager.test.ts | 10 +- .../ProviderSessionManager.ts | 120 +-- .../RunExecutionService.test.ts | 326 +++----- .../orchestration-v2/RunExecutionService.ts | 28 +- .../SelectionRestart.integration.test.ts | 701 ++++++------------ docs/internals/overview.md | 9 +- .../src/server/ProviderAdapter.ts | 7 +- 9 files changed, 406 insertions(+), 799 deletions(-) diff --git a/apps/server/src/mcp/WorktreeMcpService.ts b/apps/server/src/mcp/WorktreeMcpService.ts index 45185f858443a..9624a0eb5c95d 100644 --- a/apps/server/src/mcp/WorktreeMcpService.ts +++ b/apps/server/src/mcp/WorktreeMcpService.ts @@ -238,7 +238,6 @@ const make = Effect.gen(function* () { ); const worktreePath = worktree.worktree.path; - // Setup errors are reported after the handoff has already committed. const reportFailed = (logMessage: string) => Effect.catchCause((cause: Cause.Cause) => { const detail = errorMessage(Cause.squash(cause)); @@ -311,8 +310,6 @@ const make = Effect.gen(function* () { ), ); - // Binding and continuation events share the detach effect's commit, so - // even an immediately executing detach sees the durable queued message. const bound = yield* recheckAndBind; const queued = bound.storedEvents.some( ({ event }) => event.type === "run.created" && event.payload.status === "queued", diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index d6d7323d5c870..3fd9c675aa4d6 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -39,7 +39,6 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ providerSessionId: ProviderSessionId, detail: Schema.optional(Schema.String), reason: Schema.optional(Schema.Literal("workspace_changed")), - /** Set on terminal detaches (thread archive/delete): revoke the thread's MCP credentials. */ revokeMcpCredential: Schema.optional(Schema.Boolean), }), Schema.Struct({ diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 8398fca9cb6a5..ebb60f812257b 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -1409,8 +1409,8 @@ it.effect("ProviderSessionManagerV2 interrupts event subscriptions on server shu modelSelection, runtimePolicy, }); - const bufferedSubscription = yield* runtime.subscribeEvents!; - const activeSubscription = yield* runtime.subscribeEvents!; + const bufferedSubscription = yield* runtime.subscribeEvents!(); + const activeSubscription = yield* runtime.subscribeEvents!(); const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); assert.isDefined(adapterQueue); yield* Queue.offer(adapterQueue!, { @@ -1452,7 +1452,7 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops", modelSelection, runtimePolicy, }); - const subscription = yield* runtime.subscribeEvents!; + const subscription = yield* runtime.subscribeEvents!(); const collected = yield* subscription.events.pipe(Stream.runCollect, Effect.forkScoped); const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); assert.isDefined(adapterQueue); @@ -3185,8 +3185,8 @@ it.effect( const subscribe = firstRuntime.subscribeEvents; assert.isDefined(subscribe); if (subscribe === undefined) return; - const firstSubscription = yield* subscribe; - const secondSubscription = yield* subscribe; + const firstSubscription = yield* subscribe(); + const secondSubscription = yield* subscribe(); const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); assert.isDefined(queue); yield* Queue.offer(queue!, { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 0a021b0d4ee3b..2f4f68047d860 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -179,11 +179,6 @@ export interface ProviderSessionManagerV2Shape { readonly threadId: ThreadId; readonly detail?: string; readonly reason?: "workspace_changed"; - /** - * True for terminal detaches (thread archived or deleted): the thread's - * MCP credentials are revoked immediately instead of surviving for a - * potential re-attach. - */ readonly revokeMcpCredential?: boolean; }) => Effect.Effect; } @@ -201,9 +196,7 @@ interface LiveSessionEntry { readonly supportsMultipleProviderThreads: boolean; readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; readonly exposedRuntime: ProviderAdapter.ProviderAdapterV2SessionRuntime; - readonly eventSubscribers: Ref.Ref< - ReadonlyMap> - >; + readonly eventSubscribers: Ref.Ref>; readonly requestEventPermit: Semaphore.Semaphore; readonly scope: Scope.Closeable; readonly idleGeneration: number; @@ -219,6 +212,11 @@ interface IdleThreadUnload { readonly fiber: Fiber.Fiber | null; } +interface ProviderSessionEventSubscriber { + readonly threadId: ThreadId | undefined; + readonly queue: Queue.Queue; +} + type ProviderSessionEventSignal = | { readonly type: "event"; readonly event: ProviderAdapter.ProviderAdapterV2Event } | { @@ -496,47 +494,44 @@ export const layerWithOptions = ( ); const publishToSubscribers = ( - subscribers: Ref.Ref< - ReadonlyMap> - >, + subscribers: Ref.Ref>, signal: ProviderSessionEventSignal, ) => Ref.get(subscribers).pipe( Effect.flatMap((current) => - Effect.forEach(current.values(), (queue) => Queue.offer(queue, signal), { + Effect.forEach(current.values(), ({ queue }) => Queue.offer(queue, signal), { discard: true, }), ), ); - const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* ( + const failSubscribers = ( entry: LiveSessionEntry, - detail: string, - expectedDetach?: Pick, - ) { - const error = - expectedDetach === undefined - ? new ProviderAdapter.ProviderAdapterEventStreamError({ - driver: entry.runtime.driver, - providerSessionId: entry.runtime.providerSessionId, - cause: detail, - }) - : new ProviderAdapter.ProviderAdapterSessionDetached({ - driver: entry.runtime.driver, - providerSessionId: entry.runtime.providerSessionId, - ...expectedDetach, - }); - const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map()); - yield* Effect.forEach( - subscribers.values(), - (queue) => - Queue.offer(queue, { - type: "failure", - cause: Cause.fail(error), - }), - { discard: true }, - ); - }); + error: ProviderAdapter.ProviderAdapterV2Error, + threadId?: ThreadId, + ) => + Effect.gen(function* () { + const subscribers = yield* Ref.modify(entry.eventSubscribers, (current) => { + const failed: Array = []; + const remaining = new Map(current); + for (const [id, subscriber] of current) { + if (threadId === undefined || subscriber.threadId === threadId) { + failed.push(subscriber); + remaining.delete(id); + } + } + return [failed, remaining] as const; + }); + yield* Effect.forEach( + subscribers, + ({ queue }) => + Queue.offer(queue, { + type: "failure", + cause: Cause.fail(error), + }), + { discard: true }, + ); + }); // Server shutdown interrupts subscribers rather than ending them: a // cleanly ended stream without a terminal fails its run, while an @@ -546,7 +541,7 @@ export const layerWithOptions = ( const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map()); yield* Effect.forEach( subscribers.values(), - (queue) => Queue.clear(queue).pipe(Effect.andThen(Queue.interrupt(queue))), + ({ queue }) => Queue.clear(queue).pipe(Effect.andThen(Queue.interrupt(queue))), { discard: true }, ); }, @@ -558,7 +553,7 @@ export const layerWithOptions = ( entry: LiveSessionEntry, ) { const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map()); - yield* Effect.forEach(subscribers.values(), (queue) => Queue.end(queue), { + yield* Effect.forEach(subscribers.values(), ({ queue }) => Queue.end(queue), { discard: true, }); }); @@ -986,7 +981,6 @@ export const layerWithOptions = ( readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; readonly gracefulSubscribers?: boolean; - readonly expectedDetach?: Pick; }) => Effect.acquireUseRelease( removeLiveEntry(input), @@ -1012,8 +1006,11 @@ export const layerWithOptions = ( } else { yield* failSubscribers( entry, - input.detail ?? `Provider session released: ${input.reason}.`, - input.expectedDetach, + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: entry.runtime.driver, + providerSessionId: entry.runtime.providerSessionId, + cause: input.detail ?? `Provider session released: ${input.reason}.`, + }), ); } const closeExit = yield* closeScopeWithin(entry.scope, { @@ -1623,14 +1620,15 @@ export const layerWithOptions = ( const makeEventSubscription = Effect.fn("ProviderSessionManager.makeEventSubscription")( function* ( subscribers: Ref.Ref< - ReadonlyMap> + ReadonlyMap >, + threadId?: ThreadId, ): Effect.fn.Return { const queue = yield* Queue.unbounded(); const subscriberId = yield* Ref.getAndUpdate(nextSubscriberId, (value) => value + 1); yield* Ref.update(subscribers, (current) => { const updated = new Map(current); - updated.set(subscriberId, queue); + updated.set(subscriberId, { threadId, queue }); return updated; }); const close = Ref.modify(subscribers, (current) => { @@ -1661,12 +1659,11 @@ export const layerWithOptions = ( const decorateRuntime = ( runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime, - eventSubscribers: Ref.Ref< - ReadonlyMap> - >, + eventSubscribers: Ref.Ref>, ): ProviderAdapter.ProviderAdapterV2SessionRuntime => { const providerSessionId = runtime.providerSessionId; - const subscribeEvents = makeEventSubscription(eventSubscribers); + const subscribeEvents = (threadId?: ThreadId) => + makeEventSubscription(eventSubscribers, threadId); const turnMetrics = (operation: string, model?: string) => withMetrics({ counter: providerTurnsTotal, @@ -1681,7 +1678,7 @@ export const layerWithOptions = ( ...runtime, subscribeEvents, events: Stream.unwrap( - subscribeEvents.pipe(Effect.map((subscription) => subscription.events)), + subscribeEvents().pipe(Effect.map((subscription) => subscription.events)), ), ensureThread: (input) => observeActivity( @@ -2156,7 +2153,7 @@ export const layerWithOptions = ( }), ); const eventSubscribers = yield* Ref.make< - ReadonlyMap> + ReadonlyMap >(new Map()); const exposedRuntime = decorateRuntime(runtime, eventSubscribers); const entry: LiveSessionEntry = { @@ -2290,6 +2287,7 @@ export const layerWithOptions = ( const key = sessionKey(input.providerSessionId); const currentEntry = (yield* Ref.get(sessions)).get(key); let detachedProviderThreads: ReadonlyArray = []; + let interruptActiveTurns = Effect.void; if (currentEntry?.supportsMultipleProviderThreads === true) { const projection = yield* Effect.option( projectionStore.getThreadRecords(input.threadId, [ @@ -2307,7 +2305,7 @@ export const layerWithOptions = ( const activeTurns = projection.value.providerTurns.filter( (turn) => turn.status === "running" && providerThreads.has(turn.providerThreadId), ); - yield* Effect.forEach( + interruptActiveTurns = Effect.forEach( activeTurns, (turn) => currentEntry.exposedRuntime @@ -2387,6 +2385,19 @@ export const layerWithOptions = ( if (Option.isNone(detached)) { return; } + if (input.reason !== undefined) { + yield* failSubscribers( + detached.value, + new ProviderAdapter.ProviderAdapterSessionDetached({ + driver: detached.value.runtime.driver, + providerSessionId: input.providerSessionId, + threadId: input.threadId, + reason: input.reason, + }), + input.threadId, + ); + } + yield* interruptActiveTurns; if ( detached.value.attachedThreadIds.size === 0 && !detached.value.supportsMultipleProviderThreads @@ -2396,9 +2407,6 @@ export const layerWithOptions = ( reason: "manual_shutdown", ...(input.detail === undefined ? {} : { detail: input.detail }), detachedThreadId: input.threadId, - ...(input.reason === undefined - ? {} - : { expectedDetach: { threadId: input.threadId, reason: input.reason } }), }); return; } diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 8c9a825844d05..da5c1aefbd673 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2120,98 +2120,99 @@ it.effect( providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"), session: { events: Stream.empty, - subscribeEvents: Effect.succeed({ - events: Stream.fromIterable([ - childThreadCreatedEvent(ids), - { - type: "subagent.updated", - driver, - subagent: runningSubagent, - }, - { - type: "node.updated", - driver, - node: runningNode, - }, - { - type: "node.updated", - driver, - node: runningChildNode, - }, - { - type: "node.updated", - driver, - node: unrelatedChildNode, - }, - { - type: "turn_item.updated", - driver, - turnItem: runningTurnItem, - }, - { - type: "turn_item.updated", - driver, - turnItem: runningChildTurnItem, - }, - { - type: "turn_item.updated", - driver, - turnItem: suppressedChildAssistantTurnItem, - }, - { - type: "turn_item.updated", - driver, - turnItem: unrelatedChildTurnItem, - }, - { type: "subagent.updated", driver, subagent: runningMember }, - { type: "subagent.updated", driver, subagent: foreignSubagent }, - { type: "node.updated", driver, node: runningMemberRoot }, - { type: "node.updated", driver, node: foreignChildNode }, - { type: "turn_item.updated", driver, turnItem: foreignChildTurnItem }, - rootTerminalEvent(ids, "interrupted"), - { - type: "subagent.updated", - driver, - subagent: { - ...runningSubagent, - status: "completed" as const, - result: "should-not-apply", - completedAt: runningSubagent.updatedAt, + subscribeEvents: () => + Effect.succeed({ + events: Stream.fromIterable([ + childThreadCreatedEvent(ids), + { + type: "subagent.updated", + driver, + subagent: runningSubagent, }, - }, - { - type: "turn_item.updated", - driver, - turnItem: { - ...runningTurnItem, - status: "completed" as const, - result: "should-not-apply", - completedAt: runningTurnItem.updatedAt, - updatedAt: runningTurnItem.updatedAt, + { + type: "node.updated", + driver, + node: runningNode, }, - }, - { - type: "turn_item.updated", - driver, - turnItem: { - ...runningChildTurnItem, - status: "completed" as const, - completedAt: runningChildTurnItem.updatedAt, - updatedAt: runningChildTurnItem.updatedAt, + { + type: "node.updated", + driver, + node: runningChildNode, }, - }, - { - type: "node.updated", - driver, - node: { - ...runningChildNode, - status: "completed" as const, - completedAt: runningChildNode.startedAt, + { + type: "node.updated", + driver, + node: unrelatedChildNode, }, - }, - ] satisfies ReadonlyArray), - close: Deferred.succeed(ingestionDone, undefined), - }), + { + type: "turn_item.updated", + driver, + turnItem: runningTurnItem, + }, + { + type: "turn_item.updated", + driver, + turnItem: runningChildTurnItem, + }, + { + type: "turn_item.updated", + driver, + turnItem: suppressedChildAssistantTurnItem, + }, + { + type: "turn_item.updated", + driver, + turnItem: unrelatedChildTurnItem, + }, + { type: "subagent.updated", driver, subagent: runningMember }, + { type: "subagent.updated", driver, subagent: foreignSubagent }, + { type: "node.updated", driver, node: runningMemberRoot }, + { type: "node.updated", driver, node: foreignChildNode }, + { type: "turn_item.updated", driver, turnItem: foreignChildTurnItem }, + rootTerminalEvent(ids, "interrupted"), + { + type: "subagent.updated", + driver, + subagent: { + ...runningSubagent, + status: "completed" as const, + result: "should-not-apply", + completedAt: runningSubagent.updatedAt, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...runningTurnItem, + status: "completed" as const, + result: "should-not-apply", + completedAt: runningTurnItem.updatedAt, + updatedAt: runningTurnItem.updatedAt, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...runningChildTurnItem, + status: "completed" as const, + completedAt: runningChildTurnItem.updatedAt, + updatedAt: runningChildTurnItem.updatedAt, + }, + }, + { + type: "node.updated", + driver, + node: { + ...runningChildNode, + status: "completed" as const, + completedAt: runningChildNode.startedAt, + }, + }, + ] satisfies ReadonlyArray), + close: Deferred.succeed(ingestionDone, undefined), + }), startTurn: () => Effect.void, } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, run: { @@ -2548,99 +2549,23 @@ it.effect("does not refresh pull requests for auxiliary or stale provider termin }), ); -it.effect("finalizes a planned workspace detach before the provider announces its first turn", () => - Effect.gen(function* () { - const key = "workspace-detach-before-turn"; - const { written, observed } = yield* captureRootRunTermination({ - key, - knownProviderTurn: false, - shouldFinalizeRun: () => Effect.succeed(true), - events: (ids) => - Stream.fail( - new ProviderAdapter.ProviderAdapterSessionDetached({ - driver, - providerSessionId: ProviderSessionId.make(`session:${key}`), - threadId: ids.threadId, - reason: "workspace_changed", - }), - ), - }); - assert.deepEqual(observed, ["run:interrupted", "pull-requests-refreshed"]); - assert.deepEqual( - written.map((item) => item.type), - ["run_interrupt_result"], - ); - const interruption = written.find((item) => item.type === "run_interrupt_result"); - assert.equal(interruption?.message, "Run interrupted because the workspace changed"); - }), -); - -it.effect.each(["pending", "running"] as const)( - "interrupts %s root tool items before finalizing a planned workspace detach", - (status) => - Effect.gen(function* () { - const key = `workspace-detach-open-tools:${status}`; - const ids = backgroundScenarioIds(key); - const secondItemId = TurnItemId.make(`${ids.itemId}:second`); - const completedItemId = TurnItemId.make(`${ids.itemId}:completed`); - const { ingested, observed } = yield* captureRootRunTermination({ - key, - shouldFinalizeRun: () => Effect.succeed(true), - events: (ids) => - Stream.concat( - Stream.fromIterable([ - backgroundTurnItemEvent(ids, "dynamic_tool", status, 1), - backgroundTurnItemEvent(ids, "dynamic_tool", status, 2, secondItemId), - backgroundTurnItemEvent(ids, "dynamic_tool", "running", 3, completedItemId), - backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 3, completedItemId), - ]), - Stream.fail( - new ProviderAdapter.ProviderAdapterSessionDetached({ - driver, - providerSessionId: ProviderSessionId.make(`session:${key}`), - threadId: ids.threadId, - reason: "workspace_changed", - }), - ), - ), - }); - const interruptedItems = ingested.flatMap((event) => - event.type === "turn_item.updated" && event.turnItem.status === "interrupted" - ? [event.turnItem] - : [], - ); - assert.deepEqual( - interruptedItems.map((item) => item.id), - [ids.itemId, secondItemId], - ); - for (const item of interruptedItems) { - assert.isNotNull(item.completedAt); - assert.deepEqual(item.updatedAt, item.completedAt); - } - assert.equal(ingested.at(-1)?.type, "turn.terminal"); - assert.deepEqual(observed, ["run:interrupted", "pull-requests-refreshed"]); - }), -); - -it.effect("leaves planned-detach subagent items to the run-owned cascade", () => +it.effect("interrupts open tool items but preserves completed tools on a planned detach", () => Effect.gen(function* () { - const key = "workspace-detach-subagent"; - const { ingested, written } = yield* captureRootRunTermination({ + const key = "workspace-detach-open-tools"; + const ids = backgroundScenarioIds(key); + const runningId = TurnItemId.make(`${ids.itemId}:running`); + const completedId = TurnItemId.make(`${ids.itemId}:completed`); + const { ingested, observed } = yield* captureRootRunTermination({ key, shouldFinalizeRun: () => Effect.succeed(true), events: (ids) => Stream.concat( - Stream.make({ - type: "turn_item.updated" as const, - driver, - turnItem: makeRunOwnedSubagentTurnItemFixture({ - ids, - providerInstanceId: ProviderInstanceId.make("codex"), - childThreadId: ids.childThreadId, - driver, - status: "running", - }), - }), + Stream.fromIterable([ + backgroundTurnItemEvent(ids, "dynamic_tool", "pending", 1), + backgroundTurnItemEvent(ids, "dynamic_tool", "running", 2, runningId), + backgroundTurnItemEvent(ids, "dynamic_tool", "running", 3, completedId), + backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 3, completedId), + ]), Stream.fail( new ProviderAdapter.ProviderAdapterSessionDetached({ driver, @@ -2653,46 +2578,16 @@ it.effect("leaves planned-detach subagent items to the run-owned cascade", () => }); assert.deepEqual( ingested.flatMap((event) => - event.type === "turn_item.updated" ? [event.turnItem.status] : [], + event.type === "turn_item.updated" && event.turnItem.status === "interrupted" + ? [event.turnItem.id] + : [], ), - ["running"], - ); - assert.deepEqual( - written.filter((item) => item.type === "subagent").map((item) => item.status), - ["interrupted"], + [ids.itemId, runningId], ); + assert.deepEqual(observed, ["run:interrupted", "pull-requests-refreshed"]); }), ); -it.effect.each(["thread", "session"] as const)( - "fails rather than treating another %s's planned detach as this run's interruption", - (mismatch) => - Effect.gen(function* () { - const key = `workspace-detach-mismatched-${mismatch}`; - const { written, observed } = yield* captureRootRunTermination({ - key, - knownProviderTurn: false, - shouldFinalizeRun: () => Effect.succeed(true), - events: (ids) => - Stream.fail( - new ProviderAdapter.ProviderAdapterSessionDetached({ - driver, - providerSessionId: ProviderSessionId.make( - mismatch === "session" ? "session:other" : `session:${key}`, - ), - threadId: mismatch === "thread" ? ThreadId.make("thread:other") : ids.threadId, - reason: "workspace_changed", - }), - ), - }); - assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]); - assert.deepEqual( - written.map((item) => item.type), - ["error"], - ); - }), -); - it.effect("refreshes pull requests after a provider stream exits with an error", () => Effect.gen(function* () { const { written, observed } = yield* captureRootRunTermination({ @@ -3123,7 +3018,6 @@ function captureRootRunTermination(input: { readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"]; readonly seedOpenSubagent?: boolean; - readonly knownProviderTurn?: boolean; readonly events?: ( ids: BackgroundScenarioIds, ) => Stream.Stream< @@ -3246,7 +3140,7 @@ function captureRootRunTermination(input: { providerSessionId: ProviderSessionId.make(`session:${input.key}`), session: { events: Stream.empty, - subscribeEvents: Effect.succeed({ + subscribeEvents: () => Effect.succeed({ events: input.events?.(ids) ?? Stream.fromIterable([ @@ -3289,7 +3183,7 @@ function captureRootRunTermination(input: { } as OrchestrationV2Run, rootNode: { id: ids.rootNodeId, - providerTurnId: input.knownProviderTurn === false ? null : ids.rootProviderTurnId, + providerTurnId: ids.rootProviderTurnId, } as OrchestrationV2ExecutionNode, checkpointScope: { id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`), @@ -3301,7 +3195,7 @@ function captureRootRunTermination(input: { attempt: { id: ids.attemptId, providerTurnId: input.initialProviderTurnId === undefined - ? input.knownProviderTurn === false ? null : ids.rootProviderTurnId + ? ids.rootProviderTurnId : input.initialProviderTurnId, } as OrchestrationV2RunAttempt, attemptId: ids.attemptId, @@ -3753,7 +3647,7 @@ function runBackgroundItemScenario( providerSessionId: ProviderSessionId.make(`session:${key}`), session: { events: Stream.empty, - subscribeEvents: Effect.gen(function* () { + subscribeEvents: () => Effect.gen(function* () { yield* options?.onSubscribe ?? Effect.void; const events = Stream.fromIterable(makeEvents(ids)).pipe( Stream.concat( diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index c81b5dd54b768..8b02008ad5153 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -991,11 +991,9 @@ export const layer: Layer.Layer< const terminalEvent = yield* Ref.make(null); const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100); const latestProviderThread = yield* Ref.make(input.providerThread); - const latestRootProviderTurn = yield* Ref.make(null); + let latestRootProviderTurn: OrchestrationV2ProviderTurn | null = null; // A planned detach closes ingestion before the adapter can end its open root items. - const openRootTurnItems = yield* Ref.make< - ReadonlyMap - >(new Map()); + const openRootTurnItems = new Map(); const routeIdentity: ProviderEventRouteIdentity = { threadId: input.run.threadId, runId: input.run.id, @@ -1005,7 +1003,7 @@ export const layer: Layer.Layer< const eventSubscription = input.session.subscribeEvents === undefined ? { events: input.session.events, close: Effect.void } - : yield* input.session.subscribeEvents; + : yield* input.session.subscribeEvents(input.run.threadId); const inheritedBackgroundTurnItems = yield* ( input.loadInheritedBackgroundTurnItems?.() ?? Effect.succeed([]) ).pipe( @@ -1306,7 +1304,7 @@ export const layer: Layer.Layer< "Run interrupted because the workspace changed", ); const routing = yield* Ref.get(eventRouting); - const providerTurn = yield* Ref.get(latestRootProviderTurn); + const providerTurn = latestRootProviderTurn; const completedAt = yield* DateTime.now; const terminal = { type: "turn.terminal", @@ -1338,9 +1336,8 @@ export const layer: Layer.Layer< } : state, ); - const openItems = yield* Ref.getAndSet(openRootTurnItems, new Map()); return Stream.fromIterable([ - ...[...openItems.values()].map((turnItem) => ({ + ...[...openRootTurnItems.values()].map((turnItem) => ({ type: "turn_item.updated" as const, driver: input.providerThread.driver, turnItem: { @@ -1532,7 +1529,7 @@ export const layer: Layer.Layer< (event.providerTurn.runAttemptId === input.attempt.id || event.providerTurn.id === (yield* Ref.get(eventRouting)).rootProviderTurnId) ) { - yield* Ref.set(latestRootProviderTurn, event.providerTurn); + latestRootProviderTurn = event.providerTurn; } if ( @@ -1545,14 +1542,11 @@ export const layer: Layer.Layer< const turnItem = event.turnItem; // Subagent items are ended by the run-owned cascade. if (turnItem.type !== "subagent") { - const open = turnItem.status === "running" || turnItem.status === "pending"; - yield* Ref.update(openRootTurnItems, (items) => { - if (!open && !items.has(turnItem.id)) return items; - const next = new Map(items); - if (open) next.set(turnItem.id, turnItem); - else next.delete(turnItem.id); - return next; - }); + if (isOrchestrationV2WorkActive(turnItem.status)) { + openRootTurnItems.set(turnItem.id, turnItem); + } else { + openRootTurnItems.delete(turnItem.id); + } } } if (event.type === "turn.terminal") { diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 0458baff72c8c..ed7fcbf7414ab 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -15,7 +15,6 @@ import { ProviderTurnId, type RunId, ThreadId, - TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; @@ -30,6 +29,7 @@ import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; +import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; @@ -84,6 +84,10 @@ function makeRestartAdapter( sessionCapabilities: OrchestrationV2ProviderCapabilities = pooledCapabilities, providerInstanceId = initialSelection.instanceId, completeContinuation: Effect.Effect = Effect.void, + beforeTurn?: ( + input: ProviderAdapter.ProviderAdapterV2TurnInput, + publish: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect, + ) => Effect.Effect, ): ProviderAdapter.ProviderAdapterV2["Service"] { return { instanceId: providerInstanceId, @@ -224,6 +228,13 @@ function makeRestartAdapter( }, ], })); + if ( + beforeTurn !== undefined && + (yield* beforeTurn(input, (event) => + Queue.offer(events, event).pipe(Effect.asVoid), + )) + ) + return; const active = { input, providerTurnId: ProviderTurnId.make(`provider-turn:${input.attemptId}`), @@ -278,31 +289,6 @@ function makeRestartAdapter( }); yield* completeContinuation; const completedAt = yield* DateTime.now; - yield* Queue.offer(events, { - type: "turn_item.updated", - driver, - turnItem: { - id: TurnItemId.make(`turn-item:${input.attemptId}:result`), - threadId: input.threadId, - runId: input.runId, - nodeId: input.rootNodeId, - providerThreadId: input.providerThread.id, - providerTurnId: active.providerTurnId, - nativeItemRef: null, - parentItemId: null, - ordinal: input.providerTurnOrdinal * 100 + 2, - status: "completed", - title: null, - startedAt, - completedAt, - updatedAt: completedAt, - type: "assistant_message", - messageId: MessageId.make(`message:${input.attemptId}:result`), - text: "Completed inside the new worktree", - attachments: [], - streaming: false, - }, - }); yield* Queue.offer(events, { type: "message.updated", driver, @@ -848,15 +834,25 @@ it.live.each(["active", "idle", "selection-command", "pooled", "separate-home"] ), ); -it.live.each(["before-detach", "after-detach"] as const)( - "starts a workspace handoff continuation %s without holding its queue", - (continuationOrder) => +it.live.each(["exclusive", "shared"] as const)( + "keeps a delegated %s workspace handoff pending through its atomic continuation", + (sessionMode) => Effect.scoped( Effect.gen(function* () { - const name = `workspace-handoff-${continuationOrder}`; + const name = `delegated-workspace-handoff-${sessionMode}`; + const childReady = yield* Deferred.make<{ + input: ProviderAdapter.ProviderAdapterV2TurnInput; + publish: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect; + }>(); + const parentReady = yield* Deferred.make<{ + input: ProviderAdapter.ProviderAdapterV2TurnInput; + publish: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect; + }>(); const sourceCwd = yield* checkpointWorkspace(`${name}-source`); const targetCwd = yield* checkpointWorkspace(`${name}-target`); - const threadId = ThreadId.make(`thread:${name}`); + const parentId = ThreadId.make(`thread:${name}:parent`); + const continuationStarted = yield* Deferred.make(); + const finishContinuation = yield* Deferred.make(); const state = yield* Ref.make({ activeTurn: null, opened: [], @@ -865,18 +861,30 @@ it.live.each(["before-detach", "after-detach"] as const)( failedReplacementOpen: false, }); const registry = ProviderAdapterRegistry.layerSingle( - makeRestartAdapter(state, exclusiveCapabilities), + makeRestartAdapter( + state, + sessionMode === "shared" ? pooledCapabilities : exclusiveCapabilities, + providerInstanceId, + Deferred.succeed(continuationStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishContinuation)), + ), + (input, publish) => + input.message.text.endsWith("bind my workspace") + ? Deferred.succeed(childReady, { input, publish }).pipe(Effect.as(true)) + : Deferred.succeed(parentReady, { input, publish }).pipe(Effect.as(false)), + ), ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const sink = yield* EventSink.EventSinkV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; yield* orchestrator.dispatch({ type: "thread.create", - createdBy: "agent", - creationSource: "mcp", - commandId: CommandId.make(`${name}:create`), - threadId, + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${name}:parent:create`), + threadId: parentId, projectId: ProjectId.make(`project:${name}`), title: name, modelSelection: initialSelection, @@ -885,512 +893,221 @@ it.live.each(["before-detach", "after-detach"] as const)( branch: null, worktreePath: sourceCwd, }); - const startSequence = yield* sink.latestSequence(); - const started = yield* sink.stream({ afterSequence: startSequence, threadId }).pipe( - Stream.filter( - (stored) => - stored.event.type === "provider-turn.updated" && - stored.event.payload.status === "running", - ), - Stream.take(1), - Stream.runDrain, - Effect.forkScoped, - ); + const beforeStart = yield* sink.latestSequence(); yield* orchestrator.dispatch({ type: "message.dispatch", - createdBy: "agent", - creationSource: "mcp", - commandId: CommandId.make(`${name}:first`), - threadId, - messageId: MessageId.make(`${name}:first`), - text: "bind my workspace", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${name}:parent:start`), + threadId: parentId, + messageId: MessageId.make(`${name}:parent:start`), + text: "delegate the work", attachments: [], dispatchMode: { type: "start_immediately" }, }); yield* worker.drain(); - yield* Fiber.join(started); - const first = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; - const beforeDetach = yield* sink.latestSequence(); - const ended = yield* sink.stream({ afterSequence: beforeDetach, threadId }).pipe( + yield* sink.stream({ afterSequence: beforeStart, threadId: parentId }).pipe( Stream.filter( (stored) => - stored.event.type === "run.updated" && - stored.event.payload.id === first.id && - ["interrupted", "failed", "cancelled", "completed"].includes( - stored.event.payload.status, - ), + stored.event.type === "provider-turn.updated" && + stored.event.payload.status === "running", ), Stream.take(1), Stream.runDrain, - Effect.forkScoped, ); + const parentRun = (yield* orchestrator.getThreadProjection(parentId)).runs[0]!; yield* orchestrator.dispatch({ - type: "thread.metadata.update", - commandId: CommandId.make(`${name}:bind`), - threadId, - worktreePath: targetCwd, - }); - if (continuationOrder === "after-detach") { - yield* EffectWorker.runDaemon.pipe(Effect.forkScoped); - yield* worker.drain(); - yield* Fiber.join(ended); - assert.equal( - (yield* orchestrator.getThreadProjection(threadId)).runs[0]?.status, - "interrupted", - ); - } - const beforeContinuation = yield* sink.latestSequence(); - const continuation = yield* orchestrator.dispatch({ - type: "message.dispatch", + type: "delegated_task.request", createdBy: "agent", creationSource: "mcp", - commandId: CommandId.make(`${name}:continue`), - threadId, - messageId: MessageId.make(`${name}:continue`), - text: "handoff continuation", - attachments: [], - dispatchMode: { type: "queue_after_active" }, + commandId: CommandId.make(`${name}:delegate`), + parentThreadId: parentId, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId!, + task: "bind my workspace", + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", }); - const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; - if (continuationOrder === "before-detach") { - yield* EffectWorker.runDaemon.pipe(Effect.forkScoped); - } - if (queued.status === "queued") { - const promoted = yield* sink - .stream({ afterSequence: beforeContinuation, threadId }) - .pipe( - Stream.filter( - (stored) => - stored.event.type === "run.updated" && - stored.event.payload.id === queued.id && - (stored.event.payload.status === "starting" || - stored.event.payload.queueHeld === true), - ), - Stream.take(1), - Stream.runDrain, - Effect.forkScoped, - ); - yield* worker.drain(); - yield* Fiber.join(ended); - yield* Fiber.join(promoted); - const progress = yield* orchestrator.getThreadProjection(threadId); - assert.equal(progress.runs[1]?.queueHeld ?? false, false); - assert.equal(progress.runs[0]?.status, "interrupted"); - } - const completed = yield* sink - .stream({ afterSequence: continuation.sequence, threadId }) - .pipe( - Stream.filter( - (stored) => - stored.event.type === "run.updated" && - stored.event.payload.id === queued.id && - stored.event.payload.status === "completed", - ), - Stream.take(1), - Stream.runDrain, - Effect.forkScoped, - ); + const task = (yield* orchestrator.getThreadProjection(parentId)).subagents[0]!; + const childId = task.childThreadId!; yield* worker.drain(); - yield* Fiber.join(completed); - const settled = yield* orchestrator.getThreadProjection(threadId); + const child = yield* Deferred.await(childReady); + const parent = yield* Deferred.await(parentReady); + const parentSessionId = parent.input.providerThread.providerSessionId; + assert.isNotNull(parentSessionId); + const initialChild = yield* orchestrator.getThreadProjection(childId); + assert.isEmpty(initialChild.providerTurns); + assert.equal( + child.input.providerThread.providerSessionId === + parent.input.providerThread.providerSessionId, + sessionMode === "shared", + ); + const binding = { + type: "thread.metadata.update" as const, + commandId: CommandId.make(`${name}:bind`), + threadId: childId, + worktreePath: targetCwd, + expectedWorktreePath: sourceCwd, + worktreeContinuation: { + messageId: MessageId.make(`${name}:continue`), + text: "handoff continuation", + }, + }; + const bound = yield* orchestrator.dispatch(binding); + const boundChild = yield* orchestrator.getThreadProjection(childId); + assert.equal(boundChild.thread.worktreePath, targetCwd); assert.deepEqual( - settled.runs.map((run) => run.status), - ["interrupted", "completed"], + boundChild.runs.map((run) => run.status), + ["running", "queued"], ); - assert.equal(settled.providerTurns[0]?.status, "interrupted"); assert.equal( - settled.turnItems.find((item) => item.type === "run_interrupt_result")?.message, - "Run interrupted because the workspace changed", + boundChild.messages.find( + (message) => message.id === binding.worktreeContinuation.messageId, + )?.text, + "handoff continuation", ); - assert.isFalse(settled.runs.some((run) => run.queueHeld === true)); - assert.isFalse(settled.turnItems.some((item) => item.type === "error")); - const captured = yield* Ref.get(state); - assert.deepEqual( - captured.started.map((turn) => turn.cwd), - [sourceCwd, targetCwd], + assert.equal( + boundChild.checkpointScopes.find((scope) => scope.runId === boundChild.runs[1]?.id) + ?.cwd, + targetCwd, ); - assert.equal(captured.closedSessionCount, 1); - }).pipe( - Effect.provide( - ProviderReplayHarness.layerWithRegistry({ name }, registry, { - runEffectWorker: false, - }), - ), - ); - }), - ), -); - -it.live("keeps a delegated workspace handoff pending until its atomic continuation completes", () => - Effect.scoped( - Effect.gen(function* () { - const name = "delegated-workspace-handoff"; - const sourceCwd = yield* checkpointWorkspace(`${name}-source`); - const targetCwd = yield* checkpointWorkspace(`${name}-target`); - const parentId = ThreadId.make(`thread:${name}:parent`); - const continuationStarted = yield* Deferred.make(); - const finishContinuation = yield* Deferred.make(); - const state = yield* Ref.make({ - activeTurn: null, - opened: [], - started: [], - closedSessionCount: 0, - failedReplacementOpen: false, - }); - const registry = ProviderAdapterRegistry.layerSingle( - makeRestartAdapter( - state, - exclusiveCapabilities, - providerInstanceId, - Deferred.succeed(continuationStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishContinuation)), - ), - ), - ); - yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const sink = yield* EventSink.EventSinkV2; - yield* orchestrator.dispatch({ - type: "thread.create", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make(`${name}:parent:create`), - threadId: parentId, - projectId: ProjectId.make(`project:${name}`), - title: name, - modelSelection: initialSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: sourceCwd, - }); - const beforeStart = yield* sink.latestSequence(); - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make(`${name}:parent:start`), - threadId: parentId, - messageId: MessageId.make(`${name}:parent:start`), - text: "delegate the work", - attachments: [], - dispatchMode: { type: "start_immediately" }, - }); - yield* sink.stream({ afterSequence: beforeStart, threadId: parentId }).pipe( - Stream.filter( - (stored) => - stored.event.type === "provider-turn.updated" && - stored.event.payload.status === "running", - ), - Stream.take(1), - Stream.runDrain, - ); - const parentRun = (yield* orchestrator.getThreadProjection(parentId)).runs[0]!; - const delegated = yield* orchestrator.dispatch({ - type: "delegated_task.request", - createdBy: "agent", - creationSource: "mcp", - commandId: CommandId.make(`${name}:delegate`), - parentThreadId: parentId, - parentRunId: parentRun.id, - parentNodeId: parentRun.rootNodeId!, - task: "bind my workspace", - modelSelection: initialSelection, - runtimeMode: "full-access", - interactionMode: "default", - }); - const task = (yield* orchestrator.getThreadProjection(parentId)).subagents[0]!; - const childId = task.childThreadId!; - yield* sink.stream({ afterSequence: delegated.sequence, threadId: childId }).pipe( - Stream.filter( - (stored) => - stored.event.type === "provider-turn.updated" && - stored.event.payload.status === "running", - ), - Stream.take(1), - Stream.runDrain, - ); - const binding = { - type: "thread.metadata.update" as const, - commandId: CommandId.make(`${name}:bind`), - threadId: childId, - worktreePath: targetCwd, - expectedWorktreePath: sourceCwd, - worktreeContinuation: { - messageId: MessageId.make(`${name}:continue`), - text: "handoff continuation", - }, - }; - const bound = yield* orchestrator.dispatch(binding); - assert.lengthOf( - bound.storedEvents.filter((stored) => stored.event.type === "run.created"), - 1, - ); - assert.lengthOf( - bound.storedEvents.filter((stored) => stored.event.type === "message.updated"), - 1, - ); - // Repeating the exact handoff cannot enqueue another continuation. - yield* orchestrator.dispatch(binding); - const unchangedBinding = yield* orchestrator - .dispatch({ - ...binding, - commandId: CommandId.make(`${name}:unchanged-bind`), - expectedWorktreePath: targetCwd, - worktreeContinuation: { - messageId: MessageId.make(`${name}:unchanged-continue`), - text: "unexpected duplicate continuation", - }, - }) - .pipe(Effect.flip); - assert.include(String(unchangedBinding.cause), "changed worktree binding"); - yield* Deferred.await(continuationStarted); - const beforeCompletion = yield* orchestrator.getThreadProjection(parentId); - assert.isNull(beforeCompletion.subagents[0]?.result); - assert.isNull(beforeCompletion.subagents[0]?.completionDelivery ?? null); - assert.isFalse( - beforeCompletion.contextTransfers.some((transfer) => transfer.type === "subagent_result"), - ); - const continuing = yield* orchestrator.getThreadProjection(childId); - assert.deepEqual( - continuing.runs.map((run) => run.status), - ["interrupted", "running"], - ); - assert.isFalse(continuing.runs.some((run) => run.queueHeld)); - const beforeFinish = yield* sink.latestSequence(); - yield* Deferred.succeed(finishContinuation, undefined); - yield* sink.stream({ afterSequence: beforeFinish, threadId: parentId }).pipe( - Stream.filter( - (stored) => - stored.event.type === "subagent.updated" && - stored.event.payload.id === task.id && - stored.event.payload.status === "completed", - ), - Stream.take(1), - Stream.runDrain, - ); - const finished = yield* orchestrator.getThreadProjection(parentId); - assert.equal(finished.subagents[0]?.status, "completed"); - assert.equal(finished.subagents[0]?.result, "Completed inside the new worktree"); - assert.lengthOf( - finished.contextTransfers.filter((transfer) => transfer.type === "subagent_result"), - 1, - ); - const childFinished = yield* orchestrator.getThreadProjection(childId); - assert.deepEqual( - childFinished.runs.map((run) => run.status), - ["interrupted", "completed"], - ); - assert.lengthOf( - childFinished.messages.filter((message) => message.text === "handoff continuation"), - 1, - ); - assert.isFalse( - childFinished.messages.some( - (message) => message.text === "unexpected duplicate continuation", - ), - ); - assert.isFalse(childFinished.turnItems.some((item) => item.type === "error")); - const assistantItems = childFinished.visibleTurnItems.flatMap((row) => - row.item.type === "assistant_message" ? [row.item] : [], - ); - assert.lengthOf(assistantItems, 1); - assert.equal(assistantItems[0]?.text, "Completed inside the new worktree"); - const assistantMessage = childFinished.messages.find( - (message) => message.role === "assistant", - ); - assert.equal(assistantItems[0]?.messageId, assistantMessage?.id); - assert.equal(assistantItems[0]?.text, assistantMessage?.text); - - assert.deepEqual( - (yield* Ref.get(state)).started.map((turn) => turn.cwd), - [sourceCwd, sourceCwd, targetCwd], - ); - }).pipe(Effect.provide(ProviderReplayHarness.layerWithRegistry({ name }, registry))); - }), - ), -); - -it.live.each(["provider-crash", "explicit-stop"] as const)( - "preserves real queued input until explicitly resumed after %s", - (termination) => - Effect.scoped( - Effect.gen(function* () { - const name = `queued-control-${termination}`; - const cwd = yield* checkpointWorkspace(name); - const threadId = ThreadId.make(`thread:${name}`); - const crash = yield* Deferred.make(); - const firstSession = yield* Ref.make(true); - const state = yield* Ref.make({ - activeTurn: null, - opened: [], - started: [], - closedSessionCount: 0, - failedReplacementOpen: false, - }); - const base = makeRestartAdapter(state, exclusiveCapabilities); - const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { - ...base, - openSession: (input) => - base.openSession(input).pipe( - Effect.flatMap((session) => - Ref.getAndSet(firstSession, false).pipe( - Effect.map((isFirst) => ({ - ...session, - events: - termination === "explicit-stop" || !isFirst - ? session.events - : Stream.merge( - session.events, - Stream.fromEffect( - Deferred.await(crash).pipe( - Effect.andThen( - Effect.fail( - new ProviderAdapter.ProviderAdapterEventStreamError({ - driver, - providerSessionId: input.providerSessionId, - cause: "Simulated provider crash", - }), - ), - ), - ), - ), - ), - })), - ), - ), - ), - }; - yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const sink = yield* EventSink.EventSinkV2; - yield* orchestrator.dispatch({ - type: "thread.create", - commandId: CommandId.make(`${name}:create`), - threadId, - projectId: ProjectId.make(`project:${name}`), - title: name, - modelSelection: initialSelection, - createdBy: "user", - creationSource: "web", - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: cwd, - }); - const beforeStart = yield* sink.latestSequence(); - yield* orchestrator.dispatch({ - type: "message.dispatch", - commandId: CommandId.make(`${name}:first`), - threadId, - messageId: MessageId.make(`${name}:first`), - createdBy: "user", - creationSource: "web", - text: "keep working", - attachments: [], - dispatchMode: { type: "start_immediately" }, - }); - yield* sink.stream({ threadId, afterSequence: beforeStart }).pipe( + assert.deepEqual(yield* orchestrator.dispatch(binding), bound); + for (const worktreePath of [undefined, targetCwd]) { + const rejected = yield* orchestrator + .dispatch({ + ...binding, + commandId: CommandId.make(`${name}:reject:${worktreePath ?? "branch-only"}`), + branch: "feature/branch-only", + worktreePath, + expectedWorktreePath: targetCwd, + worktreeContinuation: { + messageId: MessageId.make(`${name}:rejected-continue`), + text: "unexpected duplicate continuation", + }, + }) + .pipe(Effect.flip); + assert.include(String(rejected.cause), "changed worktree binding"); + } + yield* worker.drain(1); + // A late provider message is a deterministic fence: a detached subscription + // must end before it can ingest another event for the old run. + const fenceAt = yield* DateTime.now; + const publishFence = ({ input, publish }: typeof child, text: string) => + publish({ + type: "message.updated", + driver, + message: { + id: MessageId.make(`${input.runId}:${text}`), + threadId: input.threadId, + runId: input.runId, + nodeId: input.rootNodeId, + role: "assistant", + text, + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "provider", + createdAt: fenceAt, + updatedAt: fenceAt, + }, + }); + yield* publishFence(child, "late-after-detach"); + yield* EffectWorker.runDaemon.pipe(Effect.forkScoped); + yield* sink.stream({ afterSequence: bound.sequence, threadId: childId }).pipe( Stream.filter( - (stored) => - stored.event.type === "provider-turn.updated" && - stored.event.payload.status === "running", + ({ event }) => + (event.type === "run.updated" && + event.payload.id === child.input.runId && + ["interrupted", "failed", "completed"].includes(event.payload.status)) || + (event.type === "message.updated" && event.payload.text === "late-after-detach"), ), Stream.take(1), Stream.runDrain, ); - const queuedCommand = { - type: "message.dispatch" as const, - commandId: CommandId.make(`${name}:queue`), - threadId, - messageId: MessageId.make(`${name}:queue`), - createdBy: "user" as const, - creationSource: "web" as const, - text: "handoff continuation", - attachments: [], - dispatchMode: { type: "queue_after_active" as const }, - }; - const queuedReceipt = yield* orchestrator.dispatch(queuedCommand); - const queuedState = yield* orchestrator.getThreadProjection(threadId); - assert.deepEqual( - queuedState.runs.map((run) => run.status), - ["running", "queued"], + const detached = yield* orchestrator.getThreadProjection(childId); + assert.equal( + detached.runs[0]?.status, + "interrupted", + "planned detach must terminate the moving run before another provider event is ingested", ); - const originalRun = queuedState.runs[0]!; - const queuedRun = queuedState.runs[1]!; - if (termination === "provider-crash") yield* Deferred.succeed(crash, undefined); - else - yield* orchestrator.dispatch({ - type: "run.interrupt", - commandId: CommandId.make(`${name}:stop`), - threadId, - runId: originalRun.id, - holdQueue: true, - }); - yield* sink.stream({ threadId, afterSequence: queuedReceipt.sequence }).pipe( + assert.equal( + detached.turnItems.find((item) => item.type === "run_interrupt_result")?.message, + "Run interrupted because the workspace changed", + ); + yield* Deferred.await(continuationStarted); + yield* publishFence(parent, "parent-still-alive"); + yield* sink.stream({ afterSequence: bound.sequence, threadId: parentId }).pipe( Stream.filter( - (stored) => - stored.event.type === "run.updated" && - stored.event.payload.id === queuedRun.id && - stored.event.payload.queueHeld === true, + ({ event }) => + (event.type === "message.updated" && event.payload.text === "parent-still-alive") || + (event.type === "run.updated" && + event.payload.id === parentRun.id && + ["interrupted", "failed", "completed"].includes(event.payload.status)), ), Stream.take(1), Stream.runDrain, ); - const held = yield* orchestrator.getThreadProjection(threadId); - assert.equal(held.runs[1]?.status, "queued"); + assert.isTrue(yield* manager.isLive(parentSessionId)); assert.equal( - held.messages.find((message) => message.id === queuedCommand.messageId)?.text, - queuedCommand.text, + (yield* orchestrator.getThreadProjection(parentId)).runs[0]?.status, + "running", ); - assert.lengthOf((yield* Ref.get(state)).started, 1); - const resumed = yield* orchestrator.dispatch({ - type: "queue.resume", - commandId: CommandId.make(`${name}:resume`), - threadId, - }); - yield* sink.stream({ threadId, afterSequence: resumed.sequence }).pipe( + const beforeCompletion = yield* orchestrator.getThreadProjection(parentId); + assert.isNull(beforeCompletion.subagents[0]?.result); + assert.isNull(beforeCompletion.subagents[0]?.completionDelivery ?? null); + assert.isFalse( + beforeCompletion.contextTransfers.some( + (transfer) => transfer.type === "subagent_result", + ), + ); + const continuing = yield* orchestrator.getThreadProjection(childId); + assert.deepEqual( + continuing.runs.map((run) => run.status), + ["interrupted", "running"], + ); + assert.isFalse(continuing.runs.some((run) => run.queueHeld)); + const beforeFinish = yield* sink.latestSequence(); + yield* Deferred.succeed(finishContinuation, undefined); + yield* sink.stream({ afterSequence: beforeFinish, threadId: parentId }).pipe( Stream.filter( (stored) => - stored.event.type === "run.updated" && - stored.event.payload.id === queuedRun.id && + stored.event.type === "subagent.updated" && + stored.event.payload.id === task.id && stored.event.payload.status === "completed", ), Stream.take(1), Stream.runDrain, ); - const completed = yield* orchestrator.getThreadProjection(threadId); + const finished = yield* orchestrator.getThreadProjection(parentId); + assert.equal(finished.subagents[0]?.status, "completed"); + assert.equal(finished.subagents[0]?.result, "Completed inside the new worktree"); + assert.lengthOf( + finished.contextTransfers.filter((transfer) => transfer.type === "subagent_result"), + 1, + ); + const childFinished = yield* orchestrator.getThreadProjection(childId); assert.deepEqual( - completed.runs.map((run) => run.status), - [termination === "provider-crash" ? "failed" : "interrupted", "completed"], + childFinished.runs.map((run) => run.status), + ["interrupted", "completed"], ); - assert.isFalse(completed.runs.some((run) => run.queueHeld)); assert.lengthOf( - completed.messages.filter((message) => message.id === queuedCommand.messageId), + childFinished.messages.filter((message) => message.text === "handoff continuation"), 1, ); - assert.lengthOf((yield* Ref.get(state)).started, 2); - assert.equal( - completed.messages.find((message) => message.role === "assistant")?.text, - "Completed inside the new worktree", + assert.isFalse( + childFinished.messages.some( + (message) => message.text === "unexpected duplicate continuation", + ), + ); + assert.isFalse(childFinished.turnItems.some((item) => item.type === "error")); + assert.deepEqual( + (yield* Ref.get(state)).started.map((turn) => turn.cwd), + [sourceCwd, sourceCwd, targetCwd], ); - if (termination === "provider-crash") - assert.isTrue( - completed.turnItems.some( - (item) => item.type === "error" && item.runId === originalRun.id, - ), - ); - else - assert.equal( - completed.turnItems.find((item) => item.type === "run_interrupt_result")?.message, - "Run interrupted by user", - ); }).pipe( Effect.provide( - ProviderReplayHarness.layerWithRegistry( - { name }, - ProviderAdapterRegistry.layerSingle(adapter), - ), + ProviderReplayHarness.layerWithRegistry({ name }, registry, { runEffectWorker: false }), ), ); }), diff --git a/docs/internals/overview.md b/docs/internals/overview.md index e8d6aacdbddc5..2a5c5f7cc4ed1 100644 --- a/docs/internals/overview.md +++ b/docs/internals/overview.md @@ -58,12 +58,9 @@ therefore means the intent committed, not that the provider, checkpoint, or othe finished. Keep external I/O out of command decisions and the database transaction. Effects tied to a lost provider process cannot simply replay; recovery retires them before admitting new work. -A worktree handoff with a continuation records the workspace binding, continuation message and -run, and provider detach effect in the same command transaction. The detach can execute immediately -after commit, so recording the continuation afterward would let a delegated child publish its result -too early. A planned exclusive workspace detach interrupts the old turn without a provider failure; -the queued continuation can then start in the new workspace. Unexpected provider failures and -explicit Stop retain queued messages until the user resumes them. +A worktree handoff commits its workspace binding, continuation and detach effect together. +The detach can execute immediately after commit; queuing the continuation afterward could let a +delegated child publish its result before its follow-up work exists. Persisted events must remain decodable on replay. Changing a schema affects old environments at startup as well as live RPC traffic. Compatibility work must account for stored history, not just diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index ef1cd6dab5a62..950c891801632 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -358,7 +358,6 @@ const ProviderAdapterFailureReason = Schema.Literals([ "attachment_unreadable", ]); export type ProviderAdapterFailureReason = typeof ProviderAdapterFailureReason.Type; -/** The app deliberately closed an exclusive session to move its thread's workspace. */ export class ProviderAdapterSessionDetached extends Schema.TaggedError()( "ProviderAdapterSessionDetached", { @@ -556,8 +555,10 @@ export interface ProviderAdapterV2SessionRuntime { readonly providerSessionId: ProviderSessionId; readonly providerSession: OrchestrationV2ProviderSession; readonly events: Stream.Stream; - /** Manager runtimes broadcast independent subscriptions; adapters may expose only a single-consumer stream. */ - readonly subscribeEvents?: Effect.Effect; + /** Manager subscriptions broadcast independently; a thread ID limits detach to its stream. */ + readonly subscribeEvents?: ( + threadId?: ThreadId, + ) => Effect.Effect; readonly hasPendingBackgroundWork?: Effect.Effect; /** Root-subscription probe; replacement checks opt in to subagents owned by the live process. */ readonly hasPendingBackgroundWorkForThread?: ( From 8b74ce3db4b0d2aea04e3dbfa1994403f73082c1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 09:15:59 +0000 Subject: [PATCH 06/14] test(orchestration): preserve startup ordering during worktree handoffs --- .../SelectionRestart.integration.test.ts | 38 +++++++++++++++---- .../testkit/ProviderReplayHarness.ts | 3 +- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index ed7fcbf7414ab..e50bf533d2664 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -25,6 +25,7 @@ import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; @@ -851,6 +852,8 @@ it.live.each(["exclusive", "shared"] as const)( const sourceCwd = yield* checkpointWorkspace(`${name}-source`); const targetCwd = yield* checkpointWorkspace(`${name}-target`); const parentId = ThreadId.make(`thread:${name}:parent`); + const contextReadStarted = yield* Deferred.make(); + const finishContextRead = yield* Deferred.make(); const continuationStarted = yield* Deferred.make(); const finishContinuation = yield* Deferred.make(); const state = yield* Ref.make({ @@ -931,18 +934,14 @@ it.live.each(["exclusive", "shared"] as const)( }); const task = (yield* orchestrator.getThreadProjection(parentId)).subagents[0]!; const childId = task.childThreadId!; - yield* worker.drain(); - const child = yield* Deferred.await(childReady); + const startingChild = yield* worker.drain(1).pipe(Effect.forkScoped); + yield* Deferred.await(contextReadStarted); + assert.lengthOf((yield* Ref.get(state)).started, 1); const parent = yield* Deferred.await(parentReady); const parentSessionId = parent.input.providerThread.providerSessionId; assert.isNotNull(parentSessionId); const initialChild = yield* orchestrator.getThreadProjection(childId); assert.isEmpty(initialChild.providerTurns); - assert.equal( - child.input.providerThread.providerSessionId === - parent.input.providerThread.providerSessionId, - sessionMode === "shared", - ); const binding = { type: "thread.metadata.update" as const, commandId: CommandId.make(`${name}:bind`), @@ -989,6 +988,18 @@ it.live.each(["exclusive", "shared"] as const)( .pipe(Effect.flip); assert.include(String(rejected.cause), "changed worktree binding"); } + // The real outbox must keep the detach behind startup, even while + // the event subscriber exists and the app-context read is suspended. + assert.isFalse(yield* worker.runOnce); + assert.isFalse(yield* Deferred.isDone(childReady)); + yield* Deferred.succeed(finishContextRead, undefined); + yield* Fiber.join(startingChild); + const child = yield* Deferred.await(childReady); + assert.equal( + child.input.providerThread.providerSessionId === + parent.input.providerThread.providerSessionId, + sessionMode === "shared", + ); yield* worker.drain(1); // A late provider message is a deterministic fence: a detached subscription // must end before it can ingest another event for the old run. @@ -1107,7 +1118,18 @@ it.live.each(["exclusive", "shared"] as const)( ); }).pipe( Effect.provide( - ProviderReplayHarness.layerWithRegistry({ name }, registry, { runEffectWorker: false }), + ProviderReplayHarness.layerWithRegistry({ name }, registry, { + runEffectWorker: false, + mcpAppModelContextLayer: Layer.mock(McpAppModelContext.McpAppModelContext)({ + forThread: (threadId) => + threadId === parentId + ? Effect.succeed([]) + : Deferred.succeed(contextReadStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishContextRead)), + Effect.as([]), + ), + }), + }), ), ); }), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 4da359f74c920..3b2dbb78269a5 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -267,6 +267,7 @@ export function layerWithRegistry( SqlClient.SqlClient, MigrationError | PlatformError.PlatformError | SqlError >; + readonly mcpAppModelContextLayer?: Layer.Layer; readonly runEffectWorker?: boolean; readonly runContinuationWorker?: boolean; readonly recoverOnStartup?: boolean; @@ -378,7 +379,7 @@ export function layerWithRegistry( const layerRunExecutionServiceProvided = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( - McpAppModelContext.layerEmpty, + options.mcpAppModelContextLayer ?? McpAppModelContext.layerEmpty, PreviewAgentActivity.layer, layerCheckpointServiceProvided, layerEventSinkProvided, From 77f2798e7226253858b84540587b14410ebf5696 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 20 Oct 2026 08:36:41 -0400 Subject: [PATCH 07/14] test(server): adapt upstream session invalidation fixture to subscriptions --- apps/server/src/orchestration-v2/ProviderSessionManager.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index ebb60f812257b..f10849bc87335 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3296,7 +3296,7 @@ it.effect( const publish = Effect.fnUntraced(function* ( event: ProviderAdapter.ProviderAdapterV2Event, ) { - const subscription = yield* failed.runtime.subscribeEvents!; + const subscription = yield* failed.runtime.subscribeEvents!(); yield* Queue.offer(queue!, event); // Observing the exact event ensures the manager has handled it before resuming. assert.deepEqual(yield* subscription.events.pipe(Stream.runHead), Option.some(event)); From 5842e15d30e7401a3da859348dc4a6c3aedef5a3 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 20 Oct 2026 09:20:27 -0400 Subject: [PATCH 08/14] fix: preserve source behavior against current upstream contracts --- .../orchestration-v2/RunExecutionService.ts | 186 +++++++++--------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 8b02008ad5153..4bc6e9885874d 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -991,9 +991,9 @@ export const layer: Layer.Layer< const terminalEvent = yield* Ref.make(null); const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100); const latestProviderThread = yield* Ref.make(input.providerThread); - let latestRootProviderTurn: OrchestrationV2ProviderTurn | null = null; - // A planned detach closes ingestion before the adapter can end its open root items. - const openRootTurnItems = new Map(); + let latestRootProviderTurn: OrchestrationV2ProviderTurn | null = null; + // A planned detach closes ingestion before the adapter can end its open root items. + const openRootTurnItems = new Map(); const routeIdentity: ProviderEventRouteIdentity = { threadId: input.run.threadId, runId: input.run.id, @@ -1003,7 +1003,7 @@ export const layer: Layer.Layer< const eventSubscription = input.session.subscribeEvents === undefined ? { events: input.session.events, close: Effect.void } - : yield* input.session.subscribeEvents(input.run.threadId); + : yield* input.session.subscribeEvents(input.run.threadId); const inheritedBackgroundTurnItems = yield* ( input.loadInheritedBackgroundTurnItems?.() ?? Effect.succeed([]) ).pipe( @@ -1051,7 +1051,7 @@ export const layer: Layer.Layer< } const providerThread = yield* Ref.get(latestProviderThread); const openSubagents = yield* Ref.get(openRunOwnedSubagents); - const message = yield* Ref.get(interruptionMessage); + const message = yield* Ref.get(interruptionMessage); yield* writeFinalRunEvents({ run: input.run, rootNode: input.rootNode, @@ -1079,7 +1079,7 @@ export const layer: Layer.Layer< : { loadRunInterruptReason: input.loadRunInterruptReason }), openRunOwnedSubagents: openSubagents, terminal, - ...(message === undefined ? {} : { interruptionMessage: message }), + ...(message === undefined ? {} : { interruptionMessage: message }), failureItemPersisted: terminal.status === "failed", refreshAfterTurn, }).pipe( @@ -1292,80 +1292,80 @@ export const layer: Layer.Layer< // as a dev server keep this consumer alive afterwards and must not pin preview focus. const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId); const providerEventFiber = yield* eventSubscription.events.pipe( - Stream.catchTag("ProviderAdapterSessionDetached", (detached) => - detached.providerSessionId !== input.providerSessionId || - detached.threadId !== input.run.threadId - ? Stream.fail(detached) - : Stream.unwrap( - Effect.gen(function* () { - if (yield* Ref.get(rootRunFinalized)) return Stream.empty; - yield* Ref.set( - interruptionMessage, - "Run interrupted because the workspace changed", - ); - const routing = yield* Ref.get(eventRouting); - const providerTurn = latestRootProviderTurn; - const completedAt = yield* DateTime.now; - const terminal = { - type: "turn.terminal", - driver: input.providerThread.driver, - providerThreadId: input.providerThread.id, - providerTurnId: - routing.rootProviderTurnId ?? - input.attempt.providerTurnId ?? - idAllocator.derive.providerTurn({ - driver: input.providerThread.driver, - nativeTurnId: `detached:${input.attempt.id}`, - }), - runOrdinal: input.run.ordinal, - status: "interrupted", - failure: null, - threadDisposition: "reusable", - } satisfies ProviderTerminalEvent; - // Before the provider's first turn event, the planned - // detach supplies this run's root identity for routing. - yield* Ref.update(eventRouting, (state) => - state.rootProviderTurnId === null - ? { - ...state, - rootProviderTurnId: terminal.providerTurnId, - ownedProviderTurnIds: new Set([ - ...state.ownedProviderTurnIds, - terminal.providerTurnId, - ]), - } - : state, - ); - return Stream.fromIterable([ - ...[...openRootTurnItems.values()].map((turnItem) => ({ - type: "turn_item.updated" as const, + Stream.catchTag("ProviderAdapterSessionDetached", (detached) => + detached.providerSessionId !== input.providerSessionId || + detached.threadId !== input.run.threadId + ? Stream.fail(detached) + : Stream.unwrap( + Effect.gen(function* () { + if (yield* Ref.get(rootRunFinalized)) return Stream.empty; + yield* Ref.set( + interruptionMessage, + "Run interrupted because the workspace changed", + ); + const routing = yield* Ref.get(eventRouting); + const providerTurn = latestRootProviderTurn; + const completedAt = yield* DateTime.now; + const terminal = { + type: "turn.terminal", + driver: input.providerThread.driver, + providerThreadId: input.providerThread.id, + providerTurnId: + routing.rootProviderTurnId ?? + input.attempt.providerTurnId ?? + idAllocator.derive.providerTurn({ driver: input.providerThread.driver, - turnItem: { - ...turnItem, - ...("streaming" in turnItem ? { streaming: false } : {}), - status: "interrupted" as const, - completedAt, - updatedAt: completedAt, - }, - })), - ...(providerTurn?.status === "running" || providerTurn?.status === "pending" - ? [ - { - type: "provider_turn.updated" as const, - driver: input.providerThread.driver, - providerTurn: { - ...providerTurn, - status: "interrupted" as const, - completedAt, - }, + nativeTurnId: `detached:${input.attempt.id}`, + }), + runOrdinal: input.run.ordinal, + status: "interrupted", + failure: null, + threadDisposition: "reusable", + } satisfies ProviderTerminalEvent; + // Before the provider's first turn event, the planned + // detach supplies this run's root identity for routing. + yield* Ref.update(eventRouting, (state) => + state.rootProviderTurnId === null + ? { + ...state, + rootProviderTurnId: terminal.providerTurnId, + ownedProviderTurns: new Map(state.ownedProviderTurns).set( + terminal.providerTurnId, + input.providerThread.id, + ), + } + : state, + ); + return Stream.fromIterable([ + ...[...openRootTurnItems.values()].map((turnItem) => ({ + type: "turn_item.updated" as const, + driver: input.providerThread.driver, + turnItem: { + ...turnItem, + ...("streaming" in turnItem ? { streaming: false } : {}), + status: "interrupted" as const, + completedAt, + updatedAt: completedAt, + }, + })), + ...(providerTurn?.status === "running" || providerTurn?.status === "pending" + ? [ + { + type: "provider_turn.updated" as const, + driver: input.providerThread.driver, + providerTurn: { + ...providerTurn, + status: "interrupted" as const, + completedAt, }, - ] - : []), - terminal, - ]); - }), - ), - ), + }, + ] + : []), + terminal, + ]); + }), + ), + ), Stream.mapEffect((event) => Effect.gen(function* () { const previous = yield* Ref.get(eventRouting); @@ -1524,13 +1524,13 @@ export const layer: Layer.Layer< yield* Ref.set(latestProviderThread, event.providerThread); } } - if ( - event.type === "provider_turn.updated" && - (event.providerTurn.runAttemptId === input.attempt.id || - event.providerTurn.id === (yield* Ref.get(eventRouting)).rootProviderTurnId) - ) { - latestRootProviderTurn = event.providerTurn; - } + if ( + event.type === "provider_turn.updated" && + (event.providerTurn.runAttemptId === input.attempt.id || + event.providerTurn.id === (yield* Ref.get(eventRouting)).rootProviderTurnId) + ) { + latestRootProviderTurn = event.providerTurn; + } if ( event.type === "turn_item.updated" && @@ -1539,15 +1539,15 @@ export const layer: Layer.Layer< yield* Ref.update(latestTurnItemOrdinal, (current) => Math.max(current, event.turnItem.ordinal), ); - const turnItem = event.turnItem; - // Subagent items are ended by the run-owned cascade. - if (turnItem.type !== "subagent") { - if (isOrchestrationV2WorkActive(turnItem.status)) { - openRootTurnItems.set(turnItem.id, turnItem); - } else { - openRootTurnItems.delete(turnItem.id); - } + const turnItem = event.turnItem; + // Subagent items are ended by the run-owned cascade. + if (turnItem.type !== "subagent") { + if (isOrchestrationV2WorkActive(turnItem.status)) { + openRootTurnItems.set(turnItem.id, turnItem); + } else { + openRootTurnItems.delete(turnItem.id); } + } } if (event.type === "turn.terminal") { yield* Ref.set(terminalEvent, event); From 3801bd626317678741f1d9c8b7544a9111cf292f Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 20 Oct 2026 09:53:56 -0400 Subject: [PATCH 09/14] fix(orchestration): preserve delegated tasks through worktree handoffs --- apps/server/src/orchestration-v2/Orchestrator.ts | 15 +++++++++++++-- .../SelectionRestart.integration.test.ts | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 96fe5d1c35eba..158365b67b319 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -4990,6 +4990,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command: Extract, events: Ref.Ref>, effects: Ref.Ref>, + worktreeContinuationOfRunId?: RunId, ) { let projection = yield* getProjectionWithPendingEvents(command.threadId, events); if (command.manualContinuationOfRunId !== undefined) { @@ -5060,7 +5061,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId); if ( !source || - !isRestartContinuationSource(source, projection.providerTurns) || + (!(source.id === worktreeContinuationOfRunId && isBlockingRun(source)) && + !isRestartContinuationSource(source, projection.providerTurns)) || projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.thread.providerInstanceId !== source.providerInstanceId || @@ -12496,7 +12498,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "provider.switch": yield* dispatchThreadMutation(command, events, effects); break; - case "thread.metadata.update": + case "thread.metadata.update": { + const sourceRun = + command.worktreeContinuation === undefined + ? undefined + : (yield* getProjectionWithPendingEvents(command.threadId, events)).runs.find( + isBlockingRun, + ); yield* dispatchThreadMutation(command, events, effects); if (command.worktreeContinuation !== undefined) { yield* dispatchMessage( @@ -12505,6 +12513,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio commandId: command.commandId, threadId: command.threadId, ...command.worktreeContinuation, + ...(sourceRun ? { restartContinuationOfRunId: sourceRun.id } : {}), attachments: [], createdBy: "agent", creationSource: "mcp", @@ -12512,9 +12521,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, events, effects, + sourceRun?.id, ); } break; + } case "thread.pull-request-watch.sync": yield* dispatchPullRequestWatchSync(command, events, effects); break; diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index e50bf533d2664..c0abf5b97df9b 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -1078,6 +1078,8 @@ it.live.each(["exclusive", "shared"] as const)( ["interrupted", "running"], ); assert.isFalse(continuing.runs.some((run) => run.queueHeld)); + assert.equal(continuing.runs[1]?.delegatedTaskId, continuing.runs[0]?.delegatedTaskId); + assert.equal(continuing.runs[1]?.restartContinuationOfRunId, continuing.runs[0]?.id); const beforeFinish = yield* sink.latestSequence(); yield* Deferred.succeed(finishContinuation, undefined); yield* sink.stream({ afterSequence: beforeFinish, threadId: parentId }).pipe( From ed16fa05b607ba38518896a6fdd4b45dcf697b79 Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Sat, 24 Oct 2026 14:10:55 -0400 Subject: [PATCH 10/14] fix(orchestration): reject stale workspace continuations --- .../src/orchestration-v2/Orchestrator.ts | 7 ++ .../ProviderSessionManager.ts | 47 +++++----- .../RunExecutionService.test.ts | 74 ++++++++------- .../orchestration-v2/RunExecutionService.ts | 12 +-- .../SelectionRestart.integration.test.ts | 91 +++++++++++++++++++ 5 files changed, 161 insertions(+), 70 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 158365b67b319..e7f0495268e64 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -5075,6 +5075,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) || (yield* stopReachedRun(command, command.threadId, source.id)) ) { + if (worktreeContinuationOfRunId !== undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "The active run changed before the workspace continuation could be queued.", + }); + } // Preserve the current row so stale automatic deliveries receive an // accepted receipt without changing work or repeatedly retrying. yield* emit( diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 2f4f68047d860..eec76fbfc2790 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -505,33 +505,32 @@ export const layerWithOptions = ( ), ); - const failSubscribers = ( + const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* ( entry: LiveSessionEntry, error: ProviderAdapter.ProviderAdapterV2Error, threadId?: ThreadId, - ) => - Effect.gen(function* () { - const subscribers = yield* Ref.modify(entry.eventSubscribers, (current) => { - const failed: Array = []; - const remaining = new Map(current); - for (const [id, subscriber] of current) { - if (threadId === undefined || subscriber.threadId === threadId) { - failed.push(subscriber); - remaining.delete(id); - } + ) { + const subscribers = yield* Ref.modify(entry.eventSubscribers, (current) => { + const failed: Array = []; + const remaining = new Map(current); + for (const [id, subscriber] of current) { + if (threadId === undefined || subscriber.threadId === threadId) { + failed.push(subscriber); + remaining.delete(id); } - return [failed, remaining] as const; - }); - yield* Effect.forEach( - subscribers, - ({ queue }) => - Queue.offer(queue, { - type: "failure", - cause: Cause.fail(error), - }), - { discard: true }, - ); + } + return [failed, remaining] as const; }); + yield* Effect.forEach( + subscribers, + ({ queue }) => + Queue.offer(queue, { + type: "failure", + cause: Cause.fail(error), + }), + { discard: true }, + ); + }); // Server shutdown interrupts subscribers rather than ending them: a // cleanly ended stream without a terminal fails its run, while an @@ -1619,9 +1618,7 @@ export const layerWithOptions = ( const makeEventSubscription = Effect.fn("ProviderSessionManager.makeEventSubscription")( function* ( - subscribers: Ref.Ref< - ReadonlyMap - >, + subscribers: Ref.Ref>, threadId?: ThreadId, ): Effect.fn.Return { const queue = yield* Queue.unbounded(); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index da5c1aefbd673..256a0075f6f01 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3116,8 +3116,10 @@ function captureRootRunTermination(input: { }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ - ingestNormalized: input.ingestNormalized ?? (({ event }) => - Ref.update(ingested, (current) => [...current, event]).pipe(Effect.as([]))), + ingestNormalized: + input.ingestNormalized ?? + (({ event }) => + Ref.update(ingested, (current) => [...current, event]).pipe(Effect.as([]))), }), ServerSettings.layerTest(), Layer.succeed(RunFinalizationService.RunFinalizationObserver, { @@ -3140,37 +3142,38 @@ function captureRootRunTermination(input: { providerSessionId: ProviderSessionId.make(`session:${input.key}`), session: { events: Stream.empty, - subscribeEvents: () => Effect.succeed({ - events: - input.events?.(ids) ?? - Stream.fromIterable([ - ...(input.seedOpenSubagent - ? [ - { type: "subagent.updated", driver, subagent: runningSubagent } as const, - { - type: "node.updated", - driver, - node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), - } as const, - { - type: "turn_item.updated", - driver, - turnItem: makeRunOwnedSubagentTurnItemFixture({ - ids, - providerInstanceId, - childThreadId: ids.childThreadId, + subscribeEvents: () => + Effect.succeed({ + events: + input.events?.(ids) ?? + Stream.fromIterable([ + ...(input.seedOpenSubagent + ? [ + { type: "subagent.updated", driver, subagent: runningSubagent } as const, + { + type: "node.updated", driver, - status: "running", - }), - } as const, - ] - : []), - rootTerminalEvent(ids, "interrupted"), - ] satisfies ReadonlyArray), - close: Deferred.succeed(ingestionDone, undefined).pipe( - Effect.andThen(input.onSubscriptionClose ?? Effect.void), - ), - }), + node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + } as const, + { + type: "turn_item.updated", + driver, + turnItem: makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + } as const, + ] + : []), + rootTerminalEvent(ids, "interrupted"), + ] satisfies ReadonlyArray), + close: Deferred.succeed(ingestionDone, undefined).pipe( + Effect.andThen(input.onSubscriptionClose ?? Effect.void), + ), + }), startTurn: input.startTurn ?? (() => Effect.void), interruptTurn: input.interruptTurn ?? (() => Effect.void), hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread, @@ -3194,9 +3197,10 @@ function captureRootRunTermination(input: { } as OrchestrationV2ProviderThread, attempt: { id: ids.attemptId, - providerTurnId: input.initialProviderTurnId === undefined - ? ids.rootProviderTurnId - : input.initialProviderTurnId, + providerTurnId: + input.initialProviderTurnId === undefined + ? ids.rootProviderTurnId + : input.initialProviderTurnId, } as OrchestrationV2RunAttempt, attemptId: ids.attemptId, providerTurnOrdinal: 1, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 4bc6e9885874d..8931c2db0251b 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -991,7 +991,7 @@ export const layer: Layer.Layer< const terminalEvent = yield* Ref.make(null); const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100); const latestProviderThread = yield* Ref.make(input.providerThread); - let latestRootProviderTurn: OrchestrationV2ProviderTurn | null = null; + const latestRootProviderTurn = yield* Ref.make(null); // A planned detach closes ingestion before the adapter can end its open root items. const openRootTurnItems = new Map(); const routeIdentity: ProviderEventRouteIdentity = { @@ -1304,7 +1304,7 @@ export const layer: Layer.Layer< "Run interrupted because the workspace changed", ); const routing = yield* Ref.get(eventRouting); - const providerTurn = latestRootProviderTurn; + const providerTurn = yield* Ref.get(latestRootProviderTurn); const completedAt = yield* DateTime.now; const terminal = { type: "turn.terminal", @@ -1524,14 +1524,6 @@ export const layer: Layer.Layer< yield* Ref.set(latestProviderThread, event.providerThread); } } - if ( - event.type === "provider_turn.updated" && - (event.providerTurn.runAttemptId === input.attempt.id || - event.providerTurn.id === (yield* Ref.get(eventRouting)).rootProviderTurnId) - ) { - latestRootProviderTurn = event.providerTurn; - } - if ( event.type === "turn_item.updated" && event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index c0abf5b97df9b..3e830b8c28dbc 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -835,6 +835,97 @@ it.live.each(["active", "idle", "selection-command", "pooled", "separate-home"] ), ); +it.live("rejects a workspace binding when a provider switch invalidates its continuation", () => + Effect.scoped( + Effect.gen(function* () { + const name = "workspace-continuation-provider-switch"; + const sourceCwd = yield* checkpointWorkspace(`${name}-source`); + const targetCwd = yield* checkpointWorkspace(`${name}-target`); + const threadId = ThreadId.make(name); + const targetId = ProviderInstanceId.make(`${name}-target`); + const state = yield* Ref.make({ + activeTurn: null, + opened: [], + started: [], + closedSessionCount: 0, + failedReplacementOpen: false, + }); + const registry = ProviderAdapterRegistry.layerFromAdapters([ + makeRestartAdapter(state), + makeRestartAdapter(state, pooledCapabilities, targetId), + ]); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sink = yield* EventSink.EventSinkV2; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${name}:create`), + threadId, + projectId: ProjectId.make(`project:${name}`), + title: name, + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: sourceCwd, + createdBy: "user", + creationSource: "web", + }); + const beforeStart = yield* sink.latestSequence({ threadId }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${name}:start`), + threadId, + messageId: MessageId.make(`${name}:start`), + text: "start", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + yield* sink.stream({ threadId, afterSequence: beforeStart }).pipe( + Stream.filter( + ({ event }) => + event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + ); + yield* orchestrator.dispatch({ + type: "provider.switch", + commandId: CommandId.make(`${name}:switch`), + threadId, + modelSelection: { instanceId: targetId, model: initialSelection.model }, + }); + const switched = yield* orchestrator.getThreadProjection(threadId); + assert.equal(switched.thread.providerInstanceId, targetId); + assert.equal(switched.runs[0]?.status, "running"); + // Leave the detach in the outbox: its old run still blocks this handoff. + const failed = yield* orchestrator + .dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`${name}:bind`), + threadId, + worktreePath: targetCwd, + expectedWorktreePath: sourceCwd, + worktreeContinuation: { + messageId: MessageId.make(`${name}:continue`), + text: "handoff continuation", + }, + }) + .pipe(Effect.flip); + assert.include(String(failed.cause), "workspace continuation could be queued"); + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal(after.thread.worktreePath, sourceCwd); + assert.lengthOf(after.runs, switched.runs.length); + assert.isFalse(after.messages.some((message) => message.id === `${name}:continue`)); + }).pipe(Effect.provide(ProviderReplayHarness.layerWithRegistry({ name }, registry))); + }), + ), +); + it.live.each(["exclusive", "shared"] as const)( "keeps a delegated %s workspace handoff pending through its atomic continuation", (sessionMode) => From 24e660ea678effe238a464887b341487ca7b2fca Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Mon, 26 Oct 2026 09:26:46 -0400 Subject: [PATCH 11/14] test(server): assert replayed handoff receipt --- .../src/orchestration-v2/SelectionRestart.integration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 3e830b8c28dbc..0ee6ed6a9a112 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -1062,7 +1062,7 @@ it.live.each(["exclusive", "shared"] as const)( ?.cwd, targetCwd, ); - assert.deepEqual(yield* orchestrator.dispatch(binding), bound); + assert.deepEqual(yield* orchestrator.dispatch(binding), { ...bound, replayed: true }); for (const worktreePath of [undefined, targetCwd]) { const rejected = yield* orchestrator .dispatch({ From 26e80781431b670cd18ef40078a835c870dfbe9a Mon Sep 17 00:00:00 2001 From: Jake Leventhal Date: Tue, 27 Oct 2026 15:11:22 -0400 Subject: [PATCH 12/14] fix(server): finish streamed messages and background items on detach --- .../RunExecutionService.test.ts | 122 +++++++++++++++--- .../orchestration-v2/RunExecutionService.ts | 25 +++- 2 files changed, 124 insertions(+), 23 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 256a0075f6f01..e5e60831601d7 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2588,6 +2588,87 @@ it.effect("interrupts open tool items but preserves completed tools on a planned }), ); +it.effect("flushes the latest buffered assistant message on a planned detach", () => + Effect.gen(function* () { + const key = "workspace-detach-message"; + const now = yield* DateTime.now; + const { ingested } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.concat( + Stream.fromIterable( + ["partial", "partial final words"].map((text) => ({ + type: "message.updated" as const, + driver, + message: { + id: MessageId.make(`message:${key}`), + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + role: "assistant" as const, + text, + attachments: [], + streaming: true, + createdBy: "agent" as const, + creationSource: "provider" as const, + createdAt: now, + updatedAt: now, + }, + })), + ), + Stream.fail( + new ProviderAdapter.ProviderAdapterSessionDetached({ + driver, + providerSessionId: ProviderSessionId.make(`session:${key}`), + threadId: ids.threadId, + reason: "workspace_changed", + }), + ), + ), + }); + const messages = ingested.flatMap((event) => + event.type === "message.updated" ? [event.message] : [], + ); + assert.equal(messages.at(-1)?.text, "partial final words"); + assert.equal(messages.at(-1)?.streaming, false); + }), +); + +it.effect( + "ends background items detached after the root has finalized without finalizing twice", + () => + Effect.gen(function* () { + const key = "workspace-detach-after-root"; + const { ingested, observed } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: (ids) => + Stream.concat( + Stream.make( + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + rootTerminalEvent(ids, "completed"), + ), + Stream.fail( + new ProviderAdapter.ProviderAdapterSessionDetached({ + driver, + providerSessionId: ProviderSessionId.make(`session:${key}`), + threadId: ids.threadId, + reason: "workspace_changed", + }), + ), + ), + }); + assert.equal( + ingested.filter( + (event) => event.type === "turn_item.updated" && event.turnItem.status === "interrupted", + ).length, + 1, + ); + assert.deepEqual(observed, ["run:waiting", "pull-requests-refreshed"]); + }), +); + it.effect("refreshes pull requests after a provider stream exits with an error", () => Effect.gen(function* () { const { written, observed } = yield* captureRootRunTermination({ @@ -3651,26 +3732,27 @@ function runBackgroundItemScenario( providerSessionId: ProviderSessionId.make(`session:${key}`), session: { events: Stream.empty, - subscribeEvents: () => Effect.gen(function* () { - yield* options?.onSubscribe ?? Effect.void; - const events = Stream.fromIterable(makeEvents(ids)).pipe( - Stream.concat( - Stream.fromEffect( - Effect.andThen( - options?.onEventsConsumed ?? Effect.void, - options?.afterEvents?.(previewActivity) ?? Effect.void, - ), - ).pipe(Stream.drain), - ), - ); - return { - events: - options?.keepEventStreamOpen === true - ? events.pipe(Stream.concat(Stream.never)) - : events, - close: Deferred.succeed(ingestionDone, undefined), - }; - }), + subscribeEvents: () => + Effect.gen(function* () { + yield* options?.onSubscribe ?? Effect.void; + const events = Stream.fromIterable(makeEvents(ids)).pipe( + Stream.concat( + Stream.fromEffect( + Effect.andThen( + options?.onEventsConsumed ?? Effect.void, + options?.afterEvents?.(previewActivity) ?? Effect.void, + ), + ).pipe(Stream.drain), + ), + ); + return { + events: + options?.keepEventStreamOpen === true + ? events.pipe(Stream.concat(Stream.never)) + : events, + close: Deferred.succeed(ingestionDone, undefined), + }; + }), startTurn: () => Effect.void, } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, run: { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 8931c2db0251b..0647d541c25f9 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -5,6 +5,8 @@ import { CommandId, type EventId, type ModelSelection, + type MessageId, + type OrchestrationV2ConversationMessage, type NodeId, type OrchestrationV2AppThread, type OrchestrationV2CheckpointScope, @@ -994,6 +996,7 @@ export const layer: Layer.Layer< const latestRootProviderTurn = yield* Ref.make(null); // A planned detach closes ingestion before the adapter can end its open root items. const openRootTurnItems = new Map(); + const streamingRootMessages = new Map(); const routeIdentity: ProviderEventRouteIdentity = { threadId: input.run.threadId, runId: input.run.id, @@ -1298,7 +1301,7 @@ export const layer: Layer.Layer< ? Stream.fail(detached) : Stream.unwrap( Effect.gen(function* () { - if (yield* Ref.get(rootRunFinalized)) return Stream.empty; + const rootFinalized = yield* Ref.get(rootRunFinalized); yield* Ref.set( interruptionMessage, "Run interrupted because the workspace changed", @@ -1348,7 +1351,13 @@ export const layer: Layer.Layer< updatedAt: completedAt, }, })), - ...(providerTurn?.status === "running" || providerTurn?.status === "pending" + ...[...streamingRootMessages.values()].map((message) => ({ + type: "message.updated" as const, + driver: input.providerThread.driver, + message: { ...message, streaming: false, updatedAt: completedAt }, + })), + ...(!rootFinalized && + (providerTurn?.status === "running" || providerTurn?.status === "pending") ? [ { type: "provider_turn.updated" as const, @@ -1361,7 +1370,7 @@ export const layer: Layer.Layer< }, ] : []), - terminal, + ...(!rootFinalized ? [terminal] : []), ]); }), ), @@ -1451,6 +1460,16 @@ export const layer: Layer.Layer< ) { yield* Ref.set(latestRootProviderTurn, event.providerTurn); } + if ( + event.type === "message.updated" && + event.message.threadId === input.run.threadId && + event.message.runId === input.run.id && + event.message.role === "assistant" + ) { + if (event.message.streaming) + streamingRootMessages.set(event.message.id, event.message); + else streamingRootMessages.delete(event.message.id); + } let storedEventCount = 0; const deliveredEvent = filterAssistantEvent( event, From 21cc55b9f38eda17d9b71883dd77a1cb06e5297a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 07:05:10 +0000 Subject: [PATCH 13/14] test(server): drop duplicate handoff rollback case and narrating comments AI-assisted; verified with the tests and tools listed above. --- .../server/src/mcp/WorktreeMcpService.test.ts | 26 ------------------- docs/user/composer.md | 3 +-- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/apps/server/src/mcp/WorktreeMcpService.test.ts b/apps/server/src/mcp/WorktreeMcpService.test.ts index 753f7dbbb53b0..cecc066b849d0 100644 --- a/apps/server/src/mcp/WorktreeMcpService.test.ts +++ b/apps/server/src/mcp/WorktreeMcpService.test.ts @@ -450,7 +450,6 @@ describe("t3_worktree_handoff", () => { }), ); expect(harness.sendToThread).not.toHaveBeenCalled(); - // The continuation must be durably queued before anything slower runs. expect(harness.dispatch.mock.invocationCallOrder[0]).toBeLessThan( harness.runForThread.mock.invocationCallOrder[0]!, ); @@ -470,26 +469,6 @@ describe("t3_worktree_handoff", () => { }); }); - it.effect( - "rolls back the worktree when the atomic bind and continuation dispatch defects", - () => { - const harness = makeHarness({ dispatchDies: true }); - return Effect.gen(function* () { - const exit = yield* Effect.exit( - runHandoff(harness, { - branch: "feature/continue-dies", - continuationPrompt: "Keep going.", - }), - ); - expectTypedFailure(exit, { _tag: "WorktreeMcpFailure", code: "operation_failed" }); - expect(harness.removeWorktree).toHaveBeenCalledTimes(1); - expect(harness.deleteLocalBranch).toHaveBeenCalledTimes(1); - expect(harness.runForThread).not.toHaveBeenCalled(); - expect(harness.sendToThread).not.toHaveBeenCalled(); - }); - }, - ); - it.effect("does not queue a continuation when the thread update fails", () => { const harness = makeHarness({ dispatchFails: true }); return Effect.gen(function* () { @@ -702,9 +681,6 @@ describe("t3_worktree_handoff", () => { const gate = yield* Deferred.make(); const harness = makeHarness({ dispatchGate: Deferred.await(gate) }); - // Interrupt arrives while the metadata dispatch is in flight; the - // binding-plus-continuation section must run to completion anyway so the - // atomic binding and continuation commit survives the pending interruption. const fiber = yield* Effect.forkChild( runHandoff(harness, { branch: "feature/interrupted", @@ -725,8 +701,6 @@ describe("t3_worktree_handoff", () => { }), ); expect(harness.sendToThread).not.toHaveBeenCalled(); - // The setup script must also survive the pending interrupt; otherwise - // the continuation run starts in a worktree that was never set up. expect(harness.runForThread).toHaveBeenCalledTimes(1); }), ); diff --git a/docs/user/composer.md b/docs/user/composer.md index 58b33b4f38726..d816807838ad5 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -74,8 +74,7 @@ your default follow-up behavior; add **Shift** to use the opposite action. This applies only to multiline messages when you choose the multiline send option. When an agent moves its thread into a worktree with a continuation message, work resumes -automatically in that worktree. An explicit Stop or a provider failure pauses queued messages -until you resume the queue; the messages stay saved. +automatically in that worktree. Use `Cmd+Shift+Enter` on macOS or `Ctrl+Shift+Enter` on Windows and Linux to send the oldest queued message as a steer. This leaves the current draft intact and From 18d3e4f6cdf53516bf4b28fac819cbaf0884e892 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 07:33:10 +0000 Subject: [PATCH 14/14] fix(server): settle a workspace detach in ingestion order The detach handler read the run's open-item maps upstream of the provider commit batcher, which can read ahead of processing. A buffered tool completion could then be overwritten by a stale interrupted snapshot. The stream now emits an in-order marker and settlement runs when it is processed. AI-assisted; verified with the tests and tools listed above. --- .../RunExecutionService.test.ts | 14 ++++++++++++++ .../src/orchestration-v2/RunExecutionService.ts | 17 ++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 1dd594f17a67f..e532e99374cb1 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2725,6 +2725,7 @@ it.effect("interrupts open tool items but preserves completed tools on a planned const { ingested, observed } = yield* captureRootRunTermination({ key, shouldFinalizeRun: () => Effect.succeed(true), + readAhead: true, events: (ids) => Stream.concat( Stream.fromIterable([ @@ -3327,6 +3328,8 @@ function captureRootRunTermination(input: { readonly afterStart?: ( closed: Deferred.Deferred, ) => Effect.Effect; + // Pull the whole provider stream before processing, as the commit batcher may read ahead. + readonly readAhead?: boolean; }) { return Effect.gen(function* () { const ids = backgroundScenarioIds(input.key); @@ -3368,6 +3371,17 @@ function captureRootRunTermination(input: { Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, mockEventSink({ + ...(input.readAhead + ? { + consumeProviderEvents: (source, process, after) => + Effect.gen(function* () { + for (const event of yield* Stream.runCollect(source)) { + yield* process(event); + if (yield* after) return; + } + }), + } + : {}), write: (payload) => Effect.gen(function* () { for (const event of payload.events) { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 043fe1ff4c200..a4e4cb3998744 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1627,7 +1627,7 @@ export const layer: Layer.Layer< yield* trackChildLifecycle(event, deliveredEvent !== null); yield* scheduleIdleFlush; }); - // A planned detach for this thread ends the stream; settle what the provider left open. + // A planned detach ends the stream. It settles in ingestion order, after buffered events. const settleDetachedRun = Effect.fnUntraced(function* () { const rootFinalized = yield* Ref.get(rootRunFinalized); yield* Ref.set(interruptionMessage, "Run interrupted because the workspace changed"); @@ -1663,7 +1663,7 @@ export const layer: Layer.Layer< } : state, ); - return Stream.fromIterable([ + const settlement: ReadonlyArray = [ ...[...openRootTurnItems.values()].map((turnItem) => ({ type: "turn_item.updated" as const, driver: input.providerThread.driver, @@ -1691,7 +1691,8 @@ export const layer: Layer.Layer< ] : []), ...(!rootFinalized ? [terminal] : []), - ]); + ]; + for (const event of settlement) yield* ingestEvent(yield* routeEvent(event)); }); const providerEventFiber = yield* eventSink .consumeProviderEvents( @@ -1700,16 +1701,18 @@ export const layer: Layer.Layer< detached.providerSessionId !== input.providerSessionId || detached.threadId !== input.run.threadId ? Stream.fail(detached) - : Stream.unwrap(settleDetachedRun()), + : Stream.succeed({ type: "workspace_detached" as const }), ), Stream.merge(Stream.fromQueue(idleFlushTicks), { haltStrategy: "left" }), ), (event) => event.type === "idle_flush" ? flushIdleAssistantText(event.at) - : Effect.gen(function* () { - yield* ingestEvent(yield* routeEvent(event)); - }), + : event.type === "workspace_detached" + ? settleDetachedRun() + : Effect.gen(function* () { + yield* ingestEvent(yield* routeEvent(event)); + }), Effect.gen(function* () { if (yield* agentTurnsSettled) yield* endPreviewActivity; return yield* shouldStopProviderEventIngestion;