From 2fb39acb8f4ca337bab8d8014b5bf28e135f8ea3 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 18 Oct 2026 06:29:33 -0400 Subject: [PATCH 01/23] fix(server): agent messages no longer cancel Claude tools Queue agent-authored updates when the live provider session declares that active steering interrupts tools. This covers automatic delivery and explicit steer requests while preserving human steering and intentional restarts. Explain the fallback in the MCP tool description and verify queued delivery, human steering, and safe-provider steering through the orchestrator. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent --- .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../src/orchestration-v2/Orchestrator.ts | 18 ++++ .../SteeringCompletion.integration.test.ts | 92 +++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 05299626c6748..37b619fcf640b 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -234,7 +234,7 @@ export const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, steers a fully active turn, or queues behind a turn that is not yet steerable. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", + "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. When the target session's steering interrupts tools (including Claude), both auto and explicit steer queue a separate turn instead; delivery='queued' confirms this fallback. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 949ef37c26c6b..3bccd497a580b 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -4721,6 +4721,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio turn.status === "completed")) ) { dispatchMode = { type: "start_immediately" }; + } else if (command.createdBy === "agent") { + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === target?.providerThreadId, + ); + if (providerThread?.providerSessionId != null) { + const session = yield* providerSessions + .get(providerThread.providerSessionId) + .pipe(mapDispatchError(command)); + if ( + Option.isSome(session) && + session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === + true + ) { + // Claude steering cancels pending tools, which appear as user refusals. + // Agent updates can wait for the durable queue's next-turn boundary. + dispatchMode = { type: "queue_after_active" }; + } + } } } if ( diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index fca2d93721281..07c8bfe934b22 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -419,6 +419,7 @@ const composerSelection = { const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* ( name: string, canRestart = false, + activeSteeringInterruptsTools = false, ) { const cwd = yield* checkpointWorkspace(name); const events = yield* Queue.unbounded(); @@ -430,6 +431,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: true, supportsSteeringByInterruptRestart: canRestart, + activeSteeringInterruptsTools, }, }; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { @@ -561,6 +563,96 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* return { events, started, steered, layer, startFirstTurn }; }); +it.effect.each([ + { actor: "agent", mode: "auto", interruptsTools: true }, + { actor: "agent", mode: "steer", interruptsTools: true }, + { actor: "user", mode: "steer", interruptsTools: true }, + { actor: "agent", mode: "auto", interruptsTools: false }, +] as const)( + "delivers $actor $mode messages safely (interruptsTools=$interruptsTools)", + ({ actor, mode, interruptsTools }) => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `agent-steering-${actor}-${mode}-${interruptsTools}`, + false, + interruptsTools, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const messageId = MessageId.make("message:update"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("update"), + threadId, + messageId, + text: "progress update", + attachments: [], + dispatchMode: + mode === "auto" + ? { type: "start_immediately" } + : { type: "steer_active", targetRunId: first.runId }, + ...(mode === "auto" ? { deliveryIntent: "auto" as const } : {}), + createdBy: actor, + creationSource: actor === "agent" ? "mcp" : "web", + }); + yield* worker.drain(); + const queued = actor === "agent" && interruptsTools; + assert.deepEqual(steered, queued ? [] : ["progress update"]); + assert.equal(started.length, 1); + const pending = yield* orchestrator.getThreadProjection(threadId); + assert.equal(pending.runs.find((run) => run.id === first.runId)?.status, "running"); + if (!queued) return; + assert.equal( + pending.runs.find((run) => run.userMessageId === messageId)?.status, + "queued", + ); + + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "waiting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const turn = pending.providerTurns[0]!; + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, + }); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(completed); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + assert.equal(started.length, 2); + assert.equal(started[1]?.message.messageId, messageId); + assert.equal(started[1]?.message.text, "progress update"); + assert.deepEqual(steered, []); + const final = yield* orchestrator.getThreadProjection(threadId); + assert.equal(final.runs.find((run) => run.id === first.runId)?.status, "completed"); + }).pipe(Effect.provide(layer)); + }), + ), +); + it.effect.each([{ canRestart: false }, { canRestart: true }])( "steers a changed turn-scoped selection into the running turn (canRestart=$canRestart)", ({ canRestart }) => From f0b61a65effac37d09c1c46a2fb235518f0e3ccd Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 18 Oct 2026 06:58:37 -0400 Subject: [PATCH 02/23] test(server): verify queued agent send results Run the steering safety cases through ThreadManagementService so they cover the delivery result returned to MCP callers and subsequent queued-turn delivery. Keep the upstream task-ownership regression on an explicitly safe native-steering fixture, preserving its assertions while Claude agent messages use the queue. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagents --- ...OrchestratorMcpToolkit.integration.test.ts | 35 ++++++++++++++++--- .../SteeringCompletion.integration.test.ts | 27 +++++++++----- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index 44d53fdbc0126..55f4739e42d6a 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -99,6 +99,7 @@ const parentThreadId = ThreadId.make("thread:mcp-orchestrator-parent"); const projectId = ProjectId.make("project:mcp-orchestrator"); const codexInstanceId = ProviderInstanceId.make("codex"); const claudeInstanceId = ProviderInstanceId.make("claudeAgent"); +const safeSteeringInstanceId = ProviderInstanceId.make("codex-native-steering"); const codexModel = "gpt-5.4"; const claudeModel = "claude-sonnet-4-6"; const parentPrompt = "Keep this parent turn active while orchestration tools are tested."; @@ -575,6 +576,22 @@ describe("orchestrator MCP toolkit", () => { : `Claude completed: ${turn.message.text}`, steerReceived: (steer) => steerReceivedGates.get(steer.message.text), }), + makeDeterministicAdapter({ + instanceId: safeSteeringInstanceId, + driver: ProviderDriverKind.make("codex"), + capabilities: { + ...CodexProviderCapabilitiesV2, + turns: { + ...CodexProviderCapabilitiesV2.turns, + supportsActiveSteering: true, + activeSteeringInterruptsTools: false, + }, + }, + capturedTurns, + shouldComplete: (turn) => turn.message.text !== cancellationPrompt, + response: (turn) => `Codex completed: ${turn.message.text}`, + steerReceived: (steer) => steerReceivedGates.get(steer.message.text), + }), ]); const continuationOffers = yield* Ref.make< ReadonlyArray @@ -620,11 +637,16 @@ describe("orchestrator MCP toolkit", () => { }, layerRegistry, ).pipe(Layer.provide(layerContinuationProbe)); + const layerSettings = ServerSettings.layerTest({ + providerInstances: { + [safeSteeringInstanceId]: { driver: "codex", enabled: true, config: {} }, + }, + }); const layerOrchestration = Layer.merge( layerOrchestrator, ThreadManagementService.layer.pipe( Layer.provide(layerOrchestrator), - Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerSettings), Layer.provide( CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), ), @@ -653,6 +675,11 @@ describe("orchestrator MCP toolkit", () => { driver: ProviderDriverKind.make("claudeAgent"), model: claudeModel, }), + makeProviderSnapshot({ + instanceId: safeSteeringInstanceId, + driver: ProviderDriverKind.make("codex"), + model: codexModel, + }), makeProviderSnapshot({ instanceId: ProviderInstanceId.make("opencode"), driver: ProviderDriverKind.make("opencode"), @@ -695,7 +722,7 @@ describe("orchestrator MCP toolkit", () => { Layer.provideMerge(layerOrchestration), Layer.provide(layerRegistry), Layer.provide(layerProviderRegistry), - Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerSettings), Layer.provide(layerScheduledTaskStub), Layer.provide(Layer.mock(ThreadSearch.ThreadSearch)({})), Layer.provide( @@ -1981,11 +2008,11 @@ describe("orchestrator MCP toolkit", () => { projection.runs.some((run) => run.id === runId && run.status === status), ); - // Claude steers into the running turn, which then ends. + // Safe native steering joins the running turn without creating a task. const steerReceived = yield* Deferred.make(); steerReceivedGates.set(childSteerText, steerReceived); const steeredSend = yield* steerIntoUntaskedChildRun( - { providerInstanceId: claudeInstanceId, model: claudeModel }, + { providerInstanceId: safeSteeringInstanceId, model: codexModel }, "steer-untasked-run", ); // A steer that misses the turn is sent again as its own run, so diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 07c8bfe934b22..15a4851be2898 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -18,14 +18,19 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import * as ThreadManagement from "./ThreadManagementService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -513,10 +518,16 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }; }), }; - const layer = ProviderReplayHarness.layerWithRegistry( + const databaseLayer = SqlitePersistence.layerMemory; + const layerRuntime = ProviderReplayHarness.layerWithRegistry( { name }, ProviderAdapterRegistry.layerSingle(adapter), - { runEffectWorker: false }, + { runEffectWorker: false, databaseLayer }, + ); + const layer = ThreadManagement.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(databaseLayer))), + Layer.provideMerge(layerRuntime), ); const startFirstTurn = Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -581,26 +592,24 @@ it.effect.each([ yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threads = yield* ThreadManagement.ThreadManagementService; const threadId = yield* startFirstTurn; const first = started[0]!; const messageId = MessageId.make("message:update"); - yield* orchestrator.dispatch({ - type: "message.dispatch", + const result = yield* threads.sendToThread({ + projectId: (yield* orchestrator.getThreadProjection(threadId)).thread.projectId, commandId: CommandId.make("update"), threadId, messageId, text: "progress update", attachments: [], - dispatchMode: - mode === "auto" - ? { type: "start_immediately" } - : { type: "steer_active", targetRunId: first.runId }, - ...(mode === "auto" ? { deliveryIntent: "auto" as const } : {}), + mode, createdBy: actor, creationSource: actor === "agent" ? "mcp" : "web", }); yield* worker.drain(); const queued = actor === "agent" && interruptsTools; + assert.equal(result.delivery, queued ? "queued" : "steered"); assert.deepEqual(steered, queued ? [] : ["progress update"]); assert.equal(started.length, 1); const pending = yield* orchestrator.getThreadProjection(threadId); From ccd3c59866d0ed24475a9588f47b1087e0c195a0 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 18 Oct 2026 07:13:15 -0400 Subject: [PATCH 03/23] fix(server): agent messages no longer cancel Claude tools Deliver Claude agent updates through root PostToolBatch and Stop hook context without interrupting running tools. Defer to the durable next-turn queue after five seconds without a boundary, or when the update contains attachments. Let human steering and immediate controls pass a waiting boundary delivery. Revoke pending updates on Stop and task disposal, and refresh completion pointers when only part of a task cohort remains unacknowledged. Keep native human steering and other providers' delivery behavior unchanged. Exercise the real send service, hook adapter, outbox ordering and ownership races with deterministic regression coverage. Upstream-PR: https://github.com/pingdotgg/t3code/pull/21158 Co-Authored-By: GPT-6.1 Sol at high effort, implementation and SDK research Co-Authored-By: GPT-6.1 Sol at xhigh effort, independent review --- ...OrchestratorMcpToolkit.integration.test.ts | 10 +- .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../Adapters/ClaudeAdapterV2.test.ts | 131 +++++ .../Adapters/ClaudeAdapterV2.ts | 148 +++++- .../src/orchestration-v2/EffectOutbox.ts | 24 +- .../src/orchestration-v2/EffectWorker.ts | 16 +- apps/server/src/orchestration-v2/EventSink.ts | 3 + .../FoundationPersistence.test.ts | 229 +++++++++ .../orchestration-v2/NotificationMailbox.ts | 17 + .../src/orchestration-v2/Orchestrator.ts | 142 +++++- .../ProviderTurnControlService.ts | 109 ++++- .../SteeringCompletion.integration.test.ts | 453 +++++++++++++++++- packages/contracts/src/orchestrationV2.ts | 3 + .../src/server/ProviderAdapter.ts | 3 + 14 files changed, 1227 insertions(+), 63 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index c7b5fc245094f..6093680306242 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -561,7 +561,15 @@ describe("orchestrator MCP toolkit", () => { makeDeterministicAdapter({ instanceId: claudeInstanceId, driver: ProviderDriverKind.make("claudeAgent"), - capabilities: ClaudeProviderCapabilitiesV2, + capabilities: { + ...ClaudeProviderCapabilitiesV2, + turns: { + ...ClaudeProviderCapabilitiesV2.turns, + // This deterministic adapter steers without interrupting tools. + // Real Claude uses boundary hooks, covered by SteeringCompletion. + activeSteeringInterruptsTools: false, + }, + }, capturedTurns, shouldComplete: (turn) => turn.message.text !== cancellationPrompt, terminalGate: (turn) => diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index f0dabc0260e37..d828c746af232 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -234,7 +234,7 @@ export const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, steers a fully active turn, or queues behind a turn that is not yet steerable. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", + "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. Claude agent updates wait for a safe tool-batch or answer boundary for up to five seconds, then queue a separate turn if no boundary arrives. Other tool-interrupting sessions queue immediately. delivery reports dispatch acceptance, not model processing. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 447272221f650..790674618e485 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2669,6 +2669,137 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + it.effect.each([ + "PostToolBatch", + "Stop", + "timeout", + "cancelled", + "completed", + "withdrawn", + "refreshed", + "aborted", + ] as const)("delivers agent messages at a safe boundary or defers them (%s)", (scenario) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ close: Queue.shutdown }); + const allocator = yield* IdAllocator.IdAllocatorV2; + const hookAbort = new AbortController(); + const attemptId = RunAttemptId.make("attempt-agent-boundary"); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId, + text: "Work through the task.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + const turn = { + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: allocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + }; + const steering = yield* harness.runtime + .steerTurn({ + ...turn, + ...(scenario === "withdrawn" || scenario === "refreshed" || scenario === "aborted" + ? { + resolveMessageText: Effect.sync(() => { + if (scenario === "aborted") hookAbort.abort(); + return scenario === "withdrawn" ? undefined : "Remaining task completed."; + }), + } + : {}), + message: { + createdBy: "agent", + creationSource: "mcp", + messageId: MessageId.make("agent-boundary-update"), + text: "The review found a missing check.", + attachments: [], + }, + }) + .pipe(Effect.forkScoped); + // Flush runnable fibers; the steer is now waiting on its boundary. + yield* TestClock.adjust("0 millis"); + const event = scenario === "Stop" ? "Stop" : "PostToolBatch"; + const hook = harness.getOpenedOptions()?.hooks?.[event]?.[0]?.hooks[0]; + assert.isDefined(hook); + const boundary = + event === "Stop" + ? { hook_event_name: "Stop" as const, stop_hook_active: false } + : { hook_event_name: "PostToolBatch" as const, tool_calls: [] }; + const hookInput = { + ...boundary, + session_id: WAKE_NATIVE_SESSION, + transcript_path: "transcript.jsonl", + cwd: "/repo", + }; + const callHook = (child = false, signal = hookAbort.signal) => + Effect.promise(() => + hook!({ ...hookInput, ...(child ? { agent_id: "child" } : {}) }, undefined, { + signal, + }), + ); + assert.deepEqual(yield* callHook(true), {}); + if (scenario === "timeout") { + yield* TestClock.adjust("5 seconds"); + } else if (scenario === "cancelled") { + yield* harness.runtime.interruptTurn(turn); + } else if (scenario === "completed") { + yield* harness.offerAndWait( + makeResultFrame({ uuid: "boundary-completed", result: "Done" }), + ); + } else { + const output = yield* callHook(); + assert.deepEqual( + output, + scenario === "withdrawn" || scenario === "aborted" + ? {} + : { + hookSpecificOutput: { + hookEventName: event, + additionalContext: + scenario === "refreshed" + ? "New message from another agent:\nRemaining task completed." + : "New message from another agent:\nThe review found a missing check.", + }, + }, + ); + if (scenario === "aborted") { + assert.deepEqual(yield* callHook(false, new AbortController().signal), { + hookSpecificOutput: { + hookEventName: event, + additionalContext: "New message from another agent:\nRemaining task completed.", + }, + }); + } + } + if ( + scenario === "timeout" || + scenario === "completed" || + scenario === "cancelled" || + scenario === "withdrawn" + ) { + const error = yield* Fiber.join(steering).pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterSteerRunError"); + assert.isTrue("deferToQueue" in error && error.deferToQueue); + } else { + yield* Fiber.join(steering); + } + assert.deepEqual(yield* callHook(), {}); + assert.equal(harness.offeredMessages.length, 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect.each([ { isError: false, title: "Check weather" }, { isError: true, title: "Check weather" }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index e969520bd05ee..d80e21a0f16f5 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -26,6 +26,7 @@ import { type EffortLevel, type ForkSessionOptions, type ForkSessionResult, + type HookCallback, getSubagentMessages, query, type Options as ClaudeQueryOptions, @@ -319,6 +320,7 @@ export const ClaudeProviderCapabilitiesV2 = { supportsInterrupt: true, supportsActiveSteering: true, activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, supportsSteeringByInterruptRestart: false, supportsQueuedMessages: true, terminalStatusQuality: "strong", @@ -3016,6 +3018,15 @@ function formatClaudeUsageLimitWait(waitMs: number): string { return minutes === 0 ? `${hours}h` : `${hours}h ${minutes}m`; } +type ClaudeAgentSteerOutcome = "delivered" | "deferred" | "cancelled"; + +interface PendingClaudeAgentSteer { + readonly text: string; + readonly completion: Deferred.Deferred; + readonly resolveMessageText?: ProviderAdapter.ProviderAdapterV2SteerInput["resolveMessageText"]; + outcome?: ClaudeAgentSteerOutcome; +} + interface ActiveClaudeTurnContext { readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly nativeThreadId: string; @@ -3073,12 +3084,26 @@ interface ActiveClaudeTurnContext { readonly subagentsByToolUseId: Map; readonly subagentNodesByTaskId: Map; readonly promptUuid: string | null; + readonly pendingAgentSteers: Map; promptEcho: "pending" | "confirmed"; promptStarted: boolean; gatedFramesBeforeEcho: number; readonly heldRootFrames: Array; } +// Claim and acknowledge synchronously so a boundary racing the timeout cannot +// both inject a message and return it to the durable queue. +function settleClaudeAgentSteers( + context: ActiveClaudeTurnContext, + outcome: ClaudeAgentSteerOutcome, +): void { + for (const pending of context.pendingAgentSteers.values()) { + pending.outcome = outcome; + Deferred.doneUnsafe(pending.completion, Effect.succeed(outcome)); + } + context.pendingAgentSteers.clear(); +} + interface ActiveClaudeProviderRetry { readonly retry: OrchestrationV2ProviderRetry; readonly failure: OrchestrationV2ProviderFailure; @@ -6066,6 +6091,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( readonly threadDisposition?: "reusable" | "broken"; readonly result?: SDKResultMessage; }) { + settleClaudeAgentSteers( + input.context, + input.status === "interrupted" || input.status === "cancelled" + ? "cancelled" + : "deferred", + ); yield* reasoningDeltas.flushTurn(input.context.nativeTurnId); if (input.context.compaction?.status === "running") { yield* emitProviderEvent({ @@ -8644,11 +8675,68 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( onUserDialog, supportedDialogKinds: ["resume_return"], }); + let boundaryQuery: ClaudeAgentSdkQuerySession | undefined; + const agentMessageBoundary: HookCallback = (hookInput, _toolUseId, options) => + runPromise( + Effect.gen(function* () { + const context = yield* Ref.get(activeTurn); + const live = yield* Ref.get(queryContext); + if ( + options.signal.aborted || + hookInput.agent_id !== undefined || + context === null || + live?.query !== boundaryQuery || + live?.stopping || + context.nativeThreadId !== hookInput.session_id || + (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) || + (hookInput.hook_event_name !== "PostToolBatch" && + hookInput.hook_event_name !== "Stop") + ) { + return {}; + } + // Resolve before claiming so an aborted hook consumes no partial batch. + const pendingSteers = yield* Effect.forEach( + Array.from(context.pendingAgentSteers), + ([id, pending]) => + (pending.resolveMessageText ?? Effect.succeed(pending.text)).pipe( + Effect.map((text) => ({ id, pending, text })), + ), + ); + const liveTurn = + (yield* Ref.get(activeTurn)) === context && + (yield* Ref.get(queryContext)) === live && + !(yield* Ref.get(interruptedTurns)).has(context.providerTurnId); + if (options.signal.aborted) return {}; + const messages: string[] = []; + for (const { id, pending, text } of pendingSteers) { + // The deadline or cancellation can win while storage is read. + if (context.pendingAgentSteers.get(id) !== pending) continue; + pending.outcome = text !== undefined && liveTurn ? "delivered" : "cancelled"; + context.pendingAgentSteers.delete(id); + Deferred.doneUnsafe(pending.completion, Effect.succeed(pending.outcome)); + if (pending.outcome === "delivered" && text !== undefined) messages.push(text); + } + return messages.length === 0 + ? {} + : { + hookSpecificOutput: { + hookEventName: hookInput.hook_event_name, + additionalContext: `New message from another agent:\n${messages.join("\n\n")}`, + }, + }; + }), + ); const querySession = yield* queryRunner .open({ threadId: turnInput.threadId, providerSessionId: input.providerSessionId, - options: queryOptions, + options: { + ...queryOptions, + hooks: { + PostToolBatch: [{ hooks: [agentMessageBoundary] }], + Stop: [{ hooks: [agentMessageBoundary] }], + }, + }, }) .pipe( Effect.onError(() => @@ -8662,6 +8750,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : Effect.void, ), ); + boundaryQuery = querySession; // Marked only after a successful open: a failed create must not // leave the runtime believing the native session exists, or the // retry would resume a session that was never created. @@ -8834,6 +8923,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( latestAssistantRateLimited: false, compactionFailure: null, steeredSegmentUsage: [], + pendingAgentSteers: new Map(), subagentsByTaskId: new Map(), subagentsByToolUseId: new Map(), subagentNodesByTaskId: new Map(), @@ -9025,6 +9115,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( next.add(turnInput.providerTurnId); return next; }); + settleClaudeAgentSteers(currentTurn, "cancelled"); yield* existing.query.interrupt; yield* existing.query.close.pipe(Effect.ignore); const closed = yield* Deferred.await(existing.closed).pipe( @@ -9150,6 +9241,44 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( detail: `Claude provider turn ${turnInput.providerTurnId} is not the active turn.`, }); } + if (turnInput.message.createdBy === "agent") { + // `priority: next` also folds after tools, but can start an unowned + // native turn if the answer ends first. Hooks keep delivery owned + // by this turn; an absent/slow boundary falls back to T3's queue. + if (turnInput.message.attachments.length > 0) { + return yield* new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + deferToQueue: true, + }); + } + const pending: PendingClaudeAgentSteer = { + text: turnInput.message.text, + completion: yield* Deferred.make(), + ...(turnInput.resolveMessageText === undefined + ? {} + : { resolveMessageText: turnInput.resolveMessageText }), + }; + currentTurn.pendingAgentSteers.set(turnInput.message.messageId, pending); + yield* Deferred.await(pending.completion).pipe( + Effect.timeoutOption("5 seconds"), + Effect.ensuring( + Effect.sync(() => { + currentTurn.pendingAgentSteers.delete(turnInput.message.messageId); + }), + ), + ); + if (pending.outcome === "delivered") return; + // A restart can interrupt this wait too. Let the caller revoke + // stopped/disposed input or retain still-owned input in its queue. + return yield* new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + deferToQueue: true, + }); + } const userMessage = yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( turnInput.message.text, @@ -9183,14 +9312,15 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, (effect, turnInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterSteerRunError({ - driver: CLAUDE_PROVIDER, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProviderAdapterSteerRunError" + ? cause + : new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + cause, + }), ), ), ); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index cb41dc7365123..91effb07d25d7 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -56,6 +56,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ providerThreadId: ProviderThreadId, providerTurnId: ProviderTurnId, messageId: MessageId, + waitForBoundary: Schema.optional(Schema.Literal(true)), }), Schema.Struct({ type: Schema.Literal("provider-turn.restart"), @@ -210,6 +211,8 @@ export interface EffectOutboxV2Shape { readonly cancelUnsettled: (input: { readonly threadId: ThreadId; readonly effectTypes: ReadonlyArray; + readonly messageIds?: ReadonlyArray; + readonly exceptCommandId?: CommandId; readonly reason: string; }) => Effect.Effect, EffectOutboxError>; readonly signalCancellations: (effectIds: ReadonlyArray) => Effect.Effect; @@ -315,6 +318,7 @@ export const layer: Layer.Layer = La ).pipe(Effect.asVoid); // An earlier effect waiting out a retry backoff still blocks later ones, so // a turn cannot start while a failed rollback is about to restore files. + // Boundary steers yield only to human steering and immediate turn controls. const claimableCandidatePredicate = ( availableBefore?: string, excludeRestartContinuations = false, @@ -330,6 +334,20 @@ export const layer: Layer.Layer = La SELECT 1 FROM orchestration_v2_effect_outbox AS active WHERE active.thread_id = candidate.thread_id + AND NOT ( + active.rowid < candidate.rowid + AND active.effect_type = 'provider-turn.steer' + AND COALESCE(json_extract(active.payload_json, '$.waitForBoundary'), 0) = 1 + AND ( + candidate.effect_type IN ( + 'provider-turn.interrupt', 'provider-turn.restart', 'runtime-request.respond' + ) + OR ( + candidate.effect_type = 'provider-turn.steer' + AND COALESCE(json_extract(candidate.payload_json, '$.waitForBoundary'), 0) != 1 + ) + ) + ) AND ( active.status = 'running' OR ( @@ -438,9 +456,9 @@ export const layer: Layer.Layer = La : new EffectOutboxError({ operation: "list", cause }), ), ), - cancelUnsettled: ({ threadId, effectTypes, reason }) => + cancelUnsettled: ({ threadId, effectTypes, messageIds, exceptCommandId, reason }) => Effect.gen(function* () { - if (effectTypes.length === 0) return []; + if (effectTypes.length === 0 || messageIds?.length === 0) return []; const now = DateTime.formatIso(yield* DateTime.now); const rows = yield* sql<{ readonly effect_id: string }>` UPDATE orchestration_v2_effect_outbox @@ -454,6 +472,8 @@ export const layer: Layer.Layer = La WHERE thread_id = ${threadId} AND status IN ('pending', 'running') AND effect_type IN ${sql.in(effectTypes)} + AND ${messageIds === undefined ? sql`1 = 1` : sql`json_extract(payload_json, '$.messageId') IN ${sql.in(messageIds)}`} + AND ${exceptCommandId === undefined ? sql`1 = 1` : sql`command_id != ${exceptCommandId}`} RETURNING effect_id `; return rows.map(({ effect_id }) => effect_id); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index a44b60c2498d1..3ff9c87cc392d 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -62,7 +62,7 @@ export function isNonRetryableProviderTurnControlFailure( export interface OrchestrationEffectExecutorV2Shape { readonly execute: ( effect: EffectOutbox.OrchestrationEffectV2, - options?: { readonly willRetry: boolean }, + options?: { readonly willRetry: boolean; readonly isCurrent?: Effect.Effect }, ) => Effect.Effect; } @@ -222,6 +222,7 @@ export const layerExecutor: Layer.Layer< case "provider-turn.steer": return providerTurnControl .steer({ + ...(options?.isCurrent === undefined ? {} : { isCurrent: options.isCurrent }), threadId: effect.threadId, providerSessionId: effect.request.providerSessionId, providerThreadId: effect.request.providerThreadId, @@ -250,9 +251,10 @@ export const layerExecutor: Layer.Layer< ), Effect.catch((error) => Effect.gen(function* () { + if ("deliveryCancelled" in error && error.deliveryCancelled) return; + const deferToQueue = "deferToQueue" in error && error.deferToQueue === true; if ( - !("turnCompleted" in error) || - !error.turnCompleted || + (!deferToQueue && (!("turnCompleted" in error) || !error.turnCompleted)) || effect.request.type !== "provider-turn.steer" ) { return yield* error; @@ -275,6 +277,7 @@ export const layerExecutor: Layer.Layer< threadId: effect.threadId, messageId: message.id, text: message.text, + ...(deferToQueue ? { steeringFallbackOfRunId: run.id } : {}), ...(message.context ? { context: message.context } : {}), attachments: message.attachments, ...(message.delegatedCompletion === undefined @@ -282,7 +285,7 @@ export const layerExecutor: Layer.Layer< : { modelSelection: run.modelSelection }), dispatchMode: { type: - message.delegatedCompletion === undefined + message.delegatedCompletion === undefined && !deferToQueue ? "start_immediately" : "queue_after_active", }, @@ -724,7 +727,10 @@ export const layerWithOptions = ( if (cancelledBeforeExecution) return true; const execution = executor - .execute(effect, { willRetry: effect.attemptCount < maxAttempts }) + .execute(effect, { + willRetry: effect.attemptCount < maxAttempts, + isCurrent: wasCancelled(effect.id).pipe(Effect.map((cancelled) => !cancelled)), + }) .pipe(Effect.as("executed" as const)); const exit = yield* Effect.exit(Effect.raceFirst(execution, cancellation)).pipe( Effect.ensuring(outbox.clearCancellation(effect.id)), diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 13916b40f9dee..a0dafae375c04 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -10,6 +10,7 @@ import { RuntimeRequestId, NodeId, type ProjectId, + type MessageId, ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -149,6 +150,7 @@ export interface EventSinkV2Shape { readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; }; }) => Effect.Effect< { @@ -627,6 +629,7 @@ const layerBase: Layer.Layer< : yield* effectOutbox.cancelUnsettled({ threadId: input.threadId, ...input.cancelUnsettledEffects, + exceptCommandId: input.commandId, }); return { receipt, storedEvents, committed: true as const, cancelledEffectIds }; }), diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 3a50b441a84ca..48e2ad05845ee 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -20,8 +20,10 @@ import { ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, RunAttemptId, RunId, + RuntimeRequestId, ThreadId, TurnItemId, } from "@t3tools/contracts"; @@ -2313,6 +2315,233 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); + const steerRequest = { + type: "provider-turn.steer", + providerSessionId: ProviderSessionId.make("session:boundary-order"), + providerThreadId: ProviderThreadId.make("provider-thread:boundary-order"), + providerTurnId: ProviderTurnId.make("provider-turn:boundary-order"), + messageId: MessageId.make("message:boundary-order"), + } as const; + const boundaryOrderCandidates = [ + { request: { ...steerRequest }, bypass: true }, + { request: { ...steerRequest, type: "provider-turn.interrupt" }, bypass: true }, + { + request: { + ...steerRequest, + type: "provider-turn.restart", + interruptedAttemptId: RunAttemptId.make("attempt:boundary-order"), + runId: RunId.make("run:boundary-order"), + }, + bypass: true, + }, + { + request: { + type: "runtime-request.respond", + providerSessionId: steerRequest.providerSessionId, + requestId: RuntimeRequestId.make("request:boundary-order"), + }, + bypass: true, + }, + { request: { ...steerRequest, waitForBoundary: true }, bypass: false }, + { + request: { type: "provider-turn.start", runId: RunId.make("run:boundary-order") }, + bypass: false, + }, + { + request: { + type: "checkpoint.capture", + runId: RunId.make("run:boundary-order"), + scopeId: CheckpointScopeId.make("scope:boundary-order"), + }, + bypass: false, + }, + { + request: { + type: "provider-thread.rollback", + providerThreadId: steerRequest.providerThreadId, + checkpointId: CheckpointId.make("checkpoint:boundary-order"), + scopeId: CheckpointScopeId.make("scope:boundary-order"), + }, + bypass: false, + }, + { request: { type: "terminal.cleanup" }, bypass: false }, + ] satisfies ReadonlyArray<{ + request: EffectOutbox.OrchestrationEffectRequestV2; + bypass: boolean; + }>; + + it.effect.each( + (["running", "pending"] as const).flatMap((status) => + boundaryOrderCandidates.map(({ request, bypass }) => ({ + status, + request, + bypass, + name: `${bypass ? "allows" : "blocks"} ${request.type}${"waitForBoundary" in request ? " at a boundary" : ""} behind a ${status} boundary steer`, + })), + ), + )("boundary steer ordering: $name", ({ status, request, bypass }) => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const commandId = CommandId.make("command:boundary-order"); + const threadId = ThreadId.make("thread:boundary-order"); + const workerId = "boundary-worker"; + const future = DateTime.add(now, { seconds: 5 }); + yield* outbox.enqueue([ + { + id: "boundary", + commandId, + threadId, + request: { ...steerRequest, waitForBoundary: true }, + }, + ]); + const boundary = Option.getOrThrow( + yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }), + ); + assert.deepEqual(boundary.request, { ...steerRequest, waitForBoundary: true }); + if (status === "pending") { + yield* outbox.retry({ + effectId: boundary.id, + workerId, + error: "Waiting for a boundary.", + delayMs: 5_000, + }); + } + yield* outbox.enqueue([{ id: "candidate", commandId, threadId, request }]); + const nextAt = yield* outbox.nextClaimableAt; + if (bypass) { + assert.equal(DateTime.formatIso(Option.getOrThrow(nextAt)), DateTime.formatIso(now)); + } else if (status === "pending") { + assert.equal(DateTime.formatIso(Option.getOrThrow(nextAt)), DateTime.formatIso(future)); + } else { + assert.isTrue(Option.isNone(nextAt)); + } + const claimed = yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); + assert.equal(Option.getOrUndefined(claimed)?.id, bypass ? "candidate" : undefined); + if (bypass) yield* outbox.succeed({ effectId: "candidate", workerId }); + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["provider-turn.steer"], + messageIds: [steerRequest.messageId], + reason: "Boundary settled.", + }); + if (!bypass && request.type !== "provider-turn.steer") { + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "candidate", + ); + } + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + + it.effect("keeps immediate controls behind ordinary steers and intervening effects", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const commandId = CommandId.make("command:ordinary-steer-order"); + const threadId = ThreadId.make("thread:ordinary-steer-order"); + const workerId = "ordinary-steer-worker"; + yield* outbox.enqueue([{ id: "ordinary", commandId, threadId, request: steerRequest }]); + yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); + yield* outbox.enqueue([ + { + id: "boundary", + commandId, + threadId, + request: { ...steerRequest, waitForBoundary: true }, + }, + { id: "cleanup", commandId, threadId, request: { type: "terminal.cleanup" } }, + { + id: "interrupt", + commandId, + threadId, + request: { ...steerRequest, type: "provider-turn.interrupt" }, + }, + ]); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "ordinary", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "boundary", + ); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "boundary", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "cleanup", + ); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "cleanup", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "interrupt", + ); + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + + it.effect( + "cancels selected messages while preserving replacement commands and other effects", + () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = ThreadId.make("thread:message-cancellation"); + const commandId = CommandId.make("command:message-cancellation"); + const replacementCommandId = CommandId.make("command:message-cancellation:replacement"); + const messageId = MessageId.make("message:cancelled"); + yield* outbox.enqueue([ + { id: "running", commandId, threadId, request: { ...steerRequest, messageId } }, + ]); + assert.equal( + Option.getOrThrow( + yield* outbox.claimNext({ workerId: "cancel-worker", leaseDurationMs: 30_000 }), + ).id, + "running", + ); + yield* outbox.enqueue([ + { id: "pending", commandId, threadId, request: { ...steerRequest, messageId } }, + { id: "other-message", commandId, threadId, request: steerRequest }, + { + id: "replacement", + commandId: replacementCommandId, + threadId, + request: { ...steerRequest, messageId }, + }, + { + id: "other-thread", + commandId, + threadId: ThreadId.make("thread:message-cancellation:other"), + request: { ...steerRequest, messageId }, + }, + { id: "cleanup", commandId, threadId, request: { type: "terminal.cleanup" } }, + ]); + const cancellation = { + threadId, + effectTypes: ["provider-turn.steer", "terminal.cleanup"], + exceptCommandId: replacementCommandId, + reason: "Message removed.", + } as const; + assert.deepEqual(yield* outbox.cancelUnsettled({ ...cancellation, messageIds: [] }), []); + assert.deepEqual( + (yield* outbox.cancelUnsettled({ ...cancellation, messageIds: [messageId] })).toSorted(), + ["pending", "running"], + ); + for (const id of ["other-message", "replacement", "other-thread", "cleanup"]) { + assert.equal(Option.getOrThrow(yield* outbox.get(id)).status, "pending"); + } + assert.deepEqual((yield* outbox.cancelUnsettled(cancellation)).toSorted(), [ + "cleanup", + "other-message", + ]); + assert.deepEqual( + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["provider-turn.steer"], + reason: "Legacy cancellation.", + }), + ["replacement"], + ); + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + it.effect("prunes settled effects past retention in batches and keeps the rest", () => Effect.gen(function* () { const outbox = yield* EffectOutbox.EffectOutboxV2; diff --git a/apps/server/src/orchestration-v2/NotificationMailbox.ts b/apps/server/src/orchestration-v2/NotificationMailbox.ts index 2afee9749e22e..28de4bc9f1546 100644 --- a/apps/server/src/orchestration-v2/NotificationMailbox.ts +++ b/apps/server/src/orchestration-v2/NotificationMailbox.ts @@ -3,8 +3,25 @@ import { type MessageId, type OrchestrationV2ConversationMessage, type OrchestrationV2ThreadProjection, + type OrchestrationV2Subagent, } from "@t3tools/contracts"; +export function delegatedCompletionWakeDetail(taskIds: ReadonlyArray): string { + const taskList = taskIds.join(", "); + return taskIds.length === 1 + ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.` + : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`; +} + +export function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean { + return ( + status === "completed" || + status === "failed" || + status === "cancelled" || + status === "interrupted" + ); +} + export function isBufferedProviderNotification( message: OrchestrationV2ConversationMessage, ): boolean { diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 96b6bab3c033d..5a466df978546 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -101,6 +101,8 @@ import { import { isBufferedProviderNotification, isUndeliveredMailboxSteer, + delegatedCompletionWakeDetail, + isTerminalDelegatedTaskStatus, } from "./NotificationMailbox.ts"; import { EventSinkV2 } from "./EventSink.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; @@ -575,22 +577,6 @@ function legacyPullRequestLink( ); } -function delegatedCompletionWakeDetail(taskIds: ReadonlyArray): string { - const taskList = taskIds.join(", "); - return taskIds.length === 1 - ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.` - : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`; -} - -function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean { - return ( - status === "completed" || - status === "failed" || - status === "cancelled" || - status === "interrupted" - ); -} - function delegatedTaskTerminalStatus( status: OrchestrationV2Run["status"], ): OrchestrationV2Subagent["status"] | null { @@ -3899,6 +3885,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly scheduledTaskId?: OrchestrationV2ConversationMessage["scheduledTaskId"]; readonly senderThreadId?: OrchestrationV2ConversationMessage["senderThreadId"]; readonly delegatedCompletion?: OrchestrationV2ConversationMessage["delegatedCompletion"]; + readonly notification?: OrchestrationV2ConversationMessage["notification"]; readonly forceRestart: boolean; }) => Effect.gen(function* () { @@ -4057,6 +4044,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...(input.delegatedCompletion === undefined ? {} : { delegatedCompletion: input.delegatedCompletion }), + ...(input.notification === undefined ? {} : { notification: input.notification }), ...(input.scheduledTaskId === undefined ? {} : { scheduledTaskId: input.scheduledTaskId }), @@ -4176,6 +4164,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio threadId: input.command.threadId, request: { type: "provider-turn.steer", + ...(input.createdBy === "agent" && + session.providerSession.capabilities.turns.supportsNonInterruptingAgentSteering === + true + ? { waitForBoundary: true as const } + : {}), providerSessionId, providerThreadId: providerThread.id, providerTurnId: providerTurn.id, @@ -4607,6 +4600,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) => Effect.gen(function* () { let projection = yield* getProjectionWithPendingEvents(command.threadId, events); + // Stop can commit after the boundary wait expires but before its fallback + // is dispatched. Do not turn that stale steer into fresh work. + if ( + command.steeringFallbackOfRunId !== undefined && + (yield* stopReachedRun(command, command.threadId, command.steeringFallbackOfRunId)) + ) { + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + return; + } if (command.manualContinuationOfRunId !== undefined) { const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId); const limited = latestRootProviderFailure(source ?? null, projection.turnItems); @@ -4832,6 +4842,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio turn.status === "completed")) ) { dispatchMode = { type: "start_immediately" }; + } else if (command.createdBy === "agent") { + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === target?.providerThreadId, + ); + if (providerThread?.providerSessionId != null) { + const session = yield* providerSessions + .get(providerThread.providerSessionId) + .pipe(mapDispatchError(command)); + if ( + Option.isSome(session) && + session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === + true && + session.value.providerSession.capabilities.turns + .supportsNonInterruptingAgentSteering !== true + ) { + // Claude steering cancels pending tools, which appear as user refusals. + // Agent updates can wait for the durable queue's next-turn boundary. + dispatchMode = { type: "queue_after_active" }; + } + } } } // Stop leaves the run reading as running until its provider stops. A plain @@ -4881,6 +4911,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (candidate) => candidate.id === requestedCompletion.parentRunId, ); const delivery = parentRun?.delegatedCompletion?.delivery; + const existingDelivery = projection.messages.find( + (message) => message.id === command.messageId, + ); + const deferredSteer = + existingDelivery !== undefined && + command.steeringFallbackOfRunId === existingDelivery.runId && + existingDelivery.delegatedCompletion?.generation === requestedCompletion.generation && + projection.runs.find((run) => run.id === existingDelivery.runId)?.userMessageId !== + command.messageId; if ( parentRun?.delegatedCompletion?.disposition !== "open" || delivery === null || @@ -4888,6 +4927,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio delivery.generation !== requestedCompletion.generation || delivery.messageId !== command.messageId || (projection.messages.some((candidate) => candidate.id === command.messageId) && + !deferredSteer && !isUndeliveredMailboxSteer(projection, command.messageId)) ) { return yield* new OrchestratorDispatchError({ @@ -4906,6 +4946,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // session's capabilities. Never interrupt/restart a turn for a notification. if ( delegatedCompletion !== undefined && + // A deferred steer is already persisted; do not attempt the hook again. + !projection.messages.some((message) => message.id === command.messageId) && delegatedCompletion.taskIds.every( (id) => projection.subagents.find((task) => task.id === id)?.completionWake === "always", ) @@ -4932,7 +4974,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ( Option.isSome(session) && session.value.providerSession.capabilities.turns.supportsActiveSteering && - session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools !== true + (session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools !== + true || + session.value.providerSession.capabilities.turns + .supportsNonInterruptingAgentSteering === true) ) { dispatchMode = { type: "steer_active", targetRunId: active.id }; } @@ -5004,6 +5049,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : (projection.runs.find((run) => run.id === dispatchMode.targetRunId) ?.modelSelection ?? modelSelection), delegatedCompletion, + ...(command.notification === undefined ? {} : { notification: command.notification }), targetRunId: dispatchMode.targetRunId, messageId: command.messageId, text: dispatchText, @@ -7838,6 +7884,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio targetRunId: command.targetRunId, messageId: queuedMessage.id, text: queuedMessage.text, + ...(queuedMessage.notification === undefined + ? {} + : { notification: queuedMessage.notification }), + ...(queuedMessage.delegatedCompletion === undefined + ? {} + : { delegatedCompletion: queuedMessage.delegatedCompletion }), attachments: queuedMessage.attachments, ...(queuedMessage.context ? { context: queuedMessage.context } : {}), createdBy: queuedMessage.createdBy, @@ -10976,6 +11028,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; }; }, OrchestratorV2Error @@ -10992,6 +11045,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio | { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; } | undefined; switch (command.type) { @@ -11249,9 +11303,41 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio yield* dispatchDelegatedTaskWakePolicy(command, events); break; case "delegated_task.completion-delivery.acknowledge": - case "delegated_task.completion-delivery.dispose": + case "delegated_task.completion-delivery.dispose": { + const before = yield* getProjectionWithPendingEvents(command.parentThreadId, events); yield* dispatchDelegatedTaskCompletionDeliveryResolution(command, events); + const task = before.subagents.find((candidate) => candidate.id === command.taskId); + const owner = before.runs.find((run) => run.id === task?.runId); + const after = yield* getProjectionWithPendingEvents(command.parentThreadId, events); + const remaining = after.runs.find((run) => run.id === owner?.id)?.delegatedCompletion + ?.delivery; + const messageIds = before.messages + .filter((message) => { + const source = message.notification?.source; + return ( + message.id.startsWith("message:delegated-task-question:") && + source?.kind === "delegated_task" && + source.taskIds.includes(command.taskId) + ); + }) + .map((message) => message.id); + const previousDelivery = owner?.delegatedCompletion?.delivery; + if ( + previousDelivery != null && + (remaining == null || + remaining.taskIds.length === 0 || + remaining.messageId !== previousDelivery.messageId) + ) { + messageIds.push(previousDelivery.messageId); + } + if (messageIds.length > 0) + cancelUnsettledEffects = { + effectTypes: ["provider-turn.steer"], + messageIds, + reason: "The delegated notification is no longer owned by this task.", + }; break; + } case "thread.created.record": yield* dispatchCreatedThreadRecord(command, events); break; @@ -11261,6 +11347,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio default: return yield* dispatchUnsupported(command); } + if (command.type === "run.interrupt" || command.type === "thread.stop") { + cancelUnsettledEffects = { + effectTypes: [...(cancelUnsettledEffects?.effectTypes ?? []), "provider-turn.steer"], + reason: cancelUnsettledEffects?.reason ?? "The user stopped this thread.", + }; + // A repeated Stop can still revoke a pending boundary delivery. Commit it + // through the event sink so cancellation and its receipt stay atomic. + if ((yield* Ref.get(events)).length === 0) { + const projection = yield* getProjectionWithPendingEvents(command.threadId, events); + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + } + } return { events: yield* Ref.get(events), effects: yield* Ref.get(effects), diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index de03594916a87..e7247430880fc 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -16,9 +16,13 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; +import { + delegatedCompletionWakeDetail, + isTerminalDelegatedTaskStatus, +} from "./NotificationMailbox.ts"; const yieldToRuntime = Effect.yieldNow.pipe( Effect.andThen( @@ -38,6 +42,8 @@ export class ProviderTurnControlError extends Schema.TaggedError; }) => Effect.Effect; readonly interruptAndAwaitTerminal: (input: { readonly threadId: ThreadId; @@ -323,21 +330,26 @@ export const layer: Layer.Layer< Effect.gen(function* () { const context = yield* projections.getProviderControlContext(input.threadId, input); const ownership = context.message?.delegatedCompletion; - if (ownership !== undefined) { - const projection = yield* projections.getThreadRecords(input.threadId, ["runs"], { - runIds: [ownership.parentRunId], - }); - const cohort = projection.runs.find( - (run) => run.id === ownership.parentRunId, - )?.delegatedCompletion; - if ( - cohort?.disposition !== "open" || - cohort.delivery?.messageId !== input.messageId || - cohort.delivery.generation !== ownership.generation || - cohort.delivery.taskIds.length === 0 - ) - return; - } + const completionTaskIds = + ownership === undefined + ? undefined + : Effect.gen(function* () { + const projection = yield* projections.getThreadRecords(input.threadId, ["runs"], { + runIds: [ownership.parentRunId], + }); + const cohort = projection.runs.find( + (run) => run.id === ownership.parentRunId, + )?.delegatedCompletion; + if ( + cohort?.disposition !== "open" || + cohort.delivery?.messageId !== input.messageId || + cohort.delivery.generation !== ownership.generation || + cohort.delivery.taskIds.length === 0 + ) + return null; + return cohort.delivery.taskIds; + }); + if (completionTaskIds !== undefined && (yield* completionTaskIds) === null) return; const loaded = yield* load({ ...input, operation: "steer" }); if (Option.isNone(loaded.session)) return; const { message, run } = loaded.context; @@ -349,18 +361,57 @@ export const layer: Layer.Layer< cause: "The persisted steering message or target run is missing.", }); } + const text = projectComposerContextForProvider({ + text: message.text, + records: message.context?.records ?? [], + }); + const resolveMessageText = Effect.gen(function* () { + if (input.isCurrent !== undefined && !(yield* input.isCurrent)) return undefined; + const source = message.notification?.source; + if ( + message.id.startsWith("message:delegated-task-question:") && + source?.kind === "delegated_task" + ) { + const parent = yield* projections.getThreadRecords(input.threadId, ["subagents"]); + const task = parent.subagents.find( + (candidate) => + candidate.origin === "app_owned" && + candidate.childThreadId === source.childThreadId && + source.taskIds.includes(candidate.id), + ); + if ( + task === undefined || + isTerminalDelegatedTaskStatus(task.status) || + task.completionDelivery?.state === "disposed" || + parent.thread.archivedAt !== null || + parent.thread.deletedAt !== null + ) + return undefined; + } + if (completionTaskIds === undefined) return text; + const taskIds = yield* completionTaskIds; + return taskIds === null ? undefined : delegatedCompletionWakeDetail(taskIds); + }); yield* loaded.session.value .steerTurn({ threadId: input.threadId, runId: run.id, providerThread: loaded.providerThread, providerTurnId: loaded.providerTurn.id, + resolveMessageText: resolveMessageText.pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: loaded.providerThread.driver, + providerThreadId: loaded.providerThread.id, + providerTurnId: loaded.providerTurn.id, + cause, + }), + ), + ), message: { messageId: message.id, - text: projectComposerContextForProvider({ - text: message.text, - records: message.context?.records ?? [], - }), + text, attachments: message.attachments, createdBy: message.createdBy, creationSource: message.creationSource, @@ -375,6 +426,14 @@ export const layer: Layer.Layer< .pipe( Effect.catch((cause) => Effect.gen(function* () { + if ((yield* resolveMessageText) === undefined) { + return yield* new ProviderTurnControlError({ + threadId: input.threadId, + operation: "steer", + providerTurnId: input.providerTurnId, + deliveryCancelled: true, + }); + } const current = yield* projections.getProviderControlContext( input.threadId, input, @@ -384,11 +443,21 @@ export const layer: Layer.Layer< operation: "steer", providerTurnId: input.providerTurnId, turnCompleted: current.providerTurn?.status === "completed", + deferToQueue: + cause._tag === "ProviderAdapterSteerRunError" && cause.deferToQueue === true, cause, }); }), ), ); + if (input.isCurrent !== undefined && !(yield* input.isCurrent)) { + return yield* new ProviderTurnControlError({ + threadId: input.threadId, + operation: "steer", + providerTurnId: input.providerTurnId, + deliveryCancelled: true, + }); + } }).pipe( Effect.mapError((cause) => isProviderTurnControlError(cause) diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 884683979fb65..61c34586cbe8f 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -18,14 +18,20 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as CommandReceiptStore from "./CommandReceiptStore.ts"; +import * as EffectOutbox from "./EffectOutbox.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import * as ThreadManagement from "./ThreadManagementService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -418,18 +424,33 @@ const composerSelection = { const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* ( name: string, - canRestart = false, + { + canRestart = false, + activeSteeringInterruptsTools = false, + supportsNonInterruptingAgentSteering = false, + deferSteering = false, + beforeBoundary = Effect.void, + }: { + canRestart?: boolean; + activeSteeringInterruptsTools?: boolean; + supportsNonInterruptingAgentSteering?: boolean; + deferSteering?: boolean; + beforeBoundary?: Effect.Effect; + } = {}, ) { const cwd = yield* checkpointWorkspace(name); const events = yield* Queue.unbounded(); const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; const steered: string[] = []; + const interrupted: ProviderTurnId[] = []; const capabilities = { ...CodexProviderCapabilitiesV2, turns: { ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: true, supportsSteeringByInterruptRestart: canRestart, + activeSteeringInterruptsTools, + supportsNonInterruptingAgentSteering, }, }; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { @@ -500,10 +521,23 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }); }), steerTurn: (turn) => + Effect.gen(function* () { + yield* beforeBoundary; + if (deferSteering) { + return yield* new ProviderAdapter.ProviderAdapterSteerRunError({ + driver, + providerThreadId: turn.providerThread.id, + providerTurnId: turn.providerTurnId, + deferToQueue: true, + }); + } + const text = yield* turn.resolveMessageText ?? Effect.succeed(turn.message.text); + if (text !== undefined) steered.push(text); + }), + interruptTurn: (turn) => Effect.sync(() => { - steered.push(turn.message.text); + interrupted.push(turn.providerTurnId); }), - interruptTurn: () => Effect.void, respondToRuntimeRequest: () => Effect.void, readThreadSnapshot: () => Effect.die("unused"), rollbackThread: () => Effect.die("unused"), @@ -511,10 +545,17 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }; }), }; - const layer = ProviderReplayHarness.layerWithRegistry( + const databaseLayer = SqlitePersistence.layerMemory; + const layerRuntime = ProviderReplayHarness.layerWithRegistry( { name }, ProviderAdapterRegistry.layerSingle(adapter), - { runEffectWorker: false }, + { runEffectWorker: false, databaseLayer }, + ); + const layer = ThreadManagement.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(databaseLayer))), + Layer.provideMerge(layerRuntime), + Layer.provideMerge(EffectOutbox.layer.pipe(Layer.provide(databaseLayer))), ); const startFirstTurn = Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -558,9 +599,111 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* yield* Fiber.join(running); return threadId; }); - return { events, started, steered, layer, startFirstTurn }; + return { events, started, steered, interrupted, layer, startFirstTurn }; }); +it.effect.each( + ( + [ + { actor: "agent", mode: "auto", interruptsTools: true }, + { actor: "agent", mode: "steer", interruptsTools: true }, + { actor: "user", mode: "steer", interruptsTools: true }, + { actor: "agent", mode: "auto", interruptsTools: false }, + { actor: "agent", mode: "steer", interruptsTools: false }, + ] as const + ) + .map((entry) => ({ ...entry, safeBoundary: false, defer: false })) + .concat([ + { actor: "agent", mode: "auto", interruptsTools: true, safeBoundary: true, defer: false }, + { actor: "agent", mode: "steer", interruptsTools: true, safeBoundary: true, defer: true }, + ]), +)( + "delivers $actor $mode messages safely (interruptsTools=$interruptsTools, boundary=$safeBoundary, defer=$defer)", + ({ actor, mode, interruptsTools, safeBoundary, defer }) => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `agent-steering-${actor}-${mode}-${interruptsTools}`, + { + activeSteeringInterruptsTools: interruptsTools, + supportsNonInterruptingAgentSteering: safeBoundary, + deferSteering: defer, + }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threads = yield* ThreadManagement.ThreadManagementService; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const messageId = MessageId.make("message:update"); + const result = yield* threads.sendToThread({ + projectId: (yield* orchestrator.getThreadProjection(threadId)).thread.projectId, + commandId: CommandId.make("update"), + threadId, + messageId, + text: "progress update", + attachments: [], + mode, + createdBy: actor, + creationSource: actor === "agent" ? "mcp" : "web", + }); + yield* worker.drain(); + const queuedAtDispatch = actor === "agent" && interruptsTools && !safeBoundary; + const queued = queuedAtDispatch || defer; + assert.equal(result.delivery, queuedAtDispatch ? "queued" : "steered"); + assert.deepEqual(steered, queued ? [] : ["progress update"]); + assert.equal(started.length, 1); + const pending = yield* orchestrator.getThreadProjection(threadId); + assert.equal(pending.runs.find((run) => run.id === first.runId)?.status, "running"); + if (!queued) return; + assert.equal( + pending.runs.find((run) => run.userMessageId === messageId)?.status, + "queued", + ); + + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "waiting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const turn = pending.providerTurns[0]!; + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, + }); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(completed); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + assert.equal(started.length, 2); + assert.equal(started[1]?.message.messageId, messageId); + assert.equal(started[1]?.message.text, "progress update"); + assert.deepEqual(steered, []); + const final = yield* orchestrator.getThreadProjection(threadId); + assert.equal(final.runs.find((run) => run.id === first.runId)?.status, "completed"); + }).pipe(Effect.provide(layer)); + }), + ), +); + it.effect.each([{ canRestart: false }, { canRestart: true }])( "steers a changed turn-scoped selection into the running turn (canRestart=$canRestart)", ({ canRestart }) => @@ -568,7 +711,7 @@ it.effect.each([{ canRestart: false }, { canRestart: true }])( Effect.gen(function* () { const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( `steering-selection-change-${canRestart}`, - canRestart, + { canRestart }, ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -636,6 +779,302 @@ it.effect.each([{ canRestart: false }, { canRestart: true }])( ), ); +it.effect("Stop suppresses a boundary timeout whose fallback has not dispatched yet", () => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const { started, interrupted, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "steering-boundary-stop-race", + { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + deferSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* startFirstTurn; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("boundary-update"), + threadId, + messageId: MessageId.make("message:boundary-update"), + text: "The review found another issue.", + attachments: [], + dispatchMode: { type: "steer_active", targetRunId: started[0]!.runId }, + createdBy: "agent", + creationSource: "mcp", + }); + const draining = yield* worker.drain().pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop-boundary-parent"), + threadId, + runId: started[0]!.runId, + holdQueue: true, + }); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(draining); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(started.length, 1); + assert.equal(projection.runs.length, 1); + const effects = yield* outbox.listByCommandId(CommandId.make("boundary-update")); + assert.lengthOf(effects, 1); + assert.include(["succeeded", "cancelled"], effects[0]?.status); + if (effects[0]?.status === "succeeded") assert.isNull(effects[0].lastError); + assert.lengthOf(interrupted, 1); + const stops = yield* outbox.listByCommandId(CommandId.make("stop-boundary-parent")); + assert.equal( + stops.find((effect) => effect.request.type === "provider-turn.interrupt")?.status, + "succeeded", + ); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect("Stop cancels pending hook delivery and executes the interrupt before a boundary", () => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const h = yield* nextTurnSelectionHarness("steering-stop-before-hook", { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("pending-hook"), + threadId, + messageId: MessageId.make("message:pending-hook"), + text: "Do not deliver after Stop.", + attachments: [], + dispatchMode: { type: "steer_active", targetRunId: h.started[0]!.runId }, + createdBy: "agent", + creationSource: "mcp", + }); + const delivery = yield* worker.runOnce.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-pending-hook"), + threadId, + }); + yield* worker.drain(); + assert.lengthOf(h.interrupted, 1); + yield* Fiber.join(delivery); + const effects = yield* outbox.listByCommandId(CommandId.make("pending-hook")); + assert.equal(effects[0]?.status, "cancelled"); + yield* Deferred.succeed(release, undefined); + yield* worker.drain(); + assert.deepEqual(h.steered, []); + assert.equal((yield* orchestrator.getThreadProjection(threadId)).runs.length, 1); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + +const dispatchClaimedCompletion = Effect.fn("dispatchClaimedCompletion")(function* ( + threadId: ThreadId, + first: ProviderAdapter.ProviderAdapterV2TurnInput, + taskIds: ReadonlyArray, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const messageId = MessageId.make("message:pending-completion"); + const current = yield* orchestrator.getThreadProjection(threadId); + const now = yield* DateTime.now; + yield* sink.write({ + events: [ + { + id: EventId.make("completion:cohort"), + type: "run.updated", + threadId, + runId: first.runId, + occurredAt: now, + payload: { + ...current.runs.find((run) => run.id === first.runId)!, + delegatedCompletion: { + disposition: "open", + nextGeneration: 2, + delivery: { generation: 1, messageId, taskIds }, + }, + }, + }, + ...taskIds.map((taskId) => ({ + id: EventId.make(`completion:${taskId}`), + type: "subagent.updated" as const, + threadId, + runId: first.runId, + nodeId: taskId, + occurredAt: now, + payload: { + id: taskId, + threadId, + runId: first.runId, + parentNodeId: first.rootNodeId, + origin: "app_owned" as const, + createdBy: "agent" as const, + driver, + providerInstanceId: instanceId, + providerThreadId: null, + childThreadId: null, + nativeTaskRef: null, + prompt: "Do background work", + title: "Background test", + model: null, + completionWake: "always" as const, + completionDelivery: { state: "claimed" as const, observedByRunId: null }, + status: "completed" as const, + result: "done", + startedAt: now, + completedAt: now, + updatedAt: now, + }, + })), + ], + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("dispatch-completion"), + threadId, + messageId, + text: `Original completion for ${taskIds.join(", ")}`, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + delegatedCompletion: { parentRunId: first.runId, generation: 1, taskIds }, + }); + return messageId; +}); + +it.effect("a live completion boundary defer creates exactly one queued follow-up", () => + Effect.scoped( + Effect.gen(function* () { + const h = yield* nextTurnSelectionHarness("steering-live-completion-defer", { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + deferSteering: true, + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + const first = h.started[0]!; + const taskId = NodeId.make("task:deferred"); + const messageId = yield* dispatchClaimedCompletion(threadId, first, [taskId]); + yield* worker.drain(); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + const followUps = projection.runs.filter((run) => run.userMessageId === messageId); + assert.lengthOf(followUps, 1); + assert.equal(followUps[0]?.status, "queued"); + assert.equal(projection.runs.find((run) => run.id === first.runId)?.status, "running"); + assert.equal(projection.messages.filter((message) => message.id === messageId).length, 1); + assert.deepEqual( + projection.messages.find((message) => message.id === messageId)?.delegatedCompletion, + { + parentRunId: first.runId, + generation: 1, + taskIds: [taskId], + }, + ); + const effects = yield* outbox.listByCommandId(CommandId.make("dispatch-completion")); + assert.equal( + effects.find((effect) => effect.request.type === "provider-turn.steer")?.status, + "succeeded", + ); + assert.equal(h.started.length, 1); + assert.deepEqual(h.steered, []); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + +it.effect.each(["dispose", "acknowledge one"] as const)( + "completion %s updates a pending boundary before delivery", + (resolution) => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const h = yield* nextTurnSelectionHarness(`steering-completion-${resolution}`, { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + const first = h.started[0]!; + const taskId = NodeId.make("task:resolved"); + const remainingTaskId = NodeId.make("task:remaining"); + const taskIds = resolution === "dispose" ? [taskId] : [taskId, remainingTaskId]; + yield* dispatchClaimedCompletion(threadId, first, taskIds); + const delivery = yield* worker.runOnce.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch( + resolution === "dispose" + ? { + type: "delegated_task.completion-delivery.dispose", + commandId: CommandId.make("resolve-completion"), + parentThreadId: threadId, + taskId, + } + : { + type: "delegated_task.completion-delivery.acknowledge", + commandId: CommandId.make("resolve-completion"), + parentThreadId: threadId, + taskId, + observedByRunId: first.runId, + }, + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(delivery); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + projection.subagents.find((task) => task.id === taskId)?.completionDelivery?.state, + resolution === "dispose" ? "disposed" : "acknowledged", + ); + if (resolution === "dispose") assert.deepEqual(h.steered, []); + else { + assert.lengthOf(h.steered, 1); + assert.include(h.steered[0]!, String(remainingTaskId)); + assert.notInclude(h.steered[0]!, String(taskId)); + assert.include(h.steered[0]!, "task_status"); + } + assert.equal(projection.runs.length, 1); + const effects = yield* outbox.listByCommandId(CommandId.make("dispatch-completion")); + assert.include(["succeeded", "cancelled"], effects[0]?.status); + if (effects[0]?.status === "succeeded") assert.isNull(effects[0].lastError); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + it.effect("starts a steer that missed the turn on the saved next-turn selection", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 94a9eca331d13..cb708647ae064 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -218,6 +218,8 @@ export const OrchestrationV2TurnCapabilities = Schema.Struct({ supportsActiveSteering: Schema.Boolean, // Some native steering mechanisms cancel pending tools before consuming the message. activeSteeringInterruptsTools: Schema.optional(Schema.Boolean), + // Agent-authored messages can use a safe boundary instead of interrupting tools. + supportsNonInterruptingAgentSteering: Schema.optional(Schema.Boolean), supportsSteeringByInterruptRestart: Schema.Boolean, supportsQueuedMessages: Schema.Boolean, terminalStatusQuality: Schema.Literals(["strong", "weak", "none"]), @@ -2779,6 +2781,7 @@ export const OrchestrationV2Command = Schema.Union([ restartContinuationOfRunId: Schema.optional(RunId), usageLimitContinuationOfRunId: Schema.optional(RunId), manualContinuationOfRunId: Schema.optional(RunId), + steeringFallbackOfRunId: Schema.optional(RunId), usageLimitRecoveryRequestId: Schema.optional(CommandId), deliveryIntent: Schema.optional(Schema.Literals(["auto", "steer", "restart"])), delegatedCompletion: Schema.optional( diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index d58a6c5904a2f..7260d6106020f 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -312,6 +312,7 @@ export class ProviderAdapterSteerRunError extends Schema.TaggedError; } export interface ProviderAdapterV2InterruptInput { From 0ced5275ecb7e5dc64ad6dd5d3b3cec637584276 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 18 Oct 2026 07:13:15 -0400 Subject: [PATCH 04/23] fix(server): agent messages no longer cancel Claude tools Deliver Claude agent updates through root PostToolBatch and Stop hook context without interrupting running tools. Defer to the durable next-turn queue after five seconds without a boundary, or when the update contains attachments. Let human steering and immediate controls pass a waiting boundary delivery. Revoke pending updates on Stop and task disposal, and refresh completion pointers when only part of a task cohort remains unacknowledged. Keep native human steering and other providers' delivery behavior unchanged. Exercise the real send service, hook adapter, outbox ordering and ownership races with deterministic regression coverage. Upstream-PR: https://github.com/pingdotgg/t3code/pull/21158 Co-Authored-By: GPT-6.1 Sol at high effort, implementation and SDK research Co-Authored-By: GPT-6.1 Sol at xhigh effort, independent review --- ...OrchestratorMcpToolkit.integration.test.ts | 10 +- .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../Adapters/ClaudeAdapterV2.test.ts | 130 +++++ .../Adapters/ClaudeAdapterV2.ts | 146 +++++- ...laudeAutomaticDelivery.integration.test.ts | 52 +- .../src/orchestration-v2/EffectOutbox.ts | 24 +- .../src/orchestration-v2/EffectWorker.ts | 30 +- apps/server/src/orchestration-v2/EventSink.ts | 9 +- .../FoundationPersistence.test.ts | 300 ++++++++++++ .../orchestration-v2/NotificationMailbox.ts | 17 + .../src/orchestration-v2/Orchestrator.ts | 155 ++++-- .../ProviderTurnControlService.ts | 150 +++++- .../SteeringCompletion.integration.test.ts | 452 +++++++++++++++++- packages/contracts/src/orchestrationV2.ts | 3 + .../src/server/ProviderAdapter.ts | 16 + 15 files changed, 1379 insertions(+), 117 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index c7b5fc245094f..6093680306242 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -561,7 +561,15 @@ describe("orchestrator MCP toolkit", () => { makeDeterministicAdapter({ instanceId: claudeInstanceId, driver: ProviderDriverKind.make("claudeAgent"), - capabilities: ClaudeProviderCapabilitiesV2, + capabilities: { + ...ClaudeProviderCapabilitiesV2, + turns: { + ...ClaudeProviderCapabilitiesV2.turns, + // This deterministic adapter steers without interrupting tools. + // Real Claude uses boundary hooks, covered by SteeringCompletion. + activeSteeringInterruptsTools: false, + }, + }, capturedTurns, shouldComplete: (turn) => turn.message.text !== cancellationPrompt, terminalGate: (turn) => diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index f0dabc0260e37..d828c746af232 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -234,7 +234,7 @@ export const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, steers a fully active turn, or queues behind a turn that is not yet steerable. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", + "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. Claude agent updates wait for a safe tool-batch or answer boundary for up to five seconds, then queue a separate turn if no boundary arrives. Other tool-interrupting sessions queue immediately. delivery reports dispatch acceptance, not model processing. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 447272221f650..228f76f0dcea9 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2669,6 +2669,136 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + it.effect.each([ + "PostToolBatch", + "Stop", + "timeout", + "cancelled", + "completed", + "withdrawn", + "refreshed", + "aborted", + ] as const)("delivers agent messages at a safe boundary or defers them (%s)", (scenario) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ close: Queue.shutdown }); + const allocator = yield* IdAllocator.IdAllocatorV2; + const hookAbort = new AbortController(); + const attemptId = RunAttemptId.make("attempt-agent-boundary"); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId, + text: "Work through the task.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + const turn = { + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: allocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + }; + const steering = yield* harness.runtime + .steerTurn({ + ...turn, + ...(scenario === "withdrawn" || scenario === "refreshed" || scenario === "aborted" + ? { + resolveMessageText: Effect.sync(() => { + if (scenario === "aborted") hookAbort.abort(); + return scenario === "withdrawn" ? undefined : "Remaining task completed."; + }), + } + : {}), + message: { + createdBy: "agent", + creationSource: "mcp", + messageId: MessageId.make("agent-boundary-update"), + text: "The review found a missing check.", + attachments: [], + }, + }) + .pipe(Effect.forkScoped); + // Flush runnable fibers; the steer is now waiting on its boundary. + yield* TestClock.adjust("0 millis"); + const event = scenario === "Stop" ? "Stop" : "PostToolBatch"; + const hook = harness.getOpenedOptions()?.hooks?.[event]?.[0]?.hooks[0]; + assert.isDefined(hook); + const boundary = + event === "Stop" + ? { hook_event_name: "Stop" as const, stop_hook_active: false } + : { hook_event_name: "PostToolBatch" as const, tool_calls: [] }; + const hookInput = { + ...boundary, + session_id: WAKE_NATIVE_SESSION, + transcript_path: "transcript.jsonl", + cwd: "/repo", + }; + const callHook = (child = false, signal = hookAbort.signal) => + Effect.promise(() => + hook!({ ...hookInput, ...(child ? { agent_id: "child" } : {}) }, undefined, { + signal, + }), + ); + assert.deepEqual(yield* callHook(true), {}); + if (scenario === "timeout") { + yield* TestClock.adjust("5 seconds"); + } else if (scenario === "cancelled") { + yield* harness.runtime.interruptTurn(turn); + } else if (scenario === "completed") { + yield* harness.offerAndWait( + makeResultFrame({ uuid: "boundary-completed", result: "Done" }), + ); + } else { + const output = yield* callHook(); + assert.deepEqual( + output, + scenario === "withdrawn" || scenario === "aborted" + ? {} + : { + hookSpecificOutput: { + hookEventName: event, + additionalContext: + scenario === "refreshed" + ? "New message from another agent:\nRemaining task completed." + : "New message from another agent:\nThe review found a missing check.", + }, + }, + ); + if (scenario === "aborted") { + assert.deepEqual(yield* callHook(false, new AbortController().signal), { + hookSpecificOutput: { + hookEventName: event, + additionalContext: "New message from another agent:\nRemaining task completed.", + }, + }); + } + } + if ( + scenario === "timeout" || + scenario === "completed" || + scenario === "cancelled" || + scenario === "withdrawn" + ) { + const error = yield* Fiber.join(steering).pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterSteerRunDeferredError"); + } else { + yield* Fiber.join(steering); + } + assert.deepEqual(yield* callHook(), {}); + assert.equal(harness.offeredMessages.length, 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect.each([ { isError: false, title: "Check weather" }, { isError: true, title: "Check weather" }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index e969520bd05ee..2faa565a9b435 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -26,6 +26,7 @@ import { type EffortLevel, type ForkSessionOptions, type ForkSessionResult, + type HookCallback, getSubagentMessages, query, type Options as ClaudeQueryOptions, @@ -319,6 +320,7 @@ export const ClaudeProviderCapabilitiesV2 = { supportsInterrupt: true, supportsActiveSteering: true, activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, supportsSteeringByInterruptRestart: false, supportsQueuedMessages: true, terminalStatusQuality: "strong", @@ -3016,6 +3018,15 @@ function formatClaudeUsageLimitWait(waitMs: number): string { return minutes === 0 ? `${hours}h` : `${hours}h ${minutes}m`; } +type ClaudeAgentSteerOutcome = "delivered" | "deferred" | "cancelled"; + +interface PendingClaudeAgentSteer { + readonly text: string; + readonly completion: Deferred.Deferred; + readonly resolveMessageText?: ProviderAdapter.ProviderAdapterV2SteerInput["resolveMessageText"]; + outcome?: ClaudeAgentSteerOutcome; +} + interface ActiveClaudeTurnContext { readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly nativeThreadId: string; @@ -3073,12 +3084,26 @@ interface ActiveClaudeTurnContext { readonly subagentsByToolUseId: Map; readonly subagentNodesByTaskId: Map; readonly promptUuid: string | null; + readonly pendingAgentSteers: Map; promptEcho: "pending" | "confirmed"; promptStarted: boolean; gatedFramesBeforeEcho: number; readonly heldRootFrames: Array; } +// Claim and acknowledge synchronously so a boundary racing the timeout cannot +// both inject a message and return it to the durable queue. +function settleClaudeAgentSteers( + context: ActiveClaudeTurnContext, + outcome: ClaudeAgentSteerOutcome, +): void { + for (const pending of context.pendingAgentSteers.values()) { + pending.outcome = outcome; + Deferred.doneUnsafe(pending.completion, Effect.succeed(outcome)); + } + context.pendingAgentSteers.clear(); +} + interface ActiveClaudeProviderRetry { readonly retry: OrchestrationV2ProviderRetry; readonly failure: OrchestrationV2ProviderFailure; @@ -6066,6 +6091,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( readonly threadDisposition?: "reusable" | "broken"; readonly result?: SDKResultMessage; }) { + settleClaudeAgentSteers( + input.context, + input.status === "interrupted" || input.status === "cancelled" + ? "cancelled" + : "deferred", + ); yield* reasoningDeltas.flushTurn(input.context.nativeTurnId); if (input.context.compaction?.status === "running") { yield* emitProviderEvent({ @@ -8644,11 +8675,68 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( onUserDialog, supportedDialogKinds: ["resume_return"], }); + let boundaryQuery: ClaudeAgentSdkQuerySession | undefined; + const agentMessageBoundary: HookCallback = (hookInput, _toolUseId, options) => + runPromise( + Effect.gen(function* () { + const context = yield* Ref.get(activeTurn); + const live = yield* Ref.get(queryContext); + if ( + options.signal.aborted || + hookInput.agent_id !== undefined || + context === null || + live?.query !== boundaryQuery || + live?.stopping || + context.nativeThreadId !== hookInput.session_id || + (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) || + (hookInput.hook_event_name !== "PostToolBatch" && + hookInput.hook_event_name !== "Stop") + ) { + return {}; + } + // Resolve before claiming so an aborted hook consumes no partial batch. + const pendingSteers = yield* Effect.forEach( + Array.from(context.pendingAgentSteers), + ([id, pending]) => + (pending.resolveMessageText ?? Effect.succeed(pending.text)).pipe( + Effect.map((text) => ({ id, pending, text })), + ), + ); + const liveTurn = + (yield* Ref.get(activeTurn)) === context && + (yield* Ref.get(queryContext)) === live && + !(yield* Ref.get(interruptedTurns)).has(context.providerTurnId); + if (options.signal.aborted) return {}; + const messages: string[] = []; + for (const { id, pending, text } of pendingSteers) { + // The deadline or cancellation can win while storage is read. + if (context.pendingAgentSteers.get(id) !== pending) continue; + pending.outcome = text !== undefined && liveTurn ? "delivered" : "cancelled"; + context.pendingAgentSteers.delete(id); + Deferred.doneUnsafe(pending.completion, Effect.succeed(pending.outcome)); + if (pending.outcome === "delivered" && text !== undefined) messages.push(text); + } + return messages.length === 0 + ? {} + : { + hookSpecificOutput: { + hookEventName: hookInput.hook_event_name, + additionalContext: `New message from another agent:\n${messages.join("\n\n")}`, + }, + }; + }), + ); const querySession = yield* queryRunner .open({ threadId: turnInput.threadId, providerSessionId: input.providerSessionId, - options: queryOptions, + options: { + ...queryOptions, + hooks: { + PostToolBatch: [{ hooks: [agentMessageBoundary] }], + Stop: [{ hooks: [agentMessageBoundary] }], + }, + }, }) .pipe( Effect.onError(() => @@ -8662,6 +8750,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : Effect.void, ), ); + boundaryQuery = querySession; // Marked only after a successful open: a failed create must not // leave the runtime believing the native session exists, or the // retry would resume a session that was never created. @@ -8834,6 +8923,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( latestAssistantRateLimited: false, compactionFailure: null, steeredSegmentUsage: [], + pendingAgentSteers: new Map(), subagentsByTaskId: new Map(), subagentsByToolUseId: new Map(), subagentNodesByTaskId: new Map(), @@ -9025,6 +9115,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( next.add(turnInput.providerTurnId); return next; }); + settleClaudeAgentSteers(currentTurn, "cancelled"); yield* existing.query.interrupt; yield* existing.query.close.pipe(Effect.ignore); const closed = yield* Deferred.await(existing.closed).pipe( @@ -9150,6 +9241,42 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( detail: `Claude provider turn ${turnInput.providerTurnId} is not the active turn.`, }); } + if (turnInput.message.createdBy === "agent") { + // `priority: next` also folds after tools, but can start an unowned + // native turn if the answer ends first. Hooks keep delivery owned + // by this turn; an absent/slow boundary falls back to T3's queue. + if (turnInput.message.attachments.length > 0) { + return yield* new ProviderAdapter.ProviderAdapterSteerRunDeferredError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + }); + } + const pending: PendingClaudeAgentSteer = { + text: turnInput.message.text, + completion: yield* Deferred.make(), + ...(turnInput.resolveMessageText === undefined + ? {} + : { resolveMessageText: turnInput.resolveMessageText }), + }; + currentTurn.pendingAgentSteers.set(turnInput.message.messageId, pending); + yield* Deferred.await(pending.completion).pipe( + Effect.timeoutOption("5 seconds"), + Effect.ensuring( + Effect.sync(() => { + currentTurn.pendingAgentSteers.delete(turnInput.message.messageId); + }), + ), + ); + if (pending.outcome === "delivered") return; + // A restart can interrupt this wait too. Let the caller revoke + // stopped/disposed input or retain still-owned input in its queue. + return yield* new ProviderAdapter.ProviderAdapterSteerRunDeferredError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + }); + } const userMessage = yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( turnInput.message.text, @@ -9183,14 +9310,15 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, (effect, turnInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterSteerRunError({ - driver: CLAUDE_PROVIDER, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProviderAdapterSteerRunDeferredError" + ? cause + : new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + cause, + }), ), ), ); diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index 9bec96d31daf1..2c384f5eebc2b 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -1,4 +1,4 @@ -import type { SDKMessage, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; +import type { HookCallback, SDKMessage, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -24,6 +24,7 @@ import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; @@ -139,6 +140,7 @@ it.effect.each(deliveries)( const offers: SDKUserMessage[] = []; const nativeQueue: SDKUserMessage[] = []; const batchAbort = new AbortController(); + let postToolBatch: HookCallback | undefined; const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ instanceId: modelSelection.instanceId, settings, @@ -150,8 +152,9 @@ it.effect.each(deliveries)( idAllocator: yield* IdAllocator.IdAllocatorV2, queryRunner: { allocateSessionId: Effect.succeed(sessionId), - open: () => - Effect.succeed({ + open: ({ options }) => { + postToolBatch = options.hooks?.PostToolBatch?.[0]?.hooks[0]; + return Effect.succeed({ messages: Stream.fromQueue(sdkMessages), offer: (message) => Effect.sync(() => { @@ -170,7 +173,8 @@ it.effect.each(deliveries)( stopTask: () => Effect.void, interrupt: Effect.die("automatic delivery must never interrupt"), close: Effect.void, - }), + }); + }, forkSession: () => Effect.die("unused"), subagentLaunchToolUseId: () => Effect.succeed(null), assertComplete: Effect.void, @@ -390,7 +394,10 @@ it.effect.each(deliveries)( assert.ok(steer?.type === "user_message"); assert.equal(steer.inputIntent, "promoted_queued_to_steer"); } - yield* worker.drain(); + const boundaryDelivery = + delivery === "child completion" ? yield* worker.drain().pipe(Effect.forkScoped) : null; + if (boundaryDelivery !== null) yield* TestClock.adjust("0 millis"); + else yield* worker.drain(); const explicitSteer = delivery === "user steering" || delivery === "promoted queued message"; assert.equal(batchAbort.signal.aborted, explicitSteer); @@ -404,13 +411,31 @@ it.effect.each(deliveries)( yield* Queue.offer(sdkMessages, toolResult("build", false)); for (const id of statusIds) yield* Queue.offer(sdkMessages, toolResult(id, batchAbort.signal.aborted)); + if (boundaryDelivery !== null) { + assert.isDefined(postToolBatch); + const output = yield* Effect.promise(() => + postToolBatch!( + { + hook_event_name: "PostToolBatch", + session_id: sessionId, + cwd, + transcript_path: "transcript.jsonl", + tool_calls: [], + }, + undefined, + { signal: batchAbort.signal }, + ), + ); + assert.include(JSON.stringify(output), "task_status"); + yield* Fiber.join(boundaryDelivery); + } if (explicitSteer) yield* Queue.offer(sdkMessages, result("aborted", true)); yield* Queue.offer(sdkMessages, result("completed", false)); yield* Fiber.join(finished); yield* worker.drain(); yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); - if (!explicitSteer) { + if (delivery === "scheduled message") { const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; const noticeFinished = yield* watch( (event) => @@ -418,18 +443,9 @@ it.effect.each(deliveries)( event.payload.id === queued.id && event.payload.status === "waiting", ); - const delivered = - delivery === "child completion" - ? yield* watch( - (event) => - event.type === "subagent.updated" && - event.payload.completionDelivery?.state === "delivered", - ) - : null; yield* Queue.offer(sdkMessages, result("notice-completed", false)); yield* Fiber.join(noticeFinished); yield* worker.drain(); - if (delivered !== null) yield* Fiber.join(delivered); } const after = yield* orchestrator.getThreadProjection(threadId); const reads = after.turnItems.filter((item) => @@ -441,13 +457,13 @@ it.effect.each(deliveries)( assert.equal(read.toolNonExecutionKind, explicitSteer ? "cancelled" : undefined); assert.equal(read.type === "dynamic_tool" && read.output === refusal, explicitSteer); } - assert.equal(offers.length, 2); + assert.equal(offers.length, delivery === "child completion" ? 1 : 2); assert.equal( offers.filter((offer) => offer.priority === "now").length, explicitSteer ? 1 : 0, ); if (!explicitSteer) { - assert.equal(after.runs.length, 2); + assert.equal(after.runs.length, delivery === "child completion" ? 1 : 2); if (delivery === "child completion") { assert.equal(after.messages.filter((message) => message.id === messageId).length, 1); assert.equal(after.subagents[0]?.completionDelivery?.state, "delivered"); @@ -461,7 +477,7 @@ it.effect.each(deliveries)( yield* orchestrator.recoverDelegatedTasks; yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); - assert.equal(offers.length, 2); + assert.equal(offers.length, delivery === "child completion" ? 1 : 2); }).pipe( Effect.provide( ProviderReplayHarness.layerWithRegistry( diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index cb41dc7365123..91effb07d25d7 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -56,6 +56,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ providerThreadId: ProviderThreadId, providerTurnId: ProviderTurnId, messageId: MessageId, + waitForBoundary: Schema.optional(Schema.Literal(true)), }), Schema.Struct({ type: Schema.Literal("provider-turn.restart"), @@ -210,6 +211,8 @@ export interface EffectOutboxV2Shape { readonly cancelUnsettled: (input: { readonly threadId: ThreadId; readonly effectTypes: ReadonlyArray; + readonly messageIds?: ReadonlyArray; + readonly exceptCommandId?: CommandId; readonly reason: string; }) => Effect.Effect, EffectOutboxError>; readonly signalCancellations: (effectIds: ReadonlyArray) => Effect.Effect; @@ -315,6 +318,7 @@ export const layer: Layer.Layer = La ).pipe(Effect.asVoid); // An earlier effect waiting out a retry backoff still blocks later ones, so // a turn cannot start while a failed rollback is about to restore files. + // Boundary steers yield only to human steering and immediate turn controls. const claimableCandidatePredicate = ( availableBefore?: string, excludeRestartContinuations = false, @@ -330,6 +334,20 @@ export const layer: Layer.Layer = La SELECT 1 FROM orchestration_v2_effect_outbox AS active WHERE active.thread_id = candidate.thread_id + AND NOT ( + active.rowid < candidate.rowid + AND active.effect_type = 'provider-turn.steer' + AND COALESCE(json_extract(active.payload_json, '$.waitForBoundary'), 0) = 1 + AND ( + candidate.effect_type IN ( + 'provider-turn.interrupt', 'provider-turn.restart', 'runtime-request.respond' + ) + OR ( + candidate.effect_type = 'provider-turn.steer' + AND COALESCE(json_extract(candidate.payload_json, '$.waitForBoundary'), 0) != 1 + ) + ) + ) AND ( active.status = 'running' OR ( @@ -438,9 +456,9 @@ export const layer: Layer.Layer = La : new EffectOutboxError({ operation: "list", cause }), ), ), - cancelUnsettled: ({ threadId, effectTypes, reason }) => + cancelUnsettled: ({ threadId, effectTypes, messageIds, exceptCommandId, reason }) => Effect.gen(function* () { - if (effectTypes.length === 0) return []; + if (effectTypes.length === 0 || messageIds?.length === 0) return []; const now = DateTime.formatIso(yield* DateTime.now); const rows = yield* sql<{ readonly effect_id: string }>` UPDATE orchestration_v2_effect_outbox @@ -454,6 +472,8 @@ export const layer: Layer.Layer = La WHERE thread_id = ${threadId} AND status IN ('pending', 'running') AND effect_type IN ${sql.in(effectTypes)} + AND ${messageIds === undefined ? sql`1 = 1` : sql`json_extract(payload_json, '$.messageId') IN ${sql.in(messageIds)}`} + AND ${exceptCommandId === undefined ? sql`1 = 1` : sql`command_id != ${exceptCommandId}`} RETURNING effect_id `; return rows.map(({ effect_id }) => effect_id); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index a44b60c2498d1..eee22cdd8f7f2 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -62,7 +62,7 @@ export function isNonRetryableProviderTurnControlFailure( export interface OrchestrationEffectExecutorV2Shape { readonly execute: ( effect: EffectOutbox.OrchestrationEffectV2, - options?: { readonly willRetry: boolean }, + options?: { readonly willRetry: boolean; readonly isCurrent?: Effect.Effect }, ) => Effect.Effect; } @@ -222,6 +222,7 @@ export const layerExecutor: Layer.Layer< case "provider-turn.steer": return providerTurnControl .steer({ + ...(options?.isCurrent === undefined ? {} : { isCurrent: options.isCurrent }), threadId: effect.threadId, providerSessionId: effect.request.providerSessionId, providerThreadId: effect.request.providerThreadId, @@ -248,15 +249,18 @@ export const layerExecutor: Layer.Layer< }); }), ), - Effect.catch((error) => + Effect.as(undefined), + Effect.catchTags({ + ProviderTurnControlSteerCancelledError: () => Effect.void, + ProviderTurnControlSteerDeferredError: (error) => Effect.succeed(error), + ProviderTurnControlError: (error) => + error.turnCompleted ? Effect.succeed(error) : Effect.fail(error), + }), + Effect.flatMap((error) => Effect.gen(function* () { - if ( - !("turnCompleted" in error) || - !error.turnCompleted || - effect.request.type !== "provider-turn.steer" - ) { - return yield* error; - } + if (error === undefined) return; + if (effect.request.type !== "provider-turn.steer") return yield* error; + const deferToQueue = error._tag === "ProviderTurnControlSteerDeferredError"; const projection = yield* threads.getThreadRecords( effect.threadId, ["messages", "runs"], @@ -275,6 +279,7 @@ export const layerExecutor: Layer.Layer< threadId: effect.threadId, messageId: message.id, text: message.text, + ...(deferToQueue ? { steeringFallbackOfRunId: run.id } : {}), ...(message.context ? { context: message.context } : {}), attachments: message.attachments, ...(message.delegatedCompletion === undefined @@ -282,7 +287,7 @@ export const layerExecutor: Layer.Layer< : { modelSelection: run.modelSelection }), dispatchMode: { type: - message.delegatedCompletion === undefined + message.delegatedCompletion === undefined && !deferToQueue ? "start_immediately" : "queue_after_active", }, @@ -724,7 +729,10 @@ export const layerWithOptions = ( if (cancelledBeforeExecution) return true; const execution = executor - .execute(effect, { willRetry: effect.attemptCount < maxAttempts }) + .execute(effect, { + willRetry: effect.attemptCount < maxAttempts, + isCurrent: wasCancelled(effect.id).pipe(Effect.map((cancelled) => !cancelled)), + }) .pipe(Effect.as("executed" as const)); const exit = yield* Effect.exit(Effect.raceFirst(execution, cancellation)).pipe( Effect.ensuring(outbox.clearCancellation(effect.id)), diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 13916b40f9dee..e050a01bdcbc4 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -10,6 +10,7 @@ import { RuntimeRequestId, NodeId, type ProjectId, + type MessageId, ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -149,6 +150,7 @@ export interface EventSinkV2Shape { readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; }; }) => Effect.Effect< { @@ -603,7 +605,11 @@ const layerBase: Layer.Layer< commandId: input.commandId, events: normalized, }); - const sequence = storedEvents.at(-1)?.sequence; + // No-op commands still commit receipts and cancellations atomically. + const sequence = + input.events.length === 0 + ? yield* eventStore.latestSequence({ threadId: input.threadId }) + : storedEvents.at(-1)?.sequence; if (sequence === undefined) { return yield* Effect.die( new Error(`Command ${input.commandId} produced no orchestration events.`), @@ -627,6 +633,7 @@ const layerBase: Layer.Layer< : yield* effectOutbox.cancelUnsettled({ threadId: input.threadId, ...input.cancelUnsettledEffects, + exceptCommandId: input.commandId, }); return { receipt, storedEvents, committed: true as const, cancelledEffectIds }; }), diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 3a50b441a84ca..453f9d8f61d7c 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -20,8 +20,10 @@ import { ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, RunAttemptId, RunId, + RuntimeRequestId, ThreadId, TurnItemId, } from "@t3tools/contracts"; @@ -1430,6 +1432,77 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); + it.effect("a no-op receipt cancels current effects once without adding events", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:noop-cancellation"); + const commandId = CommandId.make("command:noop-cancellation"); + const created = yield* sink.commitCommand({ + commandId: CommandId.make("command:noop-create"), + threadId, + commandType: "thread.create", + acceptedAt: now, + events: [ + threadCreatedEvent({ + id: "event:noop-create", + thread: makeThread(threadId, now), + now, + }), + ], + effects: [ + { + id: "effect:before-noop", + commandId: CommandId.make("command:noop-create"), + threadId, + request: { type: "terminal.cleanup" }, + }, + ], + }); + const input = { + commandId, + threadId, + commandType: "thread.stop", + acceptedAt: now, + events: [], + effects: [], + cancelUnsettledEffects: { + effectTypes: ["terminal.cleanup"], + reason: "Stopped.", + }, + } as const; + const cancellation = yield* outbox + .awaitCancellation("effect:before-noop") + .pipe(Effect.forkChild); + const stopped = yield* sink.commitCommand(input); + assert.isTrue(stopped.committed); + assert.deepEqual(stopped.storedEvents, []); + assert.equal(stopped.receipt.resultSequence, created.receipt.resultSequence); + assert.equal(stopped.cancelledEffectCount, 1); + assert.equal(Option.getOrThrow(yield* outbox.get("effect:before-noop")).status, "cancelled"); + yield* Fiber.join(cancellation); + yield* outbox.enqueue([ + { + id: "effect:after-noop", + commandId: CommandId.make("command:after-noop"), + threadId, + request: { type: "terminal.cleanup" }, + }, + ]); + const retry = yield* sink.commitCommand(input); + assert.isFalse(retry.committed); + assert.deepEqual(retry.storedEvents, []); + assert.equal(retry.cancelledEffectCount, 0); + assert.equal(Option.getOrThrow(yield* outbox.get("effect:after-noop")).status, "pending"); + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["terminal.cleanup"], + reason: "Test completed.", + }); + }), + ); + it.effect("keeps one durable effect across command retries and executes it after recovery", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; @@ -2313,6 +2386,233 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); + const steerRequest = { + type: "provider-turn.steer", + providerSessionId: ProviderSessionId.make("session:boundary-order"), + providerThreadId: ProviderThreadId.make("provider-thread:boundary-order"), + providerTurnId: ProviderTurnId.make("provider-turn:boundary-order"), + messageId: MessageId.make("message:boundary-order"), + } as const; + const boundaryOrderCandidates = [ + { request: { ...steerRequest }, bypass: true }, + { request: { ...steerRequest, type: "provider-turn.interrupt" }, bypass: true }, + { + request: { + ...steerRequest, + type: "provider-turn.restart", + interruptedAttemptId: RunAttemptId.make("attempt:boundary-order"), + runId: RunId.make("run:boundary-order"), + }, + bypass: true, + }, + { + request: { + type: "runtime-request.respond", + providerSessionId: steerRequest.providerSessionId, + requestId: RuntimeRequestId.make("request:boundary-order"), + }, + bypass: true, + }, + { request: { ...steerRequest, waitForBoundary: true }, bypass: false }, + { + request: { type: "provider-turn.start", runId: RunId.make("run:boundary-order") }, + bypass: false, + }, + { + request: { + type: "checkpoint.capture", + runId: RunId.make("run:boundary-order"), + scopeId: CheckpointScopeId.make("scope:boundary-order"), + }, + bypass: false, + }, + { + request: { + type: "provider-thread.rollback", + providerThreadId: steerRequest.providerThreadId, + checkpointId: CheckpointId.make("checkpoint:boundary-order"), + scopeId: CheckpointScopeId.make("scope:boundary-order"), + }, + bypass: false, + }, + { request: { type: "terminal.cleanup" }, bypass: false }, + ] satisfies ReadonlyArray<{ + request: EffectOutbox.OrchestrationEffectRequestV2; + bypass: boolean; + }>; + + it.effect.each( + (["running", "pending"] as const).flatMap((status) => + boundaryOrderCandidates.map(({ request, bypass }) => ({ + status, + request, + bypass, + name: `${bypass ? "allows" : "blocks"} ${request.type}${"waitForBoundary" in request ? " at a boundary" : ""} behind a ${status} boundary steer`, + })), + ), + )("boundary steer ordering: $name", ({ status, request, bypass }) => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const commandId = CommandId.make("command:boundary-order"); + const threadId = ThreadId.make("thread:boundary-order"); + const workerId = "boundary-worker"; + const future = DateTime.add(now, { seconds: 5 }); + yield* outbox.enqueue([ + { + id: "boundary", + commandId, + threadId, + request: { ...steerRequest, waitForBoundary: true }, + }, + ]); + const boundary = Option.getOrThrow( + yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }), + ); + assert.deepEqual(boundary.request, { ...steerRequest, waitForBoundary: true }); + if (status === "pending") { + yield* outbox.retry({ + effectId: boundary.id, + workerId, + error: "Waiting for a boundary.", + delayMs: 5_000, + }); + } + yield* outbox.enqueue([{ id: "candidate", commandId, threadId, request }]); + const nextAt = yield* outbox.nextClaimableAt; + if (bypass) { + assert.equal(DateTime.formatIso(Option.getOrThrow(nextAt)), DateTime.formatIso(now)); + } else if (status === "pending") { + assert.equal(DateTime.formatIso(Option.getOrThrow(nextAt)), DateTime.formatIso(future)); + } else { + assert.isTrue(Option.isNone(nextAt)); + } + const claimed = yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); + assert.equal(Option.getOrUndefined(claimed)?.id, bypass ? "candidate" : undefined); + if (bypass) yield* outbox.succeed({ effectId: "candidate", workerId }); + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["provider-turn.steer"], + messageIds: [steerRequest.messageId], + reason: "Boundary settled.", + }); + if (!bypass && request.type !== "provider-turn.steer") { + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "candidate", + ); + } + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + + it.effect("keeps immediate controls behind ordinary steers and intervening effects", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const commandId = CommandId.make("command:ordinary-steer-order"); + const threadId = ThreadId.make("thread:ordinary-steer-order"); + const workerId = "ordinary-steer-worker"; + yield* outbox.enqueue([{ id: "ordinary", commandId, threadId, request: steerRequest }]); + yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); + yield* outbox.enqueue([ + { + id: "boundary", + commandId, + threadId, + request: { ...steerRequest, waitForBoundary: true }, + }, + { id: "cleanup", commandId, threadId, request: { type: "terminal.cleanup" } }, + { + id: "interrupt", + commandId, + threadId, + request: { ...steerRequest, type: "provider-turn.interrupt" }, + }, + ]); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "ordinary", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "boundary", + ); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "boundary", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "cleanup", + ); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "cleanup", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "interrupt", + ); + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + + it.effect( + "cancels selected messages while preserving replacement commands and other effects", + () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = ThreadId.make("thread:message-cancellation"); + const commandId = CommandId.make("command:message-cancellation"); + const replacementCommandId = CommandId.make("command:message-cancellation:replacement"); + const messageId = MessageId.make("message:cancelled"); + yield* outbox.enqueue([ + { id: "running", commandId, threadId, request: { ...steerRequest, messageId } }, + ]); + assert.equal( + Option.getOrThrow( + yield* outbox.claimNext({ workerId: "cancel-worker", leaseDurationMs: 30_000 }), + ).id, + "running", + ); + yield* outbox.enqueue([ + { id: "pending", commandId, threadId, request: { ...steerRequest, messageId } }, + { id: "other-message", commandId, threadId, request: steerRequest }, + { + id: "replacement", + commandId: replacementCommandId, + threadId, + request: { ...steerRequest, messageId }, + }, + { + id: "other-thread", + commandId, + threadId: ThreadId.make("thread:message-cancellation:other"), + request: { ...steerRequest, messageId }, + }, + { id: "cleanup", commandId, threadId, request: { type: "terminal.cleanup" } }, + ]); + const cancellation = { + threadId, + effectTypes: ["provider-turn.steer", "terminal.cleanup"], + exceptCommandId: replacementCommandId, + reason: "Message removed.", + } as const; + assert.deepEqual(yield* outbox.cancelUnsettled({ ...cancellation, messageIds: [] }), []); + assert.deepEqual( + (yield* outbox.cancelUnsettled({ ...cancellation, messageIds: [messageId] })).toSorted(), + ["pending", "running"], + ); + for (const id of ["other-message", "replacement", "other-thread", "cleanup"]) { + assert.equal(Option.getOrThrow(yield* outbox.get(id)).status, "pending"); + } + assert.deepEqual((yield* outbox.cancelUnsettled(cancellation)).toSorted(), [ + "cleanup", + "other-message", + ]); + assert.deepEqual( + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["provider-turn.steer"], + reason: "Legacy cancellation.", + }), + ["replacement"], + ); + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + it.effect("prunes settled effects past retention in batches and keeps the rest", () => Effect.gen(function* () { const outbox = yield* EffectOutbox.EffectOutboxV2; diff --git a/apps/server/src/orchestration-v2/NotificationMailbox.ts b/apps/server/src/orchestration-v2/NotificationMailbox.ts index 2afee9749e22e..28de4bc9f1546 100644 --- a/apps/server/src/orchestration-v2/NotificationMailbox.ts +++ b/apps/server/src/orchestration-v2/NotificationMailbox.ts @@ -3,8 +3,25 @@ import { type MessageId, type OrchestrationV2ConversationMessage, type OrchestrationV2ThreadProjection, + type OrchestrationV2Subagent, } from "@t3tools/contracts"; +export function delegatedCompletionWakeDetail(taskIds: ReadonlyArray): string { + const taskList = taskIds.join(", "); + return taskIds.length === 1 + ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.` + : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`; +} + +export function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean { + return ( + status === "completed" || + status === "failed" || + status === "cancelled" || + status === "interrupted" + ); +} + export function isBufferedProviderNotification( message: OrchestrationV2ConversationMessage, ): boolean { diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 96b6bab3c033d..85ca597f4e26f 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -101,6 +101,8 @@ import { import { isBufferedProviderNotification, isUndeliveredMailboxSteer, + delegatedCompletionWakeDetail, + isTerminalDelegatedTaskStatus, } from "./NotificationMailbox.ts"; import { EventSinkV2 } from "./EventSink.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; @@ -575,22 +577,6 @@ function legacyPullRequestLink( ); } -function delegatedCompletionWakeDetail(taskIds: ReadonlyArray): string { - const taskList = taskIds.join(", "); - return taskIds.length === 1 - ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.` - : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`; -} - -function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean { - return ( - status === "completed" || - status === "failed" || - status === "cancelled" || - status === "interrupted" - ); -} - function delegatedTaskTerminalStatus( status: OrchestrationV2Run["status"], ): OrchestrationV2Subagent["status"] | null { @@ -3899,6 +3885,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly scheduledTaskId?: OrchestrationV2ConversationMessage["scheduledTaskId"]; readonly senderThreadId?: OrchestrationV2ConversationMessage["senderThreadId"]; readonly delegatedCompletion?: OrchestrationV2ConversationMessage["delegatedCompletion"]; + readonly notification?: OrchestrationV2ConversationMessage["notification"]; readonly forceRestart: boolean; }) => Effect.gen(function* () { @@ -4057,6 +4044,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...(input.delegatedCompletion === undefined ? {} : { delegatedCompletion: input.delegatedCompletion }), + ...(input.notification === undefined ? {} : { notification: input.notification }), ...(input.scheduledTaskId === undefined ? {} : { scheduledTaskId: input.scheduledTaskId }), @@ -4176,6 +4164,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio threadId: input.command.threadId, request: { type: "provider-turn.steer", + ...(input.createdBy === "agent" && + session.providerSession.capabilities.turns.supportsNonInterruptingAgentSteering === + true + ? { waitForBoundary: true as const } + : {}), providerSessionId, providerThreadId: providerThread.id, providerTurnId: providerTurn.id, @@ -4607,6 +4600,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) => Effect.gen(function* () { let projection = yield* getProjectionWithPendingEvents(command.threadId, events); + // Stop can commit after the boundary wait expires but before its fallback + // is dispatched. Do not turn that stale steer into fresh work. + if ( + command.steeringFallbackOfRunId !== undefined && + (yield* stopReachedRun(command, command.threadId, command.steeringFallbackOfRunId)) + ) { + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + return; + } if (command.manualContinuationOfRunId !== undefined) { const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId); const limited = latestRootProviderFailure(source ?? null, projection.turnItems); @@ -4832,6 +4842,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio turn.status === "completed")) ) { dispatchMode = { type: "start_immediately" }; + } else if (command.createdBy === "agent") { + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === target?.providerThreadId, + ); + if (providerThread?.providerSessionId != null) { + const session = yield* providerSessions + .get(providerThread.providerSessionId) + .pipe(mapDispatchError(command)); + if ( + Option.isSome(session) && + session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === + true && + session.value.providerSession.capabilities.turns + .supportsNonInterruptingAgentSteering !== true + ) { + // Claude steering cancels pending tools, which appear as user refusals. + // Agent updates can wait for the durable queue's next-turn boundary. + dispatchMode = { type: "queue_after_active" }; + } + } } } // Stop leaves the run reading as running until its provider stops. A plain @@ -4881,6 +4911,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (candidate) => candidate.id === requestedCompletion.parentRunId, ); const delivery = parentRun?.delegatedCompletion?.delivery; + const existingDelivery = projection.messages.find( + (message) => message.id === command.messageId, + ); + const deferredSteer = + existingDelivery !== undefined && + command.steeringFallbackOfRunId === existingDelivery.runId && + existingDelivery.delegatedCompletion?.generation === requestedCompletion.generation && + projection.runs.find((run) => run.id === existingDelivery.runId)?.userMessageId !== + command.messageId; if ( parentRun?.delegatedCompletion?.disposition !== "open" || delivery === null || @@ -4888,6 +4927,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio delivery.generation !== requestedCompletion.generation || delivery.messageId !== command.messageId || (projection.messages.some((candidate) => candidate.id === command.messageId) && + !deferredSteer && !isUndeliveredMailboxSteer(projection, command.messageId)) ) { return yield* new OrchestratorDispatchError({ @@ -4906,6 +4946,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // session's capabilities. Never interrupt/restart a turn for a notification. if ( delegatedCompletion !== undefined && + // A deferred steer is already persisted; do not attempt the hook again. + !projection.messages.some((message) => message.id === command.messageId) && delegatedCompletion.taskIds.every( (id) => projection.subagents.find((task) => task.id === id)?.completionWake === "always", ) @@ -4932,7 +4974,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ( Option.isSome(session) && session.value.providerSession.capabilities.turns.supportsActiveSteering && - session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools !== true + (session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools !== + true || + session.value.providerSession.capabilities.turns + .supportsNonInterruptingAgentSteering === true) ) { dispatchMode = { type: "steer_active", targetRunId: active.id }; } @@ -5004,6 +5049,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : (projection.runs.find((run) => run.id === dispatchMode.targetRunId) ?.modelSelection ?? modelSelection), delegatedCompletion, + ...(command.notification === undefined ? {} : { notification: command.notification }), targetRunId: dispatchMode.targetRunId, messageId: command.messageId, text: dispatchText, @@ -7838,6 +7884,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio targetRunId: command.targetRunId, messageId: queuedMessage.id, text: queuedMessage.text, + ...(queuedMessage.notification === undefined + ? {} + : { notification: queuedMessage.notification }), + ...(queuedMessage.delegatedCompletion === undefined + ? {} + : { delegatedCompletion: queuedMessage.delegatedCompletion }), attachments: queuedMessage.attachments, ...(queuedMessage.context ? { context: queuedMessage.context } : {}), createdBy: queuedMessage.createdBy, @@ -10976,6 +11028,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; }; }, OrchestratorV2Error @@ -10992,6 +11045,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio | { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; } | undefined; switch (command.type) { @@ -11249,9 +11303,41 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio yield* dispatchDelegatedTaskWakePolicy(command, events); break; case "delegated_task.completion-delivery.acknowledge": - case "delegated_task.completion-delivery.dispose": + case "delegated_task.completion-delivery.dispose": { + const before = yield* getProjectionWithPendingEvents(command.parentThreadId, events); yield* dispatchDelegatedTaskCompletionDeliveryResolution(command, events); + const task = before.subagents.find((candidate) => candidate.id === command.taskId); + const owner = before.runs.find((run) => run.id === task?.runId); + const after = yield* getProjectionWithPendingEvents(command.parentThreadId, events); + const remaining = after.runs.find((run) => run.id === owner?.id)?.delegatedCompletion + ?.delivery; + const messageIds = before.messages + .filter((message) => { + const source = message.notification?.source; + return ( + message.id.startsWith("message:delegated-task-question:") && + source?.kind === "delegated_task" && + source.taskIds.includes(command.taskId) + ); + }) + .map((message) => message.id); + const previousDelivery = owner?.delegatedCompletion?.delivery; + if ( + previousDelivery != null && + (remaining == null || + remaining.taskIds.length === 0 || + remaining.messageId !== previousDelivery.messageId) + ) { + messageIds.push(previousDelivery.messageId); + } + if (messageIds.length > 0) + cancelUnsettledEffects = { + effectTypes: ["provider-turn.steer"], + messageIds, + reason: "The delegated notification is no longer owned by this task.", + }; break; + } case "thread.created.record": yield* dispatchCreatedThreadRecord(command, events); break; @@ -11261,6 +11347,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio default: return yield* dispatchUnsupported(command); } + if (command.type === "run.interrupt" || command.type === "thread.stop") { + cancelUnsettledEffects = { + effectTypes: [...(cancelUnsettledEffects?.effectTypes ?? []), "provider-turn.steer"], + reason: cancelUnsettledEffects?.reason ?? "The user stopped this thread.", + }; + } return { events: yield* Ref.get(events), effects: yield* Ref.get(effects), @@ -11393,33 +11485,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); - if (plan.events.length === 0) { - // A settle that ended nothing still records its receipt: a replayed Stop - // effect then finds it instead of settling work that appeared since. - const resultSequence = yield* Effect.gen(function* () { - const sequence = yield* eventSink.latestSequence({ threadId: commandThreadId(command) }); - yield* commandReceipts.insertIfAbsent({ - commandId: command.commandId, - threadId: commandThreadId(command), - commandType: command.type, - acceptedAt: yield* DateTime.now, - resultSequence: sequence, - status: "accepted", - error: null, - }); - return sequence; - }).pipe( - Effect.mapError( - (cause) => - new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause, - }), - ), - ); - return { sequence: resultSequence, storedEvents: [] } satisfies OrchestratorV2DispatchResult; - } const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now); const committed = yield* eventSink .commitCommand({ diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index de03594916a87..1c99d0d41ae16 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -16,9 +16,13 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; +import { + delegatedCompletionWakeDetail, + isTerminalDelegatedTaskStatus, +} from "./NotificationMailbox.ts"; const yieldToRuntime = Effect.yieldNow.pipe( Effect.andThen( @@ -44,6 +48,39 @@ export class ProviderTurnControlError extends Schema.TaggedError()( + "ProviderTurnControlSteerDeferredError", + { + threadId: ThreadId, + providerTurnId: ProviderTurnId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Steering delivery was deferred to the message queue."; + } +} + +export class ProviderTurnControlSteerCancelledError extends Schema.TaggedError()( + "ProviderTurnControlSteerCancelledError", + { + threadId: ThreadId, + providerTurnId: ProviderTurnId, + }, +) { + override get message(): string { + return "Steering delivery was cancelled because the message is no longer current."; + } +} + +const isProviderTurnControlSteerError = Schema.is( + Schema.Union([ + ProviderTurnControlError, + ProviderTurnControlSteerDeferredError, + ProviderTurnControlSteerCancelledError, + ]), +); + export interface ProviderTurnControlServiceV2Shape { readonly interrupt: (input: { readonly subagent?: { @@ -65,7 +102,13 @@ export interface ProviderTurnControlServiceV2Shape { readonly providerThreadId: ProviderThreadId; readonly providerTurnId: ProviderTurnId; readonly messageId: MessageId; - }) => Effect.Effect; + readonly isCurrent?: Effect.Effect; + }) => Effect.Effect< + void, + | ProviderTurnControlError + | ProviderTurnControlSteerDeferredError + | ProviderTurnControlSteerCancelledError + >; readonly interruptAndAwaitTerminal: (input: { readonly threadId: ThreadId; readonly providerSessionId: ProviderSessionId; @@ -323,21 +366,26 @@ export const layer: Layer.Layer< Effect.gen(function* () { const context = yield* projections.getProviderControlContext(input.threadId, input); const ownership = context.message?.delegatedCompletion; - if (ownership !== undefined) { - const projection = yield* projections.getThreadRecords(input.threadId, ["runs"], { - runIds: [ownership.parentRunId], - }); - const cohort = projection.runs.find( - (run) => run.id === ownership.parentRunId, - )?.delegatedCompletion; - if ( - cohort?.disposition !== "open" || - cohort.delivery?.messageId !== input.messageId || - cohort.delivery.generation !== ownership.generation || - cohort.delivery.taskIds.length === 0 - ) - return; - } + const completionTaskIds = + ownership === undefined + ? undefined + : Effect.gen(function* () { + const projection = yield* projections.getThreadRecords(input.threadId, ["runs"], { + runIds: [ownership.parentRunId], + }); + const cohort = projection.runs.find( + (run) => run.id === ownership.parentRunId, + )?.delegatedCompletion; + if ( + cohort?.disposition !== "open" || + cohort.delivery?.messageId !== input.messageId || + cohort.delivery.generation !== ownership.generation || + cohort.delivery.taskIds.length === 0 + ) + return null; + return cohort.delivery.taskIds; + }); + if (completionTaskIds !== undefined && (yield* completionTaskIds) === null) return; const loaded = yield* load({ ...input, operation: "steer" }); if (Option.isNone(loaded.session)) return; const { message, run } = loaded.context; @@ -349,18 +397,57 @@ export const layer: Layer.Layer< cause: "The persisted steering message or target run is missing.", }); } + const text = projectComposerContextForProvider({ + text: message.text, + records: message.context?.records ?? [], + }); + const resolveMessageText = Effect.gen(function* () { + if (input.isCurrent !== undefined && !(yield* input.isCurrent)) return undefined; + const source = message.notification?.source; + if ( + message.id.startsWith("message:delegated-task-question:") && + source?.kind === "delegated_task" + ) { + const parent = yield* projections.getThreadRecords(input.threadId, ["subagents"]); + const task = parent.subagents.find( + (candidate) => + candidate.origin === "app_owned" && + candidate.childThreadId === source.childThreadId && + source.taskIds.includes(candidate.id), + ); + if ( + task === undefined || + isTerminalDelegatedTaskStatus(task.status) || + task.completionDelivery?.state === "disposed" || + parent.thread.archivedAt !== null || + parent.thread.deletedAt !== null + ) + return undefined; + } + if (completionTaskIds === undefined) return text; + const taskIds = yield* completionTaskIds; + return taskIds === null ? undefined : delegatedCompletionWakeDetail(taskIds); + }); yield* loaded.session.value .steerTurn({ threadId: input.threadId, runId: run.id, providerThread: loaded.providerThread, providerTurnId: loaded.providerTurn.id, + resolveMessageText: resolveMessageText.pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: loaded.providerThread.driver, + providerThreadId: loaded.providerThread.id, + providerTurnId: loaded.providerTurn.id, + cause, + }), + ), + ), message: { messageId: message.id, - text: projectComposerContextForProvider({ - text: message.text, - records: message.context?.records ?? [], - }), + text, attachments: message.attachments, createdBy: message.createdBy, creationSource: message.creationSource, @@ -375,6 +462,19 @@ export const layer: Layer.Layer< .pipe( Effect.catch((cause) => Effect.gen(function* () { + if ((yield* resolveMessageText) === undefined) { + return yield* new ProviderTurnControlSteerCancelledError({ + threadId: input.threadId, + providerTurnId: input.providerTurnId, + }); + } + if (cause._tag === "ProviderAdapterSteerRunDeferredError") { + return yield* new ProviderTurnControlSteerDeferredError({ + threadId: input.threadId, + providerTurnId: input.providerTurnId, + cause, + }); + } const current = yield* projections.getProviderControlContext( input.threadId, input, @@ -389,9 +489,15 @@ export const layer: Layer.Layer< }), ), ); + if (input.isCurrent !== undefined && !(yield* input.isCurrent)) { + return yield* new ProviderTurnControlSteerCancelledError({ + threadId: input.threadId, + providerTurnId: input.providerTurnId, + }); + } }).pipe( Effect.mapError((cause) => - isProviderTurnControlError(cause) + isProviderTurnControlSteerError(cause) ? cause : new ProviderTurnControlError({ threadId: input.threadId, diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 884683979fb65..bcfa2be7f2dbe 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -18,14 +18,20 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as CommandReceiptStore from "./CommandReceiptStore.ts"; +import * as EffectOutbox from "./EffectOutbox.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import * as ThreadManagement from "./ThreadManagementService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -418,18 +424,33 @@ const composerSelection = { const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* ( name: string, - canRestart = false, + { + canRestart = false, + activeSteeringInterruptsTools = false, + supportsNonInterruptingAgentSteering = false, + deferSteering = false, + beforeBoundary = Effect.void, + }: { + canRestart?: boolean; + activeSteeringInterruptsTools?: boolean; + supportsNonInterruptingAgentSteering?: boolean; + deferSteering?: boolean; + beforeBoundary?: Effect.Effect; + } = {}, ) { const cwd = yield* checkpointWorkspace(name); const events = yield* Queue.unbounded(); const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; const steered: string[] = []; + const interrupted: ProviderTurnId[] = []; const capabilities = { ...CodexProviderCapabilitiesV2, turns: { ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: true, supportsSteeringByInterruptRestart: canRestart, + activeSteeringInterruptsTools, + supportsNonInterruptingAgentSteering, }, }; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { @@ -500,10 +521,22 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }); }), steerTurn: (turn) => + Effect.gen(function* () { + yield* beforeBoundary; + if (deferSteering) { + return yield* new ProviderAdapter.ProviderAdapterSteerRunDeferredError({ + driver, + providerThreadId: turn.providerThread.id, + providerTurnId: turn.providerTurnId, + }); + } + const text = yield* turn.resolveMessageText ?? Effect.succeed(turn.message.text); + if (text !== undefined) steered.push(text); + }), + interruptTurn: (turn) => Effect.sync(() => { - steered.push(turn.message.text); + interrupted.push(turn.providerTurnId); }), - interruptTurn: () => Effect.void, respondToRuntimeRequest: () => Effect.void, readThreadSnapshot: () => Effect.die("unused"), rollbackThread: () => Effect.die("unused"), @@ -511,10 +544,17 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }; }), }; - const layer = ProviderReplayHarness.layerWithRegistry( + const databaseLayer = SqlitePersistence.layerMemory; + const layerRuntime = ProviderReplayHarness.layerWithRegistry( { name }, ProviderAdapterRegistry.layerSingle(adapter), - { runEffectWorker: false }, + { runEffectWorker: false, databaseLayer }, + ); + const layer = ThreadManagement.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(databaseLayer))), + Layer.provideMerge(layerRuntime), + Layer.provideMerge(EffectOutbox.layer.pipe(Layer.provide(databaseLayer))), ); const startFirstTurn = Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -558,9 +598,111 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* yield* Fiber.join(running); return threadId; }); - return { events, started, steered, layer, startFirstTurn }; + return { events, started, steered, interrupted, layer, startFirstTurn }; }); +it.effect.each( + ( + [ + { actor: "agent", mode: "auto", interruptsTools: true }, + { actor: "agent", mode: "steer", interruptsTools: true }, + { actor: "user", mode: "steer", interruptsTools: true }, + { actor: "agent", mode: "auto", interruptsTools: false }, + { actor: "agent", mode: "steer", interruptsTools: false }, + ] as const + ) + .map((entry) => ({ ...entry, safeBoundary: false, defer: false })) + .concat([ + { actor: "agent", mode: "auto", interruptsTools: true, safeBoundary: true, defer: false }, + { actor: "agent", mode: "steer", interruptsTools: true, safeBoundary: true, defer: true }, + ]), +)( + "delivers $actor $mode messages safely (interruptsTools=$interruptsTools, boundary=$safeBoundary, defer=$defer)", + ({ actor, mode, interruptsTools, safeBoundary, defer }) => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `agent-steering-${actor}-${mode}-${interruptsTools}`, + { + activeSteeringInterruptsTools: interruptsTools, + supportsNonInterruptingAgentSteering: safeBoundary, + deferSteering: defer, + }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threads = yield* ThreadManagement.ThreadManagementService; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const messageId = MessageId.make("message:update"); + const result = yield* threads.sendToThread({ + projectId: (yield* orchestrator.getThreadProjection(threadId)).thread.projectId, + commandId: CommandId.make("update"), + threadId, + messageId, + text: "progress update", + attachments: [], + mode, + createdBy: actor, + creationSource: actor === "agent" ? "mcp" : "web", + }); + yield* worker.drain(); + const queuedAtDispatch = actor === "agent" && interruptsTools && !safeBoundary; + const queued = queuedAtDispatch || defer; + assert.equal(result.delivery, queuedAtDispatch ? "queued" : "steered"); + assert.deepEqual(steered, queued ? [] : ["progress update"]); + assert.equal(started.length, 1); + const pending = yield* orchestrator.getThreadProjection(threadId); + assert.equal(pending.runs.find((run) => run.id === first.runId)?.status, "running"); + if (!queued) return; + assert.equal( + pending.runs.find((run) => run.userMessageId === messageId)?.status, + "queued", + ); + + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "waiting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const turn = pending.providerTurns[0]!; + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, + }); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(completed); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + assert.equal(started.length, 2); + assert.equal(started[1]?.message.messageId, messageId); + assert.equal(started[1]?.message.text, "progress update"); + assert.deepEqual(steered, []); + const final = yield* orchestrator.getThreadProjection(threadId); + assert.equal(final.runs.find((run) => run.id === first.runId)?.status, "completed"); + }).pipe(Effect.provide(layer)); + }), + ), +); + it.effect.each([{ canRestart: false }, { canRestart: true }])( "steers a changed turn-scoped selection into the running turn (canRestart=$canRestart)", ({ canRestart }) => @@ -568,7 +710,7 @@ it.effect.each([{ canRestart: false }, { canRestart: true }])( Effect.gen(function* () { const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( `steering-selection-change-${canRestart}`, - canRestart, + { canRestart }, ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -636,6 +778,302 @@ it.effect.each([{ canRestart: false }, { canRestart: true }])( ), ); +it.effect("Stop suppresses a boundary timeout whose fallback has not dispatched yet", () => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const { started, interrupted, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "steering-boundary-stop-race", + { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + deferSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* startFirstTurn; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("boundary-update"), + threadId, + messageId: MessageId.make("message:boundary-update"), + text: "The review found another issue.", + attachments: [], + dispatchMode: { type: "steer_active", targetRunId: started[0]!.runId }, + createdBy: "agent", + creationSource: "mcp", + }); + const draining = yield* worker.drain().pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop-boundary-parent"), + threadId, + runId: started[0]!.runId, + holdQueue: true, + }); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(draining); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(started.length, 1); + assert.equal(projection.runs.length, 1); + const effects = yield* outbox.listByCommandId(CommandId.make("boundary-update")); + assert.lengthOf(effects, 1); + assert.include(["succeeded", "cancelled"], effects[0]?.status); + if (effects[0]?.status === "succeeded") assert.isNull(effects[0].lastError); + assert.lengthOf(interrupted, 1); + const stops = yield* outbox.listByCommandId(CommandId.make("stop-boundary-parent")); + assert.equal( + stops.find((effect) => effect.request.type === "provider-turn.interrupt")?.status, + "succeeded", + ); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect("Stop cancels pending hook delivery and executes the interrupt before a boundary", () => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const h = yield* nextTurnSelectionHarness("steering-stop-before-hook", { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("pending-hook"), + threadId, + messageId: MessageId.make("message:pending-hook"), + text: "Do not deliver after Stop.", + attachments: [], + dispatchMode: { type: "steer_active", targetRunId: h.started[0]!.runId }, + createdBy: "agent", + creationSource: "mcp", + }); + const delivery = yield* worker.runOnce.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-pending-hook"), + threadId, + }); + yield* worker.drain(); + assert.lengthOf(h.interrupted, 1); + yield* Fiber.join(delivery); + const effects = yield* outbox.listByCommandId(CommandId.make("pending-hook")); + assert.equal(effects[0]?.status, "cancelled"); + yield* Deferred.succeed(release, undefined); + yield* worker.drain(); + assert.deepEqual(h.steered, []); + assert.equal((yield* orchestrator.getThreadProjection(threadId)).runs.length, 1); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + +const dispatchClaimedCompletion = Effect.fn("dispatchClaimedCompletion")(function* ( + threadId: ThreadId, + first: ProviderAdapter.ProviderAdapterV2TurnInput, + taskIds: ReadonlyArray, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const messageId = MessageId.make("message:pending-completion"); + const current = yield* orchestrator.getThreadProjection(threadId); + const now = yield* DateTime.now; + yield* sink.write({ + events: [ + { + id: EventId.make("completion:cohort"), + type: "run.updated", + threadId, + runId: first.runId, + occurredAt: now, + payload: { + ...current.runs.find((run) => run.id === first.runId)!, + delegatedCompletion: { + disposition: "open", + nextGeneration: 2, + delivery: { generation: 1, messageId, taskIds }, + }, + }, + }, + ...taskIds.map((taskId) => ({ + id: EventId.make(`completion:${taskId}`), + type: "subagent.updated" as const, + threadId, + runId: first.runId, + nodeId: taskId, + occurredAt: now, + payload: { + id: taskId, + threadId, + runId: first.runId, + parentNodeId: first.rootNodeId, + origin: "app_owned" as const, + createdBy: "agent" as const, + driver, + providerInstanceId: instanceId, + providerThreadId: null, + childThreadId: null, + nativeTaskRef: null, + prompt: "Do background work", + title: "Background test", + model: null, + completionWake: "always" as const, + completionDelivery: { state: "claimed" as const, observedByRunId: null }, + status: "completed" as const, + result: "done", + startedAt: now, + completedAt: now, + updatedAt: now, + }, + })), + ], + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("dispatch-completion"), + threadId, + messageId, + text: `Original completion for ${taskIds.join(", ")}`, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + delegatedCompletion: { parentRunId: first.runId, generation: 1, taskIds }, + }); + return messageId; +}); + +it.effect("a live completion boundary defer creates exactly one queued follow-up", () => + Effect.scoped( + Effect.gen(function* () { + const h = yield* nextTurnSelectionHarness("steering-live-completion-defer", { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + deferSteering: true, + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + const first = h.started[0]!; + const taskId = NodeId.make("task:deferred"); + const messageId = yield* dispatchClaimedCompletion(threadId, first, [taskId]); + yield* worker.drain(); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + const followUps = projection.runs.filter((run) => run.userMessageId === messageId); + assert.lengthOf(followUps, 1); + assert.equal(followUps[0]?.status, "queued"); + assert.equal(projection.runs.find((run) => run.id === first.runId)?.status, "running"); + assert.equal(projection.messages.filter((message) => message.id === messageId).length, 1); + assert.deepEqual( + projection.messages.find((message) => message.id === messageId)?.delegatedCompletion, + { + parentRunId: first.runId, + generation: 1, + taskIds: [taskId], + }, + ); + const effects = yield* outbox.listByCommandId(CommandId.make("dispatch-completion")); + assert.equal( + effects.find((effect) => effect.request.type === "provider-turn.steer")?.status, + "succeeded", + ); + assert.equal(h.started.length, 1); + assert.deepEqual(h.steered, []); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + +it.effect.each(["dispose", "acknowledge one"] as const)( + "completion %s updates a pending boundary before delivery", + (resolution) => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const h = yield* nextTurnSelectionHarness(`steering-completion-${resolution}`, { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + const first = h.started[0]!; + const taskId = NodeId.make("task:resolved"); + const remainingTaskId = NodeId.make("task:remaining"); + const taskIds = resolution === "dispose" ? [taskId] : [taskId, remainingTaskId]; + yield* dispatchClaimedCompletion(threadId, first, taskIds); + const delivery = yield* worker.runOnce.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch( + resolution === "dispose" + ? { + type: "delegated_task.completion-delivery.dispose", + commandId: CommandId.make("resolve-completion"), + parentThreadId: threadId, + taskId, + } + : { + type: "delegated_task.completion-delivery.acknowledge", + commandId: CommandId.make("resolve-completion"), + parentThreadId: threadId, + taskId, + observedByRunId: first.runId, + }, + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(delivery); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + projection.subagents.find((task) => task.id === taskId)?.completionDelivery?.state, + resolution === "dispose" ? "disposed" : "acknowledged", + ); + if (resolution === "dispose") assert.deepEqual(h.steered, []); + else { + assert.lengthOf(h.steered, 1); + assert.include(h.steered[0]!, String(remainingTaskId)); + assert.notInclude(h.steered[0]!, String(taskId)); + assert.include(h.steered[0]!, "task_status"); + } + assert.equal(projection.runs.length, 1); + const effects = yield* outbox.listByCommandId(CommandId.make("dispatch-completion")); + assert.include(["succeeded", "cancelled"], effects[0]?.status); + if (effects[0]?.status === "succeeded") assert.isNull(effects[0].lastError); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + it.effect("starts a steer that missed the turn on the saved next-turn selection", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 94a9eca331d13..cb708647ae064 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -218,6 +218,8 @@ export const OrchestrationV2TurnCapabilities = Schema.Struct({ supportsActiveSteering: Schema.Boolean, // Some native steering mechanisms cancel pending tools before consuming the message. activeSteeringInterruptsTools: Schema.optional(Schema.Boolean), + // Agent-authored messages can use a safe boundary instead of interrupting tools. + supportsNonInterruptingAgentSteering: Schema.optional(Schema.Boolean), supportsSteeringByInterruptRestart: Schema.Boolean, supportsQueuedMessages: Schema.Boolean, terminalStatusQuality: Schema.Literals(["strong", "weak", "none"]), @@ -2779,6 +2781,7 @@ export const OrchestrationV2Command = Schema.Union([ restartContinuationOfRunId: Schema.optional(RunId), usageLimitContinuationOfRunId: Schema.optional(RunId), manualContinuationOfRunId: Schema.optional(RunId), + steeringFallbackOfRunId: Schema.optional(RunId), usageLimitRecoveryRequestId: Schema.optional(CommandId), deliveryIntent: Schema.optional(Schema.Literals(["auto", "steer", "restart"])), delegatedCompletion: Schema.optional( diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index d58a6c5904a2f..6359a605d0951 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -320,6 +320,19 @@ export class ProviderAdapterSteerRunError extends Schema.TaggedError()( + "ProviderAdapterSteerRunDeferredError", + { + driver: ProviderDriverKind, + providerThreadId: ProviderThreadId, + providerTurnId: ProviderTurnId, + }, +) { + override get message(): string { + return "Active-run steering was deferred to the message queue."; + } +} + export class ProviderAdapterInterruptError extends Schema.TaggedError()( "ProviderAdapterInterruptError", { @@ -394,6 +407,7 @@ export const ProviderAdapterV2Error = Schema.Union([ ProviderAdapterTurnStartError, ProviderAdapterSteerRunUnsupportedError, ProviderAdapterSteerRunError, + ProviderAdapterSteerRunDeferredError, ProviderAdapterInterruptError, ProviderAdapterRuntimeRequestResponseError, ProviderAdapterEventStreamError, @@ -443,6 +457,8 @@ export interface ProviderAdapterV2SteerInput { readonly providerThread: OrchestrationV2ProviderThread; readonly providerTurnId: ProviderTurnId; readonly message: ProviderAdapterV2TurnMessage; + /** Rechecks durable delivery ownership before a delayed boundary handoff. */ + readonly resolveMessageText?: Effect.Effect; } export interface ProviderAdapterV2InterruptInput { From 5d42ba62b94de4bb378511d9a90be41448f2b5cd Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:47:57 -0700 Subject: [PATCH 05/23] test(server): agent updates queue behind tool-interrupting turns Split the mirrored four-case steering test into one unsafe-queue case (agent auto and explicit steer) and two controls (human steer on an interrupting session, agent auto on a safe native session). Each case goes through the real ThreadManagementService send path and drains the effect worker; the queued case also proves the update arrives as the next turn after the active one completes. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .../SteeringCompletion.integration.test.ts | 108 +++++++++++------- 1 file changed, 66 insertions(+), 42 deletions(-) diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 15a4851be2898..2988ac994e9a5 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -423,8 +423,10 @@ const composerSelection = { const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* ( name: string, - canRestart = false, - activeSteeringInterruptsTools = false, + { + canRestart = false, + activeSteeringInterruptsTools = false, + }: { canRestart?: boolean; activeSteeringInterruptsTools?: boolean } = {}, ) { const cwd = yield* checkpointWorkspace(name); const events = yield* Queue.unbounded(); @@ -574,51 +576,47 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* return { events, started, steered, layer, startFirstTurn }; }); -it.effect.each([ - { actor: "agent", mode: "auto", interruptsTools: true }, - { actor: "agent", mode: "steer", interruptsTools: true }, - { actor: "user", mode: "steer", interruptsTools: true }, - { actor: "agent", mode: "auto", interruptsTools: false }, -] as const)( - "delivers $actor $mode messages safely (interruptsTools=$interruptsTools)", - ({ actor, mode, interruptsTools }) => +const sendUpdate = Effect.fn("sendUpdate")(function* ( + threadId: ThreadId, + createdBy: "agent" | "user", + mode: "auto" | "steer", +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threads = yield* ThreadManagement.ThreadManagementService; + const result = yield* threads.sendToThread({ + projectId: (yield* orchestrator.getThreadProjection(threadId)).thread.projectId, + commandId: CommandId.make("update"), + threadId, + messageId: MessageId.make("message:update"), + text: "progress update", + attachments: [], + mode, + createdBy, + creationSource: createdBy === "agent" ? "mcp" : "web", + }); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + return result; +}); + +it.effect.each(["auto", "steer"] as const)( + "queues an agent %s update behind a turn whose steering cancels tools", + (mode) => Effect.scoped( Effect.gen(function* () { const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( - `agent-steering-${actor}-${mode}-${interruptsTools}`, - false, - interruptsTools, + `agent-update-queued-${mode}`, + { activeSteeringInterruptsTools: true }, ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const threads = yield* ThreadManagement.ThreadManagementService; const threadId = yield* startFirstTurn; const first = started[0]!; - const messageId = MessageId.make("message:update"); - const result = yield* threads.sendToThread({ - projectId: (yield* orchestrator.getThreadProjection(threadId)).thread.projectId, - commandId: CommandId.make("update"), - threadId, - messageId, - text: "progress update", - attachments: [], - mode, - createdBy: actor, - creationSource: actor === "agent" ? "mcp" : "web", - }); - yield* worker.drain(); - const queued = actor === "agent" && interruptsTools; - assert.equal(result.delivery, queued ? "queued" : "steered"); - assert.deepEqual(steered, queued ? [] : ["progress update"]); - assert.equal(started.length, 1); + const result = yield* sendUpdate(threadId, "agent", mode); + assert.equal(result.delivery, "queued"); + assert.deepEqual(steered, []); const pending = yield* orchestrator.getThreadProjection(threadId); assert.equal(pending.runs.find((run) => run.id === first.runId)?.status, "running"); - if (!queued) return; - assert.equal( - pending.runs.find((run) => run.userMessageId === messageId)?.status, - "queued", - ); const completed = yield* orchestrator.streamDomainEvents.pipe( Stream.filter( @@ -651,12 +649,38 @@ it.effect.each([ yield* worker.drain(); yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); - assert.equal(started.length, 2); - assert.equal(started[1]?.message.messageId, messageId); - assert.equal(started[1]?.message.text, "progress update"); + assert.deepEqual( + started.map((input) => input.message.messageId), + [MessageId.make("message:first"), MessageId.make("message:update")], + ); assert.deepEqual(steered, []); - const final = yield* orchestrator.getThreadProjection(threadId); - assert.equal(final.runs.find((run) => run.id === first.runId)?.status, "completed"); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect.each([ + { createdBy: "user", activeSteeringInterruptsTools: true }, + { createdBy: "agent", activeSteeringInterruptsTools: false }, +] as const)( + "steers an update from $createdBy when steering is allowed (interruptsTools=$activeSteeringInterruptsTools)", + ({ createdBy, activeSteeringInterruptsTools }) => + Effect.scoped( + Effect.gen(function* () { + const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `update-steered-${createdBy}`, + { activeSteeringInterruptsTools }, + ); + yield* Effect.gen(function* () { + const threadId = yield* startFirstTurn; + const result = yield* sendUpdate( + threadId, + createdBy, + createdBy === "agent" ? "auto" : "steer", + ); + assert.equal(result.delivery, "steered"); + assert.deepEqual(steered, ["progress update"]); + assert.equal(started.length, 1); }).pipe(Effect.provide(layer)); }), ), @@ -669,7 +693,7 @@ it.effect.each([{ canRestart: false }, { canRestart: true }])( Effect.gen(function* () { const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( `steering-selection-change-${canRestart}`, - canRestart, + { canRestart }, ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; From ca5ee825faa0c2455ef938d3945f25dd678febc4 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:04:38 -0700 Subject: [PATCH 06/23] fix(server): agent updates after Stop are rejected, not queued An agent auto/steer send to a tool-interrupting session was converted to a queued run before the Stop guard could reject it. With no earlier queued runs the new run carried no hold, so it started on its own once the stopped provider terminalized. Skip the queue conversion when Stop already reached the target run so the existing stopping rejection stays authoritative. Co-Authored-By: Claude Fable 5.1 --- .../src/orchestration-v2/Orchestrator.ts | 7 +++- .../SteeringCompletion.integration.test.ts | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 5559c4a66578b..63f8189393070 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -4837,7 +4837,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio turn.status === "completed")) ) { dispatchMode = { type: "start_immediately" }; - } else if (command.createdBy === "agent") { + } else if ( + command.createdBy === "agent" && + // After Stop the steer must still fail below; queueing it instead would + // start the update on its own once the provider stops. + !(yield* stopReachedRun(command, command.threadId, targetRunId)) + ) { const providerThread = projection.providerThreads.find( (candidate) => candidate.id === target?.providerThreadId, ); diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 6a7825b43b9b8..c3c8d6182c20c 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -1054,3 +1054,39 @@ it.effect("queues a plain send behind the stopped queue and releases its hold", }), ), ); + +it.effect.each(["auto", "steer"] as const)( + "rejects an agent %s update after Stop instead of queueing it behind the stopped run", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `agent-update-after-stop-${mode}`, + { activeSteeringInterruptsTools: true }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop"), + threadId, + runId: started[0]!.runId, + holdQueue: true, + }); + // Nothing was queued before Stop, so a queued update would have no hold + // and would start on its own once the provider stops. + const error = yield* sendUpdate(threadId, "agent", mode).pipe(Effect.flip); + yield* worker.drain(); + + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(error._tag, "OrchestratorDispatchError"); + assert.include(String(error.cause), "stopping"); + assert.deepEqual(steered, []); + assert.equal(started.length, 1); + assert.isFalse(projection.runs.some((run) => run.status === "queued")); + }).pipe(Effect.provide(layer)); + }), + ), +); From 78ad2e59a57abd87c585a574c1c260835bed4f61 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 18 Oct 2026 07:13:15 -0400 Subject: [PATCH 07/23] fix(server): agent messages no longer cancel Claude tools Deliver Claude agent updates through root PostToolBatch and Stop hook context without interrupting running tools. Defer to the durable next-turn queue after one minute without a boundary, or when the update contains attachments. Let human steering and immediate controls pass a waiting boundary delivery. Revoke pending updates on Stop and task disposal, and refresh completion pointers when only part of a task cohort remains unacknowledged. Keep native human steering and other providers' delivery behavior unchanged. Exercise the real send service, hook adapter, outbox ordering and ownership races with deterministic regression coverage. Upstream-PR: https://github.com/pingdotgg/t3code/pull/21158 Co-Authored-By: GPT-6.1 Sol at high effort, implementation and SDK research Co-Authored-By: GPT-6.1 Sol at xhigh effort, independent review --- ...OrchestratorMcpToolkit.integration.test.ts | 10 +- .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../Adapters/ClaudeAdapterV2.test.ts | 132 +++++ .../Adapters/ClaudeAdapterV2.ts | 147 +++++- ...laudeAutomaticDelivery.integration.test.ts | 52 +- .../src/orchestration-v2/EffectOutbox.ts | 24 +- .../src/orchestration-v2/EffectWorker.ts | 30 +- apps/server/src/orchestration-v2/EventSink.ts | 9 +- .../FoundationPersistence.test.ts | 300 ++++++++++++ .../orchestration-v2/NotificationMailbox.ts | 17 + .../src/orchestration-v2/Orchestrator.ts | 155 ++++-- .../ProviderTurnControlService.ts | 150 +++++- .../SteeringCompletion.integration.test.ts | 452 +++++++++++++++++- packages/contracts/src/orchestrationV2.ts | 3 + .../src/server/ProviderAdapter.ts | 16 + 15 files changed, 1382 insertions(+), 117 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index c7b5fc245094f..6093680306242 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -561,7 +561,15 @@ describe("orchestrator MCP toolkit", () => { makeDeterministicAdapter({ instanceId: claudeInstanceId, driver: ProviderDriverKind.make("claudeAgent"), - capabilities: ClaudeProviderCapabilitiesV2, + capabilities: { + ...ClaudeProviderCapabilitiesV2, + turns: { + ...ClaudeProviderCapabilitiesV2.turns, + // This deterministic adapter steers without interrupting tools. + // Real Claude uses boundary hooks, covered by SteeringCompletion. + activeSteeringInterruptsTools: false, + }, + }, capturedTurns, shouldComplete: (turn) => turn.message.text !== cancellationPrompt, terminalGate: (turn) => diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index f0dabc0260e37..9060160a8e42f 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -234,7 +234,7 @@ export const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, steers a fully active turn, or queues behind a turn that is not yet steerable. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", + "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. Claude agent updates wait for a safe tool-batch or answer boundary for up to one minute, then queue a separate turn if no boundary arrives. Other tool-interrupting sessions queue immediately. delivery reports dispatch acceptance, not model processing. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 447272221f650..025da377dc9a2 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2669,6 +2669,138 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + it.effect.each([ + "PostToolBatch", + "slow boundary", + "Stop", + "timeout", + "cancelled", + "completed", + "withdrawn", + "refreshed", + "aborted", + ] as const)("delivers agent messages at a safe boundary or defers them (%s)", (scenario) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ close: Queue.shutdown }); + const allocator = yield* IdAllocator.IdAllocatorV2; + const hookAbort = new AbortController(); + const attemptId = RunAttemptId.make("attempt-agent-boundary"); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId, + text: "Work through the task.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + const turn = { + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: allocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + }; + const steering = yield* harness.runtime + .steerTurn({ + ...turn, + ...(scenario === "withdrawn" || scenario === "refreshed" || scenario === "aborted" + ? { + resolveMessageText: Effect.sync(() => { + if (scenario === "aborted") hookAbort.abort(); + return scenario === "withdrawn" ? undefined : "Remaining task completed."; + }), + } + : {}), + message: { + createdBy: "agent", + creationSource: "mcp", + messageId: MessageId.make("agent-boundary-update"), + text: "The review found a missing check.", + attachments: [], + }, + }) + .pipe(Effect.forkScoped); + // Flush runnable fibers; the steer is now waiting on its boundary. + yield* TestClock.adjust("0 millis"); + const event = scenario === "Stop" ? "Stop" : "PostToolBatch"; + const hook = harness.getOpenedOptions()?.hooks?.[event]?.[0]?.hooks[0]; + assert.isDefined(hook); + const boundary = + event === "Stop" + ? { hook_event_name: "Stop" as const, stop_hook_active: false } + : { hook_event_name: "PostToolBatch" as const, tool_calls: [] }; + const hookInput = { + ...boundary, + session_id: WAKE_NATIVE_SESSION, + transcript_path: "transcript.jsonl", + cwd: "/repo", + }; + const callHook = (child = false, signal = hookAbort.signal) => + Effect.promise(() => + hook!({ ...hookInput, ...(child ? { agent_id: "child" } : {}) }, undefined, { + signal, + }), + ); + assert.deepEqual(yield* callHook(true), {}); + if (scenario === "timeout") { + yield* TestClock.adjust("1 minute"); + } else if (scenario === "cancelled") { + yield* harness.runtime.interruptTurn(turn); + } else if (scenario === "completed") { + yield* harness.offerAndWait( + makeResultFrame({ uuid: "boundary-completed", result: "Done" }), + ); + } else { + if (scenario === "slow boundary") yield* TestClock.adjust("40 seconds"); + const output = yield* callHook(); + assert.deepEqual( + output, + scenario === "withdrawn" || scenario === "aborted" + ? {} + : { + hookSpecificOutput: { + hookEventName: event, + additionalContext: + scenario === "refreshed" + ? "New message from another agent:\nRemaining task completed." + : "New message from another agent:\nThe review found a missing check.", + }, + }, + ); + if (scenario === "aborted") { + assert.deepEqual(yield* callHook(false, new AbortController().signal), { + hookSpecificOutput: { + hookEventName: event, + additionalContext: "New message from another agent:\nRemaining task completed.", + }, + }); + } + } + if ( + scenario === "timeout" || + scenario === "completed" || + scenario === "cancelled" || + scenario === "withdrawn" + ) { + const error = yield* Fiber.join(steering).pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterSteerRunDeferredError"); + } else { + yield* Fiber.join(steering); + } + assert.deepEqual(yield* callHook(), {}); + assert.equal(harness.offeredMessages.length, 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect.each([ { isError: false, title: "Check weather" }, { isError: true, title: "Check weather" }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index e969520bd05ee..9fdd29e6cc1d3 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -26,6 +26,7 @@ import { type EffortLevel, type ForkSessionOptions, type ForkSessionResult, + type HookCallback, getSubagentMessages, query, type Options as ClaudeQueryOptions, @@ -319,6 +320,7 @@ export const ClaudeProviderCapabilitiesV2 = { supportsInterrupt: true, supportsActiveSteering: true, activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, supportsSteeringByInterruptRestart: false, supportsQueuedMessages: true, terminalStatusQuality: "strong", @@ -3016,6 +3018,15 @@ function formatClaudeUsageLimitWait(waitMs: number): string { return minutes === 0 ? `${hours}h` : `${hours}h ${minutes}m`; } +type ClaudeAgentSteerOutcome = "delivered" | "deferred" | "cancelled"; + +interface PendingClaudeAgentSteer { + readonly text: string; + readonly completion: Deferred.Deferred; + readonly resolveMessageText?: ProviderAdapter.ProviderAdapterV2SteerInput["resolveMessageText"]; + outcome?: ClaudeAgentSteerOutcome; +} + interface ActiveClaudeTurnContext { readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly nativeThreadId: string; @@ -3073,12 +3084,26 @@ interface ActiveClaudeTurnContext { readonly subagentsByToolUseId: Map; readonly subagentNodesByTaskId: Map; readonly promptUuid: string | null; + readonly pendingAgentSteers: Map; promptEcho: "pending" | "confirmed"; promptStarted: boolean; gatedFramesBeforeEcho: number; readonly heldRootFrames: Array; } +// Claim and acknowledge synchronously so a boundary racing the timeout cannot +// both inject a message and return it to the durable queue. +function settleClaudeAgentSteers( + context: ActiveClaudeTurnContext, + outcome: ClaudeAgentSteerOutcome, +): void { + for (const pending of context.pendingAgentSteers.values()) { + pending.outcome = outcome; + Deferred.doneUnsafe(pending.completion, Effect.succeed(outcome)); + } + context.pendingAgentSteers.clear(); +} + interface ActiveClaudeProviderRetry { readonly retry: OrchestrationV2ProviderRetry; readonly failure: OrchestrationV2ProviderFailure; @@ -6066,6 +6091,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( readonly threadDisposition?: "reusable" | "broken"; readonly result?: SDKResultMessage; }) { + settleClaudeAgentSteers( + input.context, + input.status === "interrupted" || input.status === "cancelled" + ? "cancelled" + : "deferred", + ); yield* reasoningDeltas.flushTurn(input.context.nativeTurnId); if (input.context.compaction?.status === "running") { yield* emitProviderEvent({ @@ -8644,11 +8675,68 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( onUserDialog, supportedDialogKinds: ["resume_return"], }); + let boundaryQuery: ClaudeAgentSdkQuerySession | undefined; + const agentMessageBoundary: HookCallback = (hookInput, _toolUseId, options) => + runPromise( + Effect.gen(function* () { + const context = yield* Ref.get(activeTurn); + const live = yield* Ref.get(queryContext); + if ( + options.signal.aborted || + hookInput.agent_id !== undefined || + context === null || + live?.query !== boundaryQuery || + live?.stopping || + context.nativeThreadId !== hookInput.session_id || + (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) || + (hookInput.hook_event_name !== "PostToolBatch" && + hookInput.hook_event_name !== "Stop") + ) { + return {}; + } + // Resolve before claiming so an aborted hook consumes no partial batch. + const pendingSteers = yield* Effect.forEach( + Array.from(context.pendingAgentSteers), + ([id, pending]) => + (pending.resolveMessageText ?? Effect.succeed(pending.text)).pipe( + Effect.map((text) => ({ id, pending, text })), + ), + ); + const liveTurn = + (yield* Ref.get(activeTurn)) === context && + (yield* Ref.get(queryContext)) === live && + !(yield* Ref.get(interruptedTurns)).has(context.providerTurnId); + if (options.signal.aborted) return {}; + const messages: string[] = []; + for (const { id, pending, text } of pendingSteers) { + // The deadline or cancellation can win while storage is read. + if (context.pendingAgentSteers.get(id) !== pending) continue; + pending.outcome = text !== undefined && liveTurn ? "delivered" : "cancelled"; + context.pendingAgentSteers.delete(id); + Deferred.doneUnsafe(pending.completion, Effect.succeed(pending.outcome)); + if (pending.outcome === "delivered" && text !== undefined) messages.push(text); + } + return messages.length === 0 + ? {} + : { + hookSpecificOutput: { + hookEventName: hookInput.hook_event_name, + additionalContext: `New message from another agent:\n${messages.join("\n\n")}`, + }, + }; + }), + ); const querySession = yield* queryRunner .open({ threadId: turnInput.threadId, providerSessionId: input.providerSessionId, - options: queryOptions, + options: { + ...queryOptions, + hooks: { + PostToolBatch: [{ hooks: [agentMessageBoundary] }], + Stop: [{ hooks: [agentMessageBoundary] }], + }, + }, }) .pipe( Effect.onError(() => @@ -8662,6 +8750,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : Effect.void, ), ); + boundaryQuery = querySession; // Marked only after a successful open: a failed create must not // leave the runtime believing the native session exists, or the // retry would resume a session that was never created. @@ -8834,6 +8923,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( latestAssistantRateLimited: false, compactionFailure: null, steeredSegmentUsage: [], + pendingAgentSteers: new Map(), subagentsByTaskId: new Map(), subagentsByToolUseId: new Map(), subagentNodesByTaskId: new Map(), @@ -9025,6 +9115,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( next.add(turnInput.providerTurnId); return next; }); + settleClaudeAgentSteers(currentTurn, "cancelled"); yield* existing.query.interrupt; yield* existing.query.close.pipe(Effect.ignore); const closed = yield* Deferred.await(existing.closed).pipe( @@ -9150,6 +9241,43 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( detail: `Claude provider turn ${turnInput.providerTurnId} is not the active turn.`, }); } + if (turnInput.message.createdBy === "agent") { + // `priority: next` also folds after tools, but can start an unowned + // native turn if the answer ends first. Hooks keep delivery owned + // by this turn. Allow ordinary commands to finish before falling + // back to T3's queue. + if (turnInput.message.attachments.length > 0) { + return yield* new ProviderAdapter.ProviderAdapterSteerRunDeferredError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + }); + } + const pending: PendingClaudeAgentSteer = { + text: turnInput.message.text, + completion: yield* Deferred.make(), + ...(turnInput.resolveMessageText === undefined + ? {} + : { resolveMessageText: turnInput.resolveMessageText }), + }; + currentTurn.pendingAgentSteers.set(turnInput.message.messageId, pending); + yield* Deferred.await(pending.completion).pipe( + Effect.timeoutOption("1 minute"), + Effect.ensuring( + Effect.sync(() => { + currentTurn.pendingAgentSteers.delete(turnInput.message.messageId); + }), + ), + ); + if (pending.outcome === "delivered") return; + // A restart can interrupt this wait too. Let the caller revoke + // stopped/disposed input or retain still-owned input in its queue. + return yield* new ProviderAdapter.ProviderAdapterSteerRunDeferredError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + }); + } const userMessage = yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( turnInput.message.text, @@ -9183,14 +9311,15 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, (effect, turnInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterSteerRunError({ - driver: CLAUDE_PROVIDER, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProviderAdapterSteerRunDeferredError" + ? cause + : new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: CLAUDE_PROVIDER, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + cause, + }), ), ), ); diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index 9bec96d31daf1..2c384f5eebc2b 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -1,4 +1,4 @@ -import type { SDKMessage, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; +import type { HookCallback, SDKMessage, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -24,6 +24,7 @@ import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; @@ -139,6 +140,7 @@ it.effect.each(deliveries)( const offers: SDKUserMessage[] = []; const nativeQueue: SDKUserMessage[] = []; const batchAbort = new AbortController(); + let postToolBatch: HookCallback | undefined; const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ instanceId: modelSelection.instanceId, settings, @@ -150,8 +152,9 @@ it.effect.each(deliveries)( idAllocator: yield* IdAllocator.IdAllocatorV2, queryRunner: { allocateSessionId: Effect.succeed(sessionId), - open: () => - Effect.succeed({ + open: ({ options }) => { + postToolBatch = options.hooks?.PostToolBatch?.[0]?.hooks[0]; + return Effect.succeed({ messages: Stream.fromQueue(sdkMessages), offer: (message) => Effect.sync(() => { @@ -170,7 +173,8 @@ it.effect.each(deliveries)( stopTask: () => Effect.void, interrupt: Effect.die("automatic delivery must never interrupt"), close: Effect.void, - }), + }); + }, forkSession: () => Effect.die("unused"), subagentLaunchToolUseId: () => Effect.succeed(null), assertComplete: Effect.void, @@ -390,7 +394,10 @@ it.effect.each(deliveries)( assert.ok(steer?.type === "user_message"); assert.equal(steer.inputIntent, "promoted_queued_to_steer"); } - yield* worker.drain(); + const boundaryDelivery = + delivery === "child completion" ? yield* worker.drain().pipe(Effect.forkScoped) : null; + if (boundaryDelivery !== null) yield* TestClock.adjust("0 millis"); + else yield* worker.drain(); const explicitSteer = delivery === "user steering" || delivery === "promoted queued message"; assert.equal(batchAbort.signal.aborted, explicitSteer); @@ -404,13 +411,31 @@ it.effect.each(deliveries)( yield* Queue.offer(sdkMessages, toolResult("build", false)); for (const id of statusIds) yield* Queue.offer(sdkMessages, toolResult(id, batchAbort.signal.aborted)); + if (boundaryDelivery !== null) { + assert.isDefined(postToolBatch); + const output = yield* Effect.promise(() => + postToolBatch!( + { + hook_event_name: "PostToolBatch", + session_id: sessionId, + cwd, + transcript_path: "transcript.jsonl", + tool_calls: [], + }, + undefined, + { signal: batchAbort.signal }, + ), + ); + assert.include(JSON.stringify(output), "task_status"); + yield* Fiber.join(boundaryDelivery); + } if (explicitSteer) yield* Queue.offer(sdkMessages, result("aborted", true)); yield* Queue.offer(sdkMessages, result("completed", false)); yield* Fiber.join(finished); yield* worker.drain(); yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); - if (!explicitSteer) { + if (delivery === "scheduled message") { const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; const noticeFinished = yield* watch( (event) => @@ -418,18 +443,9 @@ it.effect.each(deliveries)( event.payload.id === queued.id && event.payload.status === "waiting", ); - const delivered = - delivery === "child completion" - ? yield* watch( - (event) => - event.type === "subagent.updated" && - event.payload.completionDelivery?.state === "delivered", - ) - : null; yield* Queue.offer(sdkMessages, result("notice-completed", false)); yield* Fiber.join(noticeFinished); yield* worker.drain(); - if (delivered !== null) yield* Fiber.join(delivered); } const after = yield* orchestrator.getThreadProjection(threadId); const reads = after.turnItems.filter((item) => @@ -441,13 +457,13 @@ it.effect.each(deliveries)( assert.equal(read.toolNonExecutionKind, explicitSteer ? "cancelled" : undefined); assert.equal(read.type === "dynamic_tool" && read.output === refusal, explicitSteer); } - assert.equal(offers.length, 2); + assert.equal(offers.length, delivery === "child completion" ? 1 : 2); assert.equal( offers.filter((offer) => offer.priority === "now").length, explicitSteer ? 1 : 0, ); if (!explicitSteer) { - assert.equal(after.runs.length, 2); + assert.equal(after.runs.length, delivery === "child completion" ? 1 : 2); if (delivery === "child completion") { assert.equal(after.messages.filter((message) => message.id === messageId).length, 1); assert.equal(after.subagents[0]?.completionDelivery?.state, "delivered"); @@ -461,7 +477,7 @@ it.effect.each(deliveries)( yield* orchestrator.recoverDelegatedTasks; yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); - assert.equal(offers.length, 2); + assert.equal(offers.length, delivery === "child completion" ? 1 : 2); }).pipe( Effect.provide( ProviderReplayHarness.layerWithRegistry( diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index cb41dc7365123..91effb07d25d7 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -56,6 +56,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ providerThreadId: ProviderThreadId, providerTurnId: ProviderTurnId, messageId: MessageId, + waitForBoundary: Schema.optional(Schema.Literal(true)), }), Schema.Struct({ type: Schema.Literal("provider-turn.restart"), @@ -210,6 +211,8 @@ export interface EffectOutboxV2Shape { readonly cancelUnsettled: (input: { readonly threadId: ThreadId; readonly effectTypes: ReadonlyArray; + readonly messageIds?: ReadonlyArray; + readonly exceptCommandId?: CommandId; readonly reason: string; }) => Effect.Effect, EffectOutboxError>; readonly signalCancellations: (effectIds: ReadonlyArray) => Effect.Effect; @@ -315,6 +318,7 @@ export const layer: Layer.Layer = La ).pipe(Effect.asVoid); // An earlier effect waiting out a retry backoff still blocks later ones, so // a turn cannot start while a failed rollback is about to restore files. + // Boundary steers yield only to human steering and immediate turn controls. const claimableCandidatePredicate = ( availableBefore?: string, excludeRestartContinuations = false, @@ -330,6 +334,20 @@ export const layer: Layer.Layer = La SELECT 1 FROM orchestration_v2_effect_outbox AS active WHERE active.thread_id = candidate.thread_id + AND NOT ( + active.rowid < candidate.rowid + AND active.effect_type = 'provider-turn.steer' + AND COALESCE(json_extract(active.payload_json, '$.waitForBoundary'), 0) = 1 + AND ( + candidate.effect_type IN ( + 'provider-turn.interrupt', 'provider-turn.restart', 'runtime-request.respond' + ) + OR ( + candidate.effect_type = 'provider-turn.steer' + AND COALESCE(json_extract(candidate.payload_json, '$.waitForBoundary'), 0) != 1 + ) + ) + ) AND ( active.status = 'running' OR ( @@ -438,9 +456,9 @@ export const layer: Layer.Layer = La : new EffectOutboxError({ operation: "list", cause }), ), ), - cancelUnsettled: ({ threadId, effectTypes, reason }) => + cancelUnsettled: ({ threadId, effectTypes, messageIds, exceptCommandId, reason }) => Effect.gen(function* () { - if (effectTypes.length === 0) return []; + if (effectTypes.length === 0 || messageIds?.length === 0) return []; const now = DateTime.formatIso(yield* DateTime.now); const rows = yield* sql<{ readonly effect_id: string }>` UPDATE orchestration_v2_effect_outbox @@ -454,6 +472,8 @@ export const layer: Layer.Layer = La WHERE thread_id = ${threadId} AND status IN ('pending', 'running') AND effect_type IN ${sql.in(effectTypes)} + AND ${messageIds === undefined ? sql`1 = 1` : sql`json_extract(payload_json, '$.messageId') IN ${sql.in(messageIds)}`} + AND ${exceptCommandId === undefined ? sql`1 = 1` : sql`command_id != ${exceptCommandId}`} RETURNING effect_id `; return rows.map(({ effect_id }) => effect_id); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index a44b60c2498d1..eee22cdd8f7f2 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -62,7 +62,7 @@ export function isNonRetryableProviderTurnControlFailure( export interface OrchestrationEffectExecutorV2Shape { readonly execute: ( effect: EffectOutbox.OrchestrationEffectV2, - options?: { readonly willRetry: boolean }, + options?: { readonly willRetry: boolean; readonly isCurrent?: Effect.Effect }, ) => Effect.Effect; } @@ -222,6 +222,7 @@ export const layerExecutor: Layer.Layer< case "provider-turn.steer": return providerTurnControl .steer({ + ...(options?.isCurrent === undefined ? {} : { isCurrent: options.isCurrent }), threadId: effect.threadId, providerSessionId: effect.request.providerSessionId, providerThreadId: effect.request.providerThreadId, @@ -248,15 +249,18 @@ export const layerExecutor: Layer.Layer< }); }), ), - Effect.catch((error) => + Effect.as(undefined), + Effect.catchTags({ + ProviderTurnControlSteerCancelledError: () => Effect.void, + ProviderTurnControlSteerDeferredError: (error) => Effect.succeed(error), + ProviderTurnControlError: (error) => + error.turnCompleted ? Effect.succeed(error) : Effect.fail(error), + }), + Effect.flatMap((error) => Effect.gen(function* () { - if ( - !("turnCompleted" in error) || - !error.turnCompleted || - effect.request.type !== "provider-turn.steer" - ) { - return yield* error; - } + if (error === undefined) return; + if (effect.request.type !== "provider-turn.steer") return yield* error; + const deferToQueue = error._tag === "ProviderTurnControlSteerDeferredError"; const projection = yield* threads.getThreadRecords( effect.threadId, ["messages", "runs"], @@ -275,6 +279,7 @@ export const layerExecutor: Layer.Layer< threadId: effect.threadId, messageId: message.id, text: message.text, + ...(deferToQueue ? { steeringFallbackOfRunId: run.id } : {}), ...(message.context ? { context: message.context } : {}), attachments: message.attachments, ...(message.delegatedCompletion === undefined @@ -282,7 +287,7 @@ export const layerExecutor: Layer.Layer< : { modelSelection: run.modelSelection }), dispatchMode: { type: - message.delegatedCompletion === undefined + message.delegatedCompletion === undefined && !deferToQueue ? "start_immediately" : "queue_after_active", }, @@ -724,7 +729,10 @@ export const layerWithOptions = ( if (cancelledBeforeExecution) return true; const execution = executor - .execute(effect, { willRetry: effect.attemptCount < maxAttempts }) + .execute(effect, { + willRetry: effect.attemptCount < maxAttempts, + isCurrent: wasCancelled(effect.id).pipe(Effect.map((cancelled) => !cancelled)), + }) .pipe(Effect.as("executed" as const)); const exit = yield* Effect.exit(Effect.raceFirst(execution, cancellation)).pipe( Effect.ensuring(outbox.clearCancellation(effect.id)), diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 13916b40f9dee..e050a01bdcbc4 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -10,6 +10,7 @@ import { RuntimeRequestId, NodeId, type ProjectId, + type MessageId, ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -149,6 +150,7 @@ export interface EventSinkV2Shape { readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; }; }) => Effect.Effect< { @@ -603,7 +605,11 @@ const layerBase: Layer.Layer< commandId: input.commandId, events: normalized, }); - const sequence = storedEvents.at(-1)?.sequence; + // No-op commands still commit receipts and cancellations atomically. + const sequence = + input.events.length === 0 + ? yield* eventStore.latestSequence({ threadId: input.threadId }) + : storedEvents.at(-1)?.sequence; if (sequence === undefined) { return yield* Effect.die( new Error(`Command ${input.commandId} produced no orchestration events.`), @@ -627,6 +633,7 @@ const layerBase: Layer.Layer< : yield* effectOutbox.cancelUnsettled({ threadId: input.threadId, ...input.cancelUnsettledEffects, + exceptCommandId: input.commandId, }); return { receipt, storedEvents, committed: true as const, cancelledEffectIds }; }), diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 3a50b441a84ca..453f9d8f61d7c 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -20,8 +20,10 @@ import { ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, RunAttemptId, RunId, + RuntimeRequestId, ThreadId, TurnItemId, } from "@t3tools/contracts"; @@ -1430,6 +1432,77 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); + it.effect("a no-op receipt cancels current effects once without adding events", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:noop-cancellation"); + const commandId = CommandId.make("command:noop-cancellation"); + const created = yield* sink.commitCommand({ + commandId: CommandId.make("command:noop-create"), + threadId, + commandType: "thread.create", + acceptedAt: now, + events: [ + threadCreatedEvent({ + id: "event:noop-create", + thread: makeThread(threadId, now), + now, + }), + ], + effects: [ + { + id: "effect:before-noop", + commandId: CommandId.make("command:noop-create"), + threadId, + request: { type: "terminal.cleanup" }, + }, + ], + }); + const input = { + commandId, + threadId, + commandType: "thread.stop", + acceptedAt: now, + events: [], + effects: [], + cancelUnsettledEffects: { + effectTypes: ["terminal.cleanup"], + reason: "Stopped.", + }, + } as const; + const cancellation = yield* outbox + .awaitCancellation("effect:before-noop") + .pipe(Effect.forkChild); + const stopped = yield* sink.commitCommand(input); + assert.isTrue(stopped.committed); + assert.deepEqual(stopped.storedEvents, []); + assert.equal(stopped.receipt.resultSequence, created.receipt.resultSequence); + assert.equal(stopped.cancelledEffectCount, 1); + assert.equal(Option.getOrThrow(yield* outbox.get("effect:before-noop")).status, "cancelled"); + yield* Fiber.join(cancellation); + yield* outbox.enqueue([ + { + id: "effect:after-noop", + commandId: CommandId.make("command:after-noop"), + threadId, + request: { type: "terminal.cleanup" }, + }, + ]); + const retry = yield* sink.commitCommand(input); + assert.isFalse(retry.committed); + assert.deepEqual(retry.storedEvents, []); + assert.equal(retry.cancelledEffectCount, 0); + assert.equal(Option.getOrThrow(yield* outbox.get("effect:after-noop")).status, "pending"); + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["terminal.cleanup"], + reason: "Test completed.", + }); + }), + ); + it.effect("keeps one durable effect across command retries and executes it after recovery", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; @@ -2313,6 +2386,233 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); + const steerRequest = { + type: "provider-turn.steer", + providerSessionId: ProviderSessionId.make("session:boundary-order"), + providerThreadId: ProviderThreadId.make("provider-thread:boundary-order"), + providerTurnId: ProviderTurnId.make("provider-turn:boundary-order"), + messageId: MessageId.make("message:boundary-order"), + } as const; + const boundaryOrderCandidates = [ + { request: { ...steerRequest }, bypass: true }, + { request: { ...steerRequest, type: "provider-turn.interrupt" }, bypass: true }, + { + request: { + ...steerRequest, + type: "provider-turn.restart", + interruptedAttemptId: RunAttemptId.make("attempt:boundary-order"), + runId: RunId.make("run:boundary-order"), + }, + bypass: true, + }, + { + request: { + type: "runtime-request.respond", + providerSessionId: steerRequest.providerSessionId, + requestId: RuntimeRequestId.make("request:boundary-order"), + }, + bypass: true, + }, + { request: { ...steerRequest, waitForBoundary: true }, bypass: false }, + { + request: { type: "provider-turn.start", runId: RunId.make("run:boundary-order") }, + bypass: false, + }, + { + request: { + type: "checkpoint.capture", + runId: RunId.make("run:boundary-order"), + scopeId: CheckpointScopeId.make("scope:boundary-order"), + }, + bypass: false, + }, + { + request: { + type: "provider-thread.rollback", + providerThreadId: steerRequest.providerThreadId, + checkpointId: CheckpointId.make("checkpoint:boundary-order"), + scopeId: CheckpointScopeId.make("scope:boundary-order"), + }, + bypass: false, + }, + { request: { type: "terminal.cleanup" }, bypass: false }, + ] satisfies ReadonlyArray<{ + request: EffectOutbox.OrchestrationEffectRequestV2; + bypass: boolean; + }>; + + it.effect.each( + (["running", "pending"] as const).flatMap((status) => + boundaryOrderCandidates.map(({ request, bypass }) => ({ + status, + request, + bypass, + name: `${bypass ? "allows" : "blocks"} ${request.type}${"waitForBoundary" in request ? " at a boundary" : ""} behind a ${status} boundary steer`, + })), + ), + )("boundary steer ordering: $name", ({ status, request, bypass }) => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const commandId = CommandId.make("command:boundary-order"); + const threadId = ThreadId.make("thread:boundary-order"); + const workerId = "boundary-worker"; + const future = DateTime.add(now, { seconds: 5 }); + yield* outbox.enqueue([ + { + id: "boundary", + commandId, + threadId, + request: { ...steerRequest, waitForBoundary: true }, + }, + ]); + const boundary = Option.getOrThrow( + yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }), + ); + assert.deepEqual(boundary.request, { ...steerRequest, waitForBoundary: true }); + if (status === "pending") { + yield* outbox.retry({ + effectId: boundary.id, + workerId, + error: "Waiting for a boundary.", + delayMs: 5_000, + }); + } + yield* outbox.enqueue([{ id: "candidate", commandId, threadId, request }]); + const nextAt = yield* outbox.nextClaimableAt; + if (bypass) { + assert.equal(DateTime.formatIso(Option.getOrThrow(nextAt)), DateTime.formatIso(now)); + } else if (status === "pending") { + assert.equal(DateTime.formatIso(Option.getOrThrow(nextAt)), DateTime.formatIso(future)); + } else { + assert.isTrue(Option.isNone(nextAt)); + } + const claimed = yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); + assert.equal(Option.getOrUndefined(claimed)?.id, bypass ? "candidate" : undefined); + if (bypass) yield* outbox.succeed({ effectId: "candidate", workerId }); + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["provider-turn.steer"], + messageIds: [steerRequest.messageId], + reason: "Boundary settled.", + }); + if (!bypass && request.type !== "provider-turn.steer") { + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "candidate", + ); + } + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + + it.effect("keeps immediate controls behind ordinary steers and intervening effects", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const commandId = CommandId.make("command:ordinary-steer-order"); + const threadId = ThreadId.make("thread:ordinary-steer-order"); + const workerId = "ordinary-steer-worker"; + yield* outbox.enqueue([{ id: "ordinary", commandId, threadId, request: steerRequest }]); + yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }); + yield* outbox.enqueue([ + { + id: "boundary", + commandId, + threadId, + request: { ...steerRequest, waitForBoundary: true }, + }, + { id: "cleanup", commandId, threadId, request: { type: "terminal.cleanup" } }, + { + id: "interrupt", + commandId, + threadId, + request: { ...steerRequest, type: "provider-turn.interrupt" }, + }, + ]); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "ordinary", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "boundary", + ); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "boundary", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "cleanup", + ); + assert.isTrue(Option.isNone(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 }))); + yield* outbox.succeed({ effectId: "cleanup", workerId }); + assert.equal( + Option.getOrThrow(yield* outbox.claimNext({ workerId, leaseDurationMs: 30_000 })).id, + "interrupt", + ); + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + + it.effect( + "cancels selected messages while preserving replacement commands and other effects", + () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = ThreadId.make("thread:message-cancellation"); + const commandId = CommandId.make("command:message-cancellation"); + const replacementCommandId = CommandId.make("command:message-cancellation:replacement"); + const messageId = MessageId.make("message:cancelled"); + yield* outbox.enqueue([ + { id: "running", commandId, threadId, request: { ...steerRequest, messageId } }, + ]); + assert.equal( + Option.getOrThrow( + yield* outbox.claimNext({ workerId: "cancel-worker", leaseDurationMs: 30_000 }), + ).id, + "running", + ); + yield* outbox.enqueue([ + { id: "pending", commandId, threadId, request: { ...steerRequest, messageId } }, + { id: "other-message", commandId, threadId, request: steerRequest }, + { + id: "replacement", + commandId: replacementCommandId, + threadId, + request: { ...steerRequest, messageId }, + }, + { + id: "other-thread", + commandId, + threadId: ThreadId.make("thread:message-cancellation:other"), + request: { ...steerRequest, messageId }, + }, + { id: "cleanup", commandId, threadId, request: { type: "terminal.cleanup" } }, + ]); + const cancellation = { + threadId, + effectTypes: ["provider-turn.steer", "terminal.cleanup"], + exceptCommandId: replacementCommandId, + reason: "Message removed.", + } as const; + assert.deepEqual(yield* outbox.cancelUnsettled({ ...cancellation, messageIds: [] }), []); + assert.deepEqual( + (yield* outbox.cancelUnsettled({ ...cancellation, messageIds: [messageId] })).toSorted(), + ["pending", "running"], + ); + for (const id of ["other-message", "replacement", "other-thread", "cleanup"]) { + assert.equal(Option.getOrThrow(yield* outbox.get(id)).status, "pending"); + } + assert.deepEqual((yield* outbox.cancelUnsettled(cancellation)).toSorted(), [ + "cleanup", + "other-message", + ]); + assert.deepEqual( + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["provider-turn.steer"], + reason: "Legacy cancellation.", + }), + ["replacement"], + ); + }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))), + ); + it.effect("prunes settled effects past retention in batches and keeps the rest", () => Effect.gen(function* () { const outbox = yield* EffectOutbox.EffectOutboxV2; diff --git a/apps/server/src/orchestration-v2/NotificationMailbox.ts b/apps/server/src/orchestration-v2/NotificationMailbox.ts index 2afee9749e22e..28de4bc9f1546 100644 --- a/apps/server/src/orchestration-v2/NotificationMailbox.ts +++ b/apps/server/src/orchestration-v2/NotificationMailbox.ts @@ -3,8 +3,25 @@ import { type MessageId, type OrchestrationV2ConversationMessage, type OrchestrationV2ThreadProjection, + type OrchestrationV2Subagent, } from "@t3tools/contracts"; +export function delegatedCompletionWakeDetail(taskIds: ReadonlyArray): string { + const taskList = taskIds.join(", "); + return taskIds.length === 1 + ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.` + : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`; +} + +export function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean { + return ( + status === "completed" || + status === "failed" || + status === "cancelled" || + status === "interrupted" + ); +} + export function isBufferedProviderNotification( message: OrchestrationV2ConversationMessage, ): boolean { diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 96b6bab3c033d..85ca597f4e26f 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -101,6 +101,8 @@ import { import { isBufferedProviderNotification, isUndeliveredMailboxSteer, + delegatedCompletionWakeDetail, + isTerminalDelegatedTaskStatus, } from "./NotificationMailbox.ts"; import { EventSinkV2 } from "./EventSink.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; @@ -575,22 +577,6 @@ function legacyPullRequestLink( ); } -function delegatedCompletionWakeDetail(taskIds: ReadonlyArray): string { - const taskList = taskIds.join(", "); - return taskIds.length === 1 - ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.` - : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`; -} - -function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean { - return ( - status === "completed" || - status === "failed" || - status === "cancelled" || - status === "interrupted" - ); -} - function delegatedTaskTerminalStatus( status: OrchestrationV2Run["status"], ): OrchestrationV2Subagent["status"] | null { @@ -3899,6 +3885,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly scheduledTaskId?: OrchestrationV2ConversationMessage["scheduledTaskId"]; readonly senderThreadId?: OrchestrationV2ConversationMessage["senderThreadId"]; readonly delegatedCompletion?: OrchestrationV2ConversationMessage["delegatedCompletion"]; + readonly notification?: OrchestrationV2ConversationMessage["notification"]; readonly forceRestart: boolean; }) => Effect.gen(function* () { @@ -4057,6 +4044,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...(input.delegatedCompletion === undefined ? {} : { delegatedCompletion: input.delegatedCompletion }), + ...(input.notification === undefined ? {} : { notification: input.notification }), ...(input.scheduledTaskId === undefined ? {} : { scheduledTaskId: input.scheduledTaskId }), @@ -4176,6 +4164,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio threadId: input.command.threadId, request: { type: "provider-turn.steer", + ...(input.createdBy === "agent" && + session.providerSession.capabilities.turns.supportsNonInterruptingAgentSteering === + true + ? { waitForBoundary: true as const } + : {}), providerSessionId, providerThreadId: providerThread.id, providerTurnId: providerTurn.id, @@ -4607,6 +4600,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) => Effect.gen(function* () { let projection = yield* getProjectionWithPendingEvents(command.threadId, events); + // Stop can commit after the boundary wait expires but before its fallback + // is dispatched. Do not turn that stale steer into fresh work. + if ( + command.steeringFallbackOfRunId !== undefined && + (yield* stopReachedRun(command, command.threadId, command.steeringFallbackOfRunId)) + ) { + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + return; + } if (command.manualContinuationOfRunId !== undefined) { const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId); const limited = latestRootProviderFailure(source ?? null, projection.turnItems); @@ -4832,6 +4842,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio turn.status === "completed")) ) { dispatchMode = { type: "start_immediately" }; + } else if (command.createdBy === "agent") { + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === target?.providerThreadId, + ); + if (providerThread?.providerSessionId != null) { + const session = yield* providerSessions + .get(providerThread.providerSessionId) + .pipe(mapDispatchError(command)); + if ( + Option.isSome(session) && + session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === + true && + session.value.providerSession.capabilities.turns + .supportsNonInterruptingAgentSteering !== true + ) { + // Claude steering cancels pending tools, which appear as user refusals. + // Agent updates can wait for the durable queue's next-turn boundary. + dispatchMode = { type: "queue_after_active" }; + } + } } } // Stop leaves the run reading as running until its provider stops. A plain @@ -4881,6 +4911,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (candidate) => candidate.id === requestedCompletion.parentRunId, ); const delivery = parentRun?.delegatedCompletion?.delivery; + const existingDelivery = projection.messages.find( + (message) => message.id === command.messageId, + ); + const deferredSteer = + existingDelivery !== undefined && + command.steeringFallbackOfRunId === existingDelivery.runId && + existingDelivery.delegatedCompletion?.generation === requestedCompletion.generation && + projection.runs.find((run) => run.id === existingDelivery.runId)?.userMessageId !== + command.messageId; if ( parentRun?.delegatedCompletion?.disposition !== "open" || delivery === null || @@ -4888,6 +4927,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio delivery.generation !== requestedCompletion.generation || delivery.messageId !== command.messageId || (projection.messages.some((candidate) => candidate.id === command.messageId) && + !deferredSteer && !isUndeliveredMailboxSteer(projection, command.messageId)) ) { return yield* new OrchestratorDispatchError({ @@ -4906,6 +4946,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // session's capabilities. Never interrupt/restart a turn for a notification. if ( delegatedCompletion !== undefined && + // A deferred steer is already persisted; do not attempt the hook again. + !projection.messages.some((message) => message.id === command.messageId) && delegatedCompletion.taskIds.every( (id) => projection.subagents.find((task) => task.id === id)?.completionWake === "always", ) @@ -4932,7 +4974,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ( Option.isSome(session) && session.value.providerSession.capabilities.turns.supportsActiveSteering && - session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools !== true + (session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools !== + true || + session.value.providerSession.capabilities.turns + .supportsNonInterruptingAgentSteering === true) ) { dispatchMode = { type: "steer_active", targetRunId: active.id }; } @@ -5004,6 +5049,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : (projection.runs.find((run) => run.id === dispatchMode.targetRunId) ?.modelSelection ?? modelSelection), delegatedCompletion, + ...(command.notification === undefined ? {} : { notification: command.notification }), targetRunId: dispatchMode.targetRunId, messageId: command.messageId, text: dispatchText, @@ -7838,6 +7884,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio targetRunId: command.targetRunId, messageId: queuedMessage.id, text: queuedMessage.text, + ...(queuedMessage.notification === undefined + ? {} + : { notification: queuedMessage.notification }), + ...(queuedMessage.delegatedCompletion === undefined + ? {} + : { delegatedCompletion: queuedMessage.delegatedCompletion }), attachments: queuedMessage.attachments, ...(queuedMessage.context ? { context: queuedMessage.context } : {}), createdBy: queuedMessage.createdBy, @@ -10976,6 +11028,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; }; }, OrchestratorV2Error @@ -10992,6 +11045,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio | { readonly effectTypes: ReadonlyArray; readonly reason: string; + readonly messageIds?: ReadonlyArray; } | undefined; switch (command.type) { @@ -11249,9 +11303,41 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio yield* dispatchDelegatedTaskWakePolicy(command, events); break; case "delegated_task.completion-delivery.acknowledge": - case "delegated_task.completion-delivery.dispose": + case "delegated_task.completion-delivery.dispose": { + const before = yield* getProjectionWithPendingEvents(command.parentThreadId, events); yield* dispatchDelegatedTaskCompletionDeliveryResolution(command, events); + const task = before.subagents.find((candidate) => candidate.id === command.taskId); + const owner = before.runs.find((run) => run.id === task?.runId); + const after = yield* getProjectionWithPendingEvents(command.parentThreadId, events); + const remaining = after.runs.find((run) => run.id === owner?.id)?.delegatedCompletion + ?.delivery; + const messageIds = before.messages + .filter((message) => { + const source = message.notification?.source; + return ( + message.id.startsWith("message:delegated-task-question:") && + source?.kind === "delegated_task" && + source.taskIds.includes(command.taskId) + ); + }) + .map((message) => message.id); + const previousDelivery = owner?.delegatedCompletion?.delivery; + if ( + previousDelivery != null && + (remaining == null || + remaining.taskIds.length === 0 || + remaining.messageId !== previousDelivery.messageId) + ) { + messageIds.push(previousDelivery.messageId); + } + if (messageIds.length > 0) + cancelUnsettledEffects = { + effectTypes: ["provider-turn.steer"], + messageIds, + reason: "The delegated notification is no longer owned by this task.", + }; break; + } case "thread.created.record": yield* dispatchCreatedThreadRecord(command, events); break; @@ -11261,6 +11347,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio default: return yield* dispatchUnsupported(command); } + if (command.type === "run.interrupt" || command.type === "thread.stop") { + cancelUnsettledEffects = { + effectTypes: [...(cancelUnsettledEffects?.effectTypes ?? []), "provider-turn.steer"], + reason: cancelUnsettledEffects?.reason ?? "The user stopped this thread.", + }; + } return { events: yield* Ref.get(events), effects: yield* Ref.get(effects), @@ -11393,33 +11485,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); - if (plan.events.length === 0) { - // A settle that ended nothing still records its receipt: a replayed Stop - // effect then finds it instead of settling work that appeared since. - const resultSequence = yield* Effect.gen(function* () { - const sequence = yield* eventSink.latestSequence({ threadId: commandThreadId(command) }); - yield* commandReceipts.insertIfAbsent({ - commandId: command.commandId, - threadId: commandThreadId(command), - commandType: command.type, - acceptedAt: yield* DateTime.now, - resultSequence: sequence, - status: "accepted", - error: null, - }); - return sequence; - }).pipe( - Effect.mapError( - (cause) => - new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause, - }), - ), - ); - return { sequence: resultSequence, storedEvents: [] } satisfies OrchestratorV2DispatchResult; - } const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now); const committed = yield* eventSink .commitCommand({ diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index de03594916a87..1c99d0d41ae16 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -16,9 +16,13 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; +import { + delegatedCompletionWakeDetail, + isTerminalDelegatedTaskStatus, +} from "./NotificationMailbox.ts"; const yieldToRuntime = Effect.yieldNow.pipe( Effect.andThen( @@ -44,6 +48,39 @@ export class ProviderTurnControlError extends Schema.TaggedError()( + "ProviderTurnControlSteerDeferredError", + { + threadId: ThreadId, + providerTurnId: ProviderTurnId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Steering delivery was deferred to the message queue."; + } +} + +export class ProviderTurnControlSteerCancelledError extends Schema.TaggedError()( + "ProviderTurnControlSteerCancelledError", + { + threadId: ThreadId, + providerTurnId: ProviderTurnId, + }, +) { + override get message(): string { + return "Steering delivery was cancelled because the message is no longer current."; + } +} + +const isProviderTurnControlSteerError = Schema.is( + Schema.Union([ + ProviderTurnControlError, + ProviderTurnControlSteerDeferredError, + ProviderTurnControlSteerCancelledError, + ]), +); + export interface ProviderTurnControlServiceV2Shape { readonly interrupt: (input: { readonly subagent?: { @@ -65,7 +102,13 @@ export interface ProviderTurnControlServiceV2Shape { readonly providerThreadId: ProviderThreadId; readonly providerTurnId: ProviderTurnId; readonly messageId: MessageId; - }) => Effect.Effect; + readonly isCurrent?: Effect.Effect; + }) => Effect.Effect< + void, + | ProviderTurnControlError + | ProviderTurnControlSteerDeferredError + | ProviderTurnControlSteerCancelledError + >; readonly interruptAndAwaitTerminal: (input: { readonly threadId: ThreadId; readonly providerSessionId: ProviderSessionId; @@ -323,21 +366,26 @@ export const layer: Layer.Layer< Effect.gen(function* () { const context = yield* projections.getProviderControlContext(input.threadId, input); const ownership = context.message?.delegatedCompletion; - if (ownership !== undefined) { - const projection = yield* projections.getThreadRecords(input.threadId, ["runs"], { - runIds: [ownership.parentRunId], - }); - const cohort = projection.runs.find( - (run) => run.id === ownership.parentRunId, - )?.delegatedCompletion; - if ( - cohort?.disposition !== "open" || - cohort.delivery?.messageId !== input.messageId || - cohort.delivery.generation !== ownership.generation || - cohort.delivery.taskIds.length === 0 - ) - return; - } + const completionTaskIds = + ownership === undefined + ? undefined + : Effect.gen(function* () { + const projection = yield* projections.getThreadRecords(input.threadId, ["runs"], { + runIds: [ownership.parentRunId], + }); + const cohort = projection.runs.find( + (run) => run.id === ownership.parentRunId, + )?.delegatedCompletion; + if ( + cohort?.disposition !== "open" || + cohort.delivery?.messageId !== input.messageId || + cohort.delivery.generation !== ownership.generation || + cohort.delivery.taskIds.length === 0 + ) + return null; + return cohort.delivery.taskIds; + }); + if (completionTaskIds !== undefined && (yield* completionTaskIds) === null) return; const loaded = yield* load({ ...input, operation: "steer" }); if (Option.isNone(loaded.session)) return; const { message, run } = loaded.context; @@ -349,18 +397,57 @@ export const layer: Layer.Layer< cause: "The persisted steering message or target run is missing.", }); } + const text = projectComposerContextForProvider({ + text: message.text, + records: message.context?.records ?? [], + }); + const resolveMessageText = Effect.gen(function* () { + if (input.isCurrent !== undefined && !(yield* input.isCurrent)) return undefined; + const source = message.notification?.source; + if ( + message.id.startsWith("message:delegated-task-question:") && + source?.kind === "delegated_task" + ) { + const parent = yield* projections.getThreadRecords(input.threadId, ["subagents"]); + const task = parent.subagents.find( + (candidate) => + candidate.origin === "app_owned" && + candidate.childThreadId === source.childThreadId && + source.taskIds.includes(candidate.id), + ); + if ( + task === undefined || + isTerminalDelegatedTaskStatus(task.status) || + task.completionDelivery?.state === "disposed" || + parent.thread.archivedAt !== null || + parent.thread.deletedAt !== null + ) + return undefined; + } + if (completionTaskIds === undefined) return text; + const taskIds = yield* completionTaskIds; + return taskIds === null ? undefined : delegatedCompletionWakeDetail(taskIds); + }); yield* loaded.session.value .steerTurn({ threadId: input.threadId, runId: run.id, providerThread: loaded.providerThread, providerTurnId: loaded.providerTurn.id, + resolveMessageText: resolveMessageText.pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterSteerRunError({ + driver: loaded.providerThread.driver, + providerThreadId: loaded.providerThread.id, + providerTurnId: loaded.providerTurn.id, + cause, + }), + ), + ), message: { messageId: message.id, - text: projectComposerContextForProvider({ - text: message.text, - records: message.context?.records ?? [], - }), + text, attachments: message.attachments, createdBy: message.createdBy, creationSource: message.creationSource, @@ -375,6 +462,19 @@ export const layer: Layer.Layer< .pipe( Effect.catch((cause) => Effect.gen(function* () { + if ((yield* resolveMessageText) === undefined) { + return yield* new ProviderTurnControlSteerCancelledError({ + threadId: input.threadId, + providerTurnId: input.providerTurnId, + }); + } + if (cause._tag === "ProviderAdapterSteerRunDeferredError") { + return yield* new ProviderTurnControlSteerDeferredError({ + threadId: input.threadId, + providerTurnId: input.providerTurnId, + cause, + }); + } const current = yield* projections.getProviderControlContext( input.threadId, input, @@ -389,9 +489,15 @@ export const layer: Layer.Layer< }), ), ); + if (input.isCurrent !== undefined && !(yield* input.isCurrent)) { + return yield* new ProviderTurnControlSteerCancelledError({ + threadId: input.threadId, + providerTurnId: input.providerTurnId, + }); + } }).pipe( Effect.mapError((cause) => - isProviderTurnControlError(cause) + isProviderTurnControlSteerError(cause) ? cause : new ProviderTurnControlError({ threadId: input.threadId, diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 884683979fb65..bcfa2be7f2dbe 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -18,14 +18,20 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as CommandReceiptStore from "./CommandReceiptStore.ts"; +import * as EffectOutbox from "./EffectOutbox.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import * as ThreadManagement from "./ThreadManagementService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -418,18 +424,33 @@ const composerSelection = { const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* ( name: string, - canRestart = false, + { + canRestart = false, + activeSteeringInterruptsTools = false, + supportsNonInterruptingAgentSteering = false, + deferSteering = false, + beforeBoundary = Effect.void, + }: { + canRestart?: boolean; + activeSteeringInterruptsTools?: boolean; + supportsNonInterruptingAgentSteering?: boolean; + deferSteering?: boolean; + beforeBoundary?: Effect.Effect; + } = {}, ) { const cwd = yield* checkpointWorkspace(name); const events = yield* Queue.unbounded(); const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; const steered: string[] = []; + const interrupted: ProviderTurnId[] = []; const capabilities = { ...CodexProviderCapabilitiesV2, turns: { ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: true, supportsSteeringByInterruptRestart: canRestart, + activeSteeringInterruptsTools, + supportsNonInterruptingAgentSteering, }, }; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { @@ -500,10 +521,22 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }); }), steerTurn: (turn) => + Effect.gen(function* () { + yield* beforeBoundary; + if (deferSteering) { + return yield* new ProviderAdapter.ProviderAdapterSteerRunDeferredError({ + driver, + providerThreadId: turn.providerThread.id, + providerTurnId: turn.providerTurnId, + }); + } + const text = yield* turn.resolveMessageText ?? Effect.succeed(turn.message.text); + if (text !== undefined) steered.push(text); + }), + interruptTurn: (turn) => Effect.sync(() => { - steered.push(turn.message.text); + interrupted.push(turn.providerTurnId); }), - interruptTurn: () => Effect.void, respondToRuntimeRequest: () => Effect.void, readThreadSnapshot: () => Effect.die("unused"), rollbackThread: () => Effect.die("unused"), @@ -511,10 +544,17 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }; }), }; - const layer = ProviderReplayHarness.layerWithRegistry( + const databaseLayer = SqlitePersistence.layerMemory; + const layerRuntime = ProviderReplayHarness.layerWithRegistry( { name }, ProviderAdapterRegistry.layerSingle(adapter), - { runEffectWorker: false }, + { runEffectWorker: false, databaseLayer }, + ); + const layer = ThreadManagement.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(databaseLayer))), + Layer.provideMerge(layerRuntime), + Layer.provideMerge(EffectOutbox.layer.pipe(Layer.provide(databaseLayer))), ); const startFirstTurn = Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -558,9 +598,111 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* yield* Fiber.join(running); return threadId; }); - return { events, started, steered, layer, startFirstTurn }; + return { events, started, steered, interrupted, layer, startFirstTurn }; }); +it.effect.each( + ( + [ + { actor: "agent", mode: "auto", interruptsTools: true }, + { actor: "agent", mode: "steer", interruptsTools: true }, + { actor: "user", mode: "steer", interruptsTools: true }, + { actor: "agent", mode: "auto", interruptsTools: false }, + { actor: "agent", mode: "steer", interruptsTools: false }, + ] as const + ) + .map((entry) => ({ ...entry, safeBoundary: false, defer: false })) + .concat([ + { actor: "agent", mode: "auto", interruptsTools: true, safeBoundary: true, defer: false }, + { actor: "agent", mode: "steer", interruptsTools: true, safeBoundary: true, defer: true }, + ]), +)( + "delivers $actor $mode messages safely (interruptsTools=$interruptsTools, boundary=$safeBoundary, defer=$defer)", + ({ actor, mode, interruptsTools, safeBoundary, defer }) => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `agent-steering-${actor}-${mode}-${interruptsTools}`, + { + activeSteeringInterruptsTools: interruptsTools, + supportsNonInterruptingAgentSteering: safeBoundary, + deferSteering: defer, + }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threads = yield* ThreadManagement.ThreadManagementService; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const messageId = MessageId.make("message:update"); + const result = yield* threads.sendToThread({ + projectId: (yield* orchestrator.getThreadProjection(threadId)).thread.projectId, + commandId: CommandId.make("update"), + threadId, + messageId, + text: "progress update", + attachments: [], + mode, + createdBy: actor, + creationSource: actor === "agent" ? "mcp" : "web", + }); + yield* worker.drain(); + const queuedAtDispatch = actor === "agent" && interruptsTools && !safeBoundary; + const queued = queuedAtDispatch || defer; + assert.equal(result.delivery, queuedAtDispatch ? "queued" : "steered"); + assert.deepEqual(steered, queued ? [] : ["progress update"]); + assert.equal(started.length, 1); + const pending = yield* orchestrator.getThreadProjection(threadId); + assert.equal(pending.runs.find((run) => run.id === first.runId)?.status, "running"); + if (!queued) return; + assert.equal( + pending.runs.find((run) => run.userMessageId === messageId)?.status, + "queued", + ); + + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "waiting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const turn = pending.providerTurns[0]!; + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, + }); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(completed); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + assert.equal(started.length, 2); + assert.equal(started[1]?.message.messageId, messageId); + assert.equal(started[1]?.message.text, "progress update"); + assert.deepEqual(steered, []); + const final = yield* orchestrator.getThreadProjection(threadId); + assert.equal(final.runs.find((run) => run.id === first.runId)?.status, "completed"); + }).pipe(Effect.provide(layer)); + }), + ), +); + it.effect.each([{ canRestart: false }, { canRestart: true }])( "steers a changed turn-scoped selection into the running turn (canRestart=$canRestart)", ({ canRestart }) => @@ -568,7 +710,7 @@ it.effect.each([{ canRestart: false }, { canRestart: true }])( Effect.gen(function* () { const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( `steering-selection-change-${canRestart}`, - canRestart, + { canRestart }, ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -636,6 +778,302 @@ it.effect.each([{ canRestart: false }, { canRestart: true }])( ), ); +it.effect("Stop suppresses a boundary timeout whose fallback has not dispatched yet", () => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const { started, interrupted, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "steering-boundary-stop-race", + { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + deferSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* startFirstTurn; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("boundary-update"), + threadId, + messageId: MessageId.make("message:boundary-update"), + text: "The review found another issue.", + attachments: [], + dispatchMode: { type: "steer_active", targetRunId: started[0]!.runId }, + createdBy: "agent", + creationSource: "mcp", + }); + const draining = yield* worker.drain().pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop-boundary-parent"), + threadId, + runId: started[0]!.runId, + holdQueue: true, + }); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(draining); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(started.length, 1); + assert.equal(projection.runs.length, 1); + const effects = yield* outbox.listByCommandId(CommandId.make("boundary-update")); + assert.lengthOf(effects, 1); + assert.include(["succeeded", "cancelled"], effects[0]?.status); + if (effects[0]?.status === "succeeded") assert.isNull(effects[0].lastError); + assert.lengthOf(interrupted, 1); + const stops = yield* outbox.listByCommandId(CommandId.make("stop-boundary-parent")); + assert.equal( + stops.find((effect) => effect.request.type === "provider-turn.interrupt")?.status, + "succeeded", + ); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect("Stop cancels pending hook delivery and executes the interrupt before a boundary", () => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const h = yield* nextTurnSelectionHarness("steering-stop-before-hook", { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("pending-hook"), + threadId, + messageId: MessageId.make("message:pending-hook"), + text: "Do not deliver after Stop.", + attachments: [], + dispatchMode: { type: "steer_active", targetRunId: h.started[0]!.runId }, + createdBy: "agent", + creationSource: "mcp", + }); + const delivery = yield* worker.runOnce.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-pending-hook"), + threadId, + }); + yield* worker.drain(); + assert.lengthOf(h.interrupted, 1); + yield* Fiber.join(delivery); + const effects = yield* outbox.listByCommandId(CommandId.make("pending-hook")); + assert.equal(effects[0]?.status, "cancelled"); + yield* Deferred.succeed(release, undefined); + yield* worker.drain(); + assert.deepEqual(h.steered, []); + assert.equal((yield* orchestrator.getThreadProjection(threadId)).runs.length, 1); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + +const dispatchClaimedCompletion = Effect.fn("dispatchClaimedCompletion")(function* ( + threadId: ThreadId, + first: ProviderAdapter.ProviderAdapterV2TurnInput, + taskIds: ReadonlyArray, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const messageId = MessageId.make("message:pending-completion"); + const current = yield* orchestrator.getThreadProjection(threadId); + const now = yield* DateTime.now; + yield* sink.write({ + events: [ + { + id: EventId.make("completion:cohort"), + type: "run.updated", + threadId, + runId: first.runId, + occurredAt: now, + payload: { + ...current.runs.find((run) => run.id === first.runId)!, + delegatedCompletion: { + disposition: "open", + nextGeneration: 2, + delivery: { generation: 1, messageId, taskIds }, + }, + }, + }, + ...taskIds.map((taskId) => ({ + id: EventId.make(`completion:${taskId}`), + type: "subagent.updated" as const, + threadId, + runId: first.runId, + nodeId: taskId, + occurredAt: now, + payload: { + id: taskId, + threadId, + runId: first.runId, + parentNodeId: first.rootNodeId, + origin: "app_owned" as const, + createdBy: "agent" as const, + driver, + providerInstanceId: instanceId, + providerThreadId: null, + childThreadId: null, + nativeTaskRef: null, + prompt: "Do background work", + title: "Background test", + model: null, + completionWake: "always" as const, + completionDelivery: { state: "claimed" as const, observedByRunId: null }, + status: "completed" as const, + result: "done", + startedAt: now, + completedAt: now, + updatedAt: now, + }, + })), + ], + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("dispatch-completion"), + threadId, + messageId, + text: `Original completion for ${taskIds.join(", ")}`, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + delegatedCompletion: { parentRunId: first.runId, generation: 1, taskIds }, + }); + return messageId; +}); + +it.effect("a live completion boundary defer creates exactly one queued follow-up", () => + Effect.scoped( + Effect.gen(function* () { + const h = yield* nextTurnSelectionHarness("steering-live-completion-defer", { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + deferSteering: true, + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + const first = h.started[0]!; + const taskId = NodeId.make("task:deferred"); + const messageId = yield* dispatchClaimedCompletion(threadId, first, [taskId]); + yield* worker.drain(); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + const followUps = projection.runs.filter((run) => run.userMessageId === messageId); + assert.lengthOf(followUps, 1); + assert.equal(followUps[0]?.status, "queued"); + assert.equal(projection.runs.find((run) => run.id === first.runId)?.status, "running"); + assert.equal(projection.messages.filter((message) => message.id === messageId).length, 1); + assert.deepEqual( + projection.messages.find((message) => message.id === messageId)?.delegatedCompletion, + { + parentRunId: first.runId, + generation: 1, + taskIds: [taskId], + }, + ); + const effects = yield* outbox.listByCommandId(CommandId.make("dispatch-completion")); + assert.equal( + effects.find((effect) => effect.request.type === "provider-turn.steer")?.status, + "succeeded", + ); + assert.equal(h.started.length, 1); + assert.deepEqual(h.steered, []); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + +it.effect.each(["dispose", "acknowledge one"] as const)( + "completion %s updates a pending boundary before delivery", + (resolution) => + Effect.scoped( + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const h = yield* nextTurnSelectionHarness(`steering-completion-${resolution}`, { + activeSteeringInterruptsTools: true, + supportsNonInterruptingAgentSteering: true, + beforeBoundary: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ), + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* h.startFirstTurn; + const first = h.started[0]!; + const taskId = NodeId.make("task:resolved"); + const remainingTaskId = NodeId.make("task:remaining"); + const taskIds = resolution === "dispose" ? [taskId] : [taskId, remainingTaskId]; + yield* dispatchClaimedCompletion(threadId, first, taskIds); + const delivery = yield* worker.runOnce.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* orchestrator.dispatch( + resolution === "dispose" + ? { + type: "delegated_task.completion-delivery.dispose", + commandId: CommandId.make("resolve-completion"), + parentThreadId: threadId, + taskId, + } + : { + type: "delegated_task.completion-delivery.acknowledge", + commandId: CommandId.make("resolve-completion"), + parentThreadId: threadId, + taskId, + observedByRunId: first.runId, + }, + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(delivery); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + projection.subagents.find((task) => task.id === taskId)?.completionDelivery?.state, + resolution === "dispose" ? "disposed" : "acknowledged", + ); + if (resolution === "dispose") assert.deepEqual(h.steered, []); + else { + assert.lengthOf(h.steered, 1); + assert.include(h.steered[0]!, String(remainingTaskId)); + assert.notInclude(h.steered[0]!, String(taskId)); + assert.include(h.steered[0]!, "task_status"); + } + assert.equal(projection.runs.length, 1); + const effects = yield* outbox.listByCommandId(CommandId.make("dispatch-completion")); + assert.include(["succeeded", "cancelled"], effects[0]?.status); + if (effects[0]?.status === "succeeded") assert.isNull(effects[0].lastError); + }).pipe(Effect.provide(h.layer)); + }), + ), +); + it.effect("starts a steer that missed the turn on the saved next-turn selection", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 94a9eca331d13..cb708647ae064 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -218,6 +218,8 @@ export const OrchestrationV2TurnCapabilities = Schema.Struct({ supportsActiveSteering: Schema.Boolean, // Some native steering mechanisms cancel pending tools before consuming the message. activeSteeringInterruptsTools: Schema.optional(Schema.Boolean), + // Agent-authored messages can use a safe boundary instead of interrupting tools. + supportsNonInterruptingAgentSteering: Schema.optional(Schema.Boolean), supportsSteeringByInterruptRestart: Schema.Boolean, supportsQueuedMessages: Schema.Boolean, terminalStatusQuality: Schema.Literals(["strong", "weak", "none"]), @@ -2779,6 +2781,7 @@ export const OrchestrationV2Command = Schema.Union([ restartContinuationOfRunId: Schema.optional(RunId), usageLimitContinuationOfRunId: Schema.optional(RunId), manualContinuationOfRunId: Schema.optional(RunId), + steeringFallbackOfRunId: Schema.optional(RunId), usageLimitRecoveryRequestId: Schema.optional(CommandId), deliveryIntent: Schema.optional(Schema.Literals(["auto", "steer", "restart"])), delegatedCompletion: Schema.optional( diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index d58a6c5904a2f..6359a605d0951 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -320,6 +320,19 @@ export class ProviderAdapterSteerRunError extends Schema.TaggedError()( + "ProviderAdapterSteerRunDeferredError", + { + driver: ProviderDriverKind, + providerThreadId: ProviderThreadId, + providerTurnId: ProviderTurnId, + }, +) { + override get message(): string { + return "Active-run steering was deferred to the message queue."; + } +} + export class ProviderAdapterInterruptError extends Schema.TaggedError()( "ProviderAdapterInterruptError", { @@ -394,6 +407,7 @@ export const ProviderAdapterV2Error = Schema.Union([ ProviderAdapterTurnStartError, ProviderAdapterSteerRunUnsupportedError, ProviderAdapterSteerRunError, + ProviderAdapterSteerRunDeferredError, ProviderAdapterInterruptError, ProviderAdapterRuntimeRequestResponseError, ProviderAdapterEventStreamError, @@ -443,6 +457,8 @@ export interface ProviderAdapterV2SteerInput { readonly providerThread: OrchestrationV2ProviderThread; readonly providerTurnId: ProviderTurnId; readonly message: ProviderAdapterV2TurnMessage; + /** Rechecks durable delivery ownership before a delayed boundary handoff. */ + readonly resolveMessageText?: Effect.Effect; } export interface ProviderAdapterV2InterruptInput { From 2418c1cdee4b349ed20cc0ba4e38e251a00eec8d Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Mon, 19 Oct 2026 23:51:23 -0400 Subject: [PATCH 08/23] fix(server): Claude takes agent updates at tool boundaries Agent updates to a Claude turn wait in the durable queue instead of steering, which would cancel running tools that Claude reads as a user refusal. The running turn now takes them at its next root PostToolBatch or Stop hook as additional context, so an update arrives right after the current tool call instead of after the whole turn. The hook claims queued updates through ProviderContinuationRequests; the worker promotes each into the running run with the existing promote-to-steer command (no steer effect, the run's own model selection, delegated completions accepted in the same commit). Held queues and stopped runs refuse the claim, so Stop wins. Unclaimed updates start the next turn as before. Updates with attachments, which hooks cannot carry, still steer natively with a note that they came from another agent. Related-PR: https://github.com/pingdotgg/t3code/pull/21158 Co-Authored-By: Claude Opus 5.5 at high effort Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent --- .../Adapters/ClaudeAdapterV2.test.ts | 125 +++++++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 55 +++++++- .../src/orchestration-v2/Orchestrator.ts | 76 +++++++++- .../ProviderContinuationService.ts | 56 ++++++++ .../SteeringCompletion.integration.test.ts | 131 +++++++++++++++++- .../testkit/ProviderReplayHarness.ts | 1 + packages/contracts/src/orchestrationV2.ts | 4 + .../server/ProviderContinuationRequests.ts | 16 +++ 8 files changed, 451 insertions(+), 13 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index dbcd849e92e01..70a7ea6b0b7b5 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2473,6 +2473,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly nativeSessionId?: string; readonly settings?: ClaudeSettings; readonly freshQueueOnReopen?: boolean; + readonly claimAgentMessages?: ProviderContinuationRequests.ClaimAgentMessages; }) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -2527,6 +2528,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { Effect.sync(() => { continuationRequests.push(request); }).pipe(Effect.andThen(Queue.offer(continuationReceipts, request))), + ...(options?.claimAgentMessages === undefined + ? {} + : { claimAgentMessages: options.claimAgentMessages }), }, queryRunner: { allocateSessionId: Effect.succeed(options?.nativeSessionId ?? WAKE_NATIVE_SESSION), @@ -3136,6 +3140,127 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("delivers claimed agent updates at root tool-batch and stop hooks", () => + Effect.scoped( + Effect.gen(function* () { + const claims: Array<{ readonly threadId: string; readonly runId: string }> = []; + const harness = yield* makeWakeHarnessWithOptions({ + claimAgentMessages: (input) => + Effect.sync(() => { + claims.push(input); + return ["Child finished."]; + }), + }); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-agent-boundary"), + text: "Run the slow build.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + const hooks = harness.getOpenedOptions()?.hooks; + const signal = new AbortController().signal; + const base = { + session_id: WAKE_NATIVE_SESSION, + transcript_path: "", + cwd: "", + }; + const batch = yield* Effect.promise(() => + hooks!.PostToolBatch![0]!.hooks[0]!( + { ...base, hook_event_name: "PostToolBatch", tool_calls: [] }, + undefined, + { signal }, + ), + ); + const stop = yield* Effect.promise(() => + hooks!.Stop![0]!.hooks[0]!( + { ...base, hook_event_name: "Stop", stop_hook_active: false }, + undefined, + { signal }, + ), + ); + // A subagent's boundary is not the root turn's. + const subagent = yield* Effect.promise(() => + hooks!.PostToolBatch![0]!.hooks[0]!( + { ...base, hook_event_name: "PostToolBatch", tool_calls: [], agent_id: "agent-1" }, + undefined, + { signal }, + ), + ); + assert.deepEqual(batch, { + hookSpecificOutput: { + hookEventName: "PostToolBatch", + additionalContext: "New message from another agent:\nChild finished.", + }, + }); + assert.deepEqual(stop, { + hookSpecificOutput: { + hookEventName: "Stop", + additionalContext: "New message from another agent:\nChild finished.", + }, + }); + assert.deepEqual(subagent, {}); + assert.deepEqual(claims, [ + { threadId: harness.threadId, runId: input.runId }, + { threadId: harness.threadId, runId: input.runId }, + ]); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect.each(["agent", "user"] as const)( + "notes that a native %s steer may have interrupted a command only for agents", + (createdBy) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const attemptId = RunAttemptId.make("attempt-agent-steer-note"); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId, + text: "Audit the settings pages.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy, + creationSource: createdBy === "agent" ? "mcp" : "web", + messageId: MessageId.make("message-agent-steer-note"), + text: "See the attached screenshot.", + attachments: [], + }, + }); + const steer = JSON.stringify(harness.offeredMessages[1]?.message.content); + assert.include(steer, "See the attached screenshot."); + assert.equal( + steer.includes("may have interrupted a running command"), + createdBy === "agent", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect.each( (["aborted_tools", "aborted_streaming"] as const).flatMap((terminalReason) => [true, false].map((steered) => ({ terminalReason, steered })), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3a00dd04e5bad..54a3da8273a7b 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -26,6 +26,7 @@ import { type EffortLevel, type ForkSessionOptions, type ForkSessionResult, + type HookCallback, getSubagentMessages, query, type Options as ClaudeQueryOptions, @@ -271,6 +272,8 @@ const CLAUDE_SCRIPT_EXTENSIONS = [".js", ".mjs", ".tsx", ".ts", ".jsx"]; // The SDK puts no deadline on control requests. The CLI answers // mcp_set_servers once each server connected or hit its own 30 s timeout. const CLAUDE_MCP_REGISTRATION_TIMEOUT = "90 seconds"; +const CLAUDE_AGENT_STEER_NOTE = + "This message came from another agent, not the user, and may have interrupted a running command; check that command's state before continuing."; // Claude Code flushes transcript writes every 100 ms, so a workflow member's // reply can land after the progress frame that settles it. Its transcript // stays the source of its answer this long after it settles. @@ -3519,6 +3522,7 @@ export interface ClaudeAdapterV2Options { readonly offer: ( request: ProviderContinuationRequests.ProviderContinuationRequest, ) => Effect.Effect; + readonly claimAgentMessages?: ProviderContinuationRequests.ClaimAgentMessages; }; } @@ -8716,11 +8720,53 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( onUserDialog, supportedDialogKinds: ["resume_return"], }); + // Native steering cancels running tools, so agent updates wait in the + // app queue. The root agent's tool-batch and stop boundaries take them + // as context: the tool has finished and, at Stop, the turn continues. + // The claim commits before the SDK reads the context: delivery is at + // most once, and a crash in that window loses the claimed updates. A + // cancelled hook interrupts the claim before its next promotion. + const takeAgentMessages: HookCallback = (hookInput, _toolUseId, { signal }) => + runPromise( + Effect.gen(function* () { + const context = yield* Ref.get(activeTurn); + if ( + (hookInput.hook_event_name !== "PostToolBatch" && + hookInput.hook_event_name !== "Stop") || + hookInput.agent_id !== undefined || + context === null || + context.nativeThreadId !== hookInput.session_id || + (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) || + continuationRequests.claimAgentMessages === undefined + ) { + return {}; + } + const texts = yield* continuationRequests.claimAgentMessages({ + threadId: context.input.threadId, + runId: context.input.runId, + }); + return texts.length === 0 + ? {} + : { + hookSpecificOutput: { + hookEventName: hookInput.hook_event_name, + additionalContext: `New message from another agent:\n${texts.join("\n\n")}`, + }, + }; + }), + { signal }, + ); const querySession = yield* queryRunner .open({ threadId: turnInput.threadId, providerSessionId: input.providerSessionId, - options: queryOptions, + options: { + ...queryOptions, + hooks: { + PostToolBatch: [{ hooks: [takeAgentMessages] }], + Stop: [{ hooks: [takeAgentMessages] }], + }, + }, }) .pipe( Effect.onError(() => @@ -9224,7 +9270,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const userMessage = yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( - turnInput.message.text, + // An agent update steers natively only when hooks cannot carry it + // (attachments). It cancels running tools; say so, or Claude reads + // the cancellation as a user refusal. + turnInput.message.createdBy === "agent" + ? `${CLAUDE_AGENT_STEER_NOTE}\n\n${turnInput.message.text}` + : turnInput.message.text, compileClaudeModelSelection(currentTurn.input.modelSelection).promptEffort, ), attachments: turnInput.message.attachments, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 63f8189393070..96c1b240b2f73 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -591,6 +591,21 @@ function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"] ); } +// An agent update that asked for prompt delivery (steer or auto, not an explicit +// queue) is queued only because the turn is busy or still starting; the running +// turn may take it at its next tool boundary. Hooks carry text only. +function agentUpdateWantsPromptDelivery( + command: Extract, +): boolean { + return ( + command.createdBy === "agent" && + command.dispatchMode.type !== "queue_after_active" && + command.attachments.length === 0 && + command.notification === undefined && + command.scheduledTaskId === undefined + ); +} + function delegatedTaskTerminalStatus( status: OrchestrationV2Run["status"], ): OrchestrationV2Subagent["status"] | null { @@ -3905,6 +3920,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly senderThreadId?: OrchestrationV2ConversationMessage["senderThreadId"]; readonly delegatedCompletion?: OrchestrationV2ConversationMessage["delegatedCompletion"]; readonly forceRestart: boolean; + /** The provider delivers the message at a tool boundary; record it without a steer effect. */ + readonly atToolBoundary?: boolean; }) => Effect.gen(function* () { const targetRun = input.projection.runs.find( @@ -4150,6 +4167,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio input.projection.thread.providerInstanceId !== input.modelSelection.instanceId; if ( input.delegatedCompletion === undefined && + input.atToolBoundary !== true && (instanceChanged || !modelSelectionsEqual(input.projection.thread.modelSelection, input.modelSelection)) ) { @@ -4173,6 +4191,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio providerThreadId: providerThread.id, providerInstanceId: targetRun.providerInstanceId, }); + if (input.atToolBoundary === true) return; yield* Ref.update(input.effects, (existing) => [ ...existing, { @@ -4820,6 +4839,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command.dispatchMode, command.deliveryIntent, ); + let atToolBoundary = false; if (dispatchMode.type === "steer_active") { const targetRunId = dispatchMode.targetRunId; const target = projection.runs.find((run) => run.id === targetRunId); @@ -4853,10 +4873,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ( Option.isSome(session) && session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === - true + true && + // Boundary hooks carry text only. An update with attachments steers + // natively; the adapter tells the agent it may have interrupted a tool. + command.attachments.length === 0 ) { // Claude steering cancels pending tools, which appear as user refusals. - // Agent updates can wait for the durable queue's next-turn boundary. + // Agent updates wait in the durable queue instead, and the running turn + // takes them at its next tool batch or Stop (ProviderContinuationService + // claimAgentMessages). Whatever it does not take starts the next turn. dispatchMode = { type: "queue_after_active" }; } } @@ -4938,6 +4963,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (id) => projection.subagents.find((task) => task.id === id)?.completionWake === "always", ) ) { + // If it queues (steering would cancel tools, or the turn is still + // starting), the running turn may take it at its next tool boundary. + atToolBoundary = true; const active = projection.runs.find((run) => run.status === "running"); const providerThread = projection.providerThreads.find( (row) => row.id === active?.providerThreadId, @@ -5295,6 +5323,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio updatedAt: now, ...(delegatedCompletion === undefined ? {} : { delegatedCompletion }), ...(command.notification === undefined ? {} : { notification: command.notification }), + ...(atToolBoundary || agentUpdateWantsPromptDelivery(command) + ? { atToolBoundary: true as const } + : {}), }; const emitEvent = emit(events, command); if (targetProviderThread === undefined) { @@ -7803,7 +7834,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Queued run ${queuedRun.id} is missing message or execution state.`, }); } - if ( + if (command.atToolBoundary === true) { + // Stop wins: a held queue or a stopped turn keeps the update queued. + if ( + queuedMessage.atToolBoundary !== true || + queuedMessage.attachments.length > 0 || + queuedRun.queueHeld === true || + (yield* stopReachedRun(command, command.threadId, command.targetRunId)) + ) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Queued run ${queuedRun.id} cannot join the running turn at a tool boundary.`, + }); + } + } else if ( queuedMessage.delegatedCompletion !== undefined || queuedMessage.notification !== undefined ) { @@ -7857,12 +7902,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, }); + const atToolBoundary = command.atToolBoundary === true; yield* dispatchSteerIntoRun({ command, events, effects, projection, - modelSelection: projection.thread.modelSelection, + // An update joining at a tool boundary keeps the running turn's model; a + // selection change could otherwise restart the turn from inside its hook. + modelSelection: atToolBoundary + ? (projection.runs.find((run) => run.id === command.targetRunId)?.modelSelection ?? + projection.thread.modelSelection) + : projection.thread.modelSelection, + ...(queuedMessage.delegatedCompletion === undefined + ? {} + : { delegatedCompletion: queuedMessage.delegatedCompletion }), + atToolBoundary, targetRunId: command.targetRunId, messageId: queuedMessage.id, text: queuedMessage.text, @@ -7878,6 +7933,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : { senderThreadId: queuedMessage.senderThreadId }), forceRestart: false, }); + if (atToolBoundary && queuedMessage.delegatedCompletion !== undefined) { + // Accept in the same commit: the cancelled queued run would otherwise + // read as a failed delivery and reserve a duplicate notification. + yield* dispatchNotificationAccepted( + { + type: "notification.delivery.accept", + commandId: command.commandId, + threadId: command.threadId, + messageId: queuedMessage.id, + }, + events, + ); + } }); const dispatchQueuedRunReorder = ( diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index 8aad65111d09d..ee278a68bdb93 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -1,4 +1,5 @@ import { CommandId, type OrchestrationV2ThreadProjection } from "@t3tools/contracts"; +import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; @@ -7,6 +8,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts"; +import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts"; const CONTINUATION_MESSAGE_TEXT = "Background task completed."; @@ -242,6 +244,60 @@ export const layer = Layer.effectDiscard( }, ); + // The provider waits on this from a tool-batch or stop hook, so each update + // joins its running turn before the hook returns the text. An update that + // cannot join (edited away, held by Stop, turn no longer running) stays + // queued for the next turn. + const claimAgentMessages: ProviderContinuationRequests.ClaimAgentMessages = (input) => + Effect.gen(function* () { + const queued = yield* threads.getThreadRecords(input.threadId, ["runs", "messages"], { + messageRoles: ["user"], + }); + const texts: Array = []; + for (const run of queuedRunsInDeliveryOrder(queued)) { + const message = queued.messages.find((row) => row.id === run.userMessageId); + if (message?.atToolBoundary !== true) continue; + const result = yield* threads + .dispatch({ + type: "queued-message.promote-to-steer", + // Fresh per try: a refusal now must not be replayed at a later boundary. + commandId: yield* ids.allocate.command({ + fixtureName: "tool-boundary", + commandName: "promote", + }), + threadId: input.threadId, + queuedRunId: run.id, + targetRunId: input.runId, + atToolBoundary: true, + }) + .pipe(Effect.catchCause(() => Effect.succeed(undefined))); + // Deliver what the promotion committed, which may postdate the snapshot. + const joined = result?.storedEvents.find( + ({ event }) => event.type === "message.updated" && event.payload.id === message.id, + )?.event; + if (joined?.type === "message.updated") { + const promoted = joined.payload; + texts.push( + projectComposerContextForProvider({ + text: promoted.text, + records: promoted.context?.records ?? [], + }), + ); + } + } + return texts; + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.provider-continuation.agent-message-claim-failed", { + threadId: input.threadId, + cause, + }).pipe(Effect.as([])), + ), + ); + if (requests.serveAgentMessages !== undefined) { + yield* requests.serveAgentMessages(claimAgentMessages); + } + yield* requests.take.pipe( Effect.flatMap((request) => dispatchContinuation(request).pipe( diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index c3c8d6182c20c..794d269179d2c 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -11,6 +11,8 @@ import { ProviderTurnId, ThreadId, TurnItemId, + type ChatAttachment, + type RunId, type ModelSelection, type OrchestrationV2DomainEvent, } from "@t3tools/contracts"; @@ -33,6 +35,7 @@ import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; const driver = ProviderDriverKind.make("codex"); const instanceId = ProviderInstanceId.make("codex"); @@ -524,7 +527,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* const layerRuntime = ProviderReplayHarness.layerWithRegistry( { name }, ProviderAdapterRegistry.layerSingle(adapter), - { runEffectWorker: false, databaseLayer }, + { runEffectWorker: false, runContinuationWorker: true, databaseLayer }, ); const layer = ThreadManagement.layer.pipe( Layer.provide(ServerSettings.layerTest()), @@ -580,6 +583,7 @@ const sendUpdate = Effect.fn("sendUpdate")(function* ( threadId: ThreadId, createdBy: "agent" | "user", mode: "auto" | "steer", + attachments: ReadonlyArray = [], ) { const orchestrator = yield* Orchestrator.OrchestratorV2; const threads = yield* ThreadManagement.ThreadManagementService; @@ -589,7 +593,7 @@ const sendUpdate = Effect.fn("sendUpdate")(function* ( threadId, messageId: MessageId.make("message:update"), text: "progress update", - attachments: [], + attachments, mode, createdBy, creationSource: createdBy === "agent" ? "mcp" : "web", @@ -659,16 +663,26 @@ it.effect.each(["auto", "steer"] as const)( ), ); +const imageAttachment: ChatAttachment = { + type: "image", + id: "attachment-1", + name: "screen.png", + mimeType: "image/png", + sizeBytes: 1, +}; + it.effect.each([ - { createdBy: "user", activeSteeringInterruptsTools: true }, - { createdBy: "agent", activeSteeringInterruptsTools: false }, + { createdBy: "user", activeSteeringInterruptsTools: true, attachments: [] }, + { createdBy: "agent", activeSteeringInterruptsTools: false, attachments: [] }, + // Boundary hooks carry text only, so this one steers natively. + { createdBy: "agent", activeSteeringInterruptsTools: true, attachments: [imageAttachment] }, ] as const)( - "steers an update from $createdBy when steering is allowed (interruptsTools=$activeSteeringInterruptsTools)", - ({ createdBy, activeSteeringInterruptsTools }) => + "steers an update from $createdBy when steering is allowed (interruptsTools=$activeSteeringInterruptsTools, attachments=$attachments.length)", + ({ createdBy, activeSteeringInterruptsTools, attachments }) => Effect.scoped( Effect.gen(function* () { const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( - `update-steered-${createdBy}`, + `update-steered-${createdBy}-${activeSteeringInterruptsTools}`, { activeSteeringInterruptsTools }, ); yield* Effect.gen(function* () { @@ -677,6 +691,7 @@ it.effect.each([ threadId, createdBy, createdBy === "agent" ? "auto" : "steer", + attachments, ); assert.equal(result.delivery, "steered"); assert.deepEqual(steered, ["progress update"]); @@ -686,6 +701,108 @@ it.effect.each([ ), ); +// The Claude adapter calls this from its root PostToolBatch and Stop hooks. +const claimAtToolBoundary = Effect.fn("claimAtToolBoundary")(function* ( + threadId: ThreadId, + runId: RunId, +) { + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + return yield* requests.claimAgentMessages!({ threadId, runId }); +}); + +it.effect("an agent update joins the running turn at its next tool boundary, once", () => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "agent-update-tool-boundary", + { activeSteeringInterruptsTools: true }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + // The update arrives while a long tool call runs; nothing reaches the provider. + assert.equal((yield* sendUpdate(threadId, "agent", "auto")).delivery, "queued"); + assert.deepEqual(steered, []); + + assert.deepEqual(yield* claimAtToolBoundary(threadId, first.runId), ["progress update"]); + assert.deepEqual(yield* claimAtToolBoundary(threadId, first.runId), []); + const joined = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + joined.messages.find((message) => message.id === MessageId.make("message:update"))?.runId, + first.runId, + ); + assert.isFalse(joined.runs.some((run) => run.status === "queued")); + + // The turn ends; the update does not start another one. + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "waiting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const turn = joined.providerTurns[0]!; + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, + }); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(completed); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + assert.equal(started.length, 1); + assert.deepEqual(steered, []); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect("Stop keeps a queued agent update out of the stopped turn's boundaries", () => + Effect.scoped( + Effect.gen(function* () { + const { started, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "agent-update-boundary-after-stop", + { activeSteeringInterruptsTools: true }, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + assert.equal((yield* sendUpdate(threadId, "agent", "auto")).delivery, "queued"); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop"), + threadId, + runId: first.runId, + holdQueue: true, + }); + + assert.deepEqual(yield* claimAtToolBoundary(threadId, first.runId), []); + const projection = yield* orchestrator.getThreadProjection(threadId); + const queued = projection.runs.find((run) => run.status === "queued"); + assert.equal(queued?.queueHeld, true); + }).pipe(Effect.provide(layer)); + }), + ), +); + it.effect.each([{ canRestart: false }, { canRestart: true }])( "steers a changed turn-scoped selection into the running turn (canRestart=$canRestart)", ({ canRestart }) => diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 0342a3ccc9d94..f82ca98f4bcbc 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -513,6 +513,7 @@ export function layerWithRegistry( layerEffectWorkerProvided, layerEventSinkProvided, layerContinuationWorkerProvided, + layerContinuationRequests, layerCommandOutputHubProvided, ).pipe(Layer.provide(ProviderTurnStartServiceTestkit.layer), Layer.provide(NodeServices.layer)); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 94a9eca331d13..709f2512011c8 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1092,6 +1092,8 @@ export const OrchestrationV2ConversationMessage = Schema.Struct({ * the workflow's opening instead and keep the script behind a disclosure. */ workflowScript: Schema.optional(Schema.Boolean), + /** A queued agent update the running turn may take at its next tool boundary. */ + atToolBoundary: Schema.optional(Schema.Literal(true)), id: MessageId, threadId: ThreadId, runId: Schema.NullOr(RunId), @@ -2852,6 +2854,8 @@ export const OrchestrationV2Command = Schema.Union([ threadId: ThreadId, queuedRunId: RunId, targetRunId: RunId, + // The provider is delivering the message itself at a tool boundary: no steer effect. + atToolBoundary: Schema.optional(Schema.Literal(true)), }), Schema.Struct({ type: Schema.Literal("queue.resume"), diff --git a/packages/provider-core/src/server/ProviderContinuationRequests.ts b/packages/provider-core/src/server/ProviderContinuationRequests.ts index ff14d5325db9c..4873e17044c05 100644 --- a/packages/provider-core/src/server/ProviderContinuationRequests.ts +++ b/packages/provider-core/src/server/ProviderContinuationRequests.ts @@ -11,6 +11,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; export interface ProviderContinuationRequest { readonly threadId: ThreadId; @@ -36,6 +37,15 @@ export interface ProviderContinuationRequest { readonly clearIfCurrent?: () => Effect.Effect; } +/** + * Moves the thread's queued agent updates into `runId`, the provider's running turn, and returns + * their text for the provider to deliver at a tool boundary. Updates it does not return stay queued. + */ +export type ClaimAgentMessages = (input: { + readonly threadId: ThreadId; + readonly runId: RunId; +}) => Effect.Effect>; + /** * The default reference drops requests, keeping adapter construction dependency-free in tests; the * live layer must be shared with the ProviderContinuationService worker that drains it. @@ -43,6 +53,9 @@ export interface ProviderContinuationRequest { export class ProviderContinuationRequests extends Context.Reference<{ readonly offer: (request: ProviderContinuationRequest) => Effect.Effect; readonly take: Effect.Effect; + readonly claimAgentMessages?: ClaimAgentMessages; + /** Installs the worker's claim; until then nothing is claimed and updates start the next turn. */ + readonly serveAgentMessages?: (claim: ClaimAgentMessages) => Effect.Effect; }>("@t3tools/provider-core/server/ProviderContinuationRequests", { defaultValue: () => ({ offer: () => Effect.void, take: Effect.never }), }) {} @@ -51,10 +64,13 @@ export const layer = Layer.effect( ProviderContinuationRequests, Effect.gen(function* () { const queue = yield* Queue.unbounded(); + const claim = yield* Ref.make(() => Effect.succeed([])); return { offer: (request: ProviderContinuationRequest) => Queue.offer(queue, request).pipe(Effect.asVoid), take: Queue.take(queue), + claimAgentMessages: (input) => Ref.get(claim).pipe(Effect.flatMap((serve) => serve(input))), + serveAgentMessages: (serve) => Ref.set(claim, serve), }; }), ); From ed47e6f58f2ba0109a08859c8972183a5eec83b1 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Tue, 20 Oct 2026 02:00:47 -0400 Subject: [PATCH 09/23] fix(server): keep send tracking when a tool boundary takes an update A Claude tool boundary that takes a queued agent update cancels the queued run that t3_thread_send returned. t3_thread_wait on that run now follows the run that took its message instead of reporting it cancelled. A queued run that owns a delegated task (a follow-up to a finished child) keeps its own turn instead, because the task settles with that run. Co-Authored-By: Claude Opus 5.5 at high effort Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent --- .../server/src/orchestration-v2/Orchestrator.ts | 3 +++ .../SteeringCompletion.integration.test.ts | 17 ++++++++++++++++- .../orchestration-v2/ThreadManagementService.ts | 15 +++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 96c1b240b2f73..03790313d79e8 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -7836,9 +7836,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } if (command.atToolBoundary === true) { // Stop wins: a held queue or a stopped turn keeps the update queued. + // A run that owns a delegated task keeps its turn: the task settles + // with its own run, which joining would cancel. if ( queuedMessage.atToolBoundary !== true || queuedMessage.attachments.length > 0 || + queuedRun.delegatedTaskId !== undefined || queuedRun.queueHeld === true || (yield* stopReachedRun(command, command.threadId, command.targetRunId)) ) { diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 794d269179d2c..d2b1e8188e1fb 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -23,6 +23,7 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; @@ -723,7 +724,8 @@ it.effect("an agent update joins the running turn at its next tool boundary, onc const threadId = yield* startFirstTurn; const first = started[0]!; // The update arrives while a long tool call runs; nothing reaches the provider. - assert.equal((yield* sendUpdate(threadId, "agent", "auto")).delivery, "queued"); + const sent = yield* sendUpdate(threadId, "agent", "auto"); + assert.equal(sent.delivery, "queued"); assert.deepEqual(steered, []); assert.deepEqual(yield* claimAtToolBoundary(threadId, first.runId), ["progress update"]); @@ -734,6 +736,19 @@ it.effect("an agent update joins the running turn at its next tool boundary, onc first.runId, ); assert.isFalse(joined.runs.some((run) => run.status === "queued")); + // The sender waits on the run it was given; the run that took the update answers. + const waiting = yield* (yield* ThreadManagement.ThreadManagementService) + .waitForThread({ + projectId: joined.thread.projectId, + threadId, + runId: sent.run.id, + timeoutMs: 1, + }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("1 second"); + const waited = yield* Fiber.join(waiting); + assert.equal(waited.run?.id, first.runId); + assert.isTrue(waited.timedOut); // The turn ends; the update does not start another one. const completed = yield* orchestrator.streamDomainEvents.pipe( diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index d83dd9578c279..7926682307135 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -862,13 +862,23 @@ const make = Effect.gen(function* () { // so a cut run is not the answer while its continuation is pending. const outcome = (runs: ReadonlyArray) => Effect.gen(function* () { - const current = runs.find((candidate) => candidate.id === selectedRun.id); - if (current === undefined) { + const selected = runs.find((candidate) => candidate.id === selectedRun.id); + if (selected === undefined) { return yield* new ThreadManagementRunNotFoundError({ threadId: input.threadId, runId: selectedRun.id, }); } + // A queued message promoted into the running turn (a steer or a tool + // boundary) cancels its own run before it starts; the run that took + // the message answers. + const takenBy = + selected.status === "cancelled" && selected.startedAt === null + ? (yield* getProjectThreadRecords(input, ["messages"], { + messageIds: [selected.userMessageId], + })).messages[0]?.runId + : undefined; + const current = runs.find((candidate) => candidate.id === takenBy) ?? selected; const run = latestRestartContinuation(runs, current); const pending = isTerminalRunStatus(run.status) && @@ -900,6 +910,7 @@ const make = Effect.gen(function* () { (stored) => stored.event.type !== "run.updated" || stored.event.payload.id === selectedRun.id || + isTerminalRunStatus(stored.event.payload.status) || stored.event.payload.restartContinuationOfRunId !== undefined, ), ), From d4ac7c1029d83062f2b24f33ddc7d0e02158ad14 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Tue, 20 Oct 2026 02:33:17 -0400 Subject: [PATCH 10/23] fix(server): stop boundary claims on hook abort and read only queued runs An aborted PostToolBatch or Stop hook interrupts its claim; the claim no longer recovers that interruption and goes on promoting updates whose hook response Claude has discarded. Only promotion refusals are ignored. The claim runs at every tool batch, so it now reads only queued runs and their messages (a new runStatuses record filter) instead of every user message and run in the thread. The t3_thread_send description says a queued Claude update may join the running turn after its current tool call, and that t3_thread_wait follows it. Co-Authored-By: Claude Opus 5.5 at high effort Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent --- .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../src/orchestration-v2/ProjectionStore.ts | 13 ++-- .../ProviderContinuationService.test.ts | 60 +++++++++++++++++++ .../ProviderContinuationService.ts | 16 +++-- 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 60dd07bf446a6..7e9c8831fc6e9 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -234,7 +234,7 @@ export const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. When the target session's steering interrupts tools (including Claude), both auto and explicit steer queue a separate turn instead; delivery='queued' confirms this fallback. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", + "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. When the target session's steering interrupts tools (including Claude), both auto and explicit steer queue instead (delivery='queued'), and the running turn may take the message after its current tool call; t3_thread_wait on the returned run then follows that turn. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 9deee8f3850a1..62de718c7f331 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -295,6 +295,7 @@ export interface ProjectionRecordFilter { readonly messageRunIds?: ReadonlyArray; readonly turnItemRunIds?: ReadonlyArray; readonly runIds?: ReadonlyArray; + readonly runStatuses?: ReadonlyArray; readonly turnItemTypes?: ReadonlyArray; readonly turnItemStatuses?: ReadonlyArray; readonly omitSettledLeafNodes?: boolean; @@ -3043,7 +3044,7 @@ export const layer: Layer.Layer = ] = yield* Effect.all([ decodeThreadPayload(threadRow.payload_json), (fields !== undefined && !fields.includes("runs")) || - (window === undefined && matchesNothing(filter?.runIds)) + (window === undefined && matchesNothing(filter?.runIds, filter?.runStatuses)) ? Effect.succeed([]) : window === undefined ? sql` @@ -3051,6 +3052,7 @@ export const layer: Layer.Layer = FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} ${filter?.runIds === undefined ? sql`` : sql`AND run_id IN (SELECT value FROM json_each(${encodeIdList(filter.runIds)}))`} + ${filter?.runStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.runStatuses)}))`} ORDER BY ordinal ASC ` : sql` @@ -6452,10 +6454,11 @@ export const layerMemory: Layer.Layer = Layer.effect( (filter?.messageIds === undefined || filter.messageIds.includes(row.id)) && (filter?.messageRoles === undefined || filter.messageRoles.includes(row.role)), ), - runs: - filter?.runIds === undefined - ? projection.runs - : projection.runs.filter((row) => filter.runIds!.includes(row.id)), + runs: projection.runs.filter( + (row) => + (filter?.runIds === undefined || filter.runIds.includes(row.id)) && + (filter?.runStatuses === undefined || filter.runStatuses.includes(row.status)), + ), turnItems: projection.turnItems.filter( (row) => (filter?.turnItemRunIds === undefined || diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index a1a720afd710a..f7e1aa69e848d 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -14,6 +14,7 @@ import * as Deferred from "effect/Deferred"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; @@ -1020,4 +1021,63 @@ describe("ProviderContinuationService", () => { ); }); }); + + it.effect("an aborted boundary hook commits no further promotions", () => + Effect.gen(function* () { + // Three queued agent updates. The fake dispatch stands in for the SQLite + // commit: the first promotion commits, the second blocks until the hook + // is aborted, and the third must never run. + const runs = [1, 2, 3].map((ordinal) => ({ + id: RunId.make(`run-boundary-${ordinal}`), + status: "queued", + ordinal, + queuePosition: ordinal, + userMessageId: MessageId.make(`message-boundary-${ordinal}`), + })); + const messages = runs.map((run) => ({ + id: run.userMessageId, + text: "update", + atToolBoundary: true, + })); + const committed: Array = []; + const blocked = yield* Deferred.make(); + const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => + Effect.succeed({ + ...projection, + runs, + messages, + } as unknown as OrchestrationV2ThreadProjection), + dispatch: (command) => + Effect.gen(function* () { + if (command.type !== "queued-message.promote-to-steer") return {} as never; + if (committed.length === 1) { + yield* Deferred.succeed(blocked, undefined); + return yield* Effect.never; + } + committed.push(command.queuedRunId); + return { sequence: 1, storedEvents: [] }; + }), + }); + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests.pipe( + Effect.provide( + ProviderContinuationService.layer.pipe( + Layer.provideMerge( + Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, layerThreads), + ), + ), + ), + ); + // The Claude adapter runs the claim with the hook's AbortSignal, which + // interrupts the claim fiber when Claude aborts the hook. + const claim = yield* requests.claimAgentMessages!({ + threadId, + runId: RunId.make("run-boundary-active"), + }).pipe(Effect.forkScoped); + yield* Deferred.await(blocked); + yield* Fiber.interrupt(claim); + assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(claim))); + assert.deepEqual(committed, [runs[0]!.id]); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index ee278a68bdb93..7e0079955139e 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -250,9 +250,15 @@ export const layer = Layer.effectDiscard( // queued for the next turn. const claimAgentMessages: ProviderContinuationRequests.ClaimAgentMessages = (input) => Effect.gen(function* () { - const queued = yield* threads.getThreadRecords(input.threadId, ["runs", "messages"], { - messageRoles: ["user"], + // This runs at every tool batch: read only queued runs and their messages. + const { runs } = yield* threads.getThreadRecords(input.threadId, ["runs"], { + runStatuses: ["queued"], }); + if (runs.length === 0) return []; + const { messages } = yield* threads.getThreadRecords(input.threadId, ["messages"], { + messageIds: runs.map((run) => run.userMessageId), + }); + const queued = { runs, messages }; const texts: Array = []; for (const run of queuedRunsInDeliveryOrder(queued)) { const message = queued.messages.find((row) => row.id === run.userMessageId); @@ -270,7 +276,9 @@ export const layer = Layer.effectDiscard( targetRunId: input.runId, atToolBoundary: true, }) - .pipe(Effect.catchCause(() => Effect.succeed(undefined))); + // A refusal leaves the update queued. An aborted hook interrupts this + // fiber instead, which must stop the loop before the next promotion. + .pipe(Effect.catch(() => Effect.succeed(undefined))); // Deliver what the promotion committed, which may postdate the snapshot. const joined = result?.storedEvents.find( ({ event }) => event.type === "message.updated" && event.payload.id === message.id, @@ -287,7 +295,7 @@ export const layer = Layer.effectDiscard( } return texts; }).pipe( - Effect.catchCause((cause) => + Effect.catch((cause) => Effect.logWarning("orchestration-v2.provider-continuation.agent-message-claim-failed", { threadId: input.threadId, cause, From 3accb79d0af5bd2eacfe847717fb83c128b88389 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 13:17:23 +0000 Subject: [PATCH 11/23] test(server): agent updates with attachments queue and aborted boundary claims lose nothing Co-Authored-By: Claude Opus 5.5 --- .../ProviderContinuationService.test.ts | 10 +++-- .../SteeringCompletion.integration.test.ts | 43 ++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index 0452cbb960670..aa758a4e044a9 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -594,11 +594,12 @@ describe("ProviderContinuationService", () => { }); }); - it.effect("an aborted boundary hook commits no further promotions", () => + it.effect("an aborted boundary hook leaves no promoted update undelivered", () => Effect.gen(function* () { // Three queued agent updates. The fake dispatch stands in for the SQLite // commit: the first promotion commits, the second blocks until the hook - // is aborted, and the third must never run. + // is aborted. Claude drops an aborted hook's output, so a committed + // promotion the claim did not return is a lost update. const runs = [1, 2, 3].map((ordinal) => ({ id: RunId.make(`run-boundary-${ordinal}`), status: "queued", @@ -648,8 +649,9 @@ describe("ProviderContinuationService", () => { }).pipe(Effect.forkScoped); yield* Deferred.await(blocked); yield* Fiber.interrupt(claim); - assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(claim))); - assert.deepEqual(committed, [runs[0]!.id]); + const exit = yield* Fiber.await(claim); + const delivered = Exit.isSuccess(exit) ? exit.value : []; + assert.equal(committed.length, delivered.length); }).pipe(Effect.scoped), ); }); diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 86f1f80007bd9..6fcc4385560ba 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -592,13 +592,27 @@ const sendUpdate = Effect.fn("sendUpdate")(function* ( return result; }); -it.effect.each(["auto", "steer"] as const)( - "queues an agent %s update behind a turn whose steering cancels tools", - (mode) => +const imageAttachment: ChatAttachment = { + type: "image", + id: "attachment-1", + name: "screen.png", + mimeType: "image/png", + sizeBytes: 1, +}; + +it.effect.each([ + { mode: "auto", attachments: [] }, + { mode: "steer", attachments: [] }, + // Steering an image natively would still cancel the running tool. + { mode: "auto", attachments: [imageAttachment] }, + { mode: "steer", attachments: [imageAttachment] }, +] as const)( + "queues an agent $mode update behind a turn whose steering cancels tools (attachments=$attachments.length)", + ({ mode, attachments }) => Effect.scoped( Effect.gen(function* () { const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( - `agent-update-queued-${mode}`, + `agent-update-queued-${mode}-${attachments.length}`, { activeSteeringInterruptsTools: true }, ); yield* Effect.gen(function* () { @@ -606,7 +620,7 @@ it.effect.each(["auto", "steer"] as const)( const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; const threadId = yield* startFirstTurn; const first = started[0]!; - const result = yield* sendUpdate(threadId, "agent", mode); + const result = yield* sendUpdate(threadId, "agent", mode, attachments); assert.equal(result.delivery, "queued"); assert.deepEqual(steered, []); const pending = yield* orchestrator.getThreadProjection(threadId); @@ -653,22 +667,12 @@ it.effect.each(["auto", "steer"] as const)( ), ); -const imageAttachment: ChatAttachment = { - type: "image", - id: "attachment-1", - name: "screen.png", - mimeType: "image/png", - sizeBytes: 1, -}; - it.effect.each([ - { createdBy: "user", activeSteeringInterruptsTools: true, attachments: [] }, - { createdBy: "agent", activeSteeringInterruptsTools: false, attachments: [] }, - // Boundary hooks carry text only, so this one steers natively. - { createdBy: "agent", activeSteeringInterruptsTools: true, attachments: [imageAttachment] }, + { createdBy: "user", activeSteeringInterruptsTools: true }, + { createdBy: "agent", activeSteeringInterruptsTools: false }, ] as const)( - "steers an update from $createdBy when steering is allowed (interruptsTools=$activeSteeringInterruptsTools, attachments=$attachments.length)", - ({ createdBy, activeSteeringInterruptsTools, attachments }) => + "steers an update from $createdBy when steering is allowed (interruptsTools=$activeSteeringInterruptsTools)", + ({ createdBy, activeSteeringInterruptsTools }) => Effect.scoped( Effect.gen(function* () { const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( @@ -681,7 +685,6 @@ it.effect.each([ threadId, createdBy, createdBy === "agent" ? "auto" : "steer", - attachments, ); assert.equal(result.delivery, "steered"); assert.deepEqual(steered, ["progress update"]); From 64ea3fa1b67adb9aaed0973a6800a3f561ec2a11 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 13:20:46 +0000 Subject: [PATCH 12/23] fix(server): agent updates to tool-cancelling turns always wait for the next turn Queue every agent auto/steer update, attachments included, when the target session's steering cancels tools. Drop the tool-boundary claim path: a committed promotion whose hook was aborted lost the update, and the attachment exemption still cancelled running tools. Co-Authored-By: Claude Opus 5.5 --- .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../Adapters/ClaudeAdapterV2.test.ts | 125 ------------------ .../Adapters/ClaudeAdapterV2.ts | 55 +------- .../src/orchestration-v2/Orchestrator.ts | 81 +----------- .../src/orchestration-v2/ProjectionStore.ts | 13 +- .../ProviderContinuationService.test.ts | 62 --------- .../ProviderContinuationService.ts | 67 ---------- .../SteeringCompletion.integration.test.ts | 121 +---------------- .../ThreadManagementService.ts | 15 +-- .../testkit/ProviderReplayHarness.ts | 1 - packages/contracts/src/orchestrationV2.ts | 4 - .../server/ProviderContinuationRequests.ts | 16 --- 12 files changed, 16 insertions(+), 546 deletions(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 6183962692dad..34bd14327d10a 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -236,7 +236,7 @@ const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. When the target session's steering interrupts tools (including Claude), both auto and explicit steer queue instead (delivery='queued'), and the running turn may take the message after its current tool call; t3_thread_wait on the returned run then follows that turn. Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", + "Send a message to any T3 thread in this environment. The target cannot have broader permission modes than the caller. Use delegate_task with the full review context and a new clientRequestId for independent review rounds. For a follow-up that needs the same child's conversation, send to its childThreadId. After the original task completes, a requested follow-up creates a fresh task in that child thread and returns taskId when it starts a turn or restarts a turn without its own task; its completion notifies the parent, and the original result stays unchanged. A steer that joins a running turn returns that turn's task, if it has one. mode='auto' starts an idle thread, safely steers a fully active turn, or queues behind a turn that is not yet steerable. When the target session's steering interrupts tools (including Claude), both auto and explicit steer queue the message for the next turn instead (delivery='queued'). Use queue for a separate follow-up turn, steer for an in-flight update, or restart to explicitly interrupt-and-restart the active turn. clientRequestId makes retries idempotent.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 8bea63a7c938d..b231fbfca9d98 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2149,7 +2149,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly nativeSessionId?: string; readonly settings?: ClaudeSettings; readonly freshQueueOnReopen?: boolean; - readonly claimAgentMessages?: ProviderContinuationRequests.ClaimAgentMessages; readonly forkSession?: ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerShape["forkSession"]; readonly allocateSessionId?: Effect.Effect; }) => @@ -2218,9 +2217,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { Effect.andThen(Queue.offer(continuationReceipts, request)), Effect.andThen(changes.notify), ), - ...(options?.claimAgentMessages === undefined - ? {} - : { claimAgentMessages: options.claimAgentMessages }), }, queryRunner: { allocateSessionId: @@ -3464,127 +3460,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect("delivers claimed agent updates at root tool-batch and stop hooks", () => - Effect.scoped( - Effect.gen(function* () { - const claims: Array<{ readonly threadId: string; readonly runId: string }> = []; - const harness = yield* makeWakeHarnessWithOptions({ - claimAgentMessages: (input) => - Effect.sync(() => { - claims.push(input); - return ["Child finished."]; - }), - }); - const input = makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("attempt-agent-boundary"), - text: "Run the slow build.", - attachments: [], - }); - yield* harness.runtime.startTurn(input); - const hooks = harness.getOpenedOptions()?.hooks; - const signal = new AbortController().signal; - const base = { - session_id: WAKE_NATIVE_SESSION, - transcript_path: "", - cwd: "", - }; - const batch = yield* Effect.promise(() => - hooks!.PostToolBatch![0]!.hooks[0]!( - { ...base, hook_event_name: "PostToolBatch", tool_calls: [] }, - undefined, - { signal }, - ), - ); - const stop = yield* Effect.promise(() => - hooks!.Stop![0]!.hooks[0]!( - { ...base, hook_event_name: "Stop", stop_hook_active: false }, - undefined, - { signal }, - ), - ); - // A subagent's boundary is not the root turn's. - const subagent = yield* Effect.promise(() => - hooks!.PostToolBatch![0]!.hooks[0]!( - { ...base, hook_event_name: "PostToolBatch", tool_calls: [], agent_id: "agent-1" }, - undefined, - { signal }, - ), - ); - assert.deepEqual(batch, { - hookSpecificOutput: { - hookEventName: "PostToolBatch", - additionalContext: "New message from another agent:\nChild finished.", - }, - }); - assert.deepEqual(stop, { - hookSpecificOutput: { - hookEventName: "Stop", - additionalContext: "New message from another agent:\nChild finished.", - }, - }); - assert.deepEqual(subagent, {}); - assert.deepEqual(claims, [ - { threadId: harness.threadId, runId: input.runId }, - { threadId: harness.threadId, runId: input.runId }, - ]); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), - ), - ); - - it.effect.each(["agent", "user"] as const)( - "notes that a native %s steer may have interrupted a command only for agents", - (createdBy) => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeWakeHarness; - const idAllocator = yield* IdAllocator.IdAllocatorV2; - const attemptId = RunAttemptId.make("attempt-agent-steer-note"); - const input = makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId, - text: "Audit the settings pages.", - attachments: [], - }); - yield* harness.runtime.startTurn(input); - yield* harness.runtime.steerTurn({ - threadId: harness.threadId, - runId: input.runId, - providerThread: harness.providerThread, - providerTurnId: idAllocator.derive.providerTurn({ - driver: ClaudeAdapterV2.CLAUDE_PROVIDER, - nativeTurnId: `turn:${attemptId}`, - }), - message: { - createdBy, - creationSource: createdBy === "agent" ? "mcp" : "web", - messageId: MessageId.make("message-agent-steer-note"), - text: "See the attached screenshot.", - attachments: [], - }, - }); - const steer = JSON.stringify(harness.offeredMessages[1]?.message.content); - assert.include(steer, "See the attached screenshot."); - assert.equal( - steer.includes("may have interrupted a running command"), - createdBy === "agent", - ); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), - ), - ); - it.effect.each( (["aborted_tools", "aborted_streaming"] as const).flatMap((terminalReason) => [true, false].map((steered) => ({ terminalReason, steered })), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 702e950c84e0e..49b344961b3fb 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -26,7 +26,6 @@ import { type EffortLevel, type ForkSessionOptions, type ForkSessionResult, - type HookCallback, getSessionMessages, getSubagentMessages, query, @@ -304,8 +303,6 @@ const CLAUDE_SCRIPT_EXTENSIONS = [".js", ".mjs", ".tsx", ".ts", ".jsx"]; // The SDK puts no deadline on control requests. The CLI answers // mcp_set_servers once each server connected or hit its own 30 s timeout. const CLAUDE_MCP_REGISTRATION_TIMEOUT = "90 seconds"; -const CLAUDE_AGENT_STEER_NOTE = - "This message came from another agent, not the user, and may have interrupted a running command; check that command's state before continuing."; // Claude can consume a subagent result before the launching turn ends, leaving // its notification with no native wake turn to answer the continuation. const CLAUDE_WAKE_TURN_START_DEADLINE = "2 minutes"; @@ -3899,7 +3896,6 @@ export interface ClaudeAdapterV2Options { readonly offer: ( request: ProviderContinuationRequests.ProviderContinuationRequest, ) => Effect.Effect; - readonly claimAgentMessages?: ProviderContinuationRequests.ClaimAgentMessages; }; } @@ -9314,53 +9310,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( supportedDialogKinds: ["resume_return"], ...(appendSystemPromptFileText === undefined ? {} : { appendSystemPromptFileText }), }); - // Native steering cancels running tools, so agent updates wait in the - // app queue. The root agent's tool-batch and stop boundaries take them - // as context: the tool has finished and, at Stop, the turn continues. - // The claim commits before the SDK reads the context: delivery is at - // most once, and a crash in that window loses the claimed updates. A - // cancelled hook interrupts the claim before its next promotion. - const takeAgentMessages: HookCallback = (hookInput, _toolUseId, { signal }) => - runPromise( - Effect.gen(function* () { - const context = yield* Ref.get(activeTurn); - if ( - (hookInput.hook_event_name !== "PostToolBatch" && - hookInput.hook_event_name !== "Stop") || - hookInput.agent_id !== undefined || - context === null || - context.nativeThreadId !== hookInput.session_id || - (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) || - continuationRequests.claimAgentMessages === undefined - ) { - return {}; - } - const texts = yield* continuationRequests.claimAgentMessages({ - threadId: context.input.threadId, - runId: context.input.runId, - }); - return texts.length === 0 - ? {} - : { - hookSpecificOutput: { - hookEventName: hookInput.hook_event_name, - additionalContext: `New message from another agent:\n${texts.join("\n\n")}`, - }, - }; - }), - { signal }, - ); const querySession = yield* queryRunner .open({ threadId: turnInput.threadId, providerSessionId: input.providerSessionId, - options: { - ...queryOptions, - hooks: { - PostToolBatch: [{ hooks: [takeAgentMessages] }], - Stop: [{ hooks: [takeAgentMessages] }], - }, - }, + options: queryOptions, }) .pipe( Effect.onError(() => @@ -9954,12 +9908,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } const userMessage = yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( - // An agent update steers natively only when hooks cannot carry it - // (attachments). It cancels running tools; say so, or Claude reads - // the cancellation as a user refusal. - turnInput.message.createdBy === "agent" - ? `${CLAUDE_AGENT_STEER_NOTE}\n\n${turnInput.message.text}` - : turnInput.message.text, + turnInput.message.text, compileClaudeModelSelection(currentTurn.input.modelSelection).promptEffort, ), attachments: turnInput.message.attachments, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 4b1142b37dad0..152387ddbebf8 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -643,21 +643,6 @@ function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"] ); } -// An agent update that asked for prompt delivery (steer or auto, not an explicit -// queue) is queued only because the turn is busy or still starting; the running -// turn may take it at its next tool boundary. Hooks carry text only. -function agentUpdateWantsPromptDelivery( - command: Extract, -): boolean { - return ( - command.createdBy === "agent" && - command.dispatchMode.type !== "queue_after_active" && - command.attachments.length === 0 && - command.notification === undefined && - command.scheduledTaskId === undefined - ); -} - function delegatedTaskTerminalStatus( status: OrchestrationV2Run["status"], ): OrchestrationV2Subagent["status"] | null { @@ -4122,8 +4107,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly senderThreadId?: OrchestrationV2ConversationMessage["senderThreadId"]; readonly delegatedCompletion?: OrchestrationV2ConversationMessage["delegatedCompletion"]; readonly forceRestart: boolean; - /** The provider delivers the message at a tool boundary; record it without a steer effect. */ - readonly atToolBoundary?: boolean; }) { const targetRun = input.projection.runs.find((candidate) => candidate.id === input.targetRunId); if (targetRun === undefined) { @@ -4364,7 +4347,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio input.projection.thread.providerInstanceId !== input.modelSelection.instanceId; if ( input.delegatedCompletion === undefined && - input.atToolBoundary !== true && (instanceChanged || !modelSelectionsEqual(input.projection.thread.modelSelection, input.modelSelection)) ) { @@ -4388,7 +4370,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio providerThreadId: providerThread.id, providerInstanceId: targetRun.providerInstanceId, }); - if (input.atToolBoundary === true) return; yield* Ref.update(input.effects, (existing) => [ ...existing, { @@ -5037,7 +5018,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command.dispatchMode, command.deliveryIntent, ); - let atToolBoundary = false; if (dispatchMode.type === "steer_active") { const targetRunId = dispatchMode.targetRunId; const target = projection.runs.find((run) => run.id === targetRunId); @@ -5070,16 +5050,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(mapDispatchError(command)); if ( Option.isSome(session) && - session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === - true && - // Boundary hooks carry text only. An update with attachments steers - // natively; the adapter tells the agent it may have interrupted a tool. - command.attachments.length === 0 + session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === true ) { // Claude steering cancels pending tools, which appear as user refusals. - // Agent updates wait in the durable queue instead, and the running turn - // takes them at its next tool batch or Stop (ProviderContinuationService - // claimAgentMessages). Whatever it does not take starts the next turn. + // Agent updates, attachments included, wait in the durable queue and + // start the next turn. dispatchMode = { type: "queue_after_active" }; } } @@ -5159,9 +5134,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (id) => projection.subagents.find((task) => task.id === id)?.completionWake === "always", ) ) { - // If it queues (steering would cancel tools, or the turn is still - // starting), the running turn may take it at its next tool boundary. - atToolBoundary = true; const active = projection.runs.find((run) => run.status === "running"); const providerThread = projection.providerThreads.find( (row) => row.id === active?.providerThreadId, @@ -5515,9 +5487,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio updatedAt: now, ...(delegatedCompletion === undefined ? {} : { delegatedCompletion }), ...(command.notification === undefined ? {} : { notification: command.notification }), - ...(atToolBoundary || agentUpdateWantsPromptDelivery(command) - ? { atToolBoundary: true as const } - : {}), }; const emitEvent = emit(events, command); if (targetProviderThread === undefined) { @@ -8028,24 +7997,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Queued run ${queuedRun.id} is missing message or execution state.`, }); } - if (command.atToolBoundary === true) { - // Stop wins: a held queue or a stopped turn keeps the update queued. - // A run that owns a delegated task keeps its turn: the task settles - // with its own run, which joining would cancel. - if ( - queuedMessage.atToolBoundary !== true || - queuedMessage.attachments.length > 0 || - queuedRun.delegatedTaskId !== undefined || - queuedRun.queueHeld === true || - (yield* stopReachedRun(command, command.threadId, command.targetRunId)) - ) { - return yield* new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause: `Queued run ${queuedRun.id} cannot join the running turn at a tool boundary.`, - }); - } - } else if ( + if ( queuedMessage.delegatedCompletion !== undefined || queuedMessage.notification !== undefined ) { @@ -8099,22 +8051,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, }); - const atToolBoundary = command.atToolBoundary === true; yield* dispatchSteerIntoRun({ command, events, effects, projection, - // An update joining at a tool boundary keeps the running turn's model; a - // selection change could otherwise restart the turn from inside its hook. - modelSelection: atToolBoundary - ? (projection.runs.find((run) => run.id === command.targetRunId)?.modelSelection ?? - projection.thread.modelSelection) - : projection.thread.modelSelection, - ...(queuedMessage.delegatedCompletion === undefined - ? {} - : { delegatedCompletion: queuedMessage.delegatedCompletion }), - atToolBoundary, + modelSelection: projection.thread.modelSelection, targetRunId: command.targetRunId, messageId: queuedMessage.id, text: queuedMessage.text, @@ -8130,19 +8072,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : { senderThreadId: queuedMessage.senderThreadId }), forceRestart: false, }); - if (atToolBoundary && queuedMessage.delegatedCompletion !== undefined) { - // Accept in the same commit: the cancelled queued run would otherwise - // read as a failed delivery and reserve a duplicate notification. - yield* dispatchNotificationAccepted( - { - type: "notification.delivery.accept", - commandId: command.commandId, - threadId: command.threadId, - messageId: queuedMessage.id, - }, - events, - ); - } }); const dispatchQueuedRunReorder = Effect.fn("Orchestrator.dispatchQueuedRunReorder")(function* ( diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 6a7fbd86ac2ee..40ff180d27495 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -323,7 +323,6 @@ export interface ProjectionRecordFilter { readonly messageRunIds?: ReadonlyArray; readonly turnItemRunIds?: ReadonlyArray; readonly runIds?: ReadonlyArray; - readonly runStatuses?: ReadonlyArray; readonly turnItemTypes?: ReadonlyArray; readonly turnItemStatuses?: ReadonlyArray; readonly omitSettledLeafNodes?: boolean; @@ -3171,7 +3170,7 @@ export const layer: Layer.Layer = ] = yield* Effect.all([ decodeThreadPayload(threadRow.payload_json), (fields !== undefined && !fields.includes("runs")) || - (window === undefined && matchesNothing(filter?.runIds, filter?.runStatuses)) + (window === undefined && matchesNothing(filter?.runIds)) ? Effect.succeed([]) : window === undefined ? sql` @@ -3179,7 +3178,6 @@ export const layer: Layer.Layer = FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} ${filter?.runIds === undefined ? sql`` : sql`AND run_id IN (SELECT value FROM json_each(${encodeIdList(filter.runIds)}))`} - ${filter?.runStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.runStatuses)}))`} ORDER BY ordinal ASC ` : sql` @@ -6842,11 +6840,10 @@ export const layerMemory: Layer.Layer = Layer.effect( (filter?.messageIds === undefined || filter.messageIds.includes(row.id)) && (filter?.messageRoles === undefined || filter.messageRoles.includes(row.role)), ), - runs: projection.runs.filter( - (row) => - (filter?.runIds === undefined || filter.runIds.includes(row.id)) && - (filter?.runStatuses === undefined || filter.runStatuses.includes(row.status)), - ), + runs: + filter?.runIds === undefined + ? projection.runs + : projection.runs.filter((row) => filter.runIds!.includes(row.id)), turnItems: projection.turnItems.filter( (row) => (filter?.turnItemRunIds === undefined || diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index aa758a4e044a9..6e2abc0c1d6b8 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -13,7 +13,6 @@ import { import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; -import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; @@ -593,65 +592,4 @@ describe("ProviderContinuationService", () => { ); }); }); - - it.effect("an aborted boundary hook leaves no promoted update undelivered", () => - Effect.gen(function* () { - // Three queued agent updates. The fake dispatch stands in for the SQLite - // commit: the first promotion commits, the second blocks until the hook - // is aborted. Claude drops an aborted hook's output, so a committed - // promotion the claim did not return is a lost update. - const runs = [1, 2, 3].map((ordinal) => ({ - id: RunId.make(`run-boundary-${ordinal}`), - status: "queued", - ordinal, - queuePosition: ordinal, - userMessageId: MessageId.make(`message-boundary-${ordinal}`), - })); - const messages = runs.map((run) => ({ - id: run.userMessageId, - text: "update", - atToolBoundary: true, - })); - const committed: Array = []; - const blocked = yield* Deferred.make(); - const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({ - getThreadRecords: () => - Effect.succeed({ - ...projection, - runs, - messages, - } as unknown as OrchestrationV2ThreadProjection), - dispatch: (command) => - Effect.gen(function* () { - if (command.type !== "queued-message.promote-to-steer") return {} as never; - if (committed.length === 1) { - yield* Deferred.succeed(blocked, undefined); - return yield* Effect.never; - } - committed.push(command.queuedRunId); - return { sequence: 1, storedEvents: [] }; - }), - }); - const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests.pipe( - Effect.provide( - ProviderContinuationService.layer.pipe( - Layer.provideMerge( - Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, layerThreads), - ), - ), - ), - ); - // The Claude adapter runs the claim with the hook's AbortSignal, which - // interrupts the claim fiber when Claude aborts the hook. - const claim = yield* requests.claimAgentMessages!({ - threadId, - runId: RunId.make("run-boundary-active"), - }).pipe(Effect.forkScoped); - yield* Deferred.await(blocked); - yield* Fiber.interrupt(claim); - const exit = yield* Fiber.await(claim); - const delivered = Exit.isSuccess(exit) ? exit.value : []; - assert.equal(committed.length, delivered.length); - }).pipe(Effect.scoped), - ); }); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index 70b2f08c45c7b..8aad65111d09d 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -1,5 +1,4 @@ import { CommandId, type OrchestrationV2ThreadProjection } from "@t3tools/contracts"; -import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; @@ -8,7 +7,6 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts"; -import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts"; const CONTINUATION_MESSAGE_TEXT = "Background task completed."; @@ -244,71 +242,6 @@ export const layer = Layer.effectDiscard( }, ); - // The provider waits on this from a tool-batch or stop hook, so each update - // joins its running turn before the hook returns the text. An update that - // cannot join (edited away, held by Stop, turn no longer running) stays - // queued for the next turn. - const claimAgentMessages = Effect.fn("ProviderContinuationService.claimAgentMessages")( - function* (input: Parameters[0]) { - // This runs at every tool batch: read only queued runs and their messages. - const { runs } = yield* threads.getThreadRecords(input.threadId, ["runs"], { - runStatuses: ["queued"], - }); - if (runs.length === 0) return []; - const { messages } = yield* threads.getThreadRecords(input.threadId, ["messages"], { - messageIds: runs.map((run) => run.userMessageId), - }); - const queued = { runs, messages }; - const texts: Array = []; - for (const run of queuedRunsInDeliveryOrder(queued)) { - const message = queued.messages.find((row) => row.id === run.userMessageId); - if (message?.atToolBoundary !== true) continue; - const result = yield* threads - .dispatch({ - type: "queued-message.promote-to-steer", - // Fresh per try: a refusal now must not be replayed at a later boundary. - commandId: yield* ids.allocate.command({ - fixtureName: "tool-boundary", - commandName: "promote", - }), - threadId: input.threadId, - queuedRunId: run.id, - targetRunId: input.runId, - atToolBoundary: true, - }) - // A refusal leaves the update queued. An aborted hook interrupts this - // fiber instead, which must stop the loop before the next promotion. - .pipe(Effect.catch(() => Effect.succeed(undefined))); - // Deliver what the promotion committed, which may postdate the snapshot. - const joined = result?.storedEvents.find( - ({ event }) => event.type === "message.updated" && event.payload.id === message.id, - )?.event; - if (joined?.type === "message.updated") { - const promoted = joined.payload; - texts.push( - projectComposerContextForProvider({ - text: promoted.text, - records: promoted.context?.records ?? [], - }), - ); - } - } - return texts; - }, - (effect, input) => - effect.pipe( - Effect.catch((cause) => - Effect.logWarning("orchestration-v2.provider-continuation.agent-message-claim-failed", { - threadId: input.threadId, - cause, - }).pipe(Effect.as([])), - ), - ), - ); - if (requests.serveAgentMessages !== undefined) { - yield* requests.serveAgentMessages(claimAgentMessages); - } - yield* requests.take.pipe( Effect.flatMap((request) => dispatchContinuation(request).pipe( diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 6fcc4385560ba..2c703cf061020 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -12,7 +12,6 @@ import { ThreadId, TurnItemId, type ChatAttachment, - type RunId, type ModelSelection, type OrchestrationV2DomainEvent, } from "@t3tools/contracts"; @@ -23,7 +22,6 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; -import * as TestClock from "effect/testing/TestClock"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; @@ -36,7 +34,6 @@ import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; -import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; const driver = ProviderDriverKind.make("codex"); const instanceId = ProviderInstanceId.make("codex"); @@ -517,7 +514,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* const layerRuntime = ProviderReplayHarness.layerWithRegistry( { name }, ProviderAdapterRegistry.layerSingle(adapter), - { runEffectWorker: false, runContinuationWorker: true, databaseLayer }, + { runEffectWorker: false, databaseLayer }, ); const layer = ThreadManagement.layer.pipe( Layer.provide(ServerSettings.layerTest()), @@ -694,122 +691,6 @@ it.effect.each([ ), ); -// The Claude adapter calls this from its root PostToolBatch and Stop hooks. -const claimAtToolBoundary = Effect.fn("claimAtToolBoundary")(function* ( - threadId: ThreadId, - runId: RunId, -) { - const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; - return yield* requests.claimAgentMessages!({ threadId, runId }); -}); - -it.effect("an agent update joins the running turn at its next tool boundary, once", () => - Effect.scoped( - Effect.gen(function* () { - const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( - "agent-update-tool-boundary", - { activeSteeringInterruptsTools: true }, - ); - yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const threadId = yield* startFirstTurn; - const first = started[0]!; - // The update arrives while a long tool call runs; nothing reaches the provider. - const sent = yield* sendUpdate(threadId, "agent", "auto"); - assert.equal(sent.delivery, "queued"); - assert.deepEqual(steered, []); - - assert.deepEqual(yield* claimAtToolBoundary(threadId, first.runId), ["progress update"]); - assert.deepEqual(yield* claimAtToolBoundary(threadId, first.runId), []); - const joined = yield* orchestrator.getThreadProjection(threadId); - assert.equal( - joined.messages.find((message) => message.id === MessageId.make("message:update"))?.runId, - first.runId, - ); - assert.isFalse(joined.runs.some((run) => run.status === "queued")); - // The sender waits on the run it was given; the run that took the update answers. - const waiting = yield* (yield* ThreadManagement.ThreadManagementService) - .waitForThread({ - projectId: joined.thread.projectId, - threadId, - runId: sent.run.id, - timeoutMs: 1, - }) - .pipe(Effect.forkScoped); - yield* TestClock.adjust("1 second"); - const waited = yield* Fiber.join(waiting); - assert.equal(waited.run?.id, first.runId); - assert.isTrue(waited.timedOut); - - // The turn ends; the update does not start another one. - const completed = yield* orchestrator.streamDomainEvents.pipe( - Stream.filter( - (event) => - event.type === "run.updated" && - event.payload.id === first.runId && - event.payload.status === "waiting", - ), - Stream.take(1), - Stream.runDrain, - Effect.forkScoped, - ); - const turn = joined.providerTurns[0]!; - yield* Queue.offer(events, { - type: "provider_turn.updated", - driver, - providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, - }); - yield* Queue.offer(events, { - type: "turn.terminal", - driver, - providerThreadId: turn.providerThreadId, - providerTurnId: turn.id, - runOrdinal: first.runOrdinal, - status: "completed", - failure: null, - threadDisposition: "reusable", - }); - yield* Fiber.join(completed); - yield* worker.drain(); - yield* orchestrator.resumeQueuedRuns; - yield* worker.drain(); - assert.equal(started.length, 1); - assert.deepEqual(steered, []); - }).pipe(Effect.provide(layer)); - }), - ), -); - -it.effect("Stop keeps a queued agent update out of the stopped turn's boundaries", () => - Effect.scoped( - Effect.gen(function* () { - const { started, layer, startFirstTurn } = yield* nextTurnSelectionHarness( - "agent-update-boundary-after-stop", - { activeSteeringInterruptsTools: true }, - ); - yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const threadId = yield* startFirstTurn; - const first = started[0]!; - assert.equal((yield* sendUpdate(threadId, "agent", "auto")).delivery, "queued"); - yield* orchestrator.dispatch({ - type: "run.interrupt", - commandId: CommandId.make("stop"), - threadId, - runId: first.runId, - holdQueue: true, - }); - - assert.deepEqual(yield* claimAtToolBoundary(threadId, first.runId), []); - const projection = yield* orchestrator.getThreadProjection(threadId); - const queued = projection.runs.find((run) => run.status === "queued"); - assert.equal(queued?.queueHeld, true); - }).pipe(Effect.provide(layer)); - }), - ), -); - it.effect("sends a selected held message once and preserves the remaining queue order", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 5662a15a56177..543b3e2f89fe1 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -864,23 +864,13 @@ const make = Effect.gen(function* () { // so a cut run is not the answer while its continuation is pending. // Runs per matching event and is already covered by the waitForThread span. const outcome = Effect.fnUntraced(function* (runs: ReadonlyArray) { - const selected = runs.find((candidate) => candidate.id === selectedRun.id); - if (selected === undefined) { + const current = runs.find((candidate) => candidate.id === selectedRun.id); + if (current === undefined) { return yield* new ThreadManagementRunNotFoundError({ threadId: input.threadId, runId: selectedRun.id, }); } - // A queued message promoted into the running turn (a steer or a tool - // boundary) cancels its own run before it starts; the run that took - // the message answers. - const takenBy = - selected.status === "cancelled" && selected.startedAt === null - ? (yield* getProjectThreadRecords(input, ["messages"], { - messageIds: [selected.userMessageId], - })).messages[0]?.runId - : undefined; - const current = runs.find((candidate) => candidate.id === takenBy) ?? selected; const run = latestRestartContinuation(runs, current); const pending = isTerminalRunStatus(run.status) && (yield* orchestrator.restartContinuationPending(run)); @@ -911,7 +901,6 @@ const make = Effect.gen(function* () { (stored) => stored.event.type !== "run.updated" || stored.event.payload.id === selectedRun.id || - isTerminalRunStatus(stored.event.payload.status) || stored.event.payload.restartContinuationOfRunId !== undefined, ), ), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index f82ca98f4bcbc..0342a3ccc9d94 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -513,7 +513,6 @@ export function layerWithRegistry( layerEffectWorkerProvided, layerEventSinkProvided, layerContinuationWorkerProvided, - layerContinuationRequests, layerCommandOutputHubProvided, ).pipe(Layer.provide(ProviderTurnStartServiceTestkit.layer), Layer.provide(NodeServices.layer)); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index d791d59f6c2d1..881a316f59ae1 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1098,8 +1098,6 @@ export const OrchestrationV2ConversationMessage = Schema.Struct({ * the workflow's opening instead and keep the script behind a disclosure. */ workflowScript: Schema.optional(Schema.Boolean), - /** A queued agent update the running turn may take at its next tool boundary. */ - atToolBoundary: Schema.optional(Schema.Literal(true)), id: MessageId, threadId: ThreadId, runId: Schema.NullOr(RunId), @@ -2881,8 +2879,6 @@ export const OrchestrationV2Command = Schema.Union([ threadId: ThreadId, queuedRunId: RunId, targetRunId: RunId, - // The provider is delivering the message itself at a tool boundary: no steer effect. - atToolBoundary: Schema.optional(Schema.Literal(true)), }), Schema.Struct({ type: Schema.Literal("queue.resume"), diff --git a/packages/provider-core/src/server/ProviderContinuationRequests.ts b/packages/provider-core/src/server/ProviderContinuationRequests.ts index 4873e17044c05..ff14d5325db9c 100644 --- a/packages/provider-core/src/server/ProviderContinuationRequests.ts +++ b/packages/provider-core/src/server/ProviderContinuationRequests.ts @@ -11,7 +11,6 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; -import * as Ref from "effect/Ref"; export interface ProviderContinuationRequest { readonly threadId: ThreadId; @@ -37,15 +36,6 @@ export interface ProviderContinuationRequest { readonly clearIfCurrent?: () => Effect.Effect; } -/** - * Moves the thread's queued agent updates into `runId`, the provider's running turn, and returns - * their text for the provider to deliver at a tool boundary. Updates it does not return stay queued. - */ -export type ClaimAgentMessages = (input: { - readonly threadId: ThreadId; - readonly runId: RunId; -}) => Effect.Effect>; - /** * The default reference drops requests, keeping adapter construction dependency-free in tests; the * live layer must be shared with the ProviderContinuationService worker that drains it. @@ -53,9 +43,6 @@ export type ClaimAgentMessages = (input: { export class ProviderContinuationRequests extends Context.Reference<{ readonly offer: (request: ProviderContinuationRequest) => Effect.Effect; readonly take: Effect.Effect; - readonly claimAgentMessages?: ClaimAgentMessages; - /** Installs the worker's claim; until then nothing is claimed and updates start the next turn. */ - readonly serveAgentMessages?: (claim: ClaimAgentMessages) => Effect.Effect; }>("@t3tools/provider-core/server/ProviderContinuationRequests", { defaultValue: () => ({ offer: () => Effect.void, take: Effect.never }), }) {} @@ -64,13 +51,10 @@ export const layer = Layer.effect( ProviderContinuationRequests, Effect.gen(function* () { const queue = yield* Queue.unbounded(); - const claim = yield* Ref.make(() => Effect.succeed([])); return { offer: (request: ProviderContinuationRequest) => Queue.offer(queue, request).pipe(Effect.asVoid), take: Queue.take(queue), - claimAgentMessages: (input) => Ref.get(claim).pipe(Effect.flatMap((serve) => serve(input))), - serveAgentMessages: (serve) => Ref.set(claim, serve), }; }), ); From 9acbcd0ac42b3353ea40586d481ae10509a586c3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 24 Oct 2026 13:31:11 +0000 Subject: [PATCH 13/23] test(server): a parent's queued update to a running Claude child becomes the task result Claude steering cancels running tools, so an agent update to a running delegated child now queues. Cover the full path with a real delegated task: the original task stays working while the queued update runs, then reports the update's turn as its result. Co-Authored-By: Claude Opus 5.5 --- ...skCancelNativeSubagent.integration.test.ts | 500 +++++++++++------- 1 file changed, 317 insertions(+), 183 deletions(-) diff --git a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts index 9c36a26045dc5..595df71a93425 100644 --- a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts +++ b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts @@ -50,188 +50,214 @@ const driver = ProviderDriverKind.make("claudeAgent"); const instanceId = ProviderInstanceId.make("claudeAgent"); const modelSelection = { instanceId, model: "claude-test" }; -// A delegated Claude child ends its turn while a background native subagent it -// started keeps running (#17154). +const watch = Effect.fn("watch")(function* ( + predicate: (event: OrchestrationV2DomainEvent) => boolean, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + return yield* orchestrator.streamDomainEvents.pipe( + Stream.filter(predicate), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); +}); -it.effect("task_cancel ends a delegated child's background native subagent", () => - Effect.scoped( - Effect.gen(function* () { - const cwd = yield* checkpointWorkspace("task-cancel-native-subagent"); - const turns: Array<{ - readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput; - readonly events: Queue.Queue; - }> = []; - const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { - instanceId, - driver, - getCapabilities: () => Effect.succeed(ClaudeProviderCapabilitiesV2), - planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), - openSession: (input) => - Effect.gen(function* () { - const now = yield* DateTime.now; - const events = yield* Queue.unbounded(); - return { - instanceId, +const delegatedClaudeHarness = Effect.fn("delegatedClaudeHarness")(function* (name: string) { + const cwd = yield* checkpointWorkspace(name); + const turns: Array<{ + readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput; + readonly events: Queue.Queue; + }> = []; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId, + driver, + getCapabilities: () => Effect.succeed(ClaudeProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const events = yield* Queue.unbounded(); + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd, + model: modelSelection.model, + capabilities: ClaudeProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId }) => + Effect.succeed({ + id: ProviderThreadId.make(`provider-thread:${threadId}`), driver, + providerInstanceId: instanceId, providerSessionId: input.providerSessionId, - providerSession: { - id: input.providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: `native:${threadId}`, strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + turns.push({ turn, events }); + yield* Queue.offer(events, { + type: "provider_turn.updated", driver, - providerInstanceId: instanceId, - status: "ready", - cwd, - model: modelSelection.model, - capabilities: ClaudeProviderCapabilitiesV2, - createdAt: now, - updatedAt: now, - lastError: null, - }, - events: Stream.fromQueue(events), - ensureThread: ({ threadId }) => - Effect.succeed({ - id: ProviderThreadId.make(`provider-thread:${threadId}`), - driver, - providerInstanceId: instanceId, - providerSessionId: input.providerSessionId, - appThreadId: threadId, - ownerNodeId: null, - nativeThreadRef: { driver, nativeId: `native:${threadId}`, strength: "strong" }, - nativeConversationHeadRef: null, - status: "idle", - firstRunOrdinal: null, - lastRunOrdinal: null, - handoffIds: [], - forkedFrom: null, - createdAt: now, - updatedAt: now, - }), - resumeThread: ({ providerThread }) => Effect.succeed(providerThread), - startTurn: (turn) => - Effect.gen(function* () { - turns.push({ turn, events }); - yield* Queue.offer(events, { - type: "provider_turn.updated", - driver, - providerTurn: { - id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), - providerThreadId: turn.providerThread.id, - nodeId: turn.rootNodeId, - runAttemptId: turn.attemptId, - nativeTurnRef: null, - ordinal: turn.providerTurnOrdinal, - status: "running", - startedAt: now, - completedAt: null, - }, - }); - }), - steerTurn: () => Effect.die("unused"), - interruptTurn: () => Effect.void, - respondToRuntimeRequest: () => Effect.die("unused"), - readThreadSnapshot: () => Effect.die("unused"), - rollbackThread: () => Effect.die("unused"), - forkThread: () => Effect.die("unused"), - }; - }), - }; - const layerAdapters = ProviderAdapterRegistry.layerSingle(adapter); - const layerOrchestrator = ProviderReplayHarness.layerWithRegistry( - { name: "task-cancel-native-subagent" }, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: null, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + }), + steerTurn: () => Effect.die("unused"), + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.die("unused"), + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + const layerAdapters = ProviderAdapterRegistry.layerSingle(adapter); + const layerOrchestrator = ProviderReplayHarness.layerWithRegistry({ name }, layerAdapters, { + runEffectWorker: false, + }); + const layerOrchestration = Layer.merge( + layerOrchestrator, + ThreadManagementService.layer.pipe( + Layer.provide(layerOrchestrator), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory))), + ), + ); + const layerTest = layerOrchestratorService.pipe( + Layer.provideMerge(layerOrchestration), + Layer.provide( + Layer.mergeAll( layerAdapters, - { runEffectWorker: false }, - ); - const layerOrchestration = Layer.merge( - layerOrchestrator, - ThreadManagementService.layer.pipe( - Layer.provide(layerOrchestrator), - Layer.provide(ServerSettings.layerTest()), - Layer.provide( - CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), - ), - ), - ); - const layerTest = layerOrchestratorService.pipe( - Layer.provideMerge(layerOrchestration), - Layer.provide( - Layer.mergeAll( - layerAdapters, - Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), - Layer.mock(ProjectService.ProjectService)({}), - Layer.mock(SecretRequests.SecretRequests)({}), - ServerSettings.layerTest(), - Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), - NodeServices.layer, - ), - ), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + ServerSettings.layerTest(), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + NodeServices.layer, + ), + ), + ); + + const startDelegatedChild = Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const parentThreadId = ThreadId.make(`thread:${name}-parent`); + const turnRunning = () => + watch( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", ); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-parent"), + threadId: parentThreadId, + projectId: ProjectId.make("project:task-cancel"), + title: "Parent", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const parentRunning = yield* turnRunning(); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("start-parent"), + threadId: parentThreadId, + messageId: MessageId.make("message:start-parent"), + text: "Delegate a review", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + yield* Fiber.join(parentRunning); + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!; + const childRunning = yield* turnRunning(); + yield* orchestrator.dispatch({ + type: "delegated_task.request", + commandId: CommandId.make("delegate"), + parentThreadId, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId!, + task: "Review the change", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + completionWake: "always", + createdBy: "agent", + creationSource: "mcp", + }); + yield* worker.drain(); + yield* Fiber.join(childRunning); + const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!; + const child = turns.find((entry) => entry.turn.threadId === task.childThreadId)!; + const childTurn = (yield* orchestrator.getThreadProjection(task.childThreadId!)) + .providerTurns[0]!; + const scope: McpInvocationScope = { + environmentId: EnvironmentId.make(`environment:${name}`), + requestNamespace: `provider-session:${name}`, + thread: { + threadId: parentThreadId, + providerSessionId: `provider-session:${name}`, + providerInstanceId: instanceId, + }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }; + return { parentThreadId, task, child, childTurn, scope }; + }); + return { turns, layerTest, startDelegatedChild }; +}); + +// A delegated Claude child ends its turn while a background native subagent it +// started keeps running (#17154). + +it.effect("task_cancel ends a delegated child's background native subagent", () => + Effect.scoped( + Effect.gen(function* () { + const { layerTest, startDelegatedChild } = yield* delegatedClaudeHarness( + "task-cancel-native-subagent", + ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; const service = yield* OrchestratorMcpService.OrchestratorMcpService; - const parentThreadId = ThreadId.make("thread:task-cancel-parent"); - const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) => - orchestrator.streamDomainEvents.pipe( - Stream.filter(predicate), - Stream.take(1), - Stream.runDrain, - Effect.forkScoped, - ); - const turnRunning = () => - watch( - (event) => event.type === "provider-turn.updated" && event.payload.status === "running", - ); - yield* orchestrator.dispatch({ - type: "thread.create", - commandId: CommandId.make("create-parent"), - threadId: parentThreadId, - projectId: ProjectId.make("project:task-cancel"), - title: "Parent", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: cwd, - createdBy: "user", - creationSource: "web", - }); - const parentRunning = yield* turnRunning(); - yield* orchestrator.dispatch({ - type: "message.dispatch", - commandId: CommandId.make("start-parent"), - threadId: parentThreadId, - messageId: MessageId.make("message:start-parent"), - text: "Delegate a review", - attachments: [], - dispatchMode: { type: "start_immediately" }, - createdBy: "user", - creationSource: "web", - }); - yield* worker.drain(); - yield* Fiber.join(parentRunning); - - const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!; - const childRunning = yield* turnRunning(); - yield* orchestrator.dispatch({ - type: "delegated_task.request", - commandId: CommandId.make("delegate"), - parentThreadId, - parentRunId: parentRun.id, - parentNodeId: parentRun.rootNodeId!, - task: "Review the change", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - completionWake: "always", - createdBy: "agent", - creationSource: "mcp", - }); - yield* worker.drain(); - yield* Fiber.join(childRunning); - const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!; - const child = turns.find((entry) => entry.turn.threadId === task.childThreadId)!; - const childTurn = (yield* orchestrator.getThreadProjection(task.childThreadId!)) - .providerTurns[0]!; + const { task, child, childTurn, scope } = yield* startDelegatedChild; const now = yield* DateTime.now; const subagentId = NodeId.make("node:native-subagent"); @@ -327,18 +353,6 @@ it.effect("task_cancel ends a delegated child's background native subagent", () yield* Fiber.join(waiting); yield* worker.drain(); - const scope: McpInvocationScope = { - environmentId: EnvironmentId.make("environment:task-cancel"), - requestNamespace: "provider-session:task-cancel", - thread: { - threadId: parentThreadId, - providerSessionId: "provider-session:task-cancel", - providerInstanceId: instanceId, - }, - client: undefined, - capabilities: new Set(["orchestration"]), - issuedAt: 1, - }; const before = yield* service.taskStatus(scope, task.id); assert.equal(before.status, "running"); assert.equal(before.workState, "waiting_for_children"); @@ -371,3 +385,123 @@ it.effect("task_cancel ends a delegated child's background native subagent", () }), ), ); + +const finishChildTurn = Effect.fn("finishChildTurn")(function* ( + entry: { + readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput; + readonly events: Queue.Queue; + }, + text: string, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const { turn, events } = entry; + const providerTurn = (yield* orchestrator.getThreadProjection(turn.threadId)).providerTurns.find( + (candidate) => candidate.runAttemptId === turn.attemptId, + )!; + const now = yield* DateTime.now; + const ended = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === turn.runId && + (event.payload.status === "waiting" || event.payload.status === "completed"), + ); + for (const event of [ + { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make(`turn-item:${turn.attemptId}`), + threadId: turn.threadId, + runId: turn.runId, + nodeId: turn.rootNodeId, + providerThreadId: providerTurn.providerThreadId, + providerTurnId: providerTurn.id, + nativeItemRef: null, + parentItemId: null, + ordinal: 0, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "assistant_message", + messageId: MessageId.make(`message:${turn.attemptId}`), + text, + streaming: false, + }, + }, + { + type: "provider_turn.updated", + driver, + providerTurn: { ...providerTurn, status: "completed", completedAt: now }, + }, + { + type: "turn.terminal", + driver, + providerThreadId: providerTurn.providerThreadId, + providerTurnId: providerTurn.id, + runOrdinal: turn.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }, + ] satisfies ReadonlyArray) { + yield* Queue.offer(events, event); + } + yield* Fiber.join(ended); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); +}); + +// Claude steering cancels running tools, so a parent's update to its running +// delegated child queues. The original task must report the update's turn. +it.effect("a parent's queued update to a running Claude child becomes the task result", () => + Effect.scoped( + Effect.gen(function* () { + const { turns, layerTest, startDelegatedChild } = + yield* delegatedClaudeHarness("queued-update-to-child"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const { parentThreadId, task, child, scope } = yield* startDelegatedChild; + + const sent = yield* service.sendToThread(scope, { + threadId: task.childThreadId!, + message: "Also check the docs", + }); + yield* worker.drain(); + assert.equal(sent.delivery, "queued"); + + const updateRunning = yield* watch( + (event) => + event.type === "provider-turn.updated" && + event.payload.status === "running" && + event.payload.runAttemptId !== child.turn.attemptId, + ); + yield* finishChildTurn(child, "initial findings"); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + yield* Fiber.join(updateRunning); + + const update = turns.find( + (entry) => + entry.turn.threadId === task.childThreadId && entry.turn.runId !== child.turn.runId, + ); + assert.isDefined(update); + const midway = yield* service.taskStatus(scope, task.id); + assert.equal(midway.workState, "working"); + assert.isNull(midway.summary); + const parentMidway = yield* orchestrator.getThreadProjection(parentThreadId); + assert.isNull(parentMidway.subagents.find((entry) => entry.id === task.id)?.result); + + yield* finishChildTurn(update!, "findings including docs"); + const done = yield* service.taskStatus(scope, task.id); + assert.equal(done.workState, "result_available"); + assert.equal(done.status, "completed"); + assert.equal(done.summary, "findings including docs"); + const parentDone = yield* orchestrator.getThreadProjection(parentThreadId); + assert.equal(parentDone.subagents.length, 1); + }).pipe(Effect.provide(layerTest)); + }), + ), +); From ef7aeaeb58bcddb13c6a7171b6449342e3762b64 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 8 Nov 2026 13:40:43 -0500 Subject: [PATCH 14/23] fix(server): keep delivery guidance within the context budget Keep the native boundary and finished-child instructions while shortening the thread-send description. The measured tool list is 24 bytes below main's limit. Co-Authored-By: GPT-6.1 Sol at high effort --- apps/server/src/mcp/toolkits/orchestrator/tools.ts | 2 +- apps/server/src/provider/agentContextBudget.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index bfe90db43f0ca..b137cf1db12fb 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -270,7 +270,7 @@ const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any thread. The target cannot run with broader modes than yours. `mode:'auto'` starts an idle thread, steers a running turn, or queues. Claude receives ordinary auto/steer updates after its current tool batch; updates after its last batch wait for the next turn. `queue`: new turn. `steer`: mid-turn update. `restart`: interrupt and restart. A send to a finished child thread starts a fresh task and returns its `taskId`. In your reply, name it as its `link` verbatim.", + "Send within your modes. `auto`: start idle, steer running, or queue. Claude auto/steer follows its tool batch, or waits for the next turn after its last batch. `queue`: next turn. `steer`: mid-turn. `restart`: interrupt/restart. Sending to a finished child starts a new task and returns `taskId`. Use its `link` verbatim in your reply.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index cc085823e2115..41116e4e0eac9 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5350, "codexAdditionalContextBytes": 7080, - "mcpToolListJsonBytes": 48243 + "mcpToolListJsonBytes": 48092 } From 7d36097efb9f97a2152ef6ddfae21134aa102cbb Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 8 Nov 2026 14:16:33 -0500 Subject: [PATCH 15/23] test(server): align Claude replay coverage with native boundaries Verify fallback deliveries finish pending tools before the next turn, including human sends and queue promotions. Retire the ordinary-message fixture that requires interrupting priority-now input; native boundaries and queued turns have dedicated coverage. Co-Authored-By: GPT-6.1 Sol at high effort --- ...laudeAutomaticDelivery.integration.test.ts | 101 ++++++++---------- .../testkit/fixtures/index.ts | 7 -- .../message_steering/claude_output.ts | 43 -------- 3 files changed, 46 insertions(+), 105 deletions(-) delete mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/message_steering/claude_output.ts diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index 46994a891f705..b715e0ce3e953 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -105,7 +105,7 @@ const toolResult = (id: string, cancelled: boolean) => ...(cancelled ? { tool_result_meta: [{ id, non_execution_kind: "cancelled" }] } : {}), }); -const result = (uuid: string, aborted: boolean) => +const result = (uuid: string) => frame({ type: "result", subtype: "success", @@ -115,7 +115,7 @@ const result = (uuid: string, aborted: boolean) => num_turns: 1, result: "", stop_reason: "end_turn", - terminal_reason: aborted ? "aborted_tools" : "completed", + terminal_reason: "completed", permission_denials: [], duration_ms: 1, duration_api_ms: 1, @@ -132,7 +132,7 @@ const deliveries = [ ] as const; it.effect.each(deliveries)( - "delivers %s with Claude's native pending-tool cancellation behavior", + "queues %s without cancelling tools when native boundary input is unavailable", (delivery) => Effect.scoped( Effect.gen(function* () { @@ -390,22 +390,15 @@ it.effect.each(deliveries)( targetRunId: parent.id, }); const promoted = yield* orchestrator.getThreadProjection(threadId); - assert.equal(promoted.runs.find((run) => run.id === queuedRun.id)?.status, "cancelled"); + assert.equal(promoted.runs.find((run) => run.id === queuedRun.id)?.status, "queued"); assert.equal( promoted.messages.find((message) => message.id === messageId)?.runId, - parent.id, + queuedRun.id, ); - const steer = promoted.turnItems.find( - (item) => item.type === "user_message" && item.messageId === messageId, - ); - assert.ok(steer?.type === "user_message"); - assert.equal(steer.inputIntent, "promoted_queued_to_steer"); } yield* worker.drain(); - const explicitSteer = - delivery === "user steering" || delivery === "promoted queued message"; - assert.equal(batchAbort.signal.aborted, explicitSteer); - assert.equal(offers.length, explicitSteer ? 2 : 1); + assert.equal(batchAbort.signal.aborted, false); + assert.equal(offers.length, 1); const finished = yield* watch( (event) => event.type === "run.updated" && @@ -415,59 +408,57 @@ it.effect.each(deliveries)( yield* Queue.offer(sdkMessages, toolResult("build", false)); for (const id of statusIds) yield* Queue.offer(sdkMessages, toolResult(id, batchAbort.signal.aborted)); - if (explicitSteer) yield* Queue.offer(sdkMessages, result("aborted", true)); - yield* Queue.offer(sdkMessages, result("completed", false)); + yield* Queue.offer(sdkMessages, result("completed")); yield* Fiber.join(finished); yield* worker.drain(); yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); - if (!explicitSteer) { - const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; - const noticeFinished = yield* watch( - (event) => - event.type === "run.updated" && - event.payload.id === queued.id && - event.payload.status === "waiting", - ); - const delivered = - delivery === "child completion" - ? yield* watch( - (event) => - event.type === "subagent.updated" && - event.payload.completionDelivery?.state === "delivered", - ) - : null; - yield* Queue.offer(sdkMessages, result("notice-completed", false)); - yield* Fiber.join(noticeFinished); - yield* worker.drain(); - if (delivered !== null) yield* Fiber.join(delivered); - } + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; + const noticeFinished = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === queued.id && + event.payload.status === "waiting", + ); + const delivered = + delivery === "child completion" + ? yield* watch( + (event) => + event.type === "subagent.updated" && + event.payload.completionDelivery?.state === "delivered", + ) + : null; + yield* Queue.offer(sdkMessages, result("notice-completed")); + yield* Fiber.join(noticeFinished); + yield* worker.drain(); + if (delivered !== null) yield* Fiber.join(delivered); const after = yield* orchestrator.getThreadProjection(threadId); const reads = after.turnItems.filter((item) => statusIds.includes(item.nativeItemRef?.nativeId ?? ""), ); assert.equal(reads.length, 4); for (const read of reads) { - assert.equal(read.status, explicitSteer ? "cancelled" : "completed"); - assert.equal(read.toolNonExecutionKind, explicitSteer ? "cancelled" : undefined); - assert.equal(read.type === "dynamic_tool" && read.output === refusal, explicitSteer); + assert.equal(read.status, "completed"); + assert.equal(read.toolNonExecutionKind, undefined); + assert.equal(read.type === "dynamic_tool" && read.output === refusal, false); } assert.equal(offers.length, 2); - assert.equal( - offers.filter((offer) => offer.priority === "now").length, - explicitSteer ? 1 : 0, - ); - if (!explicitSteer) { - assert.equal(after.runs.length, 2); - if (delivery === "child completion") { - assert.equal(after.messages.filter((message) => message.id === messageId).length, 1); - assert.equal(after.subagents[0]?.completionDelivery?.state, "delivered"); - } else { - assert.equal( - after.messages.filter((message) => message.scheduledTaskId !== undefined).length, - 1, - ); - } + assert.equal(offers.filter((offer) => offer.priority === "now").length, 0); + assert.equal(after.runs.length, 2); + if (delivery === "child completion") { + assert.equal(after.messages.filter((message) => message.id === messageId).length, 1); + assert.equal(after.subagents[0]?.completionDelivery?.state, "delivered"); + } else if (delivery === "scheduled message") { + assert.equal( + after.messages.filter((message) => message.scheduledTaskId !== undefined).length, + 1, + ); + } else { + const notice = after.messages.filter((message) => message.id === messageId); + assert.lengthOf(notice, 1); + assert.notEqual(notice[0]?.runId, parent.id); + assert.equal(notice[0]?.createdBy, "user"); + assert.equal(notice[0]?.creationSource, "web"); } yield* orchestrator.recoverDelegatedTasks; yield* orchestrator.resumeQueuedRuns; diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts index 055d8d5ce9591..7ff57db7682c4 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts @@ -70,7 +70,6 @@ import { grokPromptErrorInput } from "./grok_prompt_error/input.ts"; import { assertGrokPromptErrorOutput } from "./grok_prompt_error/output.ts"; import { grokSubagentLineageInput } from "./grok_subagent_lineage/input.ts"; import { assertGrokSubagentLineageOutput } from "./grok_subagent_lineage/output.ts"; -import { assertClaudeMessageSteeringOutput } from "./message_steering/claude_output.ts"; import { assertMessageSteeringOutput } from "./message_steering/codex_output.ts"; import { assertCursorMessageSteeringOutput } from "./message_steering/cursor_output.ts"; import { assertGrokMessageSteeringOutput } from "./message_steering/grok_output.ts"; @@ -1773,12 +1772,6 @@ export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray Date: Sun, 8 Nov 2026 14:38:10 -0500 Subject: [PATCH 16/23] test(server): isolate native-subagent replay checkpoints Use a scoped checkpoint workspace for the direct read-only replay case so it cannot snapshot the contributor's checkout. Keep the read-only assertions. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: GPT-6.1 Sol at high effort, baseline verification subagent --- .../testkit/ClaudeReplayFixtures.integration.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts index 8886daad3e017..9a4299afc5821 100644 --- a/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts @@ -8,6 +8,7 @@ import { type ProviderReplayEntry, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import { ClaudeOrchestratorReplayHarness } from "../Adapters/ClaudeAdapterV2.testkit.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; @@ -67,6 +68,7 @@ describe("Claude Agent SDK replay fixtures", () => { commands: materialized.commands, steps: materialized.steps, projectionThreadIds: materialized.projectionThreadIds, + runtimePolicyOverride: { cwd: yield* checkpointWorkspace("subagent-read-only-child") }, }; yield* Effect.gen(function* () { const result = yield* runOrchestratorV2Scenario(scenario); @@ -106,7 +108,7 @@ describe("Claude Agent SDK replay fixtures", () => { provideDeterministicTestRuntime, Effect.scoped, ); - }), + }).pipe(Effect.scoped), ); const afterRootFixture = findReplayFixture("claude_background_subagent_after_root"); From 07750aa70b7791498962e70ea1f7785dbb91ceef Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Sun, 8 Nov 2026 15:08:08 -0500 Subject: [PATCH 17/23] fix(server): preserve goal and completion receipt transitions Keep service goal commands targeted at the active turn before ordinary Claude delivery falls back to the queue. Forward nested receipt acceptance metadata so successor child completions are offered after commit and concurrent checkpoint completion cannot be overwritten by a stale parent snapshot. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: GPT-6.1 Sol at high effort, implementation subagent for goal routing Co-Authored-By: GPT-6.1 Sol at high effort, review subagent for receipt and goal transitions --- .../DelegatedCompletionDelivery.test.ts | 102 ++++++++++++++++-- .../src/orchestration-v2/Orchestrator.ts | 55 ++++++---- .../SteeringCompletion.integration.test.ts | 100 ++++++++++++----- 3 files changed, 200 insertions(+), 57 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 64a23c4bd0665..315013d3ba7f1 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -2672,11 +2672,13 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( }, ); -it.layer(layerTest)("delegated tasks across a server restart", (it) => { +const layerRestartTest = Layer.merge(layerTest, ProviderContinuationRequests.layer); +it.layer(layerRestartTest)("delegated tasks across a server restart", (it) => { it.effect("queues a child result without creating a provider history handoff", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const eventSink = yield* EventSink.EventSinkV2; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; const now = yield* DateTime.now; const threadId = ThreadId.make("thread:result-message-parent"); const projectId = ProjectId.make("project:result-message"); @@ -2843,14 +2845,75 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { afterLater.subagents.find((row) => row.id === later.taskId)?.completionDelivery?.state, "pending", ); - yield* orchestrator.dispatch({ - type: "queued-message.promote-to-steer", - commandId: CommandId.make("receipt-first-completion"), - threadId, - queuedRunId: offeredRun.id, - targetRunId: runId, - boundaryOffer: { id: "offered-completion", step: "receipt" }, + const beforeReceipt = afterLater.runs.find((run) => run.id === runId)!; + const checkpointId = CheckpointId.make("checkpoint:receipt-first-completion"); + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:receipt-first-completion-waiting"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { ...beforeReceipt, status: "waiting" }, + }, + ], }); + const commit = eventSink.commitCommand; + const receiptCommandId = CommandId.make("receipt-first-completion"); + const offeredSuccessors: MessageId[] = []; + const offer = requests.offer; + yield* Effect.acquireUseRelease( + Effect.sync(() => [ + vi.spyOn(requests, "offer").mockImplementation((request) => { + if (request.delegatedCompletion !== undefined) + offeredSuccessors.push(request.delegatedCompletion.messageId); + return offer(request); + }), + vi.spyOn(eventSink, "commitCommand").mockImplementation((input) => + Effect.gen(function* () { + if (input.commandId === receiptCommandId) { + yield* commit({ + commandId: CommandId.make("command:receipt-first-completion-checkpoint"), + threadId, + commandType: "checkpoint.capture", + acceptedAt: now, + effects: [], + events: [ + { + id: EventId.make("event:receipt-first-completion-checkpoint"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { + ...beforeReceipt, + status: "completed", + completedAt: now, + checkpointId, + }, + }, + ], + }); + } + return yield* commit(input); + }), + ), + ]), + () => + orchestrator.dispatch({ + type: "queued-message.promote-to-steer", + commandId: receiptCommandId, + threadId, + queuedRunId: offeredRun.id, + targetRunId: runId, + boundaryOffer: { id: "offered-completion", step: "receipt" }, + }), + (spies) => + Effect.sync(() => { + for (const spy of spies) spy.mockRestore(); + }), + ); const received = yield* orchestrator.getThreadProjection(threadId); assert.equal( received.subagents.find((row) => row.id === child.taskId)?.completionDelivery?.state, @@ -2864,6 +2927,14 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { received.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery?.taskIds, [later.taskId], ); + const receivedParent = received.runs.find((run) => run.id === runId)!; + assert.deepEqual(offeredSuccessors, [ + receivedParent.delegatedCompletion!.delivery!.messageId, + ]); + assert.deepEqual( + [receivedParent.status, receivedParent.completedAt, receivedParent.checkpointId], + ["completed", now, checkpointId], + ); }), ); @@ -3756,7 +3827,13 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { ...providerThread, pendingBackgroundTasks: [ ...(open - ? [{ taskId: "sleep-900", kind: "command" as const, awaited: true as const }] + ? [ + { + taskId: "sleep-900", + kind: "command" as const, + awaited: true as const, + }, + ] : []), { taskId: "log-watcher", kind: "monitor" as const }, ], @@ -3783,7 +3860,12 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { status: "completed", now, }), - payload: { ...child.run, status: "completed" as const, startedAt: now, completedAt: now }, + payload: { + ...child.run, + status: "completed" as const, + startedAt: now, + completedAt: now, + }, }); const beforeEnd = yield* sink.latestSequence(); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index b48b033c10c3a..a35b1dfea92d2 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -5860,6 +5860,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command.dispatchMode, command.deliveryIntent, ); + // Codex and Claude own /goal; elsewhere it is ordinary text that cannot steer. + const parsedGoalCommand = parseGoalCommand(command); + const goalCommand = + parsedGoalCommand === null || + (yield* goalDriverFor(command, modelSelection.instanceId)) === undefined + ? null + : parsedGoalCommand; let atToolBoundary = false; if (dispatchMode.type === "steer_active") { const targetRunId = dispatchMode.targetRunId; @@ -5881,6 +5888,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } else if ( // After Stop the steer must still fail below; queueing it instead would // start the update on its own once the provider stops. + goalCommand === null && !(yield* stopReachedRun(command, command.threadId, targetRunId)) ) { const providerThread = projection.providerThreads.find( @@ -5918,13 +5926,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) { dispatchMode = { type: "queue_after_active" }; } - // Codex and Claude own /goal; elsewhere it is ordinary text that cannot steer. - const parsedGoalCommand = parseGoalCommand(command); - const goalCommand = - parsedGoalCommand === null || - (yield* goalDriverFor(command, modelSelection.instanceId)) === undefined - ? null - : parsedGoalCommand; let goalStopRunId: RunId | undefined; if (goalCommand !== null) { yield* ensureGoalCommandSupported(command, command, modelSelection.instanceId); @@ -9812,7 +9813,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command: Extract, events: Ref.Ref>, effects: Ref.Ref>, - ) { + ): Effect.fn.Return< + | Pick< + Parameters[0], + "cancelUnsettledEffects" | "updateDelegatedCompletionOnlyRunId" + > + | undefined, + OrchestratorV2Error + > { const projection = yield* loadProjectionForCommand( command, [ @@ -9944,7 +9952,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return; } if (receipt !== undefined) { - yield* joinQueuedRunToTurn({ + const updateDelegatedCompletionOnlyRunId = yield* joinQueuedRunToTurn({ command, events, effects, @@ -9956,7 +9964,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio receipt, now, }); - return; + return updateDelegatedCompletionOnlyRunId === undefined + ? undefined + : { updateDelegatedCompletionOnlyRunId }; } const targetRun = projection.runs.find((run) => run.id === command.targetRunId); if (command.boundaryOffer?.step === "offer") { @@ -10034,7 +10044,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); const stops = isLiveRun(target) && !(yield* stopReachedRun(command, command.threadId, target.id)); - return yield* stopRunForQueuedCommand({ + const cancelUnsettledEffects = yield* stopRunForQueuedCommand({ command, events, effects, @@ -10044,6 +10054,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio reason: "Skipped compaction to send a queued message", now, }); + return cancelUnsettledEffects === undefined ? undefined : { cancelUnsettledEffects }; } if (queuedRun.freshContext === true) { return yield* new OrchestratorDispatchError({ @@ -10072,7 +10083,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio goalCommandStopsActiveRun(goalCommand, goal?.status ?? null) && (targetMessage === undefined || !isNativeMaintenanceCommand(targetMessage)) && !(yield* stopReachedRun(command, command.threadId, target.id)); - return yield* stopRunForQueuedCommand({ + const cancelUnsettledEffects = yield* stopRunForQueuedCommand({ command, events, effects, @@ -10082,6 +10093,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio reason: "Stopped to apply /goal", now, }); + return cancelUnsettledEffects === undefined ? undefined : { cancelUnsettledEffects }; } if (targetRun?.status === "running" || targetRun?.status === "waiting") { const targetProviderThread = projection.providerThreads.find( @@ -10249,7 +10261,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (receipt !== undefined && queuedMessage.delegatedCompletion !== undefined) { // Accept in the same commit: the cancelled queued run would otherwise // read as a failed delivery and reserve a duplicate notification. - yield* dispatchNotificationAccepted( + return yield* dispatchNotificationAccepted( { type: "notification.delivery.accept", commandId: command.commandId, @@ -13927,13 +13939,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "thread.stop": cancelUnsettledEffects = yield* dispatchThreadStop(command, events, effects); break; - case "queued-message.promote-to-steer": - cancelUnsettledEffects = yield* dispatchQueuedMessagePromoteToSteer( - command, - events, - effects, - ); + case "queued-message.promote-to-steer": { + const promotion = yield* dispatchQueuedMessagePromoteToSteer(command, events, effects); + cancelUnsettledEffects = promotion?.cancelUnsettledEffects; + updateDelegatedCompletionOnlyRunId = promotion?.updateDelegatedCompletionOnlyRunId; break; + } case "queued-run.start": case "queue.resume": { const projection = yield* loadProjectionForCommand( @@ -14357,7 +14368,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); } - if (command.type === "notification.delivery.accept") { + if ( + command.type === "notification.delivery.accept" || + (command.type === "queued-message.promote-to-steer" && + plan.updateDelegatedCompletionOnlyRunId !== undefined) + ) { yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.threadId)); } if ( diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 0ed4f7bea3859..d48144362735c 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -1658,11 +1658,39 @@ it.effect("queues a plain send behind the stopped queue and releases its hold", ), ); -const goalHarness = Effect.fn("goalHarness")(function* (name: string) { - const harness = yield* nextTurnSelectionHarness(name); - const send = (id: string, text: string, delivery: "queue" | "auto", freshContext = false) => +const goalHarness = Effect.fn("goalHarness")(function* ( + name: string, + activeSteeringInterruptsTools = false, +) { + const harness = yield* nextTurnSelectionHarness( + name, + false, + true, + Effect.void, + true, + activeSteeringInterruptsTools, + ); + const send = ( + id: string, + text: string, + delivery: "queue" | "auto" | "agent_auto" | "agent_steer", + freshContext = false, + ) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; + if (delivery === "agent_auto" || delivery === "agent_steer") { + return yield* (yield* ThreadManagement.ThreadManagementService).sendToThread({ + projectId: ProjectId.make(`project:${name}`), + commandId: CommandId.make(id), + threadId: ThreadId.make(`thread:${name}`), + messageId: MessageId.make(`message:${id}`), + text, + attachments: [], + mode: delivery === "agent_auto" ? "auto" : "steer", + createdBy: "agent", + creationSource: "mcp", + }); + } yield* orchestrator.dispatch({ type: "message.dispatch", commandId: CommandId.make(id), @@ -1721,26 +1749,38 @@ it.effect("stops the running turn and starts a goal sent into it next", () => ), ); -it.effect("queues a goal clear without stopping work when no goal drives it", () => - Effect.scoped( - Effect.gen(function* () { - const { steered, layer, startFirstTurn, send, read } = yield* goalHarness("goal-clear"); - yield* Effect.gen(function* () { - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const threadId = yield* startFirstTurn; - yield* send("queued", "queued", "queue"); - yield* send("clear", "/goal clear", "auto"); - yield* worker.drain(); +it.effect.each(["auto", "agent_auto", "agent_steer"] as const)( + "queues a goal clear from %s without stopping work when no goal drives it", + (delivery) => + Effect.scoped( + Effect.gen(function* () { + const { steered, layer, startFirstTurn, send, read } = yield* goalHarness( + "goal-clear", + true, + ); + yield* Effect.gen(function* () { + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + yield* send("queued", "queued", "queue"); + yield* send("clear", "/goal clear", delivery); + yield* worker.drain(); - assert.deepEqual(steered, []); - assert.deepEqual(yield* read(threadId), { - queue: ["queued", "/goal clear"], - started: ["first"], - stopRequested: false, - }); - }).pipe(Effect.provide(layer)); - }), - ), + const projection = yield* (yield* Orchestrator.OrchestratorV2).getThreadProjection( + threadId, + ); + assert.isNotTrue( + projection.messages.find((message) => message.id === MessageId.make("message:clear")) + ?.atToolBoundary, + ); + assert.deepEqual(steered, []); + assert.deepEqual(yield* read(threadId), { + queue: ["queued", "/goal clear"], + started: ["first"], + stopRequested: false, + }); + }).pipe(Effect.provide(layer)); + }), + ), ); it.effect("promotes a queued goal by stopping the running turn", () => @@ -1922,11 +1962,17 @@ it.effect("stops a turn that is still starting when a goal is sent into it", () ), ); -it.effect("stops the running turn when a goal clear ends the goal driving it", () => +it.effect.each( + (["auto", "agent_auto", "agent_steer"] as const).flatMap((delivery) => + ["/goal clear", "/goal ship the fix"].map((text) => ({ delivery, text })), + ), +)("stops the running turn when $delivery sends $text to an active goal", ({ delivery, text }) => Effect.scoped( Effect.gen(function* () { - const { events, layer, startFirstTurn, send, read, steered } = - yield* goalHarness("goal-clear-active"); + const { events, layer, startFirstTurn, send, read, steered } = yield* goalHarness( + "goal-clear-active", + true, + ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; @@ -1951,13 +1997,13 @@ it.effect("stops the running turn when a goal clear ends the goal driving it", ( }, }); yield* Fiber.join(goalSeen); - yield* send("clear", "/goal clear", "auto"); + yield* send("clear", text, delivery); yield* worker.drain(); assert.deepEqual(steered, []); assert.deepEqual(yield* read(threadId), { queue: [], - started: ["first", "/goal clear"], + started: ["first", text], stopRequested: true, }); }).pipe(Effect.provide(layer)); From bdcf05a37600bc549e11396a81311cdbbcb47cf8 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 12 Nov 2026 04:28:24 +0000 Subject: [PATCH 18/23] perf(server): read Claude boundary receipts from the handoff offset Co-Authored-By: GPT-6.1 Sol at high effort, implementing agent --- .../Adapters/ClaudeAdapterV2.ts | 16 ++ .../Adapters/ClaudeSavedMessageReceipts.ts | 59 ++++--- .../ClaudeBoundaryReceipts.test.ts | 161 +++++++++--------- .../ClaudeBoundaryReceipts.ts | 27 +-- 4 files changed, 129 insertions(+), 134 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index b199862ad5559..8c48c0ca6d0b1 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4814,6 +4814,8 @@ interface ClaudeLiveQueryContext { } interface ClaudeBoundaryBatch { + readonly transcriptPath: string; + readonly transcriptOffset: number; readonly context: ActiveClaudeTurnContext; readonly offers: ReadonlyArray; readonly saved: Set; @@ -11097,7 +11099,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), ); if (offers.length === 0) return; + const pendingBatches = live.boundaryBatches.filter((batch) => + batch.offers.some( + (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), + ), + ); const saved = yield* readClaudeBoundaryReceipts({ + transcriptPath: pendingBatches[0]!.transcriptPath, + offset: Math.min(...pendingBatches.map((batch) => batch.transcriptOffset)), projectsDirectory: path.join(homePath, "projects"), nativeThreadId: live.nativeThreadId, offers, @@ -12061,7 +12070,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( consumed: yield* Deferred.make(), }); } + const transcriptPath = hookInput.transcript_path; + const transcriptOffset = yield* fileSystem.stat(transcriptPath).pipe( + Effect.map((info) => Number(info.size)), + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(0)), + ); const batch: ClaudeBoundaryBatch = { + transcriptPath, + transcriptOffset, context, offers, saved: new Set(), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts index d6288b1965335..cd79e4ac2e81c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts @@ -14,27 +14,44 @@ export const readClaudeBoundaryReceipts = Effect.fn("readClaudeBoundaryReceipts" function* (input: { readonly projectsDirectory: string; readonly nativeThreadId: string; - readonly offers: ReadonlyArray<{ readonly id: string; readonly legacyContext?: string }>; + readonly offers: ReadonlyArray<{ readonly id: string }>; + readonly transcriptPath?: string; + readonly offset?: number; }) { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const offers = new Map(input.offers.map((offer) => [offer.id, offer])); const saved = new Set(); - const projects = yield* fileSystem - .readDirectory(input.projectsDirectory) - .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed([]))); - for (const project of projects) { - const transcript = path.join( - input.projectsDirectory, - project, - `${input.nativeThreadId}.jsonl`, - ); - if (!(yield* fileSystem.exists(transcript))) continue; - yield* fileSystem.stream(transcript).pipe( + const transcripts = + input.transcriptPath === undefined + ? yield* Effect.gen(function* () { + const projects = yield* fileSystem + .readDirectory(input.projectsDirectory) + .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed([]))); + return projects.map((project) => + path.join(input.projectsDirectory, project, `${input.nativeThreadId}.jsonl`), + ); + }) + : [input.transcriptPath]; + for (const transcript of transcripts) { + const info = yield* fileSystem + .stat(transcript) + .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.undefined)); + if (info === undefined) continue; + let offset = + input.offset !== undefined && input.offset <= Number(info.size) ? input.offset : 0; + if (offset > 0) { + const previous = yield* fileSystem + .stream(transcript, { offset: offset - 1, bytesToRead: 1 }) + .pipe(Stream.runHead); + if (Option.isNone(previous) || previous.value[0] !== 10) offset = 0; + } + yield* fileSystem.stream(transcript, { offset }).pipe( Stream.decodeText, Stream.splitLines, Stream.runForEach((line) => Effect.sync(() => { + if (!line.includes("queued_command") || !line.includes("source_uuid")) return; const decoded = decodeLine(line); if (Option.isNone(decoded)) return; const entry = decoded.value; @@ -48,24 +65,6 @@ export const readClaudeBoundaryReceipts = Effect.fn("readClaudeBoundaryReceipts" if (field(attachment, "type") === "queued_command") { const uuid = field(attachment, "source_uuid"); if (typeof uuid === "string" && offers.has(uuid)) saved.add(uuid); - } else if (field(attachment, "type") === "hook_additional_context") { - const content = field(attachment, "content"); - if (!Array.isArray(content)) return; - const contexts = content.flatMap((context) => { - if (typeof context !== "string") return []; - const prefix = "New messages:\n"; - // Legacy hooks saved the complete batch as one wrapped context string. - return context.startsWith(prefix) - ? context - .slice(prefix.length) - .split(/\n\n(?=From (?:the user|another agent) \(delivery [^\n)]+\):\n)/) - : [context]; - }); - for (const offer of offers.values()) { - if (offer.legacyContext !== undefined && contexts.includes(offer.legacyContext)) { - saved.add(offer.id); - } - } } }), ), diff --git a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts index 986326c731f13..1eaff02a8a718 100644 --- a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts @@ -119,86 +119,89 @@ it.layer(NodeServices.layer)("Claude boundary saved history", (it) => { }).pipe(Effect.scoped), ); - it.effect( - "recovers exact root receipts and legacy context while retaining ambiguous offers", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boundary-receipts-" }); - const project = path.join(directory, "project"); - yield* fs.makeDirectory(project); - const entry = (attachment: unknown, overrides: Record = {}) => ({ - type: "attachment", - sessionId: "native-session", - isSidechain: false, - attachment, - ...overrides, - }); - const legacy = "From another agent (delivery legacy):\nOriginal saved update"; - const legacyBatch = [ - "From the user (delivery batch-first):\nFirst update\n\nSecond paragraph", - "From another agent (delivery batch-middle):\nMiddle update", - "From the user (delivery batch-last):\nLast update", - ]; - const rows = [ - entry({ type: "queued_command", source_uuid: "saved" }), - entry({ type: "queued_command", source_uuid: "saved" }), - entry({ type: "queued_command", source_uuid: "foreign" }, { sessionId: "other-session" }), - entry({ type: "queued_command", source_uuid: "child" }, { isSidechain: true }), - entry({ type: "hook_additional_context", content: [`New messages:\n${legacy}`] }), - entry({ - type: "hook_additional_context", - content: [`New messages:\n${legacyBatch.join("\n\n")}`], - }), - entry({ - type: "hook_additional_context", - content: ["New messages:\nFrom the user (delivery changed-legacy):\nChanged text"], - }), - entry({ type: "hook_additional_context", content: ["changed original text"] }), - { type: "queue-operation", operation: "enqueue", content: "queued-only" }, - { type: "user", uuid: "unrelated-turn" }, - ]; - yield* fs.writeFileString( - path.join(project, "native-session.jsonl"), - `${rows.map((row) => encodeLine(row)).join("\n")}\n{"type":"attachment"`, - ); - const saved = yield* readClaudeBoundaryReceipts({ + it.effect("recovers exact root receipts while retaining ambiguous offers", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boundary-receipts-" }); + const project = path.join(directory, "project"); + yield* fs.makeDirectory(project); + const entry = (attachment: unknown, overrides: Record = {}) => ({ + type: "attachment", + sessionId: "native-session", + isSidechain: false, + attachment, + ...overrides, + }); + const legacy = "From another agent (delivery legacy):\nOriginal saved update"; + const legacyBatch = [ + "From the user (delivery batch-first):\nFirst update\n\nSecond paragraph", + "From another agent (delivery batch-middle):\nMiddle update", + "From the user (delivery batch-last):\nLast update", + ]; + const rows = [ + entry({ type: "queued_command", source_uuid: "saved" }), + entry({ type: "queued_command", source_uuid: "saved" }), + entry({ type: "queued_command", source_uuid: "foreign" }, { sessionId: "other-session" }), + entry({ type: "queued_command", source_uuid: "child" }, { isSidechain: true }), + entry({ type: "hook_additional_context", content: [`New messages:\n${legacy}`] }), + entry({ + type: "hook_additional_context", + content: [`New messages:\n${legacyBatch.join("\n\n")}`], + }), + entry({ + type: "hook_additional_context", + content: ["New messages:\nFrom the user (delivery changed-legacy):\nChanged text"], + }), + entry({ type: "hook_additional_context", content: ["changed original text"] }), + { type: "queue-operation", operation: "enqueue", content: "queued-only" }, + { type: "user", uuid: "unrelated-turn" }, + ]; + yield* fs.writeFileString( + path.join(project, "native-session.jsonl"), + `${rows.map((row) => encodeLine(row)).join("\n")}\n{"type":"attachment"`, + ); + const saved = yield* readClaudeBoundaryReceipts({ + projectsDirectory: directory, + nativeThreadId: "native-session", + offers: [ + { id: "saved" }, + { id: "foreign" }, + { id: "child" }, + { id: "queued-only" }, + { id: "missing" }, + { id: "legacy" }, + ], + }); + const transcriptPath = path.join(project, "tail.jsonl"); + const before = `${encodeLine(entry({ type: "queued_command", source_uuid: "before" }))}\n`; + const after = `${encodeLine(entry({ type: "queued_command", source_uuid: "after" }))}\n`; + yield* fs.writeFileString(transcriptPath, before); + const offset = Number((yield* fs.stat(transcriptPath)).size); + yield* fs.writeFileString(transcriptPath, before + after); + const readTail = (start: number) => + readClaudeBoundaryReceipts({ projectsDirectory: directory, nativeThreadId: "native-session", - offers: [ - { id: "saved" }, - { id: "foreign" }, - { id: "child" }, - { id: "queued-only" }, - { id: "missing" }, - { id: "legacy", legacyContext: legacy }, - ...legacyBatch.map((legacyContext, index) => ({ - id: ["batch-first", "batch-middle", "batch-last"][index]!, - legacyContext, - })), - { - id: "changed-legacy", - legacyContext: "From the user (delivery changed-legacy):\nOriginal text", - }, - { id: "wrong-legacy", legacyContext: "Original saved update" }, - { id: "partial-legacy", legacyContext: legacyBatch.slice(0, 2).join("\n\n") }, - ], - }); - assert.deepEqual( - [...saved], - ["saved", "legacy", "batch-first", "batch-middle", "batch-last"], - ); - assert.deepEqual( - [ - ...(yield* readClaudeBoundaryReceipts({ - projectsDirectory: path.join(directory, "absent"), - nativeThreadId: "native-session", - offers: [{ id: "missing" }], - })), - ], - [], - ); - }).pipe(Effect.scoped), + transcriptPath, + offset: start, + offers: [{ id: "before" }, { id: "after" }], + }).pipe(Effect.map((receipts) => [...receipts])); + assert.deepEqual(yield* readTail(offset), ["after"]); + assert.deepEqual(yield* readTail(offset - 1), ["before", "after"]); + yield* fs.writeFileString(transcriptPath, after); + assert.deepEqual(yield* readTail(offset + 1000), ["after"]); + assert.deepEqual([...saved], ["saved"]); + assert.deepEqual( + [ + ...(yield* readClaudeBoundaryReceipts({ + projectsDirectory: path.join(directory, "absent"), + nativeThreadId: "native-session", + offers: [{ id: "missing" }], + })), + ], + [], + ); + }).pipe(Effect.scoped), ); }); diff --git a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.ts b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.ts index 380112b20b3d7..712f540224af6 100644 --- a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.ts +++ b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.ts @@ -1,9 +1,4 @@ -import { - CommandId, - ClaudeSettings, - type OrchestrationV2ConversationMessage, -} from "@t3tools/contracts"; -import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; +import { CommandId, ClaudeSettings } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -21,14 +16,6 @@ import { readClaudeBoundaryReceipts } from "./Adapters/ClaudeSavedMessageReceipt const decodeClaudeSettings = Schema.decodeEffect(ClaudeSettings); -function legacyContext(message: OrchestrationV2ConversationMessage, offerId: string) { - const author = message.createdBy === "user" ? "the user" : "another agent"; - return `From ${author} (delivery ${offerId}):\n${projectComposerContextForProvider({ - text: message.text, - records: message.context?.records ?? [], - })}`; -} - export const makeClaudeBoundaryReceiptRecovery = Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; const threads = yield* ThreadManagementService.ThreadManagementService; @@ -96,17 +83,7 @@ export const makeClaudeBoundaryReceiptRecovery = Effect.gen(function* () { const saved = yield* readClaudeBoundaryReceipts({ projectsDirectory: path.join(home, "projects"), nativeThreadId, - offers: runs.map((run) => { - const message = projection.messages.find( - (candidate) => candidate.id === run.userMessageId, - ); - return { - id: run.boundaryOffer!.id, - ...(message === undefined - ? {} - : { legacyContext: legacyContext(message, run.boundaryOffer!.id) }), - }; - }), + offers: runs.map((run) => ({ id: run.boundaryOffer!.id })), }); for (const run of runs) { const offer = run.boundaryOffer!; From 328e241f378efc4dc30437a8ff8c1c7845825147 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 12 Nov 2026 04:32:30 +0000 Subject: [PATCH 19/23] fix(server): confirm Claude messages at the tool boundary Co-Authored-By: GPT-6.1 Sol at high effort, implementing agent --- .../Adapters/ClaudeAdapterV2.test.ts | 39 ++++++++++++++----- .../Adapters/ClaudeAdapterV2.ts | 14 +++++++ 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 7ab080f4163bb..19f8d185dc872 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3705,10 +3705,13 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.isDefined(store); if (hook === undefined || store === undefined) return; const signal = yield* Effect.abortSignal; + const fileSystem = yield* FileSystem.FileSystem; + const transcriptPath = (yield* Path.Path).join(harness.attachmentsDir, "boundary.jsonl"); + yield* fileSystem.writeFileString(transcriptPath, "{}\n"); const hookInput = { hook_event_name: "PostToolBatch" as const, session_id: WAKE_NATIVE_SESSION, - transcript_path: "/repo/session.jsonl", + transcript_path: transcriptPath, cwd: "/repo", tool_calls: [], }; @@ -3799,16 +3802,33 @@ describe("ClaudeAdapterV2 background wake turns", () => { harness.events.filter((event) => event.type === "boundary_messages.updated").length >= 2, ); - yield* Effect.promise(() => - store.append( - key, - offers.map((offer) => receipt(offer.id)), + yield* fileSystem.writeFileString( + transcriptPath, + `{}\n${offers.map((offer) => JSON.stringify(receipt(offer.id))).join("\n")}\n`, + ); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "assistant", + parent_tool_use_id: null, + session_id: WAKE_NATIVE_SESSION, + uuid: "boundary-reply", + message: { + role: "assistant", + content: [{ type: "text", text: "Working on the updates." }], + }, + }), + ); + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "boundary_messages.updated" && + event.step === "receipt" && + event.offers.some((offer) => offer.id === offers[1]!.id), ), ); - yield* harness.until( - () => - harness.events.filter((event) => event.type === "boundary_messages.updated").length >= - 3, + assert.lengthOf( + harness.events.filter((event) => event.type === "boundary_messages.updated"), + 3, ); yield* Effect.promise(() => store.append( @@ -3816,6 +3836,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { offers.map((offer) => receipt(offer.id)), ), ); + assert.lengthOf(harness.terminalEvents(), 0); const deliveryEvents = harness.events.filter( (event) => event.type === "boundary_messages.updated", ); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 8c48c0ca6d0b1..eb4738047c939 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -11172,6 +11172,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } if (message.type === "result") { for (const batch of liveQuery.boundaryBatches) batch.accepting = false; + } + // Transcript writes can finish after started; complete root frames retry without per-delta I/O. + if ( + message.type === "result" || + ((lifecycle?.state === "started" || + (message.type !== "stream_event" && isClaudeRootTurnFrame(message))) && + liveQuery.boundaryBatches.some( + (batch) => + batch.reserved && + batch.offers.some( + (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), + ), + )) + ) { yield* reconcileBoundaryReceipts(liveQuery); } } From 9a33e8d93dd3ca07076b6700af8097861946f56c Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 12 Nov 2026 07:34:30 +0000 Subject: [PATCH 20/23] fix(server): deliver parent follow-ups at Claude tool boundaries Settle same-owner follow-up tasks at the saved receipt without a separate completion wake. Preserve their queued task until confirmed and follow the receiving run when waiting on the send. Co-Authored-By: GPT-6.1 Sol at high effort, implementing agent --- ...skCancelNativeSubagent.integration.test.ts | 300 +++++++++++++++--- .../orchestration-v2/DelegatedBoundaryJoin.ts | 35 ++ .../src/orchestration-v2/Orchestrator.ts | 110 ++++++- .../ProviderContinuationService.ts | 13 +- .../ThreadManagementService.ts | 13 +- 5 files changed, 408 insertions(+), 63 deletions(-) create mode 100644 apps/server/src/orchestration-v2/DelegatedBoundaryJoin.ts diff --git a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts index 81a26ff6776c7..7f6fcb4eb6180 100644 --- a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts +++ b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts @@ -33,6 +33,7 @@ import * as EffectWorker from "../orchestration-v2/EffectWorker.ts"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; @@ -791,6 +792,7 @@ const delegatedClaudeHarness = Effect.fn("delegatedClaudeHarness")(function* (na const layerAdapters = ProviderAdapterRegistry.layerSingle(adapter); const layerOrchestrator = ProviderReplayHarness.layerWithRegistry({ name }, layerAdapters, { runEffectWorker: false, + runContinuationWorker: true, }); const layerOrchestration = Layer.merge( layerOrchestrator, @@ -896,56 +898,256 @@ const delegatedClaudeHarness = Effect.fn("delegatedClaudeHarness")(function* (na // A delegated Claude child ends its turn while a background native subagent it // started keeps running. -it.effect("a parent's queued update to a running Claude child becomes the task result", () => - Effect.scoped( - Effect.gen(function* () { - const { turns, layerTest, startDelegatedChild } = - yield* delegatedClaudeHarness("queued-update-to-child"); - yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const service = yield* OrchestratorMcpService.OrchestratorMcpService; - const { parentThreadId, task, child, scope } = yield* startDelegatedChild; - - const sent = yield* service.sendToThread(scope, { - threadId: task.childThreadId!, - message: "Also check the docs", - }); - yield* worker.drain(); - assert.equal(sent.delivery, "queued"); - - const updateRunning = yield* watch( - (event) => - event.type === "provider-turn.updated" && - event.payload.status === "running" && - event.payload.runAttemptId !== child.turn.attemptId, - ); - yield* finishChildTurn(child, "initial findings"); - yield* orchestrator.resumeQueuedRuns; - yield* worker.drain(); - yield* Fiber.join(updateRunning); - - const update = turns.find( - (entry) => - entry.turn.threadId === task.childThreadId && entry.turn.runId !== child.turn.runId, +it.effect.each([ + "initial", + "receipt", + "withdraw", + "unconfirmed", + "other-owner", + "other-parent", + "no-task", +] as const)( + "a parent's Claude follow-up shares completion only after a matching boundary receipt: %s", + (settlement) => + Effect.scoped( + Effect.gen(function* () { + const { turns, layerTest, startDelegatedChild } = yield* delegatedClaudeHarness( + `boundary-child-${settlement}`, ); - assert.isDefined(update); - const midway = yield* service.taskStatus(scope, task.id); - assert.equal(midway.workState, "working"); - assert.isNull(midway.summary); - const parentMidway = yield* orchestrator.getThreadProjection(parentThreadId); - assert.isNull(parentMidway.subagents.find((entry) => entry.id === task.id)?.result); - - yield* finishChildTurn(update!, "findings including docs"); - const done = yield* service.taskStatus(scope, task.id); - assert.equal(done.workState, "result_available"); - assert.equal(done.status, "completed"); - assert.equal(done.summary, "findings including docs"); - const parentDone = yield* orchestrator.getThreadProjection(parentThreadId); - assert.equal(parentDone.subagents.length, 1); - }).pipe(Effect.provide(layerTest)); - }), - ), + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const sink = yield* EventSink.EventSinkV2; + const { parentThreadId, task, child, scope } = yield* startDelegatedChild; + let active = child; + if (settlement !== "initial") { + yield* finishChildTurn(child, "initial findings"); + yield* orchestrator.recoverDelegatedTasks; + yield* service.taskStatus(scope, task.id); + const running = yield* watch( + (event) => + event.type === "provider-turn.updated" && + event.payload.status === "running" && + event.threadId === task.childThreadId, + ); + yield* service.sendToThread(scope, { + threadId: task.childThreadId!, + message: "Review again", + mode: "auto", + }); + yield* worker.drain(); + yield* Fiber.join(running); + active = turns.at(-1)!; + } + const sent = yield* service.sendToThread(scope, { + threadId: task.childThreadId!, + message: "Also check the docs", + mode: "steer", + }); + assert.equal(sent.delivery, "queued"); + const before = yield* orchestrator.getThreadProjection(task.childThreadId!); + const queued = before.runs.find((run) => run.id === sent.runId)!; + const run = before.runs.find((run) => run.id === active.turn.runId)!; + const providerTurn = before.providerTurns.find( + (turn) => turn.runAttemptId === active.turn.attemptId, + )!; + if (settlement !== "initial") assert.isDefined(queued.delegatedTaskId); + if ( + settlement === "other-owner" || + settlement === "other-parent" || + settlement === "no-task" + ) { + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + const followup = parent.subagents.find((row) => row.id === queued.delegatedTaskId)!; + const now = yield* DateTime.now; + yield* sink.write({ + commandId: CommandId.make("command:runtime-reconcile:owner"), + events: [ + { + id: EventId.make("event:owner"), + threadId: parentThreadId, + occurredAt: now, + type: "subagent.updated", + payload: { + ...followup, + ...(settlement === "other-owner" + ? { runId: RunId.make("other-owner") } + : settlement === "other-parent" + ? { threadId: ThreadId.make("other-parent") } + : {}), + }, + }, + ...(settlement !== "no-task" + ? [] + : [ + { + id: EventId.make("event:no-task"), + threadId: task.childThreadId!, + occurredAt: now, + type: "run.updated" as const, + payload: { ...run, delegatedTaskId: undefined }, + }, + ]), + ], + }); + } + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const offers = yield* requests.claimAgentMessages!({ + threadId: task.childThreadId!, + runId: run.id, + providerTurnId: providerTurn.id, + }); + const boundaryId = offers[0]?.id ?? "refused-followup"; + let promotion = 0; + const promote = (step: "offer" | "receipt" | "withdraw") => + orchestrator.dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(`boundary:${step}:${promotion++}`), + threadId: task.childThreadId!, + queuedRunId: queued.id, + targetRunId: run.id, + boundaryOffer: + step === "offer" + ? { id: boundaryId, step, providerTurnId: providerTurn.id } + : { id: boundaryId, step }, + }); + if ( + settlement === "other-owner" || + settlement === "other-parent" || + settlement === "no-task" + ) { + assert.lengthOf(offers, 0); + yield* Effect.flip(promote("offer")); + assert.equal( + (yield* orchestrator.getThreadProjection(task.childThreadId!)).runs.find( + (row) => row.id === queued.id, + )?.status, + "queued", + ); + return; + } + assert.lengthOf(offers, 1); + if (settlement === "withdraw") yield* promote("withdraw"); + const joined = settlement === "receipt" || settlement === "initial"; + if (joined) { + yield* promote("receipt"); + yield* promote("receipt"); + const received = yield* orchestrator.getThreadProjection(task.childThreadId!); + assert.equal( + received.messages.find((message) => message.id === sent.messageId)?.runId, + run.id, + ); + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + if (queued.delegatedTaskId !== undefined) { + const delivered = parent.subagents.find((row) => row.id === queued.delegatedTaskId)!; + const tracked = yield* service.taskStatus(scope, delivered.id); + assert.equal(tracked.workState, "result_available"); + assert.include(tracked.summary!, "Delivered into running run"); + assert.equal(delivered.status, "completed"); + assert.equal(delivered.completionDelivery?.state, "disposed"); + assert.include(delivered.result!, "Delivered into running run"); + assert.equal( + parent.nodes.find((node) => node.id === delivered.id)?.status, + "completed", + ); + assert.equal( + parent.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === delivered.id, + )?.status, + "completed", + ); + } + yield* finishChildTurn(active, "findings including docs"); + yield* orchestrator.recoverDelegatedTasks; + yield* orchestrator.recoverDelegatedTasks; + const done = yield* orchestrator.getThreadProjection(parentThreadId); + assert.lengthOf( + done.contextTransfers.filter( + (transfer) => + transfer.type === "subagent_result" && transfer.sourcePoint.runId === run.id, + ), + 1, + ); + const waited = yield* service.waitForThread(scope, { + threadId: task.childThreadId!, + runId: sent.runId, + timeoutMs: 0, + }); + assert.equal(waited.runId, run.id); + assert.equal(waited.status, "completed"); + const activeTaskId = run.delegatedTaskId ?? task.id; + assert.equal( + done.subagents.find((row) => row.id === activeTaskId)?.result, + "findings including docs", + ); + assert.lengthOf( + done.subagents.filter((row) => row.result === "findings including docs"), + 1, + ); + assert.isFalse( + done.subagents.some((row) => row.status === "running" || row.status === "pending"), + ); + assert.lengthOf( + done.runs + .flatMap((row) => row.delegatedCompletion?.delivery?.taskIds ?? []) + .filter((id) => id === activeTaskId), + 1, + ); + assert.isFalse( + done.runs.some( + (row) => + row.delegatedCompletion?.delivery?.taskIds.includes(queued.delegatedTaskId!) === + true, + ), + ); + assert.lengthOf( + turns.filter((entry) => entry.turn.threadId === task.childThreadId), + settlement === "initial" ? 1 : 2, + ); + } else { + const stillQueued = (yield* orchestrator.getThreadProjection( + task.childThreadId!, + )).runs.find((row) => row.id === queued.id)!; + assert.equal(stillQueued.delegatedTaskId, queued.delegatedTaskId); + assert.equal(stillQueued.boundaryOffer !== undefined, settlement === "unconfirmed"); + const updateRunning = yield* watch( + (event) => + event.type === "provider-turn.updated" && + event.payload.status === "running" && + event.payload.runAttemptId !== active.turn.attemptId && + event.threadId === task.childThreadId, + ); + yield* finishChildTurn(active, "review findings"); + if (settlement === "unconfirmed") { + assert.equal( + (yield* orchestrator.getThreadProjection(task.childThreadId!)).runs.find( + (row) => row.id === queued.id, + )?.status, + "queued", + ); + yield* orchestrator.dispatch({ + type: "queued-run.start", + commandId: CommandId.make("user-retry-unconfirmed"), + threadId: task.childThreadId!, + runId: queued.id, + requestedBy: "user", + }); + } else yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + yield* Fiber.join(updateRunning); + yield* finishChildTurn(turns.at(-1)!, "separate follow-up result"); + yield* orchestrator.recoverDelegatedTasks; + assert.equal( + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (row) => row.id === queued.delegatedTaskId, + )?.result, + "separate follow-up result", + ); + } + }).pipe(Effect.provide(layerTest)); + }), + ), ); const finishChildTurn = Effect.fn("finishChildTurn")(function* ( diff --git a/apps/server/src/orchestration-v2/DelegatedBoundaryJoin.ts b/apps/server/src/orchestration-v2/DelegatedBoundaryJoin.ts new file mode 100644 index 0000000000000..46d28d36f27ef --- /dev/null +++ b/apps/server/src/orchestration-v2/DelegatedBoundaryJoin.ts @@ -0,0 +1,35 @@ +import type { OrchestrationV2ThreadProjection } from "@t3tools/contracts"; + +type Run = OrchestrationV2ThreadProjection["runs"][number]; +type Message = OrchestrationV2ThreadProjection["messages"][number]; + +export function canJoinDelegatedBoundary( + queued: Run, + target: Run | undefined, + message: Message, + child: OrchestrationV2ThreadProjection["thread"], + tasks: OrchestrationV2ThreadProjection["subagents"], +): boolean { + if (queued.delegatedTaskId === undefined) return true; + const parentThreadId = child.lineage.parentThreadId; + const followup = tasks.find((task) => task.id === queued.delegatedTaskId); + const running = tasks.find((task) => task.id === target?.delegatedTaskId); + return ( + parentThreadId !== null && + message.senderThreadId === parentThreadId && + followup !== undefined && + running !== undefined && + followup.id !== running.id && + followup.origin === "app_owned" && + running.origin === "app_owned" && + followup.threadId === parentThreadId && + running.threadId === parentThreadId && + followup.childThreadId === child.id && + running.childThreadId === child.id && + followup.runId !== null && + followup.runId === running.runId && + followup.parentNodeId === running.parentNodeId && + followup.result === null && + running.result === null + ); +} diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index be30cff06594f..4dbe4f96967bf 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -178,6 +178,7 @@ import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as RunExecutionService from "./RunExecutionService.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts"; +import { canJoinDelegatedBoundary } from "./DelegatedBoundaryJoin.ts"; import { isAutomaticCompletionRun, queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts"; import { isRuntimePolicyAboveCeilingError, RuntimePolicyV2 } from "./RuntimePolicy.ts"; import { @@ -10601,6 +10602,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, ); + const boundaryTaskOwner = Effect.fn("Orchestrator.boundaryTaskOwner")(function* ( + projection: Parameters[0]["projection"], + command: Extract, + queuedRun: OrchestrationV2Run, + targetRunId: RunId, + message: OrchestrationV2ConversationMessage, + ) { + if (queuedRun.delegatedTaskId === undefined) return true; + const parentThreadId = projection.thread.lineage.parentThreadId; + if (parentThreadId === null) return false; + const parent = yield* projectionStore + .getThreadRecords(parentThreadId, ["subagents"]) + .pipe(mapDispatchError(command)); + return canJoinDelegatedBoundary( + queuedRun, + projection.runs.find((run) => run.id === targetRunId), + message, + projection.thread, + parent.subagents, + ); + }); + const dispatchQueuedMessagePromoteToSteer = Effect.fn( "Orchestrator.dispatchQueuedMessagePromoteToSteer", )(function* ( @@ -10693,15 +10716,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const receipt = command.boundaryOffer?.step === "receipt" ? queuedRun.boundaryOffer : undefined; if (command.boundaryOffer?.step === "offer") { // Stop wins: a held queue or a stopped turn keeps the update queued. - // A run that owns a delegated task keeps its turn: the task settles - // with its own run, which joining would cancel. + // A follow-up can share completion only with the same parent task owner. if ( queuedRun.boundaryOffer !== undefined || queuedMessage.atToolBoundary !== true || isNativeMaintenanceCommand(queuedMessage) || parseGoalCommand(queuedMessage) !== null || queuedRun.freshContext === true || - queuedRun.delegatedTaskId !== undefined || + !(yield* boundaryTaskOwner( + projection, + command, + queuedRun, + command.targetRunId, + queuedMessage, + )) || queuedRun.queueHeld === true || (yield* stopReachedRun(command, command.threadId, command.targetRunId)) ) { @@ -10936,11 +10964,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio Option.isSome(session) && session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === true ) { - // The same refusals as the boundary join: Stop wins, and a run that - // owns a delegated task keeps its own turn. + // Stop wins, and unrelated task owners keep their own turns. if ( (queuedRun.queueHeld === true && queuedRun.boundaryOffer === undefined) || - queuedRun.delegatedTaskId !== undefined || + !(yield* boundaryTaskOwner( + projection, + command, + queuedRun, + command.targetRunId, + queuedMessage, + )) || (yield* stopReachedRun(command, command.threadId, targetRun.id)) ) { return yield* new OrchestratorDispatchError({ @@ -11008,6 +11041,64 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const emitEvent = emit(events, command); const targetRunId = receipt?.targetRunId ?? command.targetRunId; const targetRun = projection.runs.find((run) => run.id === targetRunId); + let joinedRun = queuedRun; + if (receipt !== undefined && queuedRun.delegatedTaskId !== undefined) { + const parentThreadId = projection.thread.lineage.parentThreadId; + if (parentThreadId !== null) { + const parent = yield* projectionStore + .getThreadRecords(parentThreadId, ["subagents", "nodes", "turnItems"], { + turnItemTypes: ["subagent"], + }) + .pipe(mapDispatchError(command)); + const task = parent.subagents.find((task) => task.id === queuedRun.delegatedTaskId); + if (task !== undefined) { + const summary = `Delivered into running run ${targetRunId}; its task reports the result.`; + const scope = { + threadId: parentThreadId, + ...(task.runId === null ? {} : { runId: task.runId }), + nodeId: task.id, + occurredAt: now, + }; + yield* emitEvent({ + ...scope, + type: "subagent.updated", + payload: { + ...task, + status: "completed", + result: summary, + completedAt: now, + updatedAt: now, + completionDelivery: { state: "disposed", observedByRunId: null }, + }, + }); + const node = parent.nodes.find((node) => node.id === task.id); + if (node !== undefined) + yield* emitEvent({ + ...scope, + type: "node.updated", + payload: { ...node, status: "completed", completedAt: now }, + }); + const item = parent.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === task.id, + ); + if (item?.type === "subagent") + yield* emitEvent({ + ...scope, + type: "turn-item.updated", + payload: { + ...item, + status: "completed", + result: summary, + completedAt: now, + updatedAt: now, + }, + }); + // The cancelled queue row must not settle this task again during recovery. + const { delegatedTaskId: _taskId, ...unbound } = queuedRun; + joinedRun = unbound; + } + } + } yield* emitEvent({ type: "run.updated", threadId: command.threadId, @@ -11016,7 +11107,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio providerInstanceId: queuedRun.providerInstanceId, occurredAt: now, payload: { - ...queuedRun, + ...joinedRun, status: "cancelled", queuePosition: null, completedAt: now, @@ -15183,8 +15274,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? withSubagentParentLock(command.threadId, dispatch) : dispatch, ); - // Answering a message-mode question can also start a follow-up. - return yield* command.type === "runtime-request.respond" + // Answers can start follow-ups; a boundary receipt can settle their parent task. + return yield* command.type === "runtime-request.respond" || + command.type === "queued-message.promote-to-steer" ? withSubagentParentLock(command.threadId, dispatch) : dispatch; }); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index 788df109e99b1..a2bab5ce34fa6 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -17,6 +17,7 @@ import * as ProviderContinuationRequests from "@t3tools/provider-core/server/Pro import type * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts"; +import { canJoinDelegatedBoundary } from "./DelegatedBoundaryJoin.ts"; import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts"; import { isNativeMaintenanceCommand } from "./CommandPolicy.ts"; @@ -301,9 +302,15 @@ export const layer = Layer.effectDiscard( function* (input) { const unhanded: Array = []; return yield* Effect.gen(function* () { - const { runs } = yield* threads.getThreadRecords(input.threadId, ["runs"], { - runStatuses: ["queued"], + const { runs, thread } = yield* threads.getThreadRecords(input.threadId, ["runs"], { + runStatuses: ["queued", "running"], }); + const parent = + !runs.some((run) => run.status === "queued" && run.delegatedTaskId !== undefined) || + thread.lineage.parentThreadId === null + ? undefined + : yield* threads.getThreadRecords(thread.lineage.parentThreadId, ["subagents"]); + const target = runs.find((run) => run.id === input.runId); if (runs.length === 0) return []; const { messages } = yield* threads.getThreadRecords(input.threadId, ["messages"], { messageIds: runs.map((run) => run.userMessageId), @@ -316,7 +323,7 @@ export const layer = Layer.effectDiscard( run.boundaryOffer !== undefined || run.queueHeld === true || run.freshContext === true || - run.delegatedTaskId !== undefined || + !canJoinDelegatedBoundary(run, target, message, thread, parent?.subagents ?? []) || isNativeMaintenanceCommand(message) || parseGoalCommand(message) !== null ) diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 26c00b243bcf7..3c6c4f0867e65 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -955,13 +955,21 @@ const make = Effect.gen(function* () { // so a cut run is not the answer while its continuation is pending. // Runs per matching event and is already covered by the waitForThread span. const outcome = Effect.fnUntraced(function* (runs: ReadonlyArray) { - const current = runs.find((candidate) => candidate.id === selectedRun.id); - if (current === undefined) { + const selected = runs.find((candidate) => candidate.id === selectedRun.id); + if (selected === undefined) { return yield* new ThreadManagementRunNotFoundError({ threadId: input.threadId, runId: selectedRun.id, }); } + // A boundary receipt moves the message into its receiving run before cancelling the queue row. + const takenBy = + selected.status === "cancelled" && selected.startedAt === null + ? (yield* getProjectThreadRecords(input, ["messages"], { + messageIds: [selected.userMessageId], + })).messages[0]?.runId + : undefined; + const current = runs.find((candidate) => candidate.id === takenBy) ?? selected; const run = latestRestartContinuation(runs, current); const pending = isTerminalRunStatus(run.status) && (yield* orchestrator.restartContinuationPending(run)); @@ -1036,6 +1044,7 @@ const make = Effect.gen(function* () { untilDelegatedWorkSettled || stored.event.type !== "run.updated" || stored.event.payload.id === selectedRun.id || + isTerminalRunStatus(stored.event.payload.status) || stored.event.payload.restartContinuationOfRunId !== undefined, ), ), From bda1e7dd22267f0f86921e6a23f26b8d093bfc98 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 12 Nov 2026 07:52:02 +0000 Subject: [PATCH 21/23] fix(server): bound Claude receipt re-reads and recheck rewritten transcripts A reserved offer without a receipt re-read the transcript from its handoff offset on every later root message, and a transcript replaced after the handoff could hide a receipt before that offset. The tail read now advances a cursor past complete lines, and the turn-end check reads the full history once for anything still unconfirmed, then stops retrying. Co-Authored-By: Claude Fable 5.2 at medium effort, primary agent Co-Authored-By: GPT-6.1 Sol at xhigh effort, reviewing agent --- .../Adapters/ClaudeAdapterV2.ts | 94 ++++++---- .../Adapters/ClaudeSavedMessageReceipts.ts | 163 +++++++++++------- .../ClaudeBoundaryReceipts.test.ts | 37 ++-- 3 files changed, 182 insertions(+), 112 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index eb4738047c939..a169cb2526f6d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -5,7 +5,10 @@ import { toolOutputImages, } from "@t3tools/shared/toolOutput"; import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts"; -import { readClaudeBoundaryReceipts } from "./ClaudeSavedMessageReceipts.ts"; +import { + readClaudeBoundaryReceipts, + readClaudeBoundaryReceiptTail, +} from "./ClaudeSavedMessageReceipts.ts"; import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts"; import { dynamicToolTitle, @@ -4815,7 +4818,7 @@ interface ClaudeLiveQueryContext { interface ClaudeBoundaryBatch { readonly transcriptPath: string; - readonly transcriptOffset: number; + transcriptOffset: number; readonly context: ActiveClaudeTurnContext; readonly offers: ReadonlyArray; readonly saved: Set; @@ -11092,36 +11095,55 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }; const reconcileBoundaryReceipts = Effect.fn("ClaudeAdapterV2.reconcileBoundaryReceipts")( - function* (live: ClaudeLiveQueryContext) { - const offers = live.boundaryBatches.flatMap((batch) => + function* (live: ClaudeLiveQueryContext, scope: "appended" | "ended") { + const unsaved = (batch: ClaudeBoundaryBatch) => batch.offers.filter( (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), - ), - ); - if (offers.length === 0) return; - const pendingBatches = live.boundaryBatches.filter((batch) => - batch.offers.some( - (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), - ), - ); - const saved = yield* readClaudeBoundaryReceipts({ - transcriptPath: pendingBatches[0]!.transcriptPath, - offset: Math.min(...pendingBatches.map((batch) => batch.transcriptOffset)), - projectsDirectory: path.join(homePath, "projects"), - nativeThreadId: live.nativeThreadId, - offers, - }).pipe( - Effect.provideService(FileSystem.FileSystem, fileSystem), - Effect.provideService(Path.Path, path), - ); + ); + const pending = live.boundaryBatches.filter((batch) => unsaved(batch).length > 0); + const saved = new Set(); + const nextOffsets = new Map(); + if (scope === "appended") { + for (const transcriptPath of new Set(pending.map((batch) => batch.transcriptPath))) { + const batches = pending.filter((batch) => batch.transcriptPath === transcriptPath); + const tail = yield* readClaudeBoundaryReceiptTail({ + transcriptPath, + offset: Math.min(...batches.map((batch) => batch.transcriptOffset)), + nativeThreadId: live.nativeThreadId, + offers: batches.flatMap(unsaved), + }).pipe(Effect.provideService(FileSystem.FileSystem, fileSystem)); + for (const id of tail.saved) saved.add(id); + nextOffsets.set(transcriptPath, tail.nextOffset); + } + } else if (pending.length > 0) { + // The tail read trusts its offset; a rewritten transcript needs one full read. + const history = yield* readClaudeBoundaryReceipts({ + projectsDirectory: path.join(homePath, "projects"), + nativeThreadId: live.nativeThreadId, + offers: pending.flatMap(unsaved), + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + ); + for (const id of history) saved.add(id); + } yield* live.boundaryLock.withPermits(1)( Effect.gen(function* () { for (const batch of live.boundaryBatches) { for (const offer of batch.offers) if (saved.has(offer.id)) batch.saved.add(offer.id); + batch.transcriptOffset = + nextOffsets.get(batch.transcriptPath) ?? batch.transcriptOffset; yield* flushBoundaryReceipts(batch); } pruneBoundaryBatches(live.boundaryBatches); + if (scope === "ended") { + // An offer still unconfirmed here stays queued as Unconfirmed; recovery owns it. + for (let index = live.boundaryBatches.length - 1; index >= 0; index--) { + if (!live.boundaryBatches[index]!.accepting) + live.boundaryBatches.splice(index, 1); + } + } }), ); }, @@ -11172,21 +11194,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } if (message.type === "result") { for (const batch of liveQuery.boundaryBatches) batch.accepting = false; - } - // Transcript writes can finish after started; complete root frames retry without per-delta I/O. - if ( - message.type === "result" || - ((lifecycle?.state === "started" || + yield* reconcileBoundaryReceipts(liveQuery, "ended"); + } else if ( + // Transcript writes can finish after started; complete root frames retry without per-delta I/O. + (lifecycle?.state === "started" || (message.type !== "stream_event" && isClaudeRootTurnFrame(message))) && - liveQuery.boundaryBatches.some( - (batch) => - batch.reserved && - batch.offers.some( - (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), - ), - )) + liveQuery.boundaryBatches.some( + (batch) => + batch.reserved && + batch.offers.some( + (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), + ), + ) ) { - yield* reconcileBoundaryReceipts(liveQuery); + yield* reconcileBoundaryReceipts(liveQuery, "appended"); } } // The API answering again is recovery; a long text or tool input block emits nothing @@ -12380,7 +12401,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.exit, Effect.flatMap( Effect.fnUntraced(function* (exit: ClaudeQueryStreamExit) { - yield* reconcileBoundaryReceipts(context); + for (const batch of context.boundaryBatches) batch.accepting = false; + yield* reconcileBoundaryReceipts(context, "ended"); const heldContext = yield* Ref.get(activeTurn); if ( (yield* Ref.get(queryContext))?.query === querySession && diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts index cd79e4ac2e81c..4470b806a8f1c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts @@ -9,67 +9,104 @@ import { field } from "../unknownField.ts"; const decodeLine = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)); +interface ReceiptQuery { + readonly nativeThreadId: string; + readonly offers: ReadonlyArray<{ readonly id: string }>; +} + +function receiptCollector(input: ReceiptQuery) { + const offers = new Set(input.offers.map((offer) => offer.id)); + const saved = new Set(); + return { + saved, + collect: (line: string) => { + // Most lines are not receipts; skip decoding them, since transcripts reach hundreds of MB. + if (!line.includes("queued_command") || !line.includes("source_uuid")) return; + const decoded = decodeLine(line); + if (Option.isNone(decoded)) return; + const entry = decoded.value; + if ( + field(entry, "sessionId") !== input.nativeThreadId || + field(entry, "isSidechain") !== false || + field(entry, "type") !== "attachment" + ) + return; + const attachment = field(entry, "attachment"); + if (field(attachment, "type") !== "queued_command") return; + const uuid = field(attachment, "source_uuid"); + if (typeof uuid === "string" && offers.has(uuid)) saved.add(uuid); + }, + }; +} + /** Only positive root-history evidence settles an offer; missing or torn history never permits replay. */ -export const readClaudeBoundaryReceipts = Effect.fn("readClaudeBoundaryReceipts")( - function* (input: { - readonly projectsDirectory: string; - readonly nativeThreadId: string; - readonly offers: ReadonlyArray<{ readonly id: string }>; - readonly transcriptPath?: string; - readonly offset?: number; - }) { - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const offers = new Map(input.offers.map((offer) => [offer.id, offer])); - const saved = new Set(); - const transcripts = - input.transcriptPath === undefined - ? yield* Effect.gen(function* () { - const projects = yield* fileSystem - .readDirectory(input.projectsDirectory) - .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed([]))); - return projects.map((project) => - path.join(input.projectsDirectory, project, `${input.nativeThreadId}.jsonl`), - ); - }) - : [input.transcriptPath]; - for (const transcript of transcripts) { - const info = yield* fileSystem - .stat(transcript) - .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.undefined)); - if (info === undefined) continue; - let offset = - input.offset !== undefined && input.offset <= Number(info.size) ? input.offset : 0; - if (offset > 0) { - const previous = yield* fileSystem - .stream(transcript, { offset: offset - 1, bytesToRead: 1 }) - .pipe(Stream.runHead); - if (Option.isNone(previous) || previous.value[0] !== 10) offset = 0; - } - yield* fileSystem.stream(transcript, { offset }).pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - Effect.sync(() => { - if (!line.includes("queued_command") || !line.includes("source_uuid")) return; - const decoded = decodeLine(line); - if (Option.isNone(decoded)) return; - const entry = decoded.value; - if ( - field(entry, "sessionId") !== input.nativeThreadId || - field(entry, "isSidechain") !== false || - field(entry, "type") !== "attachment" - ) - return; - const attachment = field(entry, "attachment"); - if (field(attachment, "type") === "queued_command") { - const uuid = field(attachment, "source_uuid"); - if (typeof uuid === "string" && offers.has(uuid)) saved.add(uuid); - } - }), - ), - ); - } - return saved; - }, -); +export const readClaudeBoundaryReceipts = Effect.fn("readClaudeBoundaryReceipts")(function* ( + input: ReceiptQuery & { readonly projectsDirectory: string }, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { saved, collect } = receiptCollector(input); + const projects = yield* fileSystem + .readDirectory(input.projectsDirectory) + .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed([]))); + for (const project of projects) { + const transcript = path.join(input.projectsDirectory, project, `${input.nativeThreadId}.jsonl`); + if (!(yield* fileSystem.exists(transcript))) continue; + yield* fileSystem.stream(transcript).pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((line) => Effect.sync(() => collect(line))), + ); + } + return saved; +}); + +/** + * Reads the complete lines appended since `offset` and returns where the next read resumes. + * A receipt is written after its offer is handed off, so a line that straddles the offset is not one. + */ +export const readClaudeBoundaryReceiptTail = Effect.fn("readClaudeBoundaryReceiptTail")(function* ( + input: ReceiptQuery & { readonly transcriptPath: string; readonly offset: number }, +) { + const fileSystem = yield* FileSystem.FileSystem; + const { saved, collect } = receiptCollector(input); + const info = yield* fileSystem + .stat(input.transcriptPath) + .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.undefined)); + if (info === undefined) return { saved, nextOffset: 0 }; + // A file shorter than the offset was replaced, so nothing in it has been read yet. + const offset = input.offset <= Number(info.size) ? input.offset : 0; + // Starting one byte early makes the first newline end either the previous line or a straddling one. + const start = Math.max(offset - 1, 0); + const decoder = new TextDecoder(); + let skipFirstLine = start > 0; + let position = start; + let nextOffset = offset; + let carry: Array = []; + yield* fileSystem.stream(input.transcriptPath, { offset: start }).pipe( + Stream.runForEach((chunk) => + Effect.sync(() => { + let lineStart = 0; + for ( + let newline = chunk.indexOf(10); + newline !== -1; + newline = chunk.indexOf(10, lineStart) + ) { + if (skipFirstLine) { + skipFirstLine = false; + } else { + let line = ""; + for (const part of carry) line += decoder.decode(part, { stream: true }); + collect(line + decoder.decode(chunk.subarray(lineStart, newline))); + } + carry = []; + lineStart = newline + 1; + nextOffset = position + lineStart; + } + if (lineStart < chunk.length) carry.push(chunk.slice(lineStart)); + position += chunk.length; + }), + ), + ); + return { saved, nextOffset }; +}); diff --git a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts index 1eaff02a8a718..746eb408b74c0 100644 --- a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts @@ -13,7 +13,10 @@ import { } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; -import { readClaudeBoundaryReceipts } from "./Adapters/ClaudeSavedMessageReceipts.ts"; +import { + readClaudeBoundaryReceipts, + readClaudeBoundaryReceiptTail, +} from "./Adapters/ClaudeSavedMessageReceipts.ts"; import { makeClaudeBoundaryReceiptRecovery } from "./ClaudeBoundaryReceipts.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; @@ -174,23 +177,31 @@ it.layer(NodeServices.layer)("Claude boundary saved history", (it) => { ], }); const transcriptPath = path.join(project, "tail.jsonl"); - const before = `${encodeLine(entry({ type: "queued_command", source_uuid: "before" }))}\n`; - const after = `${encodeLine(entry({ type: "queued_command", source_uuid: "after" }))}\n`; - yield* fs.writeFileString(transcriptPath, before); - const offset = Number((yield* fs.stat(transcriptPath)).size); - yield* fs.writeFileString(transcriptPath, before + after); + const receipt = (id: string) => + encodeLine(entry({ type: "queued_command", source_uuid: id })); + const before = `${receipt("before")}\n`; + const after = `${receipt("after")}\n`; + const offset = new TextEncoder().encode(before).length; const readTail = (start: number) => - readClaudeBoundaryReceipts({ - projectsDirectory: directory, + readClaudeBoundaryReceiptTail({ nativeThreadId: "native-session", transcriptPath, offset: start, - offers: [{ id: "before" }, { id: "after" }], - }).pipe(Effect.map((receipts) => [...receipts])); - assert.deepEqual(yield* readTail(offset), ["after"]); - assert.deepEqual(yield* readTail(offset - 1), ["before", "after"]); + offers: [{ id: "before" }, { id: "after" }, { id: "late" }], + }).pipe(Effect.map((tail) => ({ saved: [...tail.saved], nextOffset: tail.nextOffset }))); + const torn = receipt("late"); + yield* fs.writeFileString(transcriptPath, before + after + torn.slice(0, 20)); + const appended = { + saved: ["after"], + nextOffset: new TextEncoder().encode(before + after).length, + }; + assert.deepEqual(yield* readTail(offset), appended); + // A line that straddles the offset began before the handoff, so it is not re-read. + assert.deepEqual(yield* readTail(offset - 5), appended); + yield* fs.writeFileString(transcriptPath, `${before}${after}${torn}\n`); + assert.deepEqual((yield* readTail(appended.nextOffset)).saved, ["late"]); yield* fs.writeFileString(transcriptPath, after); - assert.deepEqual(yield* readTail(offset + 1000), ["after"]); + assert.deepEqual((yield* readTail(offset + 1000)).saved, ["after"]); assert.deepEqual([...saved], ["saved"]); assert.deepEqual( [ From 146c394f4b8c36d9f410c13eac8ba6522c799a1c Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 12 Nov 2026 08:03:06 +0000 Subject: [PATCH 22/23] fix(server): retire Claude receipt batches when transcript reads fail Cover rewritten transcripts, ended unconfirmed offers, and independent transcript paths through the adapter. Retire ended batches even when history cannot be read, and skip a straddling first record at byte offset one. Co-Authored-By: GPT-6.1 Sol at high effort, implementing agent --- ...skCancelNativeSubagent.integration.test.ts | 1 - .../Adapters/ClaudeAdapterV2.test.ts | 223 +++++++++++++++++- .../Adapters/ClaudeAdapterV2.ts | 23 +- .../Adapters/ClaudeSavedMessageReceipts.ts | 2 +- .../ClaudeBoundaryReceipts.test.ts | 1 + 5 files changed, 240 insertions(+), 10 deletions(-) diff --git a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts index 7f6fcb4eb6180..8e83168c40607 100644 --- a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts +++ b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts @@ -1131,7 +1131,6 @@ it.effect.each([ commandId: CommandId.make("user-retry-unconfirmed"), threadId: task.childThreadId!, runId: queued.id, - requestedBy: "user", }); } else yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 19f8d185dc872..22ac8b4ba79a6 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3333,6 +3333,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarnessWithOptions = (options?: { + readonly fileSystem?: FileSystem.FileSystem; readonly close?: (sdkMessages: Queue.Queue) => Effect.Effect; readonly interrupt?: Effect.Effect; readonly setPermissionMode?: ClaudeAdapterV2.ClaudeAgentSdkQuerySession["setPermissionMode"]; @@ -3365,7 +3366,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly promptSuggestions?: boolean; }) => Effect.gen(function* () { - const fileSystem = yield* FileSystem.FileSystem; + const fileSystem = options?.fileSystem ?? (yield* FileSystem.FileSystem); const idAllocator = yield* IdAllocator.IdAllocatorV2; const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-claude-v2-wake-", @@ -3864,6 +3865,226 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect.each(["rewritten", "unconfirmed", "unreadable", "separate-files"] as const)( + "reconciles boundary transcript receipts without retaining ended offers: %s", + (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boundary-tail-" }); + const projects = path.join(home, "projects"); + const paths = ["first", "second"].map((project) => + path.join(projects, project, `${WAKE_NATIVE_SESSION}.jsonl`), + ); + const padding = "{}\n".repeat(256); + for (const transcript of paths) { + yield* fs.makeDirectory(path.dirname(transcript), { recursive: true }); + yield* fs.writeFileString(transcript, padding); + } + const unreadable = path.join(home, "not-a-directory"); + yield* fs.writeFileString(unreadable, ""); + const reads: Array<{ readonly path: string; readonly offset: number }> = []; + let historyReads = 0; + let failHistory = false; + const fileSystem: FileSystem.FileSystem = { + ...fs, + stream: (file, options) => { + if (paths.includes(file)) + reads.push({ + path: file, + offset: options?.offset === undefined ? 0 : Number(options.offset), + }); + return fs.stream(file, options); + }, + readDirectory: (directory, options) => { + if (directory === projects) historyReads++; + return fs.readDirectory( + directory === projects && failHistory ? unreadable : directory, + options, + ); + }, + }; + const offers: ProviderContinuationRequests.AgentMessageOffer[] = [0, 1].map((index) => ({ + id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + queuedRunId: RunId.make(`queued-tail-${index}`), + messageId: MessageId.make(`message-tail-${index}`), + text: `Follow-up ${index}`, + attachments: [], + createdBy: "user", + creationSource: "web", + })); + let claimed = 0; + const withdrawals: string[] = []; + const harness = yield* makeWakeHarnessWithOptions({ + fileSystem, + settings: { ...DEFAULT_CLAUDE_SETTINGS, homePath: home }, + claimAgentMessages: () => Effect.sync(() => [offers[claimed++]!]), + withdrawAgentMessages: ({ offers }) => + Effect.sync(() => { + withdrawals.push(...offers.map((offer) => offer.id)); + }), + }); + const turn = (attempt: string) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe("2026-11-12T00:00:00Z"), + attemptId: RunAttemptId.make(attempt), + text: "Work with tools.", + attachments: [], + }); + yield* harness.runtime.startTurn(turn("receipt-tail-first")); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + capabilities: ["interrupt_cancel_queued_v1"], + user_message_uuid: harness.promptUuid(0), + session_id: WAKE_NATIVE_SESSION, + }), + ); + const queryOptions = harness.getOpenedOptions(); + const hook = queryOptions?.hooks?.PostToolBatch?.[0]?.hooks[0]; + assert.isDefined(hook); + if (hook === undefined) return; + const signal = yield* Effect.abortSignal; + const batchCount = scenario === "separate-files" ? 2 : 1; + for (const transcript of paths.slice(0, batchCount)) { + yield* Effect.promise(() => + hook( + { + hook_event_name: "PostToolBatch", + session_id: WAKE_NATIVE_SESSION, + transcript_path: transcript, + cwd: "/repo", + tool_calls: [], + }, + undefined, + { signal }, + ), + ); + } + const receipt = (index: number) => + `${JSON.stringify({ + type: "attachment", + sessionId: WAKE_NATIVE_SESSION, + isSidechain: false, + attachment: { type: "queued_command", source_uuid: offers[index]!.id }, + })}\n`; + const received = () => + harness.events.flatMap((event) => + event.type === "boundary_messages.updated" && event.step === "receipt" + ? event.offers.map((offer) => offer.id) + : [], + ); + const root = Effect.fnUntraced(function* (text: string) { + yield* harness.offerAndWait(makeAssistantTextFrame({ uuid: text, text })); + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.text.includes(text), + ), + ); + }); + if (scenario === "rewritten") yield* fs.writeFileString(paths[0]!, receipt(0) + padding); + for (const offer of offers.slice(0, batchCount)) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "command_lifecycle", + command_uuid: offer.id, + state: "started", + session_id: WAKE_NATIVE_SESSION, + }), + ); + } + if (scenario === "separate-files") { + yield* harness.until( + () => + harness.events.filter( + (event) => event.type === "boundary_messages.updated" && event.step === "reserve", + ).length === batchCount, + ); + for (let index = 0; index < batchCount; index++) + yield* fs.writeFileString(paths[index]!, receipt(index), { flag: "a" }); + } + yield* root("mid-turn receipt check"); + if (scenario === "separate-files") { + assert.deepEqual( + received(), + offers.map((offer) => offer.id), + ); + assert.isTrue( + paths.every((transcript) => reads.some((read) => read.path === transcript)), + ); + assert.equal(historyReads, 0); + return; + } + assert.deepEqual(received(), []); + assert.equal(historyReads, 0); + const previousOffset = reads.at(-1)?.offset; + yield* fs.writeFileString(paths[0]!, "{}\n", { flag: "a" }); + yield* root("advancing the receipt cursor"); + yield* root("checking the advanced receipt cursor"); + const advancedOffset = reads.at(-1)?.offset; + failHistory = scenario === "unreadable"; + yield* harness.offerAndWait( + makeResultFrame({ uuid: "receipt-tail-result", result: "Finished." }), + ); + yield* harness.awaitTerminal(1); + failHistory = false; + if (scenario === "rewritten") { + assert.deepEqual(received(), [offers[0]!.id]); + assert.equal(historyReads, 1); + return; + } + assert.deepEqual(received(), []); + const readsAtEnd = reads.length; + const historyReadsAtEnd = historyReads; + yield* harness.offerAndWait( + makeAssistantTextFrame({ uuid: "late-root", text: "Late root output" }), + ); + yield* harness.runtime.startTurn(turn("receipt-tail-later")); + assert.strictEqual(harness.getOpenedOptions(), queryOptions); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "status", + status: null, + user_message_uuid: harness.promptUuid(2), + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* root("later turn receipt check"); + yield* harness.offerAndWait( + makeResultFrame({ uuid: "receipt-tail-later-result", result: "Later work." }), + ); + yield* harness.awaitTerminal(2); + assert.equal(reads.length, readsAtEnd); + assert.equal(historyReads, historyReadsAtEnd); + assert.equal(historyReads, 1); + assert.isAbove(Number(advancedOffset), Number(previousOffset)); + assert.deepEqual(received(), []); + assert.deepEqual( + harness.offeredMessages + .filter((message) => message.uuid === offers[0]!.id) + .map((message) => message.priority), + ["next"], + ); + assert.deepEqual(withdrawals, []); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("withdraws a never-pulled input discarded after its hook aborts", () => Effect.gen(function* () { const offers: ProviderContinuationRequests.AgentMessageOffer[] = [0, 1].map((index) => ({ diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index a169cb2526f6d..de85b148fbf38 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -11137,16 +11137,25 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( yield* flushBoundaryReceipts(batch); } pruneBoundaryBatches(live.boundaryBatches); - if (scope === "ended") { - // An offer still unconfirmed here stays queued as Unconfirmed; recovery owns it. - for (let index = live.boundaryBatches.length - 1; index >= 0; index--) { - if (!live.boundaryBatches[index]!.accepting) - live.boundaryBatches.splice(index, 1); - } - } }), ); }, + (effect, live, scope) => + effect.pipe( + Effect.ensuring( + scope === "ended" + ? live.boundaryLock.withPermits(1)( + Effect.sync(() => { + // Even unreadable history leaves ended offers Unconfirmed; only recovery may retry them. + for (let index = live.boundaryBatches.length - 1; index >= 0; index--) { + if (!live.boundaryBatches[index]!.accepting) + live.boundaryBatches.splice(index, 1); + } + }), + ) + : Effect.void, + ), + ), Effect.catchTags({ PlatformError: (cause) => Effect.logWarning("orchestration-v2.claude-boundary-history-read-failed", { cause }), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts index 4470b806a8f1c..6c864955f22fb 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts @@ -79,7 +79,7 @@ export const readClaudeBoundaryReceiptTail = Effect.fn("readClaudeBoundaryReceip // Starting one byte early makes the first newline end either the previous line or a straddling one. const start = Math.max(offset - 1, 0); const decoder = new TextDecoder(); - let skipFirstLine = start > 0; + let skipFirstLine = offset > 0; let position = start; let nextOffset = offset; let carry: Array = []; diff --git a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts index 746eb408b74c0..2968b7290637f 100644 --- a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts @@ -198,6 +198,7 @@ it.layer(NodeServices.layer)("Claude boundary saved history", (it) => { assert.deepEqual(yield* readTail(offset), appended); // A line that straddles the offset began before the handoff, so it is not re-read. assert.deepEqual(yield* readTail(offset - 5), appended); + assert.deepEqual(yield* readTail(1), appended); yield* fs.writeFileString(transcriptPath, `${before}${after}${torn}\n`); assert.deepEqual((yield* readTail(appended.nextOffset)).saved, ["late"]); yield* fs.writeFileString(transcriptPath, after); From c9aed41eaaa169c68a91a157740c3800368dceac Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 12 Nov 2026 08:23:56 +0000 Subject: [PATCH 23/23] fix(server): keep tool-boundary delivery to providers that deliver there Upstream now marks Cursor's steering as tool-interrupting so its mailbox wakes stay queued. This branch used the same capability to queue explicit steers for delivery at a tool boundary, which only the Claude adapter does, so Cursor steers were queued and never delivered. Boundary delivery now has its own capability. Co-Authored-By: Claude Fable 5.2 at medium effort, primary agent --- apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts | 1 + apps/server/src/orchestration-v2/Orchestrator.ts | 4 ++-- .../orchestration-v2/SteeringCompletion.integration.test.ts | 2 ++ packages/contracts/src/orchestrationV2.ts | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index de85b148fbf38..e34defe621602 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -488,6 +488,7 @@ export const ClaudeProviderCapabilitiesV2 = { supportsInterrupt: true, supportsActiveSteering: true, activeSteeringInterruptsTools: true, + deliversUpdatesAtToolBoundary: true, supportsSteeringByInterruptRestart: false, supportsQueuedMessages: true, terminalStatusQuality: "strong", diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 4dbe4f96967bf..f1c88e1c85775 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -6551,7 +6551,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(mapDispatchError(command)); if ( Option.isSome(session) && - session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === true + session.value.providerSession.capabilities.turns.deliversUpdatesAtToolBoundary === true ) { dispatchMode = { type: "queue_after_active" }; atToolBoundary = true; @@ -10962,7 +10962,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(mapDispatchError(command)); if ( Option.isSome(session) && - session.value.providerSession.capabilities.turns.activeSteeringInterruptsTools === true + session.value.providerSession.capabilities.turns.deliversUpdatesAtToolBoundary === true ) { // Stop wins, and unrelated task owners keep their own turns. if ( diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index cd78feb26c15d..efec46281a5b9 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -78,6 +78,7 @@ it.effect.each( ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: timing !== "without native steering", activeSteeringInterruptsTools: timing === "with interrupting native steering", + deliversUpdatesAtToolBoundary: timing === "with interrupting native steering", }, }; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { @@ -515,6 +516,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* supportsActiveSteering: canSteer, supportsSteeringByInterruptRestart: canRestart, activeSteeringInterruptsTools, + deliversUpdatesAtToolBoundary: activeSteeringInterruptsTools, activeSteeringAcceptsAttachments: steeringAcceptsAttachments, ...(nativeGoals === undefined ? {} : { supportsNativeGoals: nativeGoals }), }, diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 9a50d2b02df9b..b09331869aa3f 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -286,6 +286,8 @@ export const OrchestrationV2TurnCapabilities = Schema.Struct({ supportsActiveSteering: Schema.Boolean, // Some native steering mechanisms cancel pending tools before consuming the message. activeSteeringInterruptsTools: Schema.optional(Schema.Boolean), + // The adapter hands queued updates to the provider between tool batches, so a steer queues instead. + deliversUpdatesAtToolBoundary: Schema.optional(Schema.Boolean), // Some take text only, so a steer with attachments restarts the turn instead. activeSteeringAcceptsAttachments: Schema.optional(Schema.Boolean), // Only where a steer leaves a running automatic compaction intact; otherwise clients queue.