diff --git a/apps/mobile/src/features/threads/ThreadQueueControl.tsx b/apps/mobile/src/features/threads/ThreadQueueControl.tsx index 58129ebc0cece..2288dfa5e95f4 100644 --- a/apps/mobile/src/features/threads/ThreadQueueControl.tsx +++ b/apps/mobile/src/features/threads/ThreadQueueControl.tsx @@ -5,6 +5,7 @@ import { queuedRunEditBlockedReason, } from "@t3tools/client-runtime/queued-run-edit"; import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { UNCONFIRMED_DELIVERY_HINT } from "@t3tools/client-runtime/state/thread-workflows"; import { resolveUserMessagePresentation } from "@t3tools/client-runtime/user-message"; import type { ChatAttachment, EnvironmentId, RunId, ThreadId } from "@t3tools/contracts"; import { Atom } from "effect/reactivity"; @@ -151,9 +152,9 @@ export function ThreadQueueSheet({ route }: StaticScreenProps) { if (busyRef.current) return; const index = queuedRuns.findIndex(({ run }) => run.id === runId); if (index < 0) return; + const entry = queuedRuns[index]!; if (action === "edit") { - if (editBlockedReason !== null) return; - const entry = queuedRuns[index]!; + if (editBlockedReason !== null || entry.editBlockedReason !== null) return; void Haptics.selectionAsync(); beginQueuedRunEdit( threadKey, @@ -177,6 +178,7 @@ export function ThreadQueueSheet({ route }: StaticScreenProps) { return; } if (action !== "steer" && action !== "remove") return; + if (action === "steer" && entry.automatic && workflow?.canSendNow !== true) return; busyRef.current = true; setBusyRunId(runId); void Haptics.selectionAsync(); @@ -227,20 +229,26 @@ export function ThreadQueueSheet({ route }: StaticScreenProps) { {editBlockedReason !== null && queuedRuns.length > 0 ? ( {editBlockedReason} ) : null} + {queuedRuns.some((entry) => entry.delivery === "unconfirmed") ? ( + {UNCONFIRMED_DELIVERY_HINT} + ) : null} {queuedRuns.length === 0 ? ( No messages waiting in this queue. ) : null} - {queuedRuns.map(({ run, text, attachments, context, message }, index) => { + {queuedRuns.map((entry, index) => { + const { run, text, attachments, context, message } = entry; + const rowEditBlockedReason = editBlockedReason ?? entry.editBlockedReason; const controls = resolveThreadQueueRowControls({ busy: busyRunId !== null, // A fresh message starts its own turn: it may skip compaction, but not steer into a turn. canPromoteToSteer: + !entry.automatic && (workflow?.canPromoteToSteer ?? false) && (run.freshContext !== true || workflow?.promoteSkipsCompaction === true), canSendNow: workflow?.canSendNow ?? false, - editBlocked: editBlockedReason !== null, + editBlocked: rowEditBlockedReason !== null, isEditing: editing?.runId === run.id, isGuestMessage: context?.guest !== undefined, text, @@ -292,7 +300,7 @@ export function ThreadQueueSheet({ route }: StaticScreenProps) { accessibilityHint={ controls.canEdit ? "Opens this message in the composer for editing" - : (editBlockedReason ?? undefined) + : (rowEditBlockedReason ?? undefined) } disabled={!controls.canEdit} onPress={() => void act(run.id, "edit")} @@ -321,6 +329,11 @@ export function ThreadQueueSheet({ route }: StaticScreenProps) { )} {title} + {entry.delivery !== "queued" ? ( + + {entry.deliveryLabel} + + ) : null} {controls.isEditing ? ( diff --git a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts index bbc3fbf06940d..8e83168c40607 100644 --- a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts +++ b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts @@ -33,6 +33,7 @@ import * as EffectWorker from "../orchestration-v2/EffectWorker.ts"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; @@ -694,3 +695,525 @@ it.effect("a refused descendant stop does not settle or dispose its parent task" it.effect("task_cancel stops a remote descendant without reacquiring its parent lock", () => cancelBackgroundOnlyTask("command", { remote: true }), ); + +const watch = Effect.fn("watch")(function* ( + predicate: (event: OrchestrationV2DomainEvent) => boolean, +) { + const sink = yield* EventSink.EventSinkV2; + const afterSequence = yield* sink.latestSequence(); + return yield* sink.stream({ afterSequence }).pipe( + Stream.map((stored) => stored.event), + Stream.filter(predicate), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); +}); + +const delegatedClaudeHarness = Effect.fn("delegatedClaudeHarness")(function* (name: string) { + const cwd = yield* checkpointWorkspace(name); + const turns: Array<{ + readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput; + readonly events: Queue.Queue; + }> = []; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId, + driver, + getCapabilities: () => Effect.succeed(ClaudeProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const events = yield* Queue.unbounded(); + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd, + model: modelSelection.model, + capabilities: ClaudeProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId }) => + Effect.succeed({ + id: ProviderThreadId.make(`provider-thread:${threadId}`), + driver, + providerInstanceId: instanceId, + providerSessionId: input.providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: `native:${threadId}`, strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + turns.push({ turn, events }); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: null, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + }), + steerTurn: () => Effect.die("unused"), + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.die("unused"), + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + const layerAdapters = ProviderAdapterRegistry.layerSingle(adapter); + const layerOrchestrator = ProviderReplayHarness.layerWithRegistry({ name }, layerAdapters, { + runEffectWorker: false, + runContinuationWorker: true, + }); + const layerOrchestration = Layer.merge( + layerOrchestrator, + ThreadManagementService.layer.pipe( + Layer.provide(layerOrchestrator), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory))), + ), + ); + const layerTest = layerOrchestratorService.pipe( + Layer.provideMerge(layerOrchestration), + Layer.provide( + Layer.mergeAll( + layerAdapters, + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({}), + ServerSettings.layerTest(), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ); + + const startDelegatedChild = Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const parentThreadId = ThreadId.make(`thread:${name}-parent`); + const turnRunning = () => + watch( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-parent"), + threadId: parentThreadId, + projectId: ProjectId.make("project:task-cancel"), + title: "Parent", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const parentRunning = yield* turnRunning(); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("start-parent"), + threadId: parentThreadId, + messageId: MessageId.make("message:start-parent"), + text: "Delegate a review", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + yield* Fiber.join(parentRunning); + + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!; + const childRunning = yield* turnRunning(); + yield* orchestrator.dispatch({ + type: "delegated_task.request", + commandId: CommandId.make("delegate"), + parentThreadId, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId!, + task: "Review the change", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + completionWake: "always", + createdBy: "agent", + creationSource: "mcp", + }); + yield* worker.drain(); + yield* Fiber.join(childRunning); + const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!; + const child = turns.find((entry) => entry.turn.threadId === task.childThreadId)!; + const childTurn = (yield* orchestrator.getThreadProjection(task.childThreadId!)) + .providerTurns[0]!; + const scope: McpInvocationScope = { + environmentId: EnvironmentId.make(`environment:${name}`), + requestNamespace: `provider-session:${name}`, + thread: { + threadId: parentThreadId, + providerSessionId: `provider-session:${name}`, + providerInstanceId: instanceId, + }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }; + return { parentThreadId, task, child, childTurn, scope }; + }); + return { turns, layerTest, startDelegatedChild }; +}); + +// A delegated Claude child ends its turn while a background native subagent it +// started keeps running. + +it.effect.each([ + "initial", + "receipt", + "withdraw", + "unconfirmed", + "other-owner", + "other-parent", + "no-task", +] as const)( + "a parent's Claude follow-up shares completion only after a matching boundary receipt: %s", + (settlement) => + Effect.scoped( + Effect.gen(function* () { + const { turns, layerTest, startDelegatedChild } = yield* delegatedClaudeHarness( + `boundary-child-${settlement}`, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const sink = yield* EventSink.EventSinkV2; + const { parentThreadId, task, child, scope } = yield* startDelegatedChild; + let active = child; + if (settlement !== "initial") { + yield* finishChildTurn(child, "initial findings"); + yield* orchestrator.recoverDelegatedTasks; + yield* service.taskStatus(scope, task.id); + const running = yield* watch( + (event) => + event.type === "provider-turn.updated" && + event.payload.status === "running" && + event.threadId === task.childThreadId, + ); + yield* service.sendToThread(scope, { + threadId: task.childThreadId!, + message: "Review again", + mode: "auto", + }); + yield* worker.drain(); + yield* Fiber.join(running); + active = turns.at(-1)!; + } + const sent = yield* service.sendToThread(scope, { + threadId: task.childThreadId!, + message: "Also check the docs", + mode: "steer", + }); + assert.equal(sent.delivery, "queued"); + const before = yield* orchestrator.getThreadProjection(task.childThreadId!); + const queued = before.runs.find((run) => run.id === sent.runId)!; + const run = before.runs.find((run) => run.id === active.turn.runId)!; + const providerTurn = before.providerTurns.find( + (turn) => turn.runAttemptId === active.turn.attemptId, + )!; + if (settlement !== "initial") assert.isDefined(queued.delegatedTaskId); + if ( + settlement === "other-owner" || + settlement === "other-parent" || + settlement === "no-task" + ) { + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + const followup = parent.subagents.find((row) => row.id === queued.delegatedTaskId)!; + const now = yield* DateTime.now; + yield* sink.write({ + commandId: CommandId.make("command:runtime-reconcile:owner"), + events: [ + { + id: EventId.make("event:owner"), + threadId: parentThreadId, + occurredAt: now, + type: "subagent.updated", + payload: { + ...followup, + ...(settlement === "other-owner" + ? { runId: RunId.make("other-owner") } + : settlement === "other-parent" + ? { threadId: ThreadId.make("other-parent") } + : {}), + }, + }, + ...(settlement !== "no-task" + ? [] + : [ + { + id: EventId.make("event:no-task"), + threadId: task.childThreadId!, + occurredAt: now, + type: "run.updated" as const, + payload: { ...run, delegatedTaskId: undefined }, + }, + ]), + ], + }); + } + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const offers = yield* requests.claimAgentMessages!({ + threadId: task.childThreadId!, + runId: run.id, + providerTurnId: providerTurn.id, + }); + const boundaryId = offers[0]?.id ?? "refused-followup"; + let promotion = 0; + const promote = (step: "offer" | "receipt" | "withdraw") => + orchestrator.dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(`boundary:${step}:${promotion++}`), + threadId: task.childThreadId!, + queuedRunId: queued.id, + targetRunId: run.id, + boundaryOffer: + step === "offer" + ? { id: boundaryId, step, providerTurnId: providerTurn.id } + : { id: boundaryId, step }, + }); + if ( + settlement === "other-owner" || + settlement === "other-parent" || + settlement === "no-task" + ) { + assert.lengthOf(offers, 0); + yield* Effect.flip(promote("offer")); + assert.equal( + (yield* orchestrator.getThreadProjection(task.childThreadId!)).runs.find( + (row) => row.id === queued.id, + )?.status, + "queued", + ); + return; + } + assert.lengthOf(offers, 1); + if (settlement === "withdraw") yield* promote("withdraw"); + const joined = settlement === "receipt" || settlement === "initial"; + if (joined) { + yield* promote("receipt"); + yield* promote("receipt"); + const received = yield* orchestrator.getThreadProjection(task.childThreadId!); + assert.equal( + received.messages.find((message) => message.id === sent.messageId)?.runId, + run.id, + ); + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + if (queued.delegatedTaskId !== undefined) { + const delivered = parent.subagents.find((row) => row.id === queued.delegatedTaskId)!; + const tracked = yield* service.taskStatus(scope, delivered.id); + assert.equal(tracked.workState, "result_available"); + assert.include(tracked.summary!, "Delivered into running run"); + assert.equal(delivered.status, "completed"); + assert.equal(delivered.completionDelivery?.state, "disposed"); + assert.include(delivered.result!, "Delivered into running run"); + assert.equal( + parent.nodes.find((node) => node.id === delivered.id)?.status, + "completed", + ); + assert.equal( + parent.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === delivered.id, + )?.status, + "completed", + ); + } + yield* finishChildTurn(active, "findings including docs"); + yield* orchestrator.recoverDelegatedTasks; + yield* orchestrator.recoverDelegatedTasks; + const done = yield* orchestrator.getThreadProjection(parentThreadId); + assert.lengthOf( + done.contextTransfers.filter( + (transfer) => + transfer.type === "subagent_result" && transfer.sourcePoint.runId === run.id, + ), + 1, + ); + const waited = yield* service.waitForThread(scope, { + threadId: task.childThreadId!, + runId: sent.runId, + timeoutMs: 0, + }); + assert.equal(waited.runId, run.id); + assert.equal(waited.status, "completed"); + const activeTaskId = run.delegatedTaskId ?? task.id; + assert.equal( + done.subagents.find((row) => row.id === activeTaskId)?.result, + "findings including docs", + ); + assert.lengthOf( + done.subagents.filter((row) => row.result === "findings including docs"), + 1, + ); + assert.isFalse( + done.subagents.some((row) => row.status === "running" || row.status === "pending"), + ); + assert.lengthOf( + done.runs + .flatMap((row) => row.delegatedCompletion?.delivery?.taskIds ?? []) + .filter((id) => id === activeTaskId), + 1, + ); + assert.isFalse( + done.runs.some( + (row) => + row.delegatedCompletion?.delivery?.taskIds.includes(queued.delegatedTaskId!) === + true, + ), + ); + assert.lengthOf( + turns.filter((entry) => entry.turn.threadId === task.childThreadId), + settlement === "initial" ? 1 : 2, + ); + } else { + const stillQueued = (yield* orchestrator.getThreadProjection( + task.childThreadId!, + )).runs.find((row) => row.id === queued.id)!; + assert.equal(stillQueued.delegatedTaskId, queued.delegatedTaskId); + assert.equal(stillQueued.boundaryOffer !== undefined, settlement === "unconfirmed"); + const updateRunning = yield* watch( + (event) => + event.type === "provider-turn.updated" && + event.payload.status === "running" && + event.payload.runAttemptId !== active.turn.attemptId && + event.threadId === task.childThreadId, + ); + yield* finishChildTurn(active, "review findings"); + if (settlement === "unconfirmed") { + assert.equal( + (yield* orchestrator.getThreadProjection(task.childThreadId!)).runs.find( + (row) => row.id === queued.id, + )?.status, + "queued", + ); + yield* orchestrator.dispatch({ + type: "queued-run.start", + commandId: CommandId.make("user-retry-unconfirmed"), + threadId: task.childThreadId!, + runId: queued.id, + }); + } else yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + yield* Fiber.join(updateRunning); + yield* finishChildTurn(turns.at(-1)!, "separate follow-up result"); + yield* orchestrator.recoverDelegatedTasks; + assert.equal( + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (row) => row.id === queued.delegatedTaskId, + )?.result, + "separate follow-up result", + ); + } + }).pipe(Effect.provide(layerTest)); + }), + ), +); + +const finishChildTurn = Effect.fn("finishChildTurn")(function* ( + entry: { + readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput; + readonly events: Queue.Queue; + }, + text: string, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const { turn, events } = entry; + const providerTurn = (yield* orchestrator.getThreadProjection(turn.threadId)).providerTurns.find( + (candidate) => candidate.runAttemptId === turn.attemptId, + )!; + const now = yield* DateTime.now; + const ended = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === turn.runId && + (event.payload.status === "waiting" || event.payload.status === "completed"), + ); + for (const event of [ + { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make(`turn-item:${turn.attemptId}`), + threadId: turn.threadId, + runId: turn.runId, + nodeId: turn.rootNodeId, + providerThreadId: providerTurn.providerThreadId, + providerTurnId: providerTurn.id, + nativeItemRef: null, + parentItemId: null, + ordinal: 0, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "assistant_message", + messageId: MessageId.make(`message:${turn.attemptId}`), + text, + streaming: false, + }, + }, + { + type: "provider_turn.updated", + driver, + providerTurn: { ...providerTurn, status: "completed", completedAt: now }, + }, + { + type: "turn.terminal", + driver, + providerThreadId: providerTurn.providerThreadId, + providerTurnId: providerTurn.id, + runOrdinal: turn.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }, + ] satisfies ReadonlyArray) { + yield* Queue.offer(events, event); + } + yield* Fiber.join(ended); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); +}); + +// Claude steering cancels running tools, so a parent's update to its running +// delegated child queues. The original task must report the update's turn. diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 38d8fb4e0ada5..518d36dd86cb8 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -277,7 +277,7 @@ const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any thread. The target cannot run with broader modes than yours. `mode:'auto'` starts an idle thread, steers a running turn, or queues. `queue`: new turn. `steer`: mid-turn update. `restart`: interrupt and restart. A send to a finished child thread starts a fresh task and returns its `taskId`. In your reply, name it as its `link` verbatim.", + "Send within your modes. `auto`: start idle, steer running, or queue. Claude auto/steer follows its tool batch, or waits for the next turn after its last batch. `queue`: next turn. `steer`: mid-turn. `restart`: interrupt/restart. Sending to a finished child starts a new task and returns `taskId`. Use its `link` verbatim in your reply.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/mcp/toolkits/thread/handlers.ts b/apps/server/src/mcp/toolkits/thread/handlers.ts index 32e45b5d0a1c0..b7c714c3e8335 100644 --- a/apps/server/src/mcp/toolkits/thread/handlers.ts +++ b/apps/server/src/mcp/toolkits/thread/handlers.ts @@ -320,6 +320,7 @@ export const layer = McpToolAccess.toLayer(ThreadToolkit, { type: "queued-message.promote-to-steer", queuedRunId: runId, targetRunId, + requestedBy: "agent", })); } default: diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index ec2873f387fe8..e10bd4dda1dde 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -110,6 +110,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderInstallationGate from "../../provider/ProviderInstallationGate.ts"; import { ClaudeOrchestratorReplayHarness } from "./ClaudeAdapterV2.testkit.ts"; import { provideDeterministicTestRuntime } from "../testkit/DeterministicRuntime.ts"; +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import { runOrchestratorV2Scenario } from "../testkit/OrchestratorScenario.ts"; import { layerProviderReplay } from "../testkit/ProviderReplayHarness.ts"; import { @@ -3382,6 +3383,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarnessWithOptions = (options?: { + readonly fileSystem?: FileSystem.FileSystem; readonly close?: (sdkMessages: Queue.Queue) => Effect.Effect; readonly interrupt?: Effect.Effect; readonly setPermissionMode?: ClaudeAdapterV2.ClaudeAgentSdkQuerySession["setPermissionMode"]; @@ -3406,11 +3408,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly freshQueueOnReopen?: boolean; readonly forkSession?: ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerShape["forkSession"]; readonly allocateSessionId?: Effect.Effect; + readonly claimAgentMessages?: ProviderContinuationRequests.ClaimAgentMessages; + readonly withdrawAgentMessages?: ProviderContinuationRequests.AcknowledgeAgentMessages; + readonly onHandoff?: ( + handoff: NonNullable[2]>, + ) => void; readonly offer?: (message: SDKUserMessage) => Effect.Effect; readonly promptSuggestions?: boolean; }) => Effect.gen(function* () { - const fileSystem = yield* FileSystem.FileSystem; + const fileSystem = options?.fileSystem ?? (yield* FileSystem.FileSystem); const idAllocator = yield* IdAllocator.IdAllocatorV2; const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-claude-v2-wake-", @@ -3430,6 +3437,25 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const changes = yield* makeChangeSignal; const offeredMessages: Array = []; + const offerPrompt: ClaudeAdapterV2.ClaudeAgentSdkQuerySession["offer"] = ( + message, + onQueued, + handoff, + ) => + Effect.sync(() => { + offeredMessages.push(message); + onQueued?.(); + if (handoff !== undefined) { + if (options?.onHandoff !== undefined) options.onHandoff(handoff); + else if (handoff.isCurrent()) { + handoff.onPulled(); + handoff.onConsumed(); + } else handoff.onDiscarded(); + } + }).pipe( + Effect.andThen(changes.notify), + Effect.andThen(options?.offer?.(message) ?? Effect.void), + ); const permissionModeChanges: Array = []; const liveSelectionChanges: Array = []; const continuationRequests: Array = @@ -3472,6 +3498,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { random: yield* SecureRandom.SecureRandom, idAllocator, continuationRequests: { + ...(options?.claimAgentMessages === undefined + ? {} + : { claimAgentMessages: options.claimAgentMessages }), + ...(options?.withdrawAgentMessages === undefined + ? {} + : { withdrawAgentMessages: options.withdrawAgentMessages }), offer: (request) => Effect.sync(() => { continuationRequests.push(request); @@ -3511,10 +3543,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { processQueues.push(processMessages); return { messages, - offer: (message: SDKUserMessage) => - Effect.sync(() => { - offeredMessages.push(message); - }).pipe(Effect.andThen(changes.notify)), + offer: offerPrompt, setModel: () => Effect.void, applyFlagSettings: () => Effect.void, setPermissionMode: (mode) => @@ -3536,13 +3565,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { processQueues.push(sdkMessages); return { messages, - offer: (message) => - Effect.sync(() => { - offeredMessages.push(message); - }).pipe( - Effect.andThen(changes.notify), - Effect.andThen(options?.offer?.(message) ?? Effect.void), - ), + offer: offerPrompt, setModel: (model) => Effect.sync(() => { liveSelectionChanges.push({ model }); @@ -3664,6 +3687,544 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + it.effect( + "hands off a mixed boundary batch before returning and reserves before early receipts", + () => + Effect.gen(function* () { + const image = ChatImageAttachment.make({ + type: "image", + id: ChatAttachmentId.make("boundary-image"), + name: "diagram.png", + mimeType: "image/png", + sizeBytes: 4, + }); + const offers: ProviderContinuationRequests.AgentMessageOffer[] = Array.from( + { length: 40 }, + (_, index) => ({ + id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + queuedRunId: RunId.make(`queued-boundary-${index}`), + messageId: MessageId.make(`boundary-message-${index}`), + text: `UPDATE_${index}`, + attachments: index === 2 ? [image] : [], + createdBy: index % 2 === 0 ? "user" : "agent", + creationSource: index % 2 === 0 ? "web" : "server", + }), + ); + const handoffs: Array< + NonNullable[2]> + > = []; + const claims: Parameters[0][] = []; + const withdrawals: Parameters[0][] = + []; + const harness = yield* makeWakeHarnessWithOptions({ + claimAgentMessages: (claim) => + Effect.sync(() => { + claims.push(claim); + return offers; + }), + withdrawAgentMessages: (withdrawal) => + Effect.sync(() => { + withdrawals.push(withdrawal); + }), + onHandoff: (handoff) => { + handoffs.push(handoff); + }, + }); + yield* (yield* FileSystem.FileSystem).writeFile( + (yield* Path.Path).join(harness.attachmentsDir, attachmentRelativePath(image)!), + new Uint8Array([137, 80, 78, 71]), + ); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("mixed-boundary-attempt"), + text: "Work with tools.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + capabilities: ["interrupt_cancel_queued_v1"], + session_id: WAKE_NATIVE_SESSION, + }), + ); + const options = harness.getOpenedOptions(); + const hook = options?.hooks?.PostToolBatch?.[0]?.hooks[0]; + const store = options?.sessionStore; + assert.isDefined(hook); + assert.isDefined(store); + if (hook === undefined || store === undefined) return; + const signal = yield* Effect.abortSignal; + const fileSystem = yield* FileSystem.FileSystem; + const transcriptPath = (yield* Path.Path).join(harness.attachmentsDir, "boundary.jsonl"); + yield* fileSystem.writeFileString(transcriptPath, "{}\n"); + const hookInput = { + hook_event_name: "PostToolBatch" as const, + session_id: WAKE_NATIVE_SESSION, + transcript_path: transcriptPath, + cwd: "/repo", + tool_calls: [], + }; + yield* Effect.promise(() => hook(hookInput, undefined, { signal })); + assert.lengthOf(claims, 0); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "status", + status: null, + user_message_uuid: harness.promptUuid(0), + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Effect.promise(() => + hook({ ...hookInput, agent_id: "child" }, undefined, { signal }), + ); + assert.lengthOf(claims, 0); + let returned = false; + const pendingHook = yield* Effect.promise(() => + hook(hookInput, undefined, { signal }).then((result) => { + returned = true; + return result; + }), + ).pipe(Effect.forkChild); + yield* Effect.raceFirst( + harness.until(() => handoffs.length === 40), + Fiber.join(pendingHook).pipe( + Effect.andThen(Effect.die("Hook returned before input handoff")), + ), + ); + assert.isFalse(returned); + assert.deepEqual( + harness.offeredMessages.slice(1).map((message) => message.uuid), + offers.map((offer) => offer.id), + ); + assert.isTrue( + harness.offeredMessages.slice(1).every((message) => message.priority === "next"), + ); + assert.deepEqual( + harness.offeredMessages.slice(1, 3).map((message) => message.message.content), + ["UPDATE_0", "From another agent:\nUPDATE_1"], + ); + const imageContent = harness.offeredMessages[3]!.message.content; + assert.isArray(imageContent); + if (Array.isArray(imageContent)) assert.equal(imageContent[0]?.type, "image"); + for (const handoff of handoffs.slice(0, -1)) { + handoff.onPulled(); + handoff.onConsumed(); + } + assert.isFalse(returned); + handoffs.at(-1)!.onPulled(); + handoffs.at(-1)!.onConsumed(); + assert.deepEqual(yield* Fiber.join(pendingHook), {}); + assert.lengthOf(claims, 1); + assert.deepEqual( + withdrawals.flatMap((withdrawal) => withdrawal.offers), + [], + ); + const receipt = (id: string) => ({ + type: "attachment", + isSidechain: false, + sessionId: WAKE_NATIVE_SESSION, + attachment: { type: "queued_command", source_uuid: id }, + }); + const key = { projectKey: "-repo", sessionId: WAKE_NATIVE_SESSION }; + yield* Effect.promise(() => + store.append({ ...key, sessionId: "foreign" }, [receipt(offers[0]!.id)]), + ); + yield* Effect.promise(() => + store.append({ ...key, subpath: "subagents/child" }, [receipt(offers[0]!.id)]), + ); + yield* Effect.promise(() => store.append(key, [receipt(offers[0]!.id)])); + assert.lengthOf( + harness.events.filter((event) => event.type === "boundary_messages.updated"), + 0, + ); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "command_lifecycle", + command_uuid: offers[0]!.id, + state: "started", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* harness.until( + () => + harness.events.filter((event) => event.type === "boundary_messages.updated").length >= + 2, + ); + yield* fileSystem.writeFileString( + transcriptPath, + `{}\n${offers.map((offer) => JSON.stringify(receipt(offer.id))).join("\n")}\n`, + ); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "assistant", + parent_tool_use_id: null, + session_id: WAKE_NATIVE_SESSION, + uuid: "boundary-reply", + message: { + role: "assistant", + content: [{ type: "text", text: "Working on the updates." }], + }, + }), + ); + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "boundary_messages.updated" && + event.step === "receipt" && + event.offers.some((offer) => offer.id === offers[1]!.id), + ), + ); + assert.lengthOf( + harness.events.filter((event) => event.type === "boundary_messages.updated"), + 3, + ); + yield* Effect.promise(() => + store.append( + key, + offers.map((offer) => receipt(offer.id)), + ), + ); + assert.lengthOf(harness.terminalEvents(), 0); + const deliveryEvents = harness.events.filter( + (event) => event.type === "boundary_messages.updated", + ); + assert.deepEqual( + deliveryEvents.map((event) => event.step), + ["reserve", "receipt", "receipt"], + ); + assert.deepEqual( + deliveryEvents[0]!.offers.map((offer) => offer.messageId), + offers.map((offer) => offer.messageId), + ); + assert.deepEqual( + deliveryEvents.slice(1).flatMap((event) => event.offers.map((offer) => offer.id)), + offers.map((offer) => offer.id), + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect.each(["rewritten", "unconfirmed", "unreadable", "separate-files"] as const)( + "reconciles boundary transcript receipts without retaining ended offers: %s", + (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boundary-tail-" }); + const projects = path.join(home, "projects"); + const paths = ["first", "second"].map((project) => + path.join(projects, project, `${WAKE_NATIVE_SESSION}.jsonl`), + ); + const padding = "{}\n".repeat(256); + for (const transcript of paths) { + yield* fs.makeDirectory(path.dirname(transcript), { recursive: true }); + yield* fs.writeFileString(transcript, padding); + } + const unreadable = path.join(home, "not-a-directory"); + yield* fs.writeFileString(unreadable, ""); + const reads: Array<{ readonly path: string; readonly offset: number }> = []; + let historyReads = 0; + let failHistory = false; + const fileSystem: FileSystem.FileSystem = { + ...fs, + stream: (file, options) => { + if (paths.includes(file)) + reads.push({ + path: file, + offset: options?.offset === undefined ? 0 : Number(options.offset), + }); + return fs.stream(file, options); + }, + readDirectory: (directory, options) => { + if (directory === projects) historyReads++; + return fs.readDirectory( + directory === projects && failHistory ? unreadable : directory, + options, + ); + }, + }; + const offers: ProviderContinuationRequests.AgentMessageOffer[] = [0, 1].map((index) => ({ + id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + queuedRunId: RunId.make(`queued-tail-${index}`), + messageId: MessageId.make(`message-tail-${index}`), + text: `Follow-up ${index}`, + attachments: [], + createdBy: "user", + creationSource: "web", + })); + let claimed = 0; + const withdrawals: string[] = []; + const harness = yield* makeWakeHarnessWithOptions({ + fileSystem, + settings: { ...DEFAULT_CLAUDE_SETTINGS, homePath: home }, + claimAgentMessages: () => Effect.sync(() => [offers[claimed++]!]), + withdrawAgentMessages: ({ offers }) => + Effect.sync(() => { + withdrawals.push(...offers.map((offer) => offer.id)); + }), + }); + const turn = (attempt: string) => + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: DateTime.makeUnsafe("2026-11-12T00:00:00Z"), + attemptId: RunAttemptId.make(attempt), + text: "Work with tools.", + attachments: [], + }); + yield* harness.runtime.startTurn(turn("receipt-tail-first")); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + capabilities: ["interrupt_cancel_queued_v1"], + user_message_uuid: harness.promptUuid(0), + session_id: WAKE_NATIVE_SESSION, + }), + ); + const queryOptions = harness.getOpenedOptions(); + const hook = queryOptions?.hooks?.PostToolBatch?.[0]?.hooks[0]; + assert.isDefined(hook); + if (hook === undefined) return; + const signal = yield* Effect.abortSignal; + const batchCount = scenario === "separate-files" ? 2 : 1; + for (const transcript of paths.slice(0, batchCount)) { + yield* Effect.promise(() => + hook( + { + hook_event_name: "PostToolBatch", + session_id: WAKE_NATIVE_SESSION, + transcript_path: transcript, + cwd: "/repo", + tool_calls: [], + }, + undefined, + { signal }, + ), + ); + } + const receipt = (index: number) => + `${JSON.stringify({ + type: "attachment", + sessionId: WAKE_NATIVE_SESSION, + isSidechain: false, + attachment: { type: "queued_command", source_uuid: offers[index]!.id }, + })}\n`; + const received = () => + harness.events.flatMap((event) => + event.type === "boundary_messages.updated" && event.step === "receipt" + ? event.offers.map((offer) => offer.id) + : [], + ); + const root = Effect.fnUntraced(function* (text: string) { + yield* harness.offerAndWait(makeAssistantTextFrame({ uuid: text, text })); + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.text.includes(text), + ), + ); + }); + if (scenario === "rewritten") yield* fs.writeFileString(paths[0]!, receipt(0) + padding); + for (const offer of offers.slice(0, batchCount)) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "command_lifecycle", + command_uuid: offer.id, + state: "started", + session_id: WAKE_NATIVE_SESSION, + }), + ); + } + if (scenario === "separate-files") { + yield* harness.until( + () => + harness.events.filter( + (event) => event.type === "boundary_messages.updated" && event.step === "reserve", + ).length === batchCount, + ); + for (let index = 0; index < batchCount; index++) + yield* fs.writeFileString(paths[index]!, receipt(index), { flag: "a" }); + } + yield* root("mid-turn receipt check"); + if (scenario === "separate-files") { + assert.deepEqual( + received(), + offers.map((offer) => offer.id), + ); + assert.isTrue( + paths.every((transcript) => reads.some((read) => read.path === transcript)), + ); + assert.equal(historyReads, 0); + return; + } + assert.deepEqual(received(), []); + assert.equal(historyReads, 0); + const previousOffset = reads.at(-1)?.offset; + yield* fs.writeFileString(paths[0]!, "{}\n", { flag: "a" }); + yield* root("advancing the receipt cursor"); + yield* root("checking the advanced receipt cursor"); + const advancedOffset = reads.at(-1)?.offset; + failHistory = scenario === "unreadable"; + yield* harness.offerAndWait( + makeResultFrame({ uuid: "receipt-tail-result", result: "Finished." }), + ); + yield* harness.awaitTerminal(1); + failHistory = false; + if (scenario === "rewritten") { + assert.deepEqual(received(), [offers[0]!.id]); + assert.equal(historyReads, 1); + return; + } + assert.deepEqual(received(), []); + const readsAtEnd = reads.length; + const historyReadsAtEnd = historyReads; + yield* harness.offerAndWait( + makeAssistantTextFrame({ uuid: "late-root", text: "Late root output" }), + ); + yield* harness.runtime.startTurn(turn("receipt-tail-later")); + assert.strictEqual(harness.getOpenedOptions(), queryOptions); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "status", + status: null, + user_message_uuid: harness.promptUuid(2), + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* root("later turn receipt check"); + yield* harness.offerAndWait( + makeResultFrame({ uuid: "receipt-tail-later-result", result: "Later work." }), + ); + yield* harness.awaitTerminal(2); + assert.equal(reads.length, readsAtEnd); + assert.equal(historyReads, historyReadsAtEnd); + assert.equal(historyReads, 1); + assert.isAbove(Number(advancedOffset), Number(previousOffset)); + assert.deepEqual(received(), []); + assert.deepEqual( + harness.offeredMessages + .filter((message) => message.uuid === offers[0]!.id) + .map((message) => message.priority), + ["next"], + ); + assert.deepEqual(withdrawals, []); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect("withdraws a never-pulled input discarded after its hook aborts", () => + Effect.gen(function* () { + const offers: ProviderContinuationRequests.AgentMessageOffer[] = [0, 1].map((index) => ({ + id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + queuedRunId: RunId.make(`abort-queued-${index}`), + messageId: MessageId.make(`abort-message-${index}`), + text: `UPDATE_${index}`, + attachments: [], + createdBy: "user", + creationSource: "web", + })); + const handoffs: Array< + NonNullable[2]> + > = []; + const withdrawals = + yield* Queue.unbounded< + Parameters[0] + >(); + const harness = yield* makeWakeHarnessWithOptions({ + claimAgentMessages: () => Effect.succeed(offers), + withdrawAgentMessages: (withdrawal) => + Queue.offer(withdrawals, withdrawal).pipe(Effect.asVoid), + onHandoff: (handoff) => { + handoffs.push(handoff); + }, + }); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("abort-boundary-attempt"), + text: "Work.", + attachments: [], + }), + ); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + capabilities: ["interrupt_cancel_queued_v1"], + user_message_uuid: harness.promptUuid(0), + session_id: WAKE_NATIVE_SESSION, + }), + ); + const hook = harness.getOpenedOptions()?.hooks?.PostToolBatch?.[0]?.hooks[0]; + assert.isDefined(hook); + if (hook === undefined) return; + const finished = Promise.withResolvers(); + const pending = yield* Effect.promise((signal) => + hook( + { + hook_event_name: "PostToolBatch", + session_id: WAKE_NATIVE_SESSION, + transcript_path: "/repo/session.jsonl", + cwd: "/repo", + tool_calls: [], + }, + undefined, + { signal }, + ).finally(() => { + finished.resolve(); + }), + ).pipe(Effect.forkChild); + yield* harness.until(() => handoffs.length === 2); + handoffs[0]!.onPulled(); + handoffs[0]!.onConsumed(); + yield* Fiber.interrupt(pending); + yield* Effect.promise(() => finished.promise); + assert.isFalse(handoffs[1]!.isCurrent()); + handoffs[1]!.onDiscarded(); + const withdrawal = yield* Queue.take(withdrawals); + assert.deepEqual( + withdrawal.offers.map((offer) => offer.id), + [offers[1]!.id], + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect.each(["release-after", "cancel-after", "release-before", "resume-pending"] as const)( "keeps attachment reservations across compaction: %s boundary", (outcome) => @@ -10307,7 +10868,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { driver: ProviderDriverKind.make("claudeAgent"), modelSelection: CLAUDE_MODEL_SELECTION, }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime); - const scenario = { name: transcript.scenario, transcript, ...materialized }; + const scenario = { + name: transcript.scenario, + transcript, + ...materialized, + runtimePolicyOverride: { cwd: yield* checkpointWorkspace(transcript.scenario) }, + }; yield* Effect.gen(function* () { const result = yield* runOrchestratorV2Scenario(scenario); assertBaseProjection({ result, transcript, runCount: 1, runStatuses: ["completed"] }); @@ -10328,7 +10894,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { provideDeterministicTestRuntime, Effect.scoped, ); - }), + }).pipe(Effect.scoped), ); it.effect( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index afa0925fbb4c2..13b14c6481c61 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -5,6 +5,10 @@ import { toolOutputImages, } from "@t3tools/shared/toolOutput"; import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts"; +import { + readClaudeBoundaryReceipts, + readClaudeBoundaryReceiptTail, +} from "./ClaudeSavedMessageReceipts.ts"; import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts"; import { dynamicToolTitle, @@ -488,6 +492,7 @@ export const ClaudeProviderCapabilitiesV2 = { supportsInterrupt: true, supportsActiveSteering: true, activeSteeringInterruptsTools: true, + deliversUpdatesAtToolBoundary: true, supportsSteeringByInterruptRestart: false, supportsQueuedMessages: true, terminalStatusQuality: "strong", @@ -618,6 +623,12 @@ export interface ClaudeAgentSdkQuerySession { readonly offer: ( message: SDKUserMessage, onQueued?: () => void, + handoff?: { + readonly isCurrent: () => boolean; + readonly onPulled: () => void; + readonly onConsumed: () => void; + readonly onDiscarded: () => void; + }, ) => Effect.Effect; readonly setModel: (model: string) => Effect.Effect; readonly applyFlagSettings: ( @@ -789,6 +800,17 @@ function queryRunnerError(cause: unknown, method: string): ClaudeAgentSdkQueryRu : new ClaudeAgentSdkQueryRunnerError({ cause, method }); } +// A plain interrupt leaves uuid-stamped queued input queued, and the CLI then runs it as a new +// turn. SDK 0.3.316 sends cancel_queued for interrupt({ cancelQueued: true }) but its declaration +// omits the parameter; CLIs advertise support as interrupt_cancel_queued_v1. +function interruptClaudeQuery(queryRuntime: ClaudeQuery, cancelQueued: boolean) { + if (!cancelQueued) return queryRuntime.interrupt(); + const interrupt: (options: { + readonly cancelQueued: true; + }) => ReturnType = queryRuntime.interrupt.bind(queryRuntime); + return interrupt({ cancelQueued: true }); +} + const CLAUDE_PROCESS_EXIT_ERROR_CLASSES: ReadonlySet = new Set([ "process_killed_by_signal", "process_exited_nonzero", @@ -1062,6 +1084,7 @@ export type ClaudeAgentSdkProtocolLogEvent = readonly stage: "decoded"; readonly payload: { readonly type: "query.interrupt"; + readonly cancelQueued: boolean; }; } | { @@ -1308,13 +1331,49 @@ export const layerQueryRunner: Layer.Layer< const logProtocolEvent = (event: ClaudeAgentSdkProtocolLogEvent) => protocolLogger === undefined ? Effect.void : protocolLogger(event); // A failed open shuts down its queue before retrying, so the old CLI cannot take the prompt. - const promptQueue = yield* Queue.unbounded(); - const prompt = Stream.fromQueue(promptQueue).pipe( + type QueuedPrompt = { + readonly message: SDKUserMessage; + readonly handoff?: Parameters[2]; + }; + const promptQueue = yield* Queue.unbounded(); + const promptLock = yield* Semaphore.make(1); + let promptFenced = false; + let inFlightPrompt: Promise | undefined; + const queuedPrompts = Stream.fromPull( + Effect.succeed(Queue.take(promptQueue).pipe(Effect.map((queued) => [queued] as const))), + ).pipe( Stream.catchCause((cause) => Cause.hasInterruptsOnly(cause) ? Stream.empty : Stream.failCause(cause), ), Stream.toAsyncIterable, ); + const prompt: AsyncIterable = { + async *[Symbol.asyncIterator]() { + for await (const queued of queuedPrompts) { + if (promptFenced || queued.handoff?.isCurrent() === false) { + queued.handoff?.onDiscarded(); + continue; + } + const consumed = Promise.withResolvers(); + inFlightPrompt = consumed.promise; + queued.handoff?.onPulled(); + yield queued.message; + // SDK 0.3.316 writes the yielded input before requesting the next one. + queued.handoff?.onConsumed(); + consumed.resolve(); + inFlightPrompt = undefined; + } + }, + }; + const fencePrompts = promptLock.withPermits(1)( + Effect.gen(function* () { + if (promptFenced) return; + promptFenced = true; + for (const queued of yield* Queue.clear(promptQueue)) { + queued.handoff?.onDiscarded(); + } + }), + ); // Send MCP credentials over stdin to keep them out of SDK argv and inherited command environments. const { mcpServers, ...sdkOptions } = input.options; // Jobs the agent starts inherit the thread, so port discovery can attribute their servers. @@ -1355,6 +1414,7 @@ export const layerQueryRunner: Layer.Layer< let stderrTail = ""; let stderrClosed: Promise = Promise.resolve(); let answered = false; + let cancelsQueuedOnInterrupt = false; const appendStderr = (text: string) => { if (answered) return; stderrTail = (stderrTail + text).slice(-CLAUDE_STDERR_TAIL_LENGTH); @@ -1537,9 +1597,13 @@ export const layerQueryRunner: Layer.Layer< }), ), ), - Stream.tap(() => + Stream.tap((message) => Effect.sync(() => { answered = true; + if (message.type === "system" && message.subtype === "init") { + cancelsQueuedOnInterrupt = + message.capabilities?.includes("interrupt_cancel_queued_v1") === true; + } }), ), Stream.tap((message) => @@ -1550,24 +1614,42 @@ export const layerQueryRunner: Layer.Layer< }), ), ), - offer: (message, onQueued) => - Effect.uninterruptibleMask((restore) => - restore(Queue.offer(promptQueue, message)).pipe( - Effect.tap(() => Effect.sync(() => onQueued?.())), - ), - ).pipe( - Effect.asVoid, - Effect.tap(() => - logProtocolEvent({ - direction: "outgoing", - stage: "decoded", - payload: { - type: "prompt.offer", - message, - }, - }), + offer: (message, onQueued, handoff) => + promptLock + .withPermits(1)( + Effect.uninterruptible( + Effect.gen(function* () { + if (promptFenced || handoff?.isCurrent() === false) { + handoff?.onDiscarded(); + if (handoff === undefined) { + return yield* queryRunnerError( + new Error("Claude input is closed."), + "offer", + ); + } + return; + } + yield* Queue.offer(promptQueue, { + message, + ...(handoff === undefined ? {} : { handoff }), + }); + onQueued?.(); + }), + ), + ) + .pipe( + Effect.asVoid, + Effect.tap(() => + logProtocolEvent({ + direction: "outgoing", + stage: "decoded", + payload: { + type: "prompt.offer", + message, + }, + }), + ), ), - ), setModel: (model) => Effect.tryPromise({ try: () => queryRuntime.setModel(model), @@ -1676,22 +1758,30 @@ export const layerQueryRunner: Layer.Layer< Effect.map(({ state }) => state.rules), Effect.mapError((cause) => queryRunnerError(cause, "listPermissionRules")), ), - interrupt: Effect.tryPromise({ - try: () => queryRuntime.interrupt(), - catch: (cause) => queryRunnerError(cause, "interrupt"), - }).pipe( + interrupt: fencePrompts.pipe( + Effect.andThen(Effect.promise(() => inFlightPrompt ?? Promise.resolve())), + Effect.andThen( + Effect.tryPromise({ + // Stop is the only interrupt, and it must not leave queued input to run on its own. + try: () => interruptClaudeQuery(queryRuntime, cancelsQueuedOnInterrupt), + catch: (cause) => queryRunnerError(cause, "interrupt"), + }), + ), Effect.tap(() => logProtocolEvent({ direction: "outgoing", stage: "decoded", payload: { type: "query.interrupt", + cancelQueued: cancelsQueuedOnInterrupt, }, }), ), ), - // A job started since the last poll is recorded before the CLI is told to exit. - close: Effect.suspend(() => (processExited ? Effect.void : refreshDescendants)).pipe( + close: fencePrompts.pipe( + Effect.andThen( + Effect.suspend(() => (processExited ? Effect.void : refreshDescendants)), + ), Effect.andThen(Queue.shutdown(promptQueue)), Effect.andThen(closeClaudeQuery(queryRuntime)), Effect.tap(() => @@ -4790,6 +4880,20 @@ interface ClaudeLiveQueryContext { // Per process, not per session: a relative temp dir follows the cwd this CLI // was launched in, and a later turn's cwd can differ. readonly taskOutputRoot: string | undefined; + readonly boundaryBatches: Array; + readonly boundaryLock: Semaphore.Semaphore; +} + +interface ClaudeBoundaryBatch { + readonly transcriptPath: string; + transcriptOffset: number; + readonly context: ActiveClaudeTurnContext; + readonly offers: ReadonlyArray; + readonly saved: Set; + readonly emitted: Set; + readonly withdrawn: Set; + reserved: boolean; + accepting: boolean; } interface ActiveClaudeToolCall { @@ -5197,6 +5301,8 @@ export interface ClaudeAdapterV2Options { readonly offer: ( request: ProviderContinuationRequests.ProviderContinuationRequest, ) => Effect.Effect; + readonly claimAgentMessages?: ProviderContinuationRequests.ClaimAgentMessages; + readonly withdrawAgentMessages?: ProviderContinuationRequests.AcknowledgeAgentMessages; }; readonly installationGate?: ProviderInstallationGate.ProviderInstallationGate["Service"]; } @@ -11068,6 +11174,108 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } }); + const flushBoundaryReceipts = Effect.fnUntraced(function* (batch: ClaudeBoundaryBatch) { + if (!batch.reserved) return; + const offers = batch.offers.filter( + (offer) => batch.saved.has(offer.id) && !batch.emitted.has(offer.id), + ); + if (offers.length === 0) return; + yield* emitProviderEvent({ + type: "boundary_messages.updated", + driver: CLAUDE_PROVIDER, + threadId: batch.context.input.threadId, + runId: batch.context.input.runId, + providerTurnId: batch.context.providerTurnId, + step: "receipt", + offers: offers.map(({ id, queuedRunId, messageId }) => ({ + id, + queuedRunId, + messageId, + })), + }); + for (const offer of offers) batch.emitted.add(offer.id); + }); + + const pruneBoundaryBatches = (batches: Array) => { + for (let index = batches.length - 1; index >= 0; index--) { + const batch = batches[index]!; + if ( + batch.offers.every( + (offer) => batch.emitted.has(offer.id) || batch.withdrawn.has(offer.id), + ) + ) + batches.splice(index, 1); + } + }; + + const reconcileBoundaryReceipts = Effect.fn("ClaudeAdapterV2.reconcileBoundaryReceipts")( + function* (live: ClaudeLiveQueryContext, scope: "appended" | "ended") { + const unsaved = (batch: ClaudeBoundaryBatch) => + batch.offers.filter( + (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), + ); + const pending = live.boundaryBatches.filter((batch) => unsaved(batch).length > 0); + const saved = new Set(); + const nextOffsets = new Map(); + if (scope === "appended") { + for (const transcriptPath of new Set(pending.map((batch) => batch.transcriptPath))) { + const batches = pending.filter((batch) => batch.transcriptPath === transcriptPath); + const tail = yield* readClaudeBoundaryReceiptTail({ + transcriptPath, + offset: Math.min(...batches.map((batch) => batch.transcriptOffset)), + nativeThreadId: live.nativeThreadId, + offers: batches.flatMap(unsaved), + }).pipe(Effect.provideService(FileSystem.FileSystem, fileSystem)); + for (const id of tail.saved) saved.add(id); + nextOffsets.set(transcriptPath, tail.nextOffset); + } + } else if (pending.length > 0) { + // The tail read trusts its offset; a rewritten transcript needs one full read. + const history = yield* readClaudeBoundaryReceipts({ + projectsDirectory: path.join(homePath, "projects"), + nativeThreadId: live.nativeThreadId, + offers: pending.flatMap(unsaved), + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + ); + for (const id of history) saved.add(id); + } + yield* live.boundaryLock.withPermits(1)( + Effect.gen(function* () { + for (const batch of live.boundaryBatches) { + for (const offer of batch.offers) + if (saved.has(offer.id)) batch.saved.add(offer.id); + batch.transcriptOffset = + nextOffsets.get(batch.transcriptPath) ?? batch.transcriptOffset; + yield* flushBoundaryReceipts(batch); + } + pruneBoundaryBatches(live.boundaryBatches); + }), + ); + }, + (effect, live, scope) => + effect.pipe( + Effect.ensuring( + scope === "ended" + ? live.boundaryLock.withPermits(1)( + Effect.sync(() => { + // Even unreadable history leaves ended offers Unconfirmed; only recovery may retry them. + for (let index = live.boundaryBatches.length - 1; index >= 0; index--) { + if (!live.boundaryBatches[index]!.accepting) + live.boundaryBatches.splice(index, 1); + } + }), + ) + : Effect.void, + ), + ), + Effect.catchTags({ + PlatformError: (cause) => + Effect.logWarning("orchestration-v2.claude-boundary-history-read-failed", { cause }), + }), + ); + const handleSdkMessage = Effect.fnUntraced(function* (input: { readonly query: ClaudeAgentSdkQuerySession; readonly message: SDKMessage; @@ -11075,6 +11283,56 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const message = input.message; const context = yield* Ref.get(activeTurn); const liveQuery = yield* Ref.get(queryContext); + if (liveQuery?.query === input.query) { + const lifecycle = claudePromptLifecycle(message); + if (lifecycle?.state === "started") { + yield* liveQuery.boundaryLock.withPermits(1)( + Effect.gen(function* () { + for (const batch of liveQuery.boundaryBatches) { + if ( + batch.reserved || + batch.context !== context || + !batch.offers.some((offer) => offer.id === lifecycle.uuid) + ) + continue; + yield* emitProviderEvent({ + type: "boundary_messages.updated", + driver: CLAUDE_PROVIDER, + threadId: batch.context.input.threadId, + runId: batch.context.input.runId, + providerTurnId: batch.context.providerTurnId, + step: "reserve", + offers: batch.offers.map(({ id, queuedRunId, messageId }) => ({ + id, + queuedRunId, + messageId, + })), + }); + batch.reserved = true; + yield* flushBoundaryReceipts(batch); + } + pruneBoundaryBatches(liveQuery.boundaryBatches); + }), + ); + } + if (message.type === "result") { + for (const batch of liveQuery.boundaryBatches) batch.accepting = false; + yield* reconcileBoundaryReceipts(liveQuery, "ended"); + } else if ( + // Transcript writes can finish after started; complete root frames retry without per-delta I/O. + (lifecycle?.state === "started" || + (message.type !== "stream_event" && isClaudeRootTurnFrame(message))) && + liveQuery.boundaryBatches.some( + (batch) => + batch.reserved && + batch.offers.some( + (offer) => !batch.saved.has(offer.id) && !batch.withdrawn.has(offer.id), + ), + ) + ) { + yield* reconcileBoundaryReceipts(liveQuery, "appended"); + } + } // The API answering again is recovery; a long text or tool input block emits nothing // else. Judged here, in arrival order: held and buffered frames are handled later. if ( @@ -11855,6 +12113,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); const closedExistingNativeThreadId = existing !== null ? existing.nativeThreadId : null; if (existing !== null) { + existing.stopping = true; yield* existing.query.close.pipe(Effect.ignore); if (existing.nativeThreadId !== nativeThreadId) { yield* clearWakeStateForNativeThread(existing.nativeThreadId); @@ -11874,6 +12133,218 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( turnInput.runtimePolicy.cwd, launchArgSettings, ); + const boundaryBatches: Array = []; + const boundaryLock = yield* Semaphore.make(1); + const hookLock = yield* Semaphore.make(1); + let ownedQuery: ClaudeLiveQueryContext | undefined; + const postToolBatch: HookCallback = (hookInput, _toolUseId, { signal }) => + runPromise( + hookLock.withPermits(1)( + Effect.scoped( + Effect.gen(function* () { + const live = ownedQuery; + const context = yield* Ref.get(activeTurn); + const claim = continuationRequests.claimAgentMessages; + if ( + claim === undefined || + signal.aborted || + hookInput.agent_id !== undefined || + hookInput.session_id !== nativeThreadId || + live === undefined || + live.stopping || + !live.capabilities.has("interrupt_cancel_queued_v1") || + (yield* Ref.get(queryContext)) !== live || + context === null || + context.promptEcho !== "confirmed" || + (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) + ) + return {}; + const offers = yield* claim({ + threadId: context.input.threadId, + runId: context.input.runId, + providerTurnId: context.providerTurnId, + }); + if (offers.length === 0) return {}; + const neverHandedOff = new Set(offers.map((offer) => offer.id)); + const withdrawn = new Set(); + yield* Effect.addFinalizer(() => { + const withdraw = continuationRequests.withdrawAgentMessages; + const unsent = offers.filter((offer) => neverHandedOff.has(offer.id)); + return withdraw === undefined || neverHandedOff.size === 0 + ? Effect.void + : withdraw({ + threadId: context.input.threadId, + runId: context.input.runId, + offers: unsent, + }).pipe( + Effect.andThen( + boundaryLock.withPermits(1)( + Effect.sync(() => { + for (const offer of unsent) withdrawn.add(offer.id); + pruneBoundaryBatches(boundaryBatches); + }), + ), + ), + ); + }); + if ( + signal.aborted || + live.stopping || + (yield* Ref.get(queryContext)) !== live || + (yield* Ref.get(activeTurn)) !== context || + (yield* Ref.get(interruptedTurns)).has(context.providerTurnId) + ) + return {}; + const prepared = []; + for (const offer of offers) { + const reservation = yield* reserveClaudePdfBudget( + pdfBudgets.get(nativeThreadId) ?? + makeClaudeAttachmentBudget({ + pagesUsed: CLAUDE_ATTACHMENT_MAX_PAGES, + bytesUsed: CLAUDE_ATTACHMENT_MAX_BYTES, + }), + persistAttachmentBudget(nativeThreadId), + ); + const message = yield* makeClaudeUserMessageWithAttachments({ + text: + offer.createdBy === "agent" + ? `From another agent:\n${offer.text}` + : offer.text, + attachments: offer.attachments, + priority: "next", + uuid: offer.id, + attachmentsDir, + fileSystem, + pdfReservation: reservation, + skills: yield* userInvocableSkills(context.input.runtimePolicy.cwd), + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(Path.Path, path), + ); + prepared.push({ + offer, + reservation, + message, + consumed: yield* Deferred.make(), + }); + } + const transcriptPath = hookInput.transcript_path; + const transcriptOffset = yield* fileSystem.stat(transcriptPath).pipe( + Effect.map((info) => Number(info.size)), + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(0)), + ); + const batch: ClaudeBoundaryBatch = { + transcriptPath, + transcriptOffset, + context, + offers, + saved: new Set(), + emitted: new Set(), + withdrawn, + reserved: false, + accepting: true, + }; + boundaryBatches.push(batch); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + batch.accepting = false; + }), + ); + const isCurrent = () => + !signal.aborted && + !live.stopping && + live.lastTurn === context && + batch.accepting; + for (const preparedOffer of prepared) { + yield* live.query.offer( + preparedOffer.message, + () => { + neverHandedOff.delete(preparedOffer.offer.id); + }, + { + isCurrent, + onPulled: () => preparedOffer.reservation.commit(), + onConsumed: () => { + void runPromise(Deferred.succeed(preparedOffer.consumed, undefined)); + }, + onDiscarded: () => { + neverHandedOff.add(preparedOffer.offer.id); + void runPromise(Deferred.succeed(preparedOffer.consumed, undefined)); + const withdraw = continuationRequests.withdrawAgentMessages; + if (withdraw !== undefined) { + void runPromise( + withdraw({ + threadId: context.input.threadId, + runId: context.input.runId, + offers: [preparedOffer.offer], + }).pipe( + Effect.andThen( + boundaryLock.withPermits(1)( + Effect.sync(() => { + withdrawn.add(preparedOffer.offer.id); + pruneBoundaryBatches(boundaryBatches); + }), + ), + ), + Effect.catchCause((cause) => + Effect.logWarning( + "orchestration-v2.claude-boundary-withdraw-failed", + { offerId: preparedOffer.offer.id, cause }, + ), + ), + ), + ); + } + }, + }, + ); + } + const aborted = Effect.callback((resume) => { + const abort = () => resume(Effect.void); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + return Effect.sync(() => signal.removeEventListener("abort", abort)); + }); + yield* Effect.raceFirst( + Effect.forEach(prepared, (entry) => Deferred.await(entry.consumed), { + discard: true, + }), + aborted, + ); + return {}; + }), + ), + ), + ); + const receiptStore: NonNullable = { + load: async () => null, + append: (key, entries) => + runPromise( + boundaryLock.withPermits(1)( + Effect.gen(function* () { + if (key.sessionId !== nativeThreadId || key.subpath !== undefined) return; + for (const entry of entries) { + if ( + entry.type !== "attachment" || + entry.isSidechain !== false || + entry.sessionId !== nativeThreadId + ) + continue; + const attachment = field(entry, "attachment"); + if (field(attachment, "type") !== "queued_command") continue; + const sourceUuid = field(attachment, "source_uuid"); + if (typeof sourceUuid !== "string") continue; + for (const batch of boundaryBatches) { + if (!batch.offers.some((offer) => offer.id === sourceUuid)) continue; + batch.saved.add(sourceUuid); + } + } + for (const batch of boundaryBatches) yield* flushBoundaryReceipts(batch); + pruneBoundaryBatches(boundaryBatches); + }), + ), + ), + }; const queryOptions = makeClaudeQueryOptions({ modelSelection: turnInput.modelSelection, modelCatalog, @@ -11919,6 +12390,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ...(appendSystemPromptFileText === undefined ? {} : { appendSystemPromptFileText }), ...(toolSearchOverride === undefined ? {} : { toolSearchOverride }), }); + queryOptions.hooks = { + ...queryOptions.hooks, + PostToolBatch: [{ timeout: 120, hooks: [postToolBatch] }], + }; + queryOptions.sessionStore = receiptStore; + queryOptions.sessionStoreFlush = "eager"; const resolvedTurnCwd = yield* fileSystem .realPath(turnInput.runtimePolicy.cwd ?? ".") .pipe(Effect.orElseSucceed(() => turnInput.runtimePolicy.cwd)); @@ -11987,7 +12464,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( resolvedTurnCwd, hostPlatform, ), + boundaryBatches, + boundaryLock, }; + ownedQuery = context; // One step, so the replaced process cannot exit between the install and the handoff. yield* Ref.update(queryContext, (replaced) => { if (replaced !== null) { @@ -12045,6 +12525,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.exit, Effect.flatMap( Effect.fnUntraced(function* (exit: ClaudeQueryStreamExit) { + for (const batch of context.boundaryBatches) batch.accepting = false; + yield* reconcileBoundaryReceipts(context, "ended"); const heldContext = yield* Ref.get(activeTurn); if ( (yield* Ref.get(queryContext))?.query === querySession && @@ -12477,6 +12959,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( detail: `Claude provider turn ${turnInput.providerTurnId} is not the active turn.`, }); } + existing.stopping = true; yield* Ref.update(interruptedTurns, (current) => { const next = new Set(current); next.add(turnInput.providerTurnId); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts index 4716ec06175a6..a8ca93a248be7 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts @@ -298,6 +298,103 @@ it.effect("reopens a stalled MCP registration before delivering the resumed prom ), ); +it.effect.each([ + { case: "a CLI that cancels queued input", capabilities: ["interrupt_cancel_queued_v1"] }, + { case: "an older CLI", capabilities: undefined }, +])("Stop cancels input the CLI still has queued: $case", ({ capabilities }) => + Effect.scoped( + Effect.gen(function* () { + const interrupt = vi.fn(async (_options?: { readonly cancelQueued: true }) => undefined); + const frames = [{ type: "system", subtype: "init", capabilities }]; + const querySpy = vi.spyOn(ClaudeSdk, "query").mockImplementation( + () => + ({ + close: vi.fn(), + interrupt, + next: async () => + frames.length > 0 + ? { done: false, value: frames.shift() } + : { done: true, value: undefined }, + }) as unknown as ClaudeSdk.Query, + ); + yield* Effect.addFinalizer(() => Effect.sync(() => querySpy.mockRestore())); + const runner = yield* ClaudeAgentSdkQueryRunner; + const { mcpServers: _mcpServers, ...options } = resumedMcpQueryInput.options; + const session = yield* runner.open({ ...resumedMcpQueryInput, options }); + yield* Effect.addFinalizer(() => session.close.pipe(Effect.orDie)); + yield* Stream.runDrain(session.messages); + yield* session.interrupt; + assert.deepEqual(interrupt.mock.calls, [ + capabilities === undefined ? [] : [{ cancelQueued: true }], + ]); + }).pipe(Effect.provide(realQueryRunner)), + ), +); + +it.effect("Stop fences the input producer before cancelling a partly written batch", () => + Effect.scoped( + Effect.gen(function* () { + const interrupt = vi.fn(async () => undefined); + const querySpy = vi + .spyOn(ClaudeSdk, "query") + .mockImplementation(() => ({ close: vi.fn(), interrupt }) as unknown as ClaudeSdk.Query); + yield* Effect.addFinalizer(() => Effect.sync(() => querySpy.mockRestore())); + const runner = yield* ClaudeAgentSdkQueryRunner; + const { mcpServers: _mcpServers, ...options } = resumedMcpQueryInput.options; + const session = yield* runner.open({ ...resumedMcpQueryInput, options }); + yield* Effect.addFinalizer(() => session.close.pipe(Effect.orDie)); + const prompt = querySpy.mock.calls[0]![0].prompt; + if (typeof prompt === "string") throw new Error("Expected streaming input"); + const input = prompt[Symbol.asyncIterator](); + const discarded = Promise.withResolvers(); + const consumed: string[] = []; + const pulled: string[] = []; + const message = (content: string): ClaudeSdk.SDKUserMessage => ({ + type: "user", + message: { role: "user", content }, + parent_tool_use_id: null, + session_id: FORK_SESSION_ID, + }); + const handoff = (id: string) => ({ + isCurrent: () => true, + onPulled: () => { + pulled.push(id); + }, + onConsumed: () => { + consumed.push(id); + }, + onDiscarded: () => { + discarded.resolve(); + }, + }); + yield* session.offer(message("first"), undefined, handoff("first")); + yield* session.offer(message("second"), undefined, handoff("second")); + yield* session.offer(message("third"), undefined, handoff("third")); + assert.equal((yield* Effect.promise(() => input.next())).value.message.content, "first"); + assert.deepEqual(pulled, ["first"]); + assert.deepEqual(consumed, []); + assert.equal((yield* Effect.promise(() => input.next())).value.message.content, "second"); + assert.deepEqual(consumed, ["first"]); + const stopping = yield* session.interrupt.pipe(Effect.forkChild); + yield* Effect.promise(() => discarded.promise); + assert.lengthOf(interrupt.mock.calls, 0); + const nextInput = yield* Effect.promise(() => input.next()).pipe(Effect.forkChild); + yield* Fiber.join(stopping); + assert.deepEqual(consumed, ["first", "second"]); + assert.lengthOf(interrupt.mock.calls, 1); + const afterStop = vi.fn(); + yield* session.offer(message("fourth"), undefined, { + ...handoff("fourth"), + onDiscarded: afterStop, + }); + assert.equal(afterStop.mock.calls.length, 1); + yield* session.close; + assert.isTrue((yield* Fiber.join(nextInput)).done); + assert.deepEqual(pulled, ["first", "second"]); + }).pipe(Effect.provide(realQueryRunner)), + ), +); + it.effect.each([ { case: "a readable rule list", errors: undefined, readable: true }, { case: "a settings file Claude skipped", errors: [{ file: "settings.json" }], readable: false }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts new file mode 100644 index 0000000000000..6c864955f22fb --- /dev/null +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeSavedMessageReceipts.ts @@ -0,0 +1,112 @@ +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import { field } from "../unknownField.ts"; + +const decodeLine = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)); + +interface ReceiptQuery { + readonly nativeThreadId: string; + readonly offers: ReadonlyArray<{ readonly id: string }>; +} + +function receiptCollector(input: ReceiptQuery) { + const offers = new Set(input.offers.map((offer) => offer.id)); + const saved = new Set(); + return { + saved, + collect: (line: string) => { + // Most lines are not receipts; skip decoding them, since transcripts reach hundreds of MB. + if (!line.includes("queued_command") || !line.includes("source_uuid")) return; + const decoded = decodeLine(line); + if (Option.isNone(decoded)) return; + const entry = decoded.value; + if ( + field(entry, "sessionId") !== input.nativeThreadId || + field(entry, "isSidechain") !== false || + field(entry, "type") !== "attachment" + ) + return; + const attachment = field(entry, "attachment"); + if (field(attachment, "type") !== "queued_command") return; + const uuid = field(attachment, "source_uuid"); + if (typeof uuid === "string" && offers.has(uuid)) saved.add(uuid); + }, + }; +} + +/** Only positive root-history evidence settles an offer; missing or torn history never permits replay. */ +export const readClaudeBoundaryReceipts = Effect.fn("readClaudeBoundaryReceipts")(function* ( + input: ReceiptQuery & { readonly projectsDirectory: string }, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { saved, collect } = receiptCollector(input); + const projects = yield* fileSystem + .readDirectory(input.projectsDirectory) + .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed([]))); + for (const project of projects) { + const transcript = path.join(input.projectsDirectory, project, `${input.nativeThreadId}.jsonl`); + if (!(yield* fileSystem.exists(transcript))) continue; + yield* fileSystem.stream(transcript).pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((line) => Effect.sync(() => collect(line))), + ); + } + return saved; +}); + +/** + * Reads the complete lines appended since `offset` and returns where the next read resumes. + * A receipt is written after its offer is handed off, so a line that straddles the offset is not one. + */ +export const readClaudeBoundaryReceiptTail = Effect.fn("readClaudeBoundaryReceiptTail")(function* ( + input: ReceiptQuery & { readonly transcriptPath: string; readonly offset: number }, +) { + const fileSystem = yield* FileSystem.FileSystem; + const { saved, collect } = receiptCollector(input); + const info = yield* fileSystem + .stat(input.transcriptPath) + .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.undefined)); + if (info === undefined) return { saved, nextOffset: 0 }; + // A file shorter than the offset was replaced, so nothing in it has been read yet. + const offset = input.offset <= Number(info.size) ? input.offset : 0; + // Starting one byte early makes the first newline end either the previous line or a straddling one. + const start = Math.max(offset - 1, 0); + const decoder = new TextDecoder(); + let skipFirstLine = offset > 0; + let position = start; + let nextOffset = offset; + let carry: Array = []; + yield* fileSystem.stream(input.transcriptPath, { offset: start }).pipe( + Stream.runForEach((chunk) => + Effect.sync(() => { + let lineStart = 0; + for ( + let newline = chunk.indexOf(10); + newline !== -1; + newline = chunk.indexOf(10, lineStart) + ) { + if (skipFirstLine) { + skipFirstLine = false; + } else { + let line = ""; + for (const part of carry) line += decoder.decode(part, { stream: true }); + collect(line + decoder.decode(chunk.subarray(lineStart, newline))); + } + carry = []; + lineStart = newline + 1; + nextOffset = position + lineStart; + } + if (lineStart < chunk.length) carry.push(chunk.slice(lineStart)); + position += chunk.length; + }), + ), + ); + return { saved, nextOffset }; +}); diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index faa8d8b8b3cbf..fa43186ffcbf1 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -107,7 +107,7 @@ const toolResult = (id: string, cancelled: boolean) => ...(cancelled ? { tool_result_meta: [{ id, non_execution_kind: "cancelled" }] } : {}), }); -const result = (uuid: string, aborted: boolean) => +const result = (uuid: string, aborted = false) => frame({ type: "result", subtype: "success", @@ -134,7 +134,7 @@ const deliveries = [ ] as const; it.effect.each(deliveries)( - "delivers %s with Claude's native pending-tool cancellation behavior", + "queues %s without cancelling tools when native boundary input is unavailable", (delivery) => Effect.scoped( Effect.gen(function* () { @@ -393,22 +393,15 @@ it.effect.each(deliveries)( targetRunId: parent.id, }); const promoted = yield* orchestrator.getThreadProjection(threadId); - assert.equal(promoted.runs.find((run) => run.id === queuedRun.id)?.status, "cancelled"); + assert.equal(promoted.runs.find((run) => run.id === queuedRun.id)?.status, "queued"); assert.equal( promoted.messages.find((message) => message.id === messageId)?.runId, - parent.id, + queuedRun.id, ); - const steer = promoted.turnItems.find( - (item) => item.type === "user_message" && item.messageId === messageId, - ); - assert.ok(steer?.type === "user_message"); - assert.equal(steer.inputIntent, "promoted_queued_to_steer"); } yield* worker.drain(); - const explicitSteer = - delivery === "user steering" || delivery === "promoted queued message"; - assert.equal(batchAbort.signal.aborted, explicitSteer); - assert.equal(offers.length, explicitSteer ? 2 : 1); + assert.equal(batchAbort.signal.aborted, false); + assert.equal(offers.length, 1); const finished = yield* watch( (event) => event.type === "run.updated" && @@ -418,59 +411,57 @@ it.effect.each(deliveries)( yield* Queue.offer(sdkMessages, toolResult("build", false)); for (const id of statusIds) yield* Queue.offer(sdkMessages, toolResult(id, batchAbort.signal.aborted)); - if (explicitSteer) yield* Queue.offer(sdkMessages, result("aborted", true)); - yield* Queue.offer(sdkMessages, result("completed", false)); + yield* Queue.offer(sdkMessages, result("completed")); yield* Fiber.join(finished); yield* worker.drain(); yield* orchestrator.resumeQueuedRuns; yield* worker.drain(); - if (!explicitSteer) { - const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; - const noticeFinished = yield* watch( - (event) => - event.type === "run.updated" && - event.payload.id === queued.id && - event.payload.status === "waiting", - ); - const delivered = - delivery === "child completion" - ? yield* watch( - (event) => - event.type === "subagent.updated" && - event.payload.completionDelivery?.state === "delivered", - ) - : null; - yield* Queue.offer(sdkMessages, result("notice-completed", false)); - yield* Fiber.join(noticeFinished); - yield* worker.drain(); - if (delivered !== null) yield* Fiber.join(delivered); - } + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs[1]!; + const noticeFinished = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === queued.id && + event.payload.status === "waiting", + ); + const delivered = + delivery === "child completion" + ? yield* watch( + (event) => + event.type === "subagent.updated" && + event.payload.completionDelivery?.state === "delivered", + ) + : null; + yield* Queue.offer(sdkMessages, result("notice-completed")); + yield* Fiber.join(noticeFinished); + yield* worker.drain(); + if (delivered !== null) yield* Fiber.join(delivered); const after = yield* orchestrator.getThreadProjection(threadId); const reads = after.turnItems.filter((item) => statusIds.includes(item.nativeItemRef?.nativeId ?? ""), ); assert.equal(reads.length, 4); for (const read of reads) { - assert.equal(read.status, explicitSteer ? "cancelled" : "completed"); - assert.equal(read.toolNonExecutionKind, explicitSteer ? "cancelled" : undefined); - assert.equal(read.type === "dynamic_tool" && read.output === refusal, explicitSteer); + assert.equal(read.status, "completed"); + assert.equal(read.toolNonExecutionKind, undefined); + assert.equal(read.type === "dynamic_tool" && read.output === refusal, false); } assert.equal(offers.length, 2); - assert.equal( - offers.filter((offer) => offer.priority === "now").length, - explicitSteer ? 1 : 0, - ); - if (!explicitSteer) { - assert.equal(after.runs.length, 2); - if (delivery === "child completion") { - assert.equal(after.messages.filter((message) => message.id === messageId).length, 1); - assert.equal(after.subagents[0]?.completionDelivery?.state, "delivered"); - } else { - assert.equal( - after.messages.filter((message) => message.scheduledTaskId !== undefined).length, - 1, - ); - } + assert.equal(offers.filter((offer) => offer.priority === "now").length, 0); + assert.equal(after.runs.length, 2); + if (delivery === "child completion") { + assert.equal(after.messages.filter((message) => message.id === messageId).length, 1); + assert.equal(after.subagents[0]?.completionDelivery?.state, "delivered"); + } else if (delivery === "scheduled message") { + assert.equal( + after.messages.filter((message) => message.scheduledTaskId !== undefined).length, + 1, + ); + } else { + const notice = after.messages.filter((message) => message.id === messageId); + assert.lengthOf(notice, 1); + assert.notEqual(notice[0]?.runId, parent.id); + assert.equal(notice[0]?.createdBy, "user"); + assert.equal(notice[0]?.creationSource, "web"); } yield* orchestrator.recoverDelegatedTasks; yield* orchestrator.resumeQueuedRuns; diff --git a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts new file mode 100644 index 0000000000000..2968b7290637f --- /dev/null +++ b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.test.ts @@ -0,0 +1,219 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Layer from "effect/Layer"; +import { + DEFAULT_SERVER_SETTINGS, + ProviderDriverKind, + ThreadId, + type OrchestrationV2ThreadProjection, +} from "@t3tools/contracts"; +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; + +import { + readClaudeBoundaryReceipts, + readClaudeBoundaryReceiptTail, +} from "./Adapters/ClaudeSavedMessageReceipts.ts"; +import { makeClaudeBoundaryReceiptRecovery } from "./ClaudeBoundaryReceipts.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ThreadManagementService from "./ThreadManagementService.ts"; +import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +import * as ServerSettings from "../serverSettings.ts"; + +const encodeLine = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); + +it.layer(NodeServices.layer)("Claude boundary saved history", (it) => { + it.effect( + "settles saved offers after restart in their original run and leaves missing offers held", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boundary-recovery-" }); + const project = path.join(directory, "projects", "project"); + yield* fs.makeDirectory(project, { recursive: true }); + yield* fs.writeFileString( + path.join(project, "native-session.jsonl"), + encodeLine({ + type: "attachment", + sessionId: "native-session", + isSidechain: false, + attachment: { type: "queued_command", source_uuid: "saved" }, + }), + ); + const threadId = ThreadId.make("recovery-thread"); + const projection = { + thread: { id: threadId, archivedAt: null, deletedAt: null }, + runs: ["saved", "missing"].map((id) => ({ + id: `queued-${id}`, + userMessageId: `message-${id}`, + status: "queued", + queueHeld: true, + boundaryOffer: { id, targetRunId: "original-run", providerTurnId: "original-turn" }, + })), + messages: [], + providerTurns: [{ id: "original-turn", providerThreadId: "provider-thread" }], + providerThreads: [ + { + id: "provider-thread", + providerInstanceId: "claude", + nativeThreadRef: { nativeId: "native-session" }, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + const operations: unknown[] = []; + const recover = yield* makeClaudeBoundaryReceiptRecovery.pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => Effect.succeed(projection), + dispatch: (command) => + Effect.sync(() => { + operations.push(command); + return {} as never; + }), + }), + Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({ + reserveBoundary: (reservation) => + Effect.sync(() => { + operations.push(reservation); + }), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.succeed({ + ...DEFAULT_SERVER_SETTINGS, + providerInstances: { + claude: { + driver: ProviderDriverKind.make("claudeAgent"), + config: { homePath: directory }, + }, + }, + }), + }), + ), + ), + ); + yield* recover; + assert.deepEqual(operations, [ + { + threadId, + turnItemId: "turn-item:message:message-saved", + runId: "original-run", + queuedRunId: "queued-saved", + offerId: "saved", + providerTurnId: "original-turn", + }, + { + type: "queued-message.promote-to-steer", + commandId: "tool-boundary-receipt:saved", + threadId, + queuedRunId: "queued-saved", + targetRunId: "original-run", + boundaryOffer: { id: "saved", step: "receipt" }, + }, + ]); + }).pipe(Effect.scoped), + ); + + it.effect("recovers exact root receipts while retaining ambiguous offers", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boundary-receipts-" }); + const project = path.join(directory, "project"); + yield* fs.makeDirectory(project); + const entry = (attachment: unknown, overrides: Record = {}) => ({ + type: "attachment", + sessionId: "native-session", + isSidechain: false, + attachment, + ...overrides, + }); + const legacy = "From another agent (delivery legacy):\nOriginal saved update"; + const legacyBatch = [ + "From the user (delivery batch-first):\nFirst update\n\nSecond paragraph", + "From another agent (delivery batch-middle):\nMiddle update", + "From the user (delivery batch-last):\nLast update", + ]; + const rows = [ + entry({ type: "queued_command", source_uuid: "saved" }), + entry({ type: "queued_command", source_uuid: "saved" }), + entry({ type: "queued_command", source_uuid: "foreign" }, { sessionId: "other-session" }), + entry({ type: "queued_command", source_uuid: "child" }, { isSidechain: true }), + entry({ type: "hook_additional_context", content: [`New messages:\n${legacy}`] }), + entry({ + type: "hook_additional_context", + content: [`New messages:\n${legacyBatch.join("\n\n")}`], + }), + entry({ + type: "hook_additional_context", + content: ["New messages:\nFrom the user (delivery changed-legacy):\nChanged text"], + }), + entry({ type: "hook_additional_context", content: ["changed original text"] }), + { type: "queue-operation", operation: "enqueue", content: "queued-only" }, + { type: "user", uuid: "unrelated-turn" }, + ]; + yield* fs.writeFileString( + path.join(project, "native-session.jsonl"), + `${rows.map((row) => encodeLine(row)).join("\n")}\n{"type":"attachment"`, + ); + const saved = yield* readClaudeBoundaryReceipts({ + projectsDirectory: directory, + nativeThreadId: "native-session", + offers: [ + { id: "saved" }, + { id: "foreign" }, + { id: "child" }, + { id: "queued-only" }, + { id: "missing" }, + { id: "legacy" }, + ], + }); + const transcriptPath = path.join(project, "tail.jsonl"); + const receipt = (id: string) => + encodeLine(entry({ type: "queued_command", source_uuid: id })); + const before = `${receipt("before")}\n`; + const after = `${receipt("after")}\n`; + const offset = new TextEncoder().encode(before).length; + const readTail = (start: number) => + readClaudeBoundaryReceiptTail({ + nativeThreadId: "native-session", + transcriptPath, + offset: start, + offers: [{ id: "before" }, { id: "after" }, { id: "late" }], + }).pipe(Effect.map((tail) => ({ saved: [...tail.saved], nextOffset: tail.nextOffset }))); + const torn = receipt("late"); + yield* fs.writeFileString(transcriptPath, before + after + torn.slice(0, 20)); + const appended = { + saved: ["after"], + nextOffset: new TextEncoder().encode(before + after).length, + }; + assert.deepEqual(yield* readTail(offset), appended); + // A line that straddles the offset began before the handoff, so it is not re-read. + assert.deepEqual(yield* readTail(offset - 5), appended); + assert.deepEqual(yield* readTail(1), appended); + yield* fs.writeFileString(transcriptPath, `${before}${after}${torn}\n`); + assert.deepEqual((yield* readTail(appended.nextOffset)).saved, ["late"]); + yield* fs.writeFileString(transcriptPath, after); + assert.deepEqual((yield* readTail(offset + 1000)).saved, ["after"]); + assert.deepEqual([...saved], ["saved"]); + assert.deepEqual( + [ + ...(yield* readClaudeBoundaryReceipts({ + projectsDirectory: path.join(directory, "absent"), + nativeThreadId: "native-session", + offers: [{ id: "missing" }], + })), + ], + [], + ); + }).pipe(Effect.scoped), + ); +}); diff --git a/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.ts b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.ts new file mode 100644 index 0000000000000..712f540224af6 --- /dev/null +++ b/apps/server/src/orchestration-v2/ClaudeBoundaryReceipts.ts @@ -0,0 +1,123 @@ +import { CommandId, ClaudeSettings } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; +import { resolveClaudeHomePath } from "../provider/Drivers/ClaudeHome.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ThreadManagementService from "./ThreadManagementService.ts"; +import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +import { readClaudeBoundaryReceipts } from "./Adapters/ClaudeSavedMessageReceipts.ts"; + +const decodeClaudeSettings = Schema.decodeEffect(ClaudeSettings); + +export const makeClaudeBoundaryReceiptRecovery = Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const positions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; + const ids = yield* IdAllocator.IdAllocatorV2; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const environment = yield* HostProcess.Environment; + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + return Effect.gen(function* () { + const settings = yield* serverSettings.getSettings; + for (const threadId of yield* projections.getRecoveryThreadIds("boundary-offers")) { + yield* Effect.gen(function* () { + const queued = yield* threads.getThreadRecords(threadId, ["runs"], { + runStatuses: ["queued"], + }); + if (queued.thread.archivedAt !== null || queued.thread.deletedAt !== null) return; + const offered = queued.runs.filter( + (run) => run.status === "queued" && run.boundaryOffer !== undefined, + ); + const projection = yield* threads.getThreadRecords( + threadId, + ["messages", "providerTurns", "providerThreads"], + { + messageIds: offered.map((run) => run.userMessageId), + }, + ); + const grouped = new Map< + string, + { home: string; nativeThreadId: string; runs: typeof offered } + >(); + for (const run of offered) { + const turn = projection.providerTurns.find( + (candidate) => candidate.id === run.boundaryOffer!.providerTurnId, + ); + if (turn === undefined) continue; + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === turn.providerThreadId, + ); + const nativeId = providerThread?.nativeThreadRef?.nativeId; + const instance = + providerThread === undefined + ? undefined + : settings.providerInstances[providerThread.providerInstanceId]; + if (nativeId == null || instance?.driver !== "claudeAgent") continue; + // A secret config-directory override cannot be reconstructed from redacted settings. + if ( + instance.environment?.some( + (variable) => + variable.name.toUpperCase() === "CLAUDE_CONFIG_DIR" && + typeof variable.value !== "string", + ) + ) + continue; + const config = yield* decodeClaudeSettings(instance.config ?? {}); + const merged = yield* mergeProviderInstanceEnvironment(instance.environment, environment); + const home = yield* resolveClaudeHomePath(config, merged); + const key = `${home}\0${nativeId}`; + const group = grouped.get(key); + if (group === undefined) + grouped.set(key, { home, nativeThreadId: nativeId, runs: [run] }); + else group.runs.push(run); + } + for (const { home, nativeThreadId, runs } of grouped.values()) { + const saved = yield* readClaudeBoundaryReceipts({ + projectsDirectory: path.join(home, "projects"), + nativeThreadId, + offers: runs.map((run) => ({ id: run.boundaryOffer!.id })), + }); + for (const run of runs) { + const offer = run.boundaryOffer!; + if (!saved.has(offer.id)) continue; + yield* positions.reserveBoundary({ + threadId, + turnItemId: ids.derive.userTurnItem({ messageId: run.userMessageId }), + runId: offer.targetRunId, + queuedRunId: run.id, + offerId: offer.id, + providerTurnId: offer.providerTurnId, + }); + yield* threads.dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(`tool-boundary-receipt:${offer.id}`), + threadId, + queuedRunId: run.id, + targetRunId: offer.targetRunId, + boundaryOffer: { id: offer.id, step: "receipt" }, + }); + } + } + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Claude saved-message recovery left offers unconfirmed", { + threadId, + cause, + }), + ), + ); + } + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.withSpan("recoverClaudeBoundaryReceipts"), + ); +}); diff --git a/apps/server/src/orchestration-v2/DelegatedBoundaryJoin.ts b/apps/server/src/orchestration-v2/DelegatedBoundaryJoin.ts new file mode 100644 index 0000000000000..46d28d36f27ef --- /dev/null +++ b/apps/server/src/orchestration-v2/DelegatedBoundaryJoin.ts @@ -0,0 +1,35 @@ +import type { OrchestrationV2ThreadProjection } from "@t3tools/contracts"; + +type Run = OrchestrationV2ThreadProjection["runs"][number]; +type Message = OrchestrationV2ThreadProjection["messages"][number]; + +export function canJoinDelegatedBoundary( + queued: Run, + target: Run | undefined, + message: Message, + child: OrchestrationV2ThreadProjection["thread"], + tasks: OrchestrationV2ThreadProjection["subagents"], +): boolean { + if (queued.delegatedTaskId === undefined) return true; + const parentThreadId = child.lineage.parentThreadId; + const followup = tasks.find((task) => task.id === queued.delegatedTaskId); + const running = tasks.find((task) => task.id === target?.delegatedTaskId); + return ( + parentThreadId !== null && + message.senderThreadId === parentThreadId && + followup !== undefined && + running !== undefined && + followup.id !== running.id && + followup.origin === "app_owned" && + running.origin === "app_owned" && + followup.threadId === parentThreadId && + running.threadId === parentThreadId && + followup.childThreadId === child.id && + running.childThreadId === child.id && + followup.runId !== null && + followup.runId === running.runId && + followup.parentNodeId === running.parentNodeId && + followup.result === null && + running.result === null + ); +} diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 426da22121e36..50e1d5280d845 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -19,6 +19,8 @@ import { ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, + RunAttemptId, RunId, RuntimeRequestId, ThreadId, @@ -3706,7 +3708,8 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( }, ); -it.layer(layerTest)("delegated tasks across a server restart", (it) => { +const layerRestartTest = Layer.merge(layerTest, ProviderContinuationRequests.layer); +it.layer(layerRestartTest)("delegated tasks across a server restart", (it) => { it.effect.each([ { name: "releases", key: "", pendingBackgroundTasks: [], released: true }, { @@ -3842,6 +3845,274 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { }), ); + it.effect( + "a boundary-offered completion preserves its message and offers one successor after receipt", + () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:boundary-result-message-parent"); + const projectId = ProjectId.make("project:boundary-result-message"); + const runId = RunId.make("run:boundary-result-message-parent"); + const rootNodeId = NodeId.make("node:boundary-result-message-parent"); + yield* seedParentWithTerminalTask({ + threadId, + projectId, + runId, + rootNodeId, + taskId: NodeId.make("node:boundary-result-message-seed"), + deliveryState: "acknowledged", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: "boundary-result-message-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + const parent = yield* orchestrator.getThreadProjection(threadId); + const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId); + const providerThreadId = ProviderThreadId.make( + "provider-thread:boundary-result-message-child", + ); + yield* eventSink.write({ + commandId: CommandId.make("command:boundary-result-message-child-provider"), + events: [ + { + id: EventId.make("event:boundary-result-message-child-provider"), + type: "provider-thread.updated", + threadId: child.childThreadId, + occurredAt: now, + payload: { + ...parent.providerThreads[0]!, + id: providerThreadId, + appThreadId: child.childThreadId, + }, + }, + { + id: EventId.make("event:boundary-result-message-child-run"), + type: "run.updated", + threadId: child.childThreadId, + runId: child.childRunId, + occurredAt: now, + payload: { ...childProjection.runs[0]!, providerThreadId }, + }, + ], + }); + + yield* orchestrator.recoverDelegatedTasks; + const recovered = yield* orchestrator.getThreadProjection(threadId); + const task = recovered.subagents.find((candidate) => candidate.id === child.taskId)!; + assert.equal(task.status, "completed"); + assert.isNotNull(task.result); + assert.isEmpty(recovered.contextHandoffs); + assert.equal(recovered.contextTransfers.length, 1); + + const delivery = recovered.runs.find((candidate) => candidate.id === runId)! + .delegatedCompletion!.delivery!; + const messageId = delivery.messageId; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("command:boundary-result-message-delivery"), + threadId, + messageId, + text: "Delegated task finished.", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + delegatedCompletion: { + parentRunId: runId, + generation: delivery.generation, + taskIds: [child.taskId], + }, + }); + const queued = yield* orchestrator.getThreadProjection(threadId); + assert.include( + queued.messages.find((message) => message.id === messageId)?.text ?? "", + task.result!, + ); + const offeredRun = queued.runs.find((run) => run.userMessageId === messageId)!; + const offeredMessage = queued.messages.find((message) => message.id === messageId)!; + const offeredTurnId = ProviderTurnId.make("provider-turn:offered-completion"); + const offeredAttemptId = RunAttemptId.make("attempt:offered-completion"); + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:offer-completion"), + type: "run.updated", + threadId, + runId: offeredRun.id, + occurredAt: now, + payload: { + ...offeredRun, + boundaryOffer: { + id: "offered-completion", + targetRunId: runId, + providerTurnId: offeredTurnId, + }, + }, + }, + { + id: EventId.make("event:offered-completion-attempt"), + type: "run-attempt.updated", + threadId, + runId, + occurredAt: now, + payload: { + id: offeredAttemptId, + runId, + attemptOrdinal: 1, + rootNodeId, + providerInstanceId: modelSelection.instanceId, + providerThreadId: parent.providerThreads[0]!.id, + providerTurnId: offeredTurnId, + reason: "initial", + status: "running", + startedAt: now, + completedAt: null, + }, + }, + { + id: EventId.make("event:offered-completion-turn"), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { + id: offeredTurnId, + providerThreadId: parent.providerThreads[0]!.id, + nodeId: rootNodeId, + runAttemptId: offeredAttemptId, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ], + }); + const later = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: "boundary-result-message-later-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.recoverDelegatedTasks; + const afterLater = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + afterLater.messages.find((message) => message.id === messageId), + offeredMessage, + ); + assert.equal( + afterLater.subagents.find((row) => row.id === later.taskId)?.completionDelivery?.state, + "pending", + ); + const beforeReceipt = afterLater.runs.find((run) => run.id === runId)!; + const checkpointId = CheckpointId.make("checkpoint:receipt-first-completion"); + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:receipt-first-completion-waiting"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { ...beforeReceipt, status: "waiting" }, + }, + ], + }); + const commit = eventSink.commitCommand; + const receiptCommandId = CommandId.make("receipt-first-completion"); + const offeredSuccessors: MessageId[] = []; + const offer = requests.offer; + yield* Effect.acquireUseRelease( + Effect.sync(() => [ + vi.spyOn(requests, "offer").mockImplementation((request) => { + if (request.delegatedCompletion !== undefined) + offeredSuccessors.push(request.delegatedCompletion.messageId); + return offer(request); + }), + vi.spyOn(eventSink, "commitCommand").mockImplementation((input) => + Effect.gen(function* () { + if (input.commandId === receiptCommandId) { + yield* commit({ + commandId: CommandId.make("command:receipt-first-completion-checkpoint"), + threadId, + commandType: "checkpoint.capture", + acceptedAt: now, + effects: [], + events: [ + { + id: EventId.make("event:receipt-first-completion-checkpoint"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { + ...beforeReceipt, + status: "completed", + completedAt: now, + checkpointId, + }, + }, + ], + }); + } + return yield* commit(input); + }), + ), + ]), + () => + orchestrator.dispatch({ + type: "queued-message.promote-to-steer", + commandId: receiptCommandId, + threadId, + queuedRunId: offeredRun.id, + targetRunId: runId, + boundaryOffer: { id: "offered-completion", step: "receipt" }, + }), + (spies) => + Effect.sync(() => { + for (const spy of spies) spy.mockRestore(); + }), + ); + const received = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + received.subagents.find((row) => row.id === child.taskId)?.completionDelivery?.state, + "delivered", + ); + assert.equal( + received.subagents.find((row) => row.id === later.taskId)?.completionDelivery?.state, + "claimed", + ); + assert.deepEqual( + received.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery?.taskIds, + [later.taskId], + ); + const receivedParent = received.runs.find((run) => run.id === runId)!; + assert.deepEqual(offeredSuccessors, [ + receivedParent.delegatedCompletion!.delivery!.messageId, + ]); + assert.deepEqual( + [receivedParent.status, receivedParent.completedAt, receivedParent.checkpointId], + ["completed", now, checkpointId], + ); + }), + ); + it.effect("holds a restart-cancelled child for its continuation's result", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -4795,7 +5066,13 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { ...providerThread, pendingBackgroundTasks: [ ...(open - ? [{ taskId: "sleep-900", kind: "command" as const, awaited: true as const }] + ? [ + { + taskId: "sleep-900", + kind: "command" as const, + awaited: true as const, + }, + ] : []), { taskId: "log-watcher", kind: "monitor" as const }, ], @@ -4822,7 +5099,12 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { status: "completed", now, }), - payload: { ...child.run, status: "completed" as const, startedAt: now, completedAt: now }, + payload: { + ...child.run, + status: "completed" as const, + startedAt: now, + completedAt: now, + }, }); const beforeEnd = yield* sink.latestSequence(); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 6a585abc5e5d5..c00c4291a6114 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -178,6 +178,7 @@ import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as RunExecutionService from "./RunExecutionService.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts"; +import { canJoinDelegatedBoundary } from "./DelegatedBoundaryJoin.ts"; import { isAutomaticCompletionRun, queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts"; import { isRuntimePolicyAboveCeilingError, RuntimePolicyV2 } from "./RuntimePolicy.ts"; import { @@ -872,7 +873,9 @@ function worktreeSetupPending( function nextQueuedRun( projection: Pick, ): OrchestrationV2Run | undefined { - return queuedRunsInDeliveryOrder(projection).find((run) => run.queueHeld !== true); + return queuedRunsInDeliveryOrder(projection).find( + (run) => run.queueHeld !== true && run.boundaryOffer === undefined, + ); } function followsCurrentConversation( @@ -2099,7 +2102,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio options?.queuedRunId === undefined ? nextQueuedRun(projection) : projection.runs.find( - (run) => run.id === options.queuedRunId && run.status === "queued", + (run) => + run.id === options.queuedRunId && + run.status === "queued" && + run.boundaryOffer === undefined, ); while ( storedQueuedRun !== undefined && @@ -5468,6 +5474,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly delegatedCompletion?: OrchestrationV2ConversationMessage["delegatedCompletion"]; readonly notification?: OrchestrationV2ConversationMessage["notification"]; readonly forceRestart: boolean; + readonly deliveredInProviderTurnId?: OrchestrationV2ProviderTurn["id"]; }) { const targetRun = input.projection.runs.find((candidate) => candidate.id === input.targetRunId); if (targetRun === undefined) { @@ -5477,6 +5484,144 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Target run ${input.targetRunId} was not found.`, }); } + const now = yield* DateTime.now; + const emitEvent = emit(input.events, input.command); + const appendSteeringMessage = Effect.fn("Orchestrator.appendSteeringMessage")( + function* (messageInput: { + readonly runId: OrchestrationV2Run["id"]; + readonly nodeId: OrchestrationV2ExecutionNode["id"]; + readonly providerTurnId: OrchestrationV2ProviderTurn["id"] | null; + readonly providerThreadId: OrchestrationV2ProviderThread["id"]; + readonly providerInstanceId: ProviderInstanceId; + readonly modelSwitch?: OrchestrationV2ModelSwitch | undefined; + }) { + const message: OrchestrationV2ConversationMessage = { + createdBy: input.createdBy, + creationSource: input.creationSource, + ...(input.delegatedCompletion === undefined + ? {} + : { delegatedCompletion: input.delegatedCompletion }), + ...(input.notification === undefined ? {} : { notification: input.notification }), + ...(input.scheduledTaskId === undefined + ? {} + : { scheduledTaskId: input.scheduledTaskId }), + ...(input.senderThreadId === undefined ? {} : { senderThreadId: input.senderThreadId }), + ...(input.linkedSender === undefined ? {} : { linkedSender: input.linkedSender }), + ...(input.idleCompaction === undefined ? {} : { idleCompaction: input.idleCompaction }), + id: input.messageId, + threadId: input.command.threadId, + runId: messageInput.runId, + nodeId: messageInput.nodeId, + role: "user", + text: input.text, + attachments: input.attachments, + ...(input.context ? { context: input.context } : {}), + streaming: false, + createdAt: now, + updatedAt: now, + }; + const turnItem: OrchestrationV2TurnItem = { + createdBy: input.createdBy, + creationSource: input.creationSource, + ...(input.scheduledTaskId === undefined + ? {} + : { scheduledTaskId: input.scheduledTaskId }), + ...(input.senderThreadId === undefined ? {} : { senderThreadId: input.senderThreadId }), + ...(input.linkedSender === undefined ? {} : { linkedSender: input.linkedSender }), + ...(input.idleCompaction === undefined ? {} : { idleCompaction: input.idleCompaction }), + id: idAllocator.derive.userTurnItem({ messageId: input.messageId }), + threadId: input.command.threadId, + runId: messageInput.runId, + nodeId: messageInput.nodeId, + providerThreadId: messageInput.providerThreadId, + providerTurnId: messageInput.providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: yield* nextTurnItemOrdinal(input.projection), + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + messageId: input.messageId, + inputIntent: + input.command.type === "queued-message.promote-to-steer" + ? "promoted_queued_to_steer" + : "steer", + text: input.text, + attachments: input.attachments, + ...(input.context ? { context: input.context } : {}), + ...(messageInput.modelSwitch === undefined + ? {} + : { modelSwitch: messageInput.modelSwitch }), + }; + yield* emitEvent({ + type: "message.updated", + threadId: input.command.threadId, + runId: messageInput.runId, + nodeId: messageInput.nodeId, + providerInstanceId: messageInput.providerInstanceId, + occurredAt: now, + payload: message, + }); + yield* emitEvent({ + type: "turn-item.updated", + threadId: input.command.threadId, + runId: messageInput.runId, + nodeId: messageInput.nodeId, + providerInstanceId: messageInput.providerInstanceId, + occurredAt: now, + payload: notificationTurnItem( + turnItem, + message, + yield* notificationTasks(input.command.threadId, message, input.projection.subagents), + ), + }); + }, + ); + + if (input.deliveredInProviderTurnId !== undefined) { + const deliveredTurn = input.projection.providerTurns.find( + (candidate) => candidate.id === input.deliveredInProviderTurnId, + ); + const deliveredAttempt = input.projection.attempts.find( + (candidate) => candidate.id === deliveredTurn?.runAttemptId, + ); + if (deliveredTurn === undefined || deliveredAttempt?.runId !== targetRun.id) { + return yield* new OrchestratorDispatchError({ + commandId: input.command.commandId, + commandType: input.command.type, + cause: `Provider turn ${input.deliveredInProviderTurnId} does not belong to run ${targetRun.id}.`, + }); + } + yield* appendSteeringMessage({ + runId: targetRun.id, + nodeId: deliveredTurn.nodeId, + providerTurnId: deliveredTurn.id, + providerThreadId: deliveredTurn.providerThreadId, + providerInstanceId: deliveredAttempt.providerInstanceId, + }); + return; + } + if (isNativeMaintenanceCommand(input)) { + return yield* new OrchestratorDispatchError({ + commandId: input.command.commandId, + commandType: input.command.type, + cause: + input.text.trim().toLowerCase() === "/compact" + ? "Context compaction must run as a separate turn. Queue it or wait for the active turn to finish." + : "Signing out must run as a separate turn. Queue it or wait for the active turn to finish.", + }); + } + if (parseGoalCommand(input) !== null) { + return yield* new OrchestratorDispatchError({ + commandId: input.command.commandId, + commandType: input.command.type, + cause: + "Goal commands must run as a separate turn. Queue it or wait for the active turn to finish.", + }); + } yield* ensureSteeringMessage(input.command, input); const targetMessage = input.projection.messages.find( (message) => message.id === targetRun.userMessageId, @@ -5556,8 +5701,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } const session = sessionOption.value; - const now = yield* DateTime.now; - const emitEvent = emit(input.events, input.command); const selectionChanged = !modelSelectionsEqual(targetRun.modelSelection, input.modelSelection); const providerInstanceChanged = targetRun.providerInstanceId !== input.modelSelection.instanceId; @@ -5588,101 +5731,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: selectionTransition.reason, }); } - const appendSteeringMessage = Effect.fn("Orchestrator.appendSteeringMessage")( - function* (messageInput: { - readonly runId: OrchestrationV2Run["id"]; - readonly nodeId: OrchestrationV2ExecutionNode["id"]; - readonly providerTurnId: typeof providerTurn.id | null; - readonly providerThreadId: OrchestrationV2ProviderThread["id"]; - readonly providerInstanceId: ProviderInstanceId; - readonly modelSwitch?: OrchestrationV2ModelSwitch | undefined; - }) { - const message: OrchestrationV2ConversationMessage = { - createdBy: input.createdBy, - creationSource: input.creationSource, - ...(input.delegatedCompletion === undefined - ? {} - : { delegatedCompletion: input.delegatedCompletion }), - ...(input.notification === undefined ? {} : { notification: input.notification }), - ...(input.scheduledTaskId === undefined - ? {} - : { scheduledTaskId: input.scheduledTaskId }), - ...(input.senderThreadId === undefined ? {} : { senderThreadId: input.senderThreadId }), - ...(input.linkedSender === undefined ? {} : { linkedSender: input.linkedSender }), - ...(input.idleCompaction === undefined ? {} : { idleCompaction: input.idleCompaction }), - id: input.messageId, - threadId: input.command.threadId, - runId: messageInput.runId, - nodeId: messageInput.nodeId, - role: "user", - text: input.text, - attachments: input.attachments, - ...(input.context ? { context: input.context } : {}), - streaming: false, - createdAt: now, - updatedAt: now, - }; - const turnItem: OrchestrationV2TurnItem = { - createdBy: input.createdBy, - creationSource: input.creationSource, - ...(input.scheduledTaskId === undefined - ? {} - : { scheduledTaskId: input.scheduledTaskId }), - ...(input.senderThreadId === undefined ? {} : { senderThreadId: input.senderThreadId }), - ...(input.linkedSender === undefined ? {} : { linkedSender: input.linkedSender }), - ...(input.idleCompaction === undefined ? {} : { idleCompaction: input.idleCompaction }), - id: idAllocator.derive.userTurnItem({ messageId: input.messageId }), - threadId: input.command.threadId, - runId: messageInput.runId, - nodeId: messageInput.nodeId, - providerThreadId: messageInput.providerThreadId, - providerTurnId: messageInput.providerTurnId, - nativeItemRef: null, - parentItemId: null, - ordinal: yield* nextTurnItemOrdinal(input.projection), - status: "completed", - title: null, - startedAt: now, - completedAt: now, - updatedAt: now, - type: "user_message", - messageId: input.messageId, - inputIntent: - input.command.type === "queued-message.promote-to-steer" - ? "promoted_queued_to_steer" - : "steer", - text: input.text, - attachments: input.attachments, - ...(input.context ? { context: input.context } : {}), - ...(messageInput.modelSwitch === undefined - ? {} - : { modelSwitch: messageInput.modelSwitch }), - }; - yield* emitEvent({ - type: "message.updated", - threadId: input.command.threadId, - runId: messageInput.runId, - nodeId: messageInput.nodeId, - providerInstanceId: messageInput.providerInstanceId, - occurredAt: now, - payload: message, - }); - yield* emitEvent({ - type: "turn-item.updated", - threadId: input.command.threadId, - runId: messageInput.runId, - nodeId: messageInput.nodeId, - providerInstanceId: messageInput.providerInstanceId, - occurredAt: now, - payload: notificationTurnItem( - turnItem, - message, - yield* notificationTasks(input.command.threadId, message, input.projection.subagents), - ), - }); - }, - ); - // A selection the provider applies on its next turn joins the running // turn as a steer and waits for the next turn. Restarting the turn for it // would end the turn's native subagents, which a steer must never do. @@ -6464,6 +6512,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command.dispatchMode, command.deliveryIntent, ); + // Codex and Claude own /goal; elsewhere it is ordinary text that cannot steer. + const parsedGoalCommand = parseGoalCommand(command); + const goalCommand = + parsedGoalCommand === null || + (yield* goalDriverFor(command, modelSelection.instanceId)) === undefined + ? null + : parsedGoalCommand; + let atToolBoundary = false; if (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") { const targetRunId = dispatchMode.targetRunId; const target = projection.runs.find((run) => run.id === targetRunId); @@ -6485,6 +6541,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio turn.status !== "running")))) ) { dispatchMode = { type: "start_immediately" }; + } else if ( + // After Stop the steer must still fail below; queueing it instead would + // start the update on its own once the provider stops. + dispatchMode.type === "steer_active" && + goalCommand === null && + !(yield* stopReachedRun(command, command.threadId, targetRunId)) + ) { + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === target?.providerThreadId, + ); + if (providerThread?.providerSessionId != null) { + const session = yield* providerSessions + .get(providerThread.providerSessionId) + .pipe(mapDispatchError(command)); + if ( + Option.isSome(session) && + session.value.providerSession.capabilities.turns.deliversUpdatesAtToolBoundary === true + ) { + dispatchMode = { type: "queue_after_active" }; + atToolBoundary = true; + } + } } } // Stop leaves the run reading as running until its provider stops. A plain @@ -6492,6 +6570,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // text the agent never receives, and like any user send it releases the // hold; an explicit steer still fails below. if ( + command.createdBy === "user" && command.deliveryIntent === "auto" && (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") && (yield* stopReachedRun(command, command.threadId, dispatchMode.targetRunId)) @@ -6513,13 +6592,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) { dispatchMode = { type: "queue_after_active" }; } - // Native goal commands must not steer an existing turn. - const parsedGoalCommand = parseGoalCommand(command); - const goalCommand = - parsedGoalCommand === null || - (yield* goalDriverFor(command, modelSelection.instanceId)) === undefined - ? null - : parsedGoalCommand; let queuedRunStop: | { readonly runId: RunId; readonly reason: string; readonly skipsHeldQueue?: true } | undefined; @@ -6871,6 +6943,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Thread ${command.threadId} has a pending merge-back transfer; queued merge-back consumption is not implemented yet.`, }); } + const deliverAtToolBoundary = + !freshContext && + goalCommand === null && + !isNativeMaintenanceCommand(command) && + command.scheduledTaskId === undefined && + (atToolBoundary || + command.deliveryIntent === "auto" || + command.deliveryIntent === "steer" || + command.dispatchMode.type === "steer_active" || + command.delegatedCompletion !== undefined || + command.notification?.source.kind === "delegated_task"); const queueProviderThread = activeProviderThread ?? projection.providerThreads.find( @@ -7098,6 +7181,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio updatedAt: now, ...(delegatedCompletion === undefined ? {} : { delegatedCompletion }), ...(command.notification === undefined ? {} : { notification: command.notification }), + ...(deliverAtToolBoundary ? { atToolBoundary: true as const } : {}), }; const emitEvent = emit(events, command); if (targetProviderThread === undefined) { @@ -10537,13 +10621,42 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, ); + const boundaryTaskOwner = Effect.fn("Orchestrator.boundaryTaskOwner")(function* ( + projection: Parameters[0]["projection"], + command: Extract, + queuedRun: OrchestrationV2Run, + targetRunId: RunId, + message: OrchestrationV2ConversationMessage, + ) { + if (queuedRun.delegatedTaskId === undefined) return true; + const parentThreadId = projection.thread.lineage.parentThreadId; + if (parentThreadId === null) return false; + const parent = yield* projectionStore + .getThreadRecords(parentThreadId, ["subagents"]) + .pipe(mapDispatchError(command)); + return canJoinDelegatedBoundary( + queuedRun, + projection.runs.find((run) => run.id === targetRunId), + message, + projection.thread, + parent.subagents, + ); + }); + const dispatchQueuedMessagePromoteToSteer = Effect.fn( "Orchestrator.dispatchQueuedMessagePromoteToSteer", )(function* ( command: Extract, events: Ref.Ref>, effects: Ref.Ref>, - ) { + ): Effect.fn.Return< + | Pick< + Parameters[0], + "cancelUnsettledEffects" | "updateDelegatedCompletionOnlyRunId" + > + | undefined, + OrchestratorV2Error + > { const projection = yield* loadProjectionForCommand( command, [ @@ -10561,6 +10674,27 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // User messages measure a restart's model switch. { turnItemTypes: ["user_message"], messageRoles: ["user"] }, ); + const queuedRun = projection.runs.find((candidate) => candidate.id === command.queuedRunId); + // Offer ids are never reused, so a receipt for an offer that was cancelled or sent again is + // stale: it changes nothing and cannot resurrect the message. + if ( + (command.boundaryOffer?.step === "receipt" || command.boundaryOffer?.step === "withdraw") && + (queuedRun?.status !== "queued" || + queuedRun.boundaryOffer?.id !== command.boundaryOffer.id || + queuedRun.boundaryOffer.targetRunId !== command.targetRunId) + ) { + // This dispatch path requires an event even for an idempotent stale receipt. + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + return; + } if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, @@ -10568,7 +10702,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Thread ${command.threadId} is not active.`, }); } - const queuedRun = projection.runs.find((candidate) => candidate.id === command.queuedRunId); if (queuedRun === undefined || queuedRun.status !== "queued") { return yield* new OrchestratorDispatchError({ commandId: command.commandId, @@ -10598,9 +10731,36 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Queued run ${queuedRun.id} is missing message or execution state.`, }); } - if ( - queuedMessage.delegatedCompletion !== undefined || - queuedMessage.notification !== undefined + // The provider saved the offer in its history, so Stop no longer refuses its receipt. + const receipt = command.boundaryOffer?.step === "receipt" ? queuedRun.boundaryOffer : undefined; + if (command.boundaryOffer?.step === "offer") { + // Stop wins: a held queue or a stopped turn keeps the update queued. + // A follow-up can share completion only with the same parent task owner. + if ( + queuedRun.boundaryOffer !== undefined || + queuedMessage.atToolBoundary !== true || + isNativeMaintenanceCommand(queuedMessage) || + parseGoalCommand(queuedMessage) !== null || + queuedRun.freshContext === true || + !(yield* boundaryTaskOwner( + projection, + command, + queuedRun, + command.targetRunId, + queuedMessage, + )) || + queuedRun.queueHeld === true || + (yield* stopReachedRun(command, command.threadId, command.targetRunId)) + ) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Queued run ${queuedRun.id} cannot join the running turn at a tool boundary.`, + }); + } + } else if ( + command.boundaryOffer === undefined && + (queuedMessage.delegatedCompletion !== undefined || queuedMessage.notification !== undefined) ) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, @@ -10608,9 +10768,93 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: "Automatic completion deliveries cannot be promoted to Steer.", }); } + // The agent may already have read an offered update; only a person may send it again. + if (command.requestedBy === "agent" && queuedRun.boundaryOffer !== undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Queued run ${queuedRun.id} may already have reached the agent; it is not sent again automatically.`, + }); + } const now = yield* DateTime.now; const emitEvent = emit(events, command); + if (command.boundaryOffer?.step === "withdraw") { + const { boundaryOffer: _boundaryOffer, ...unoffered } = queuedRun; + yield* emitEvent({ + type: "run.updated", + threadId: command.threadId, + runId: queuedRun.id, + nodeId: queuedRootNode.id, + providerInstanceId: queuedRun.providerInstanceId, + occurredAt: now, + payload: unoffered, + }); + return; + } + if (receipt !== undefined) { + const updateDelegatedCompletionOnlyRunId = yield* joinQueuedRunToTurn({ + command, + events, + effects, + projection, + queuedRun, + queuedRootNode, + queuedAttempt, + queuedMessage, + receipt, + now, + }); + return updateDelegatedCompletionOnlyRunId === undefined + ? undefined + : { updateDelegatedCompletionOnlyRunId }; + } + const targetRun = projection.runs.find((run) => run.id === command.targetRunId); + if (command.boundaryOffer?.step === "offer") { + // The run stays queued with its message until the provider confirms saving the offer. A + // hook from a replaced or restarted turn cannot offer into the run's newer turn. + const offeredTurnId = command.boundaryOffer.providerTurnId; + const providerTurn = projection.providerTurns.find( + (candidate) => + candidate.id === offeredTurnId && + candidate.runAttemptId === targetRun?.activeAttemptId && + candidate.nodeId === targetRun?.rootNodeId && + candidate.providerThreadId === targetRun?.providerThreadId && + candidate.status === "running", + ); + const targetMessage = projection.messages.find( + (message) => message.id === targetRun?.userMessageId, + ); + if ( + targetRun?.status !== "running" || + providerTurn === undefined || + targetRun.providerInstanceId !== queuedRun.providerInstanceId || + (targetMessage !== undefined && isNativeMaintenanceCommand(targetMessage)) + ) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Provider turn ${offeredTurnId} is not the running turn of run ${command.targetRunId}.`, + }); + } + yield* emitEvent({ + type: "run.updated", + threadId: command.threadId, + runId: queuedRun.id, + nodeId: queuedRootNode.id, + providerInstanceId: queuedRun.providerInstanceId, + occurredAt: now, + payload: { + ...queuedRun, + boundaryOffer: { + id: command.boundaryOffer.id, + targetRunId: targetRun.id, + providerTurnId: providerTurn.id, + }, + }, + }); + return; + } const parsedGoalCommand = parseGoalCommand(queuedMessage); const goalCommand = parsedGoalCommand === null || @@ -10641,7 +10885,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); const stops = isLiveRun(target) && !(yield* stopReachedRun(command, command.threadId, target.id)); - return yield* stopRunForQueuedCommand({ + const cancelUnsettledEffects = yield* stopRunForQueuedCommand({ command, events, effects, @@ -10651,8 +10895,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio reason: "Skipped compaction to send a queued message", now, }); + return cancelUnsettledEffects === undefined ? undefined : { cancelUnsettledEffects }; } - // Skipping compaction starts the message as its own turn; steering would join the current one. if (queuedRun.freshContext === true) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, @@ -10680,7 +10924,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio goalCommandStopsActiveRun(goalCommand, goal?.status ?? null) && (targetMessage === undefined || !isNativeMaintenanceCommand(targetMessage)) && !(yield* stopReachedRun(command, command.threadId, target.id)); - return yield* stopRunForQueuedCommand({ + const cancelUnsettledEffects = yield* stopRunForQueuedCommand({ command, events, effects, @@ -10690,6 +10934,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio reason: "Stopped to apply /goal", now, }); + return cancelUnsettledEffects === undefined ? undefined : { cancelUnsettledEffects }; } if ( (target?.status === "preparing" || @@ -10724,6 +10969,155 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); return; } + if (targetRun?.status === "running" || targetRun?.status === "waiting") { + const targetProviderThread = projection.providerThreads.find( + (candidate) => candidate.id === targetRun.providerThreadId, + ); + const session = + targetProviderThread?.providerSessionId == null + ? Option.none() + : yield* providerSessions + .get(targetProviderThread.providerSessionId) + .pipe(mapDispatchError(command)); + if ( + Option.isSome(session) && + session.value.providerSession.capabilities.turns.deliversUpdatesAtToolBoundary === true + ) { + // Stop wins, and unrelated task owners keep their own turns. + if ( + (queuedRun.queueHeld === true && queuedRun.boundaryOffer === undefined) || + !(yield* boundaryTaskOwner( + projection, + command, + queuedRun, + command.targetRunId, + queuedMessage, + )) || + (yield* stopReachedRun(command, command.threadId, targetRun.id)) + ) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Queued run ${queuedRun.id} cannot wait for the running turn's next tool boundary.`, + }); + } + if (queuedRun.boundaryOffer !== undefined) { + const { boundaryOffer: _boundaryOffer, ...retryRun } = queuedRun; + yield* emitEvent({ + type: "run.updated", + threadId: command.threadId, + runId: queuedRun.id, + nodeId: queuedRootNode.id, + providerInstanceId: queuedRun.providerInstanceId, + occurredAt: now, + payload: { ...retryRun, queueHeld: false }, + }); + } + yield* emitEvent({ + type: "message.updated", + threadId: command.threadId, + runId: queuedRun.id, + nodeId: queuedRootNode.id, + providerInstanceId: queuedRun.providerInstanceId, + occurredAt: now, + payload: { ...queuedMessage, atToolBoundary: true, updatedAt: now }, + }); + return; + } + } + yield* joinQueuedRunToTurn({ + command, + events, + effects, + projection, + queuedRun, + queuedRootNode, + queuedAttempt, + queuedMessage, + receipt: undefined, + now, + }); + }); + + const joinQueuedRunToTurn = Effect.fn("Orchestrator.joinQueuedRunToTurn")(function* (input: { + readonly command: Extract< + OrchestrationV2Command, + { readonly type: "queued-message.promote-to-steer" } + >; + readonly events: Ref.Ref>; + readonly effects: Ref.Ref>; + readonly projection: Parameters[0]["projection"]; + readonly queuedRun: OrchestrationV2Run; + readonly queuedRootNode: OrchestrationV2ExecutionNode; + readonly queuedAttempt: OrchestrationV2RunAttempt; + readonly queuedMessage: OrchestrationV2ConversationMessage; + readonly receipt: NonNullable | undefined; + readonly now: DateTime.Utc; + }) { + const { command, events, effects, projection, queuedRun, queuedRootNode, queuedAttempt } = + input; + const { queuedMessage, receipt, now } = input; + const emitEvent = emit(events, command); + const targetRunId = receipt?.targetRunId ?? command.targetRunId; + const targetRun = projection.runs.find((run) => run.id === targetRunId); + let joinedRun = queuedRun; + if (receipt !== undefined && queuedRun.delegatedTaskId !== undefined) { + const parentThreadId = projection.thread.lineage.parentThreadId; + if (parentThreadId !== null) { + const parent = yield* projectionStore + .getThreadRecords(parentThreadId, ["subagents", "nodes", "turnItems"], { + turnItemTypes: ["subagent"], + }) + .pipe(mapDispatchError(command)); + const task = parent.subagents.find((task) => task.id === queuedRun.delegatedTaskId); + if (task !== undefined) { + const summary = `Delivered into running run ${targetRunId}; its task reports the result.`; + const scope = { + threadId: parentThreadId, + ...(task.runId === null ? {} : { runId: task.runId }), + nodeId: task.id, + occurredAt: now, + }; + yield* emitEvent({ + ...scope, + type: "subagent.updated", + payload: { + ...task, + status: "completed", + result: summary, + completedAt: now, + updatedAt: now, + completionDelivery: { state: "disposed", observedByRunId: null }, + }, + }); + const node = parent.nodes.find((node) => node.id === task.id); + if (node !== undefined) + yield* emitEvent({ + ...scope, + type: "node.updated", + payload: { ...node, status: "completed", completedAt: now }, + }); + const item = parent.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === task.id, + ); + if (item?.type === "subagent") + yield* emitEvent({ + ...scope, + type: "turn-item.updated", + payload: { + ...item, + status: "completed", + result: summary, + completedAt: now, + updatedAt: now, + }, + }); + // The cancelled queue row must not settle this task again during recovery. + const { delegatedTaskId: _taskId, ...unbound } = queuedRun; + joinedRun = unbound; + } + } + } yield* emitEvent({ type: "run.updated", threadId: command.threadId, @@ -10732,7 +11126,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio providerInstanceId: queuedRun.providerInstanceId, occurredAt: now, payload: { - ...queuedRun, + ...joinedRun, status: "cancelled", queuePosition: null, completedAt: now, @@ -10770,8 +11164,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio events, effects, projection, - modelSelection: queuedRun.modelSelection, - targetRunId: command.targetRunId, + // An update joining at a tool boundary keeps the running turn's model; a + // selection change could otherwise restart the turn from inside its hook. + modelSelection: + receipt === undefined + ? queuedRun.modelSelection + : (targetRun?.modelSelection ?? projection.thread.modelSelection), + ...(queuedMessage.delegatedCompletion === undefined + ? {} + : { delegatedCompletion: queuedMessage.delegatedCompletion }), + ...(queuedMessage.notification === undefined + ? {} + : { notification: queuedMessage.notification }), + ...(receipt === undefined ? {} : { deliveredInProviderTurnId: receipt.providerTurnId }), + targetRunId, messageId: queuedMessage.id, text: queuedMessage.text, attachments: queuedMessage.attachments, @@ -10792,6 +11198,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : { idleCompaction: queuedMessage.idleCompaction }), forceRestart: false, }); + if (receipt !== undefined && queuedMessage.delegatedCompletion !== undefined) { + // Accept in the same commit: the cancelled queued run would otherwise + // read as a failed delivery and reserve a duplicate notification. + return yield* dispatchNotificationAccepted( + { + type: "notification.delivery.accept", + commandId: command.commandId, + threadId: command.threadId, + messageId: queuedMessage.id, + }, + events, + ); + } }); const dispatchQueuedRunReorder = Effect.fn("Orchestrator.dispatchQueuedRunReorder")(function* ( @@ -10813,6 +11232,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const movingMessage = projection.messages.find( (candidate) => candidate.id === moving.userMessageId, ); + if (moving.boundaryOffer !== undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "A message already offered to the agent cannot be reordered.", + }); + } if ( movingMessage?.delegatedCompletion !== undefined || movingMessage?.notification !== undefined @@ -10853,6 +11279,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio moving, ...withoutMoving.slice(beforeIndex), ]; + if ( + reordered.some( + (run, index) => run.boundaryOffer !== undefined && queuedRuns[index]?.id !== run.id, + ) + ) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Reordering cannot move a message already offered to the agent.", + }); + } const now = yield* DateTime.now; const emitEvent = emit(events, command); yield* Effect.forEach( @@ -10957,6 +11394,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Run ${command.runId} is not queued.`, }); } + if (queuedRun.boundaryOffer !== undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "This message may already have reached the agent; cancel or send it instead.", + }); + } const queuedMessage = projection.messages.find( (candidate) => candidate.id === queuedRun.userMessageId, ); @@ -13365,7 +13809,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const delivery = cohort?.delivery ?? null; const deliveryRun = completionDeliveryRun(input.parentProjection, delivery); if (delivery !== null) { - if (deliveryRun?.status === "queued" && deliveryRun.userMessageId === delivery.messageId) { + if ( + deliveryRun?.status === "queued" && + deliveryRun.boundaryOffer === undefined && + deliveryRun.userMessageId === delivery.messageId + ) { const taskIds = Array.from(new Set([...delivery.taskIds, input.task.id])); const message = completionDeliveryMessage(input.parentProjection, delivery); const nextCohort = { @@ -14371,13 +14819,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "thread.stop": cancelUnsettledEffects = yield* dispatchThreadStop(command, events, effects); break; - case "queued-message.promote-to-steer": - cancelUnsettledEffects = yield* dispatchQueuedMessagePromoteToSteer( - command, - events, - effects, - ); + case "queued-message.promote-to-steer": { + const promotion = yield* dispatchQueuedMessagePromoteToSteer(command, events, effects); + cancelUnsettledEffects = promotion?.cancelUnsettledEffects; + updateDelegatedCompletionOnlyRunId = promotion?.updateDelegatedCompletionOnlyRunId; break; + } case "queued-run.start": case "queue.resume": { const projection = yield* loadProjectionForCommand( @@ -14404,8 +14851,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ( selectedRun === undefined || selectedMessage === undefined || - selectedMessage.delegatedCompletion !== undefined || - selectedMessage.notification !== undefined || + ((selectedMessage.delegatedCompletion !== undefined || + selectedMessage.notification !== undefined) && + selectedRun.boundaryOffer === undefined) || + (command.type === "queued-run.start" && + command.requestedBy === "agent" && + selectedRun.boundaryOffer !== undefined) || projection.runs.some(isActiveRun) ) { return yield* new OrchestratorDispatchError({ @@ -14433,6 +14884,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const now = yield* DateTime.now; const queued = projection.runs.filter((run) => run.status === "queued"); for (const run of queued) { + const { boundaryOffer: _boundaryOffer, ...retryRun } = run; yield* emit( events, command, @@ -14442,7 +14894,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio runId: run.id, providerInstanceId: run.providerInstanceId, occurredAt: now, - payload: { ...run, queueHeld: false }, + payload: { + ...(run.id === selectedRunId ? retryRun : run), + queueHeld: false, + }, }); } if (queued.length === 0) { @@ -14816,7 +15271,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ({ event }) => event.type === "subagent.updated" && event.payload.completionDelivery?.state === "claimed", - )) + )) || + (command.type === "queued-message.promote-to-steer" && + plan.updateDelegatedCompletionOnlyRunId !== undefined) ) { yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.threadId)); } @@ -14885,8 +15342,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? withSubagentParentLock(command.threadId, dispatch) : dispatch, ); - // Answering a message-mode question can also start a follow-up. - return yield* command.type === "runtime-request.respond" + // Answers can start follow-ups; a boundary receipt can settle their parent task. + return yield* command.type === "runtime-request.respond" || + command.type === "queued-message.promote-to-steer" ? withSubagentParentLock(command.threadId, dispatch) : dispatch; }); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 797427ede42ee..8b09fd89fb808 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -170,6 +170,7 @@ export type ProjectionStoreV2Error = typeof ProjectionStoreV2Error.Type; export type ProjectionRecoveryKind = | "queued-runs" + | "boundary-offers" | "runtime" | "restart-continuations" | "subagent-results" @@ -4444,6 +4445,12 @@ export const layer: Layer.Layer = ) { const candidates = (() => { switch (kind) { + case "boundary-offers": + return sql` + SELECT thread_id FROM orchestration_v2_projection_runs + WHERE status = 'queued' + AND json_type(payload_json, '$.boundaryOffer') = 'object' + `; case "queued-runs": return sql` SELECT thread_id FROM orchestration_v2_projection_runs @@ -5824,7 +5831,8 @@ export const layer: Layer.Layer = WHERE thread.thread_id = ${threadId} AND thread.archived_at IS NULL AND thread.deleted_at IS NULL AND EXISTS (SELECT 1 FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} AND status = 'queued' - AND json_extract(payload_json, '$.queueHeld') IS NOT 1) + AND json_extract(payload_json, '$.queueHeld') IS NOT 1 + AND json_extract(payload_json, '$.boundaryOffer') IS NULL) AND NOT EXISTS (SELECT 1 FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} AND status IN ('preparing', 'starting', 'running', 'waiting'))`; diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index 1827700ca16c5..643349ee4c3f2 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -6,6 +6,7 @@ import { ProviderDriverKind, ProviderInstanceId, ProviderThreadId, + ProviderTurnId, RunId, ThreadId, type OrchestrationV2ThreadProjection, @@ -14,6 +15,7 @@ import { import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; @@ -618,4 +620,191 @@ describe("ProviderContinuationService", () => { ); }); }); + it.effect("an aborted boundary hook commits no further promotions", () => + Effect.gen(function* () { + // Three queued agent updates. The fake dispatch stands in for the SQLite + // commit: the first promotion commits, the second blocks until the hook + // is aborted, and the third must never run. + const runs = [1, 2, 3].map((ordinal) => ({ + id: RunId.make(`run-boundary-${ordinal}`), + status: "queued", + ordinal, + queuePosition: ordinal, + userMessageId: MessageId.make(`message-boundary-${ordinal}`), + })); + const messages = runs.map((run) => ({ + id: run.userMessageId, + text: "update", + atToolBoundary: true, + attachments: [], + createdBy: "agent", + creationSource: "mcp", + })); + const committed: Array = []; + const withdrawn: Array = []; + const blocked = yield* Deferred.make(); + const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => + Effect.succeed({ + ...projection, + runs, + messages, + } as unknown as OrchestrationV2ThreadProjection), + dispatch: (command) => + Effect.gen(function* () { + if (command.type !== "queued-message.promote-to-steer") return {} as never; + if (command.boundaryOffer?.step === "withdraw") { + withdrawn.push(command.queuedRunId); + return { sequence: 1, storedEvents: [] }; + } + if (committed.length === 1) { + yield* Deferred.succeed(blocked, undefined); + return yield* Effect.never; + } + committed.push(command.queuedRunId); + return { sequence: 1, storedEvents: [] }; + }), + }); + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests.pipe( + Effect.provide( + ProviderContinuationService.layer.pipe( + Layer.provideMerge( + Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, layerThreads), + ), + ), + ), + ); + // The Claude adapter runs the claim with the hook's AbortSignal, which + // interrupts the claim fiber when Claude aborts the hook. + const claim = yield* requests.claimAgentMessages!({ + threadId, + runId: RunId.make("run-boundary-active"), + providerTurnId: ProviderTurnId.make("provider-turn-boundary-active"), + }).pipe(Effect.forkScoped); + yield* Deferred.await(blocked); + yield* Fiber.interrupt(claim); + assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(claim))); + assert.deepEqual(committed, [runs[0]!.id]); + assert.deepEqual(withdrawn, [runs[0]!.id, runs[1]!.id]); + }).pipe(Effect.scoped), + ); + + it.effect("a receipt that fails to commit fails the transcript mirror so Claude retries it", () => + Effect.gen(function* () { + const commandIds: Array = []; + const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({ + dispatch: (command) => + Effect.gen(function* () { + commandIds.push(command.commandId); + if (commandIds.length === 1) { + return yield* new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "database is locked", + }); + } + return { sequence: 1, storedEvents: [] }; + }), + }); + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests.pipe( + Effect.provide( + ProviderContinuationService.layer.pipe( + Layer.provideMerge( + Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, layerThreads), + ), + ), + ), + ); + const receipt = requests.acknowledgeAgentMessages!({ + threadId, + runId: RunId.make("run-boundary-active"), + offers: [{ id: "offer-1", queuedRunId: RunId.make("run-boundary-1") }], + }); + assert.isTrue(Exit.isFailure(yield* Effect.exit(receipt))); + yield* receipt; + // The retry reuses the receipt's command id, so a commit that did land replays. + assert.deepEqual(commandIds, [ + "tool-boundary-receipt:offer-1", + "tool-boundary-receipt:offer-1", + ]); + }).pipe(Effect.scoped), + ); + + it.effect.each(["offer", "read"] as const)( + "returns the earlier boundary offers and withdraws the failed %s before later messages", + (stage) => + Effect.gen(function* () { + const runs = [1, 2, 3].map((ordinal) => ({ + id: RunId.make(`run-partial-${ordinal}`), + status: "queued", + ordinal, + queuePosition: ordinal, + userMessageId: MessageId.make(`message-partial-${ordinal}`), + })); + const messages = runs.map((run, index) => ({ + id: run.userMessageId, + text: `update ${index}`, + atToolBoundary: true, + attachments: [], + createdBy: "agent", + creationSource: "mcp", + })); + const commands: Array<{ readonly runId: string; readonly step: string | undefined }> = []; + let reads = 0; + const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => + Effect.gen(function* () { + reads++; + if (stage === "read" && reads === 4) { + return yield* new Orchestrator.OrchestratorDispatchError({ + commandId: CommandId.make("failed-read"), + commandType: "queued-message.promote-to-steer", + cause: "database is locked", + }); + } + return { + ...projection, + runs, + messages, + } as unknown as OrchestrationV2ThreadProjection; + }), + dispatch: (command) => + Effect.gen(function* () { + if (command.type !== "queued-message.promote-to-steer") return {} as never; + commands.push({ runId: command.queuedRunId, step: command.boundaryOffer?.step }); + if (stage === "offer" && commands.length === 2) { + return yield* new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "database is locked", + }); + } + return { sequence: 1, storedEvents: [] }; + }), + }); + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests.pipe( + Effect.provide( + ProviderContinuationService.layer.pipe( + Layer.provideMerge( + Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, layerThreads), + ), + ), + ), + ); + const claimed = yield* requests.claimAgentMessages!({ + threadId, + runId: RunId.make("active-run"), + providerTurnId: ProviderTurnId.make("active-turn"), + }); + assert.deepEqual( + claimed.map((offer) => offer.messageId), + [runs[0]!.userMessageId], + ); + assert.deepEqual(commands, [ + { runId: runs[0]!.id, step: "offer" }, + { runId: runs[1]!.id, step: "offer" }, + { runId: runs[1]!.id, step: "withdraw" }, + ]); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index 01f6e0acb989d..a2bab5ce34fa6 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -1,5 +1,6 @@ import { CommandId, + parseGoalCommand, type MessageId, type OrchestrationV2ThreadProjection, } from "@t3tools/contracts"; @@ -7,13 +8,18 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; +import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import * as Stream from "effect/Stream"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import type * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts"; +import { canJoinDelegatedBoundary } from "./DelegatedBoundaryJoin.ts"; +import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts"; +import { isNativeMaintenanceCommand } from "./CommandPolicy.ts"; const CONTINUATION_MESSAGE_TEXT = "Background task completed."; @@ -290,6 +296,165 @@ export const layer = Layer.effectDiscard( }, ); + const claimAgentMessages: ProviderContinuationRequests.ClaimAgentMessages = Effect.fn( + "ProviderContinuationService.claimAgentMessages", + )( + function* (input) { + const unhanded: Array = []; + return yield* Effect.gen(function* () { + const { runs, thread } = yield* threads.getThreadRecords(input.threadId, ["runs"], { + runStatuses: ["queued", "running"], + }); + const parent = + !runs.some((run) => run.status === "queued" && run.delegatedTaskId !== undefined) || + thread.lineage.parentThreadId === null + ? undefined + : yield* threads.getThreadRecords(thread.lineage.parentThreadId, ["subagents"]); + const target = runs.find((run) => run.id === input.runId); + if (runs.length === 0) return []; + const { messages } = yield* threads.getThreadRecords(input.threadId, ["messages"], { + messageIds: runs.map((run) => run.userMessageId), + }); + const offers: Array = []; + for (const run of queuedRunsInDeliveryOrder({ runs, messages })) { + const message = messages.find((row) => row.id === run.userMessageId); + if ( + message?.atToolBoundary !== true || + run.boundaryOffer !== undefined || + run.queueHeld === true || + run.freshContext === true || + !canJoinDelegatedBoundary(run, target, message, thread, parent?.subagents ?? []) || + isNativeMaintenanceCommand(message) || + parseGoalCommand(message) !== null + ) + continue; + const offerId = yield* randomUuidV4; + unhanded.push({ id: offerId, queuedRunId: run.id }); + const offered = yield* Effect.gen(function* () { + yield* threads.dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(`tool-boundary-offer:${offerId}`), + threadId: input.threadId, + queuedRunId: run.id, + targetRunId: input.runId, + boundaryOffer: { id: offerId, step: "offer", providerTurnId: input.providerTurnId }, + }); + const current = yield* threads.getThreadRecords(input.threadId, ["messages"], { + messageIds: [message.id], + }); + const committed = current.messages.find((row) => row.id === message.id)!; + return { + id: offerId, + queuedRunId: run.id, + messageId: committed.id, + text: projectComposerContextForProvider({ + text: committed.text, + records: committed.context?.records ?? [], + }), + attachments: committed.attachments, + createdBy: committed.createdBy, + creationSource: committed.creationSource, + }; + }).pipe( + Effect.catch((cause) => + Effect.gen(function* () { + yield* Effect.logWarning( + "orchestration-v2.provider-continuation.agent-message-offer-failed", + { + threadId: input.threadId, + offerId, + cause, + }, + ); + // No transport has received this offer; undo it before returning the earlier batch. + yield* threads + .dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(`tool-boundary-withdraw:${offerId}`), + threadId: input.threadId, + queuedRunId: run.id, + targetRunId: input.runId, + boundaryOffer: { id: offerId, step: "withdraw" }, + }) + .pipe( + Effect.catch((withdrawCause) => + Effect.logWarning( + "orchestration-v2.provider-continuation.agent-message-withdraw-failed", + { + threadId: input.threadId, + offerId, + cause: withdrawCause, + }, + ), + ), + ); + return undefined; + }), + ), + ); + if (offered === undefined) break; + offers.push(offered); + } + return offers; + }).pipe( + Effect.onInterrupt(() => + settleAgentMessages("withdraw")({ + threadId: input.threadId, + runId: input.runId, + offers: unhanded, + }), + ), + ); + }, + (effect, input) => + effect.pipe( + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.provider-continuation.agent-message-claim-failed", { + threadId: input.threadId, + cause, + }).pipe(Effect.as([])), + ), + ), + ); + + const settleAgentMessages = ( + step: "receipt" | "withdraw", + ): ProviderContinuationRequests.AcknowledgeAgentMessages => + Effect.fn(`ProviderContinuationService.${step}AgentMessages`)(function* (input) { + for (const offer of input.offers) { + yield* threads + .dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(`tool-boundary-${step}:${offer.id}`), + threadId: input.threadId, + queuedRunId: offer.queuedRunId, + targetRunId: input.runId, + boundaryOffer: { id: offer.id, step }, + }) + .pipe( + Effect.tapError((cause) => + Effect.logWarning( + "orchestration-v2.provider-continuation.agent-message-settlement-failed", + { + threadId: input.threadId, + offerId: offer.id, + step, + cause, + }, + ), + ), + Effect.orDie, + ); + } + }); + if (requests.serveAgentMessages !== undefined) { + yield* requests.serveAgentMessages({ + claim: claimAgentMessages, + acknowledge: settleAgentMessages("receipt"), + withdraw: settleAgentMessages("withdraw"), + }); + } + yield* requests.take.pipe( Effect.flatMap((request) => dispatchContinuation(request).pipe( diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 623fddb491108..4ed747aa2eed4 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -646,6 +646,7 @@ export const layer: Layer.Layer< ]; } case "background_work.updated": + case "boundary_messages.updated": return []; case "command_output.delta": case "prompt_suggestion.updated": diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 0e584bcf0c0a5..1f8c89b12d810 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -52,6 +52,7 @@ import type { PendingOrchestrationEffectV2 } from "./EffectOutbox.ts"; import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; @@ -193,6 +194,26 @@ it.effect("routes shared-runtime events only to their owning root run", () => assert.isFalse( RunExecutionService.routeProviderEvent(terminalEvent, second, secondAfterTurn)[0], ); + for (const step of ["reserve", "receipt"] as const) { + const boundary: ProviderAdapter.ProviderAdapterV2Event = { + type: "boundary_messages.updated", + driver, + threadId: first.threadId, + runId: first.runId, + providerTurnId: firstTurnId, + step, + offers: [], + }; + assert.isTrue(RunExecutionService.routeProviderEvent(boundary, first, firstAfterTurn)[0]); + assert.isFalse(RunExecutionService.routeProviderEvent(boundary, second, secondAfterTurn)[0]); + assert.isFalse( + RunExecutionService.routeProviderEvent( + { ...boundary, providerTurnId: ProviderTurnId.make("provider-turn:replacement") }, + first, + firstAfterTurn, + )[0], + ); + } }), ); @@ -3421,6 +3442,153 @@ it.effect("keeps completed runs completed when pull request refresh fails", () = }), ); +it.effect( + "reserves boundary message slots after committed tools and before a delayed receipt", + () => + Effect.gen(function* () { + const commitStarted = yield* Deferred.make(); + const releaseCommit = yield* Deferred.make(); + const positions = new Map(); + const sequence: string[] = []; + const messageIds = [MessageId.make("boundary:first"), MessageId.make("boundary:second")]; + const messageItemIds = messageIds.map((messageId) => + IdAllocator.derive.userTurnItem({ messageId }), + ); + let firstCommit = true; + let toolsCommitted = false; + const coordinator = yield* ProviderCommitBatch.make( + (work) => + Effect.gen(function* () { + if (firstCommit) { + firstCommit = false; + yield* Deferred.succeed(commitStarted, undefined); + yield* Deferred.await(releaseCommit); + } + yield* work; + toolsCommitted = true; + }), + true, + ); + const allocate = ( + input: Parameters[0], + ) => + Effect.sync(() => { + if (messageItemIds.includes(input.turnItemId)) { + assert.isTrue(toolsCommitted); + } + const prior = positions.get(input.turnItemId); + if (prior !== undefined) return prior; + const ordinal = positions.size + 1; + positions.set(input.turnItemId, ordinal); + sequence.push(input.turnItemId); + return ordinal; + }); + yield* captureRootRunTermination({ + key: "boundary-placement", + shouldFinalizeRun: () => Effect.succeed(true), + consumeProviderEvents: coordinator.consume, + turnItemPositions: { + allocate, + reserveBoundary: (input) => + Effect.gen(function* () { + assert.equal( + input.providerTurnId, + ProviderTurnId.make("provider-turn:boundary-placement"), + ); + yield* allocate(input); + }), + normalize: (item) => + allocate({ threadId: item.threadId, turnItemId: item.id, runId: item.runId }).pipe( + Effect.map((ordinal) => ({ ...item, ordinal })), + ), + }, + ingestNormalized: ({ event }) => + event.type === "turn_item.updated" + ? allocate({ + threadId: event.turnItem.threadId, + turnItemId: event.turnItem.id, + runId: event.turnItem.runId, + }).pipe(Effect.as([])) + : Effect.succeed([]), + events: (ids) => { + const offers = messageIds.map((messageId, index) => ({ + id: `offer:${index}`, + queuedRunId: RunId.make(`queued:${index}`), + messageId, + })); + const boundary = { + type: "boundary_messages.updated", + driver, + threadId: ids.threadId, + runId: ids.runId, + providerTurnId: ids.rootProviderTurnId, + offers, + } as const; + return Stream.make( + backgroundTurnItemEvent(ids, "command_execution", "completed", 1), + { ...boundary, step: "reserve" }, + { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("assistant-after-boundary"), + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + providerThreadId: ids.providerThreadId, + providerTurnId: ids.rootProviderTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "completed", + title: "Reply", + startedAt: DateTime.makeUnsafe("2026-11-08T15:00:00Z"), + completedAt: DateTime.makeUnsafe("2026-11-08T15:00:00Z"), + updatedAt: DateTime.makeUnsafe("2026-11-08T15:00:00Z"), + type: "assistant_message", + messageId: MessageId.make("assistant-after-boundary"), + text: "After both updates", + streaming: false, + }, + }, + { ...boundary, step: "receipt" }, + rootTerminalEvent(ids, "completed"), + ); + }, + afterStart: () => + Effect.gen(function* () { + yield* Deferred.await(commitStarted); + assert.equal(positions.size, 0); + yield* Deferred.succeed(releaseCommit, undefined); + }), + }).pipe( + Effect.provideService(ProviderContinuationRequests.ProviderContinuationRequests, { + offer: () => Effect.void, + take: Effect.never, + acknowledgeAgentMessages: ({ offers }) => + Effect.sync(() => { + assert.deepEqual( + offers.map((offer) => offer.id), + ["offer:0", "offer:1"], + ); + assert.deepEqual( + messageItemIds.map((itemId) => positions.get(itemId)), + [2, 3], + ); + assert.equal(positions.get(TurnItemId.make("assistant-after-boundary")), 4); + sequence.push("receipt"); + }), + }), + ); + assert.deepEqual(sequence, [ + "turn-item:boundary-placement", + ...messageItemIds, + "assistant-after-boundary", + "receipt", + ]); + }).pipe(Effect.scoped), +); + function captureRootRunTermination(input: { readonly key: string; readonly shouldFinalizeRun: () => Effect.Effect; @@ -3448,6 +3616,8 @@ function captureRootRunTermination(input: { readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"]; readonly appContext?: ReturnType; readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"]; + readonly turnItemPositions?: TurnItemPositionStore.TurnItemPositionStoreV2["Service"]; + readonly consumeProviderEvents?: EventSink.EventSinkV2["Service"]["consumeProviderEvents"]; readonly beforeTerminalWrite?: ( request: Parameters[0], ) => Effect.Effect; @@ -3492,8 +3662,13 @@ function captureRootRunTermination(input: { forThread: () => input.appContext ?? Effect.succeed([]), }), Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), - layerTurnItemPositionsTest, + input.turnItemPositions === undefined + ? layerTurnItemPositionsTest + : Layer.succeed(TurnItemPositionStore.TurnItemPositionStoreV2, input.turnItemPositions), mockEventSink({ + ...(input.consumeProviderEvents === undefined + ? {} + : { consumeProviderEvents: input.consumeProviderEvents }), write: (payload) => Effect.gen(function* () { for (const event of payload.events) { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 612a130e019aa..0eb6714147f88 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -46,6 +46,7 @@ import * as CheckpointService from "./CheckpointService.ts"; import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; @@ -522,6 +523,13 @@ export function routeProviderEvent( ]; case "background_work.updated": return [state.ownedProviderTurns.has(event.providerTurnId), state]; + case "boundary_messages.updated": + return [ + event.threadId === input.threadId && + event.runId === input.runId && + event.providerTurnId === state.rootProviderTurnId, + state, + ]; case "command_output.delta": case "prompt_suggestion.updated": // The session manager serves these to clients directly; runs never see them. @@ -628,6 +636,7 @@ export const layer: Layer.Layer< const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext; const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity; const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; + const continuationRequests = yield* ProviderContinuationRequests.ProviderContinuationRequests; const finalizationObserver = yield* RunFinalizationService.RunFinalizationObserver; const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics; const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor; @@ -1581,6 +1590,32 @@ export const layer: Layer.Layer< accepted: boolean; }) { if (!accepted) return; + if (event.type === "boundary_messages.updated") { + for (const offer of event.offers) { + yield* turnItemPositions.reserveBoundary({ + threadId: event.threadId, + turnItemId: IdAllocator.derive.userTurnItem({ messageId: offer.messageId }), + runId: event.runId, + runOrdinal: input.run.ordinal, + queuedRunId: offer.queuedRunId, + offerId: offer.id, + providerTurnId: event.providerTurnId, + }); + } + if (event.step === "receipt") { + // The control-event barrier commits earlier items before a saved receipt joins its slot. + const acknowledge = continuationRequests.acknowledgeAgentMessages; + if (acknowledge === undefined) { + return yield* Effect.die("Native message receipt handler is not installed."); + } + yield* acknowledge({ + threadId: event.threadId, + runId: event.runId, + offers: event.offers, + }); + } + return; + } if ( event.type === "provider_turn.updated" && event.providerTurn.runAttemptId === input.attempt.id && diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 8d784a91f59cb..efec46281a5b9 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -11,8 +11,10 @@ import { ProviderInstanceId, ProviderThreadId, ProviderTurnId, + RunAttemptId, ThreadId, TurnItemId, + type ChatAttachment, type ModelSelection, type OrchestrationV2DomainEvent, } from "@t3tools/contracts"; @@ -23,7 +25,10 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; import * as EventSink from "./EventSink.ts"; @@ -31,9 +36,6 @@ import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; -import * as ServerSettings from "../serverSettings.ts"; -import * as SqlitePersistence from "../persistence/Sqlite.ts"; -import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; @@ -76,6 +78,7 @@ it.effect.each( ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: timing !== "without native steering", activeSteeringInterruptsTools: timing === "with interrupting native steering", + deliversUpdatesAtToolBoundary: timing === "with interrupting native steering", }, }; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { @@ -493,6 +496,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* canSteer = true, interruptTurn: Effect.Effect = Effect.void, steeringAcceptsAttachments = true, + activeSteeringInterruptsTools = false, nativeGoals?: boolean, onSteer: ( turn: ProviderAdapter.ProviderAdapterV2SteerInput, @@ -511,6 +515,8 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* ...CodexProviderCapabilitiesV2.turns, supportsActiveSteering: canSteer, supportsSteeringByInterruptRestart: canRestart, + activeSteeringInterruptsTools, + deliversUpdatesAtToolBoundary: activeSteeringInterruptsTools, activeSteeringAcceptsAttachments: steeringAcceptsAttachments, ...(nativeGoals === undefined ? {} : { supportsNativeGoals: nativeGoals }), }, @@ -617,10 +623,16 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* }; }), }; - const layer = ProviderReplayHarness.layerWithRegistry( + const databaseLayer = SqlitePersistence.layerMemory; + const layerRuntime = ProviderReplayHarness.layerWithRegistry( { name }, ProviderAdapterRegistry.layerSingle(adapter), - { runEffectWorker: false }, + { runEffectWorker: false, runContinuationWorker: activeSteeringInterruptsTools, databaseLayer }, + ); + const layer = ThreadManagementService.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(databaseLayer))), + Layer.provideMerge(layerRuntime), ); const startFirstTurn = Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -1391,6 +1403,7 @@ it.effect.each(["complete", "stop-before-decline", "stop-after-decline", "queued true, Effect.void, true, + false, undefined, (turn) => Effect.suspend(() => beforeDecline).pipe( @@ -2499,18 +2512,41 @@ it.effect("queues a plain send behind the stopped queue and releases its hold", ), ); -const goalHarness = Effect.fn("goalHarness")(function* (name: string, nativeGoals?: boolean) { +const goalHarness = Effect.fn("goalHarness")(function* ( + name: string, + activeSteeringInterruptsTools = false, + nativeGoals?: boolean, +) { const harness = yield* nextTurnSelectionHarness( name, false, true, Effect.void, true, + activeSteeringInterruptsTools, nativeGoals, ); - const send = (id: string, text: string, delivery: "queue" | "auto", freshContext = false) => + const send = ( + id: string, + text: string, + delivery: "queue" | "auto" | "agent_auto" | "agent_steer", + freshContext = false, + ) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; + if (delivery === "agent_auto" || delivery === "agent_steer") { + return yield* (yield* ThreadManagementService.ThreadManagementService).sendToThread({ + projectId: ProjectId.make(`project:${name}`), + commandId: CommandId.make(id), + threadId: ThreadId.make(`thread:${name}`), + messageId: MessageId.make(`message:${id}`), + text, + attachments: [], + mode: delivery === "agent_auto" ? "auto" : "steer", + createdBy: "agent", + creationSource: "mcp", + }); + } yield* orchestrator.dispatch({ type: "message.dispatch", commandId: CommandId.make(id), @@ -2554,6 +2590,7 @@ it.effect.each([undefined, false, true])( Effect.gen(function* () { const { steered, layer, startFirstTurn, send, read } = yield* goalHarness( "goal-steer", + false, nativeGoals, ); yield* Effect.gen(function* () { @@ -2758,26 +2795,38 @@ it.effect("reports a restart that stops the running turn as restarted", () => ), ); -it.effect("queues a goal clear without stopping work when no goal drives it", () => - Effect.scoped( - Effect.gen(function* () { - const { steered, layer, startFirstTurn, send, read } = yield* goalHarness("goal-clear"); - yield* Effect.gen(function* () { - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const threadId = yield* startFirstTurn; - yield* send("queued", "queued", "queue"); - yield* send("clear", "/goal clear", "auto"); - yield* worker.drain(); +it.effect.each(["auto", "agent_auto", "agent_steer"] as const)( + "queues a goal clear from %s without stopping work when no goal drives it", + (delivery) => + Effect.scoped( + Effect.gen(function* () { + const { steered, layer, startFirstTurn, send, read } = yield* goalHarness( + "goal-clear", + true, + ); + yield* Effect.gen(function* () { + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + yield* send("queued", "queued", "queue"); + yield* send("clear", "/goal clear", delivery); + yield* worker.drain(); - assert.deepEqual(steered, []); - assert.deepEqual(yield* read(threadId), { - queue: ["queued", "/goal clear"], - started: ["first"], - stopRequested: false, - }); - }).pipe(Effect.provide(layer)); - }), - ), + const projection = yield* (yield* Orchestrator.OrchestratorV2).getThreadProjection( + threadId, + ); + assert.isNotTrue( + projection.messages.find((message) => message.id === MessageId.make("message:clear")) + ?.atToolBoundary, + ); + assert.deepEqual(steered, []); + assert.deepEqual(yield* read(threadId), { + queue: ["queued", "/goal clear"], + started: ["first"], + stopRequested: false, + }); + }).pipe(Effect.provide(layer)); + }), + ), ); it.effect("promotes a queued goal by stopping the running turn", () => @@ -2959,11 +3008,17 @@ it.effect("stops a turn that is still starting when a goal is sent into it", () ), ); -it.effect("stops the running turn when a goal clear ends the goal driving it", () => +it.effect.each( + (["auto", "agent_auto", "agent_steer"] as const).flatMap((delivery) => + ["/goal clear", "/goal ship the fix"].map((text) => ({ delivery, text })), + ), +)("stops the running turn when $delivery sends $text to an active goal", ({ delivery, text }) => Effect.scoped( Effect.gen(function* () { - const { events, layer, startFirstTurn, send, read, steered } = - yield* goalHarness("goal-clear-active"); + const { events, layer, startFirstTurn, send, read, steered } = yield* goalHarness( + "goal-clear-active", + true, + ); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; @@ -2988,13 +3043,13 @@ it.effect("stops the running turn when a goal clear ends the goal driving it", ( }, }); yield* Fiber.join(goalSeen); - yield* send("clear", "/goal clear", "auto"); + yield* send("clear", text, delivery); yield* worker.drain(); assert.deepEqual(steered, []); assert.deepEqual(yield* read(threadId), { queue: [], - started: ["first", "/goal clear"], + started: ["first", text], stopRequested: true, }); }).pipe(Effect.provide(layer)); @@ -3080,3 +3135,739 @@ it.effect("a side chat that opens with /goal gets its context on the next questi }), ), ); + +it.effect.each(["auto", "steer"] as const)( + "rejects an agent %s update after Stop instead of queueing it behind the stopped run", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `agent-update-after-stop-${mode}`, + false, + true, + Effect.void, + true, + true, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop"), + threadId, + runId: started[0]!.runId, + holdQueue: true, + }); + // Nothing was queued before Stop, so a queued update would have no hold + // and would start on its own once the provider stops. + const error = yield* sendUpdate(threadId, "agent", mode).pipe(Effect.flip); + yield* worker.drain(); + + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(error._tag, "OrchestratorDispatchError"); + assert.include(String(error.cause), "stopping"); + assert.deepEqual(steered, []); + assert.equal(started.length, 1); + assert.isFalse(projection.runs.some((run) => run.status === "queued")); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect.each(["receipt", "withdraw", "cancel", "retry", "resume"] as const)( + "keeps native boundary ownership through Stop and %s", + (settlement) => + Effect.scoped( + Effect.gen(function* () { + const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `native-boundary-${settlement}`, + false, + true, + Effect.void, + true, + true, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const sent = yield* sendUpdate(threadId, "user", "auto", [imageAttachment]); + const turn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!; + assert.deepEqual( + yield* requests.claimAgentMessages!({ + threadId, + runId: first.runId, + providerTurnId: ProviderTurnId.make("replaced-turn"), + }), + [], + ); + let offers = yield* requests.claimAgentMessages!({ + threadId, + runId: first.runId, + providerTurnId: turn.id, + }); + assert.lengthOf(offers, 1); + assert.deepEqual(offers[0]?.attachments, [imageAttachment]); + assert.equal(offers[0]?.messageId, "message:update"); + assert.equal(offers[0]?.createdBy, "user"); + assert.equal(offers[0]?.text, "progress update"); + assert.deepEqual( + yield* requests.claimAgentMessages!({ + threadId, + runId: first.runId, + providerTurnId: turn.id, + }), + [], + ); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === sent.run.id, + )?.status, + "queued", + ); + const edit = yield* orchestrator + .dispatch({ + type: "queued-run.edit", + commandId: CommandId.make("edit-offered"), + threadId, + runId: sent.run.id, + text: "changed", + }) + .pipe(Effect.flip); + assert.equal(edit._tag, "OrchestratorCommandRejectedError"); + const reordered = yield* orchestrator + .dispatch({ + type: "queued-run.reorder", + commandId: CommandId.make("reorder-offered"), + threadId, + runId: sent.run.id, + beforeRunId: null, + }) + .pipe(Effect.flip); + assert.equal(reordered._tag, "OrchestratorDispatchError"); + if (settlement === "receipt") { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("late-queued"), + threadId, + messageId: MessageId.make("message:late-queued"), + text: "later", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + const later = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === "message:late-queued", + )!; + const crossed = yield* orchestrator + .dispatch({ + type: "queued-run.reorder", + commandId: CommandId.make("cross-offered"), + threadId, + runId: later.id, + beforeRunId: sent.run.id, + }) + .pipe(Effect.flip); + assert.equal(crossed._tag, "OrchestratorDispatchError"); + const originalOffers = offers; + const agentRetry = yield* orchestrator + .dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make("agent-reoffer"), + threadId, + queuedRunId: sent.run.id, + targetRunId: first.runId, + requestedBy: "agent", + }) + .pipe(Effect.flip); + assert.equal(agentRetry._tag, "OrchestratorDispatchError"); + for (const commandId of ["human-reoffer", "human-reoffer-again"]) { + yield* orchestrator.dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(commandId), + threadId, + queuedRunId: sent.run.id, + targetRunId: first.runId, + }); + } + assert.isUndefined( + (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === sent.run.id, + )?.boundaryOffer, + ); + yield* requests.acknowledgeAgentMessages!({ + threadId, + runId: first.runId, + offers: originalOffers, + }); + offers = yield* requests.claimAgentMessages!({ + threadId, + runId: first.runId, + providerTurnId: turn.id, + }); + assert.lengthOf(offers, 1); + assert.notEqual(offers[0]!.id, originalOffers[0]!.id); + } + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop"), + threadId, + runId: first.runId, + holdQueue: true, + }); + const ended = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "interrupted", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* worker.drain(); + yield* Fiber.join(ended); + yield* worker.drain(); + if (settlement === "withdraw") { + yield* requests.withdrawAgentMessages!({ threadId, runId: first.runId, offers }); + } else if (settlement === "cancel") { + yield* orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("cancel"), + threadId, + runId: sent.run.id, + }); + } else if (settlement === "retry") { + const refused = yield* orchestrator + .dispatch({ + type: "queued-run.start", + commandId: CommandId.make("agent-retry"), + threadId, + runId: sent.run.id, + requestedBy: "agent", + }) + .pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorDispatchError"); + yield* orchestrator.dispatch({ + type: "queued-run.start", + commandId: CommandId.make("retry"), + threadId, + runId: sent.run.id, + }); + yield* worker.drain(); + } else if (settlement === "resume") { + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("resume"), + threadId, + }); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + } + if (settlement !== "resume" && settlement !== "withdraw") { + yield* requests.acknowledgeAgentMessages!({ threadId, runId: first.runId, offers }); + yield* requests.acknowledgeAgentMessages!({ threadId, runId: first.runId, offers }); + } + const settled = yield* orchestrator.getThreadProjection(threadId); + const run = settled.runs.find((run) => run.id === sent.run.id)!; + const message = settled.messages.find((message) => message.id === "message:update")!; + if (settlement === "receipt") { + assert.equal(message.runId, first.runId); + const item = settled.turnItems.find( + (item) => item.type === "user_message" && item.messageId === message.id, + )!; + assert.equal(item.providerTurnId, turn.id); + assert.equal(item.nodeId, turn.nodeId); + assert.equal(message.createdBy, "user"); + assert.deepEqual(message.attachments, [imageAttachment]); + } else { + assert.equal(message.runId, sent.run.id); + } + if (settlement === "resume" || settlement === "withdraw") { + assert.equal(run.status, "queued"); + assert.equal( + run.boundaryOffer?.id, + settlement === "resume" ? offers[0]!.id : undefined, + ); + assert.equal(run.queueHeld, settlement === "withdraw"); + } + if (settlement === "retry") assert.notEqual(run.status, "queued"); + assert.deepEqual(steered, []); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect( + "keeps forty simultaneous human and agent boundary updates in committed queue order", + () => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "native-boundary-mixed-batch", + false, + true, + Effect.void, + true, + true, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const threadId = yield* startFirstTurn; + const projectId = (yield* orchestrator.getThreadProjection(threadId)).thread.projectId; + const first = started[0]!; + yield* Effect.forEach( + Array.from({ length: 40 }, (_, index) => index), + (index) => + threads.sendToThread({ + projectId, + threadId, + commandId: CommandId.make(`mixed:${index}`), + messageId: MessageId.make(`message:mixed:${index}`), + text: `mixed update ${index}`, + attachments: index === 17 ? [imageAttachment] : [], + mode: index % 3 === 0 ? "steer" : "auto", + createdBy: index % 2 === 0 ? "user" : "agent", + creationSource: index % 2 === 0 ? "web" : "mcp", + }), + { concurrency: "unbounded", discard: true }, + ); + const pending = yield* orchestrator.getThreadProjection(threadId); + const expected = pending.runs + .filter((run) => run.status === "queued") + .toSorted((left, right) => left.queuePosition! - right.queuePosition!) + .map((run) => pending.messages.find((message) => message.id === run.userMessageId)!); + assert.lengthOf(expected, 40); + const turn = pending.providerTurns[0]!; + const offers = yield* requests.claimAgentMessages!({ + threadId, + runId: first.runId, + providerTurnId: turn.id, + }); + assert.deepEqual( + offers.map((offer) => offer.messageId), + expected.map((message) => message.id), + ); + assert.deepEqual( + offers.map((offer) => offer.createdBy), + expected.map((message) => message.createdBy), + ); + assert.deepEqual( + offers.map((offer) => offer.text), + expected.map((message) => message.text), + ); + assert.deepEqual( + offers.map((offer) => offer.attachments), + expected.map((message) => message.attachments), + ); + assert.lengthOf(new Set(offers.map((offer) => offer.id)), 40); + const now = yield* DateTime.now; + const itemBase = { + threadId, + runId: first.runId, + nodeId: first.rootNodeId, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + nativeItemRef: null, + parentItemId: null, + ordinal: 0, + status: "completed" as const, + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + }; + const replyId = TurnItemId.make("native-batch-reply"); + const replyCommitted = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "turn-item.updated" && event.payload.id === replyId, + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const received = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "message.updated" && + event.payload.id === offers.at(-1)!.messageId && + event.payload.runId === first.runId, + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Queue.offer(events, { + type: "turn_item.updated", + driver, + turnItem: { + ...itemBase, + id: TurnItemId.make("native-batch-tool"), + type: "command_execution", + input: "synthetic noisy tool", + exitCode: 0, + output: Array.from({ length: 100 }, (_, index) => `noise ${index}`).join("\n"), + }, + }); + yield* Queue.offer(events, { + type: "boundary_messages.updated", + driver, + threadId, + runId: first.runId, + providerTurnId: turn.id, + step: "reserve", + offers, + }); + yield* Queue.offer(events, { + type: "turn_item.updated", + driver, + turnItem: { + ...itemBase, + id: replyId, + type: "assistant_message", + messageId: MessageId.make("message:native-batch-reply"), + text: "Received all forty human and agent updates in order.", + streaming: false, + }, + }); + yield* Fiber.join(replyCommitted); + yield* Queue.offer(events, { + type: "boundary_messages.updated", + driver, + threadId, + runId: first.runId, + providerTurnId: turn.id, + step: "receipt", + offers, + }); + yield* Fiber.join(received); + yield* requests.acknowledgeAgentMessages!({ threadId, runId: first.runId, offers }); + const settled = yield* orchestrator.getThreadProjection(threadId); + assert.isFalse(settled.runs.some((run) => run.status === "queued")); + assert.deepEqual( + settled.turnItems + .filter((item) => item.runId === first.runId) + .toSorted((left, right) => left.ordinal - right.ordinal) + .map((item) => (item.type === "user_message" ? item.messageId : item.id)), + [ + MessageId.make("message:first"), + TurnItemId.make("native-batch-tool"), + ...offers.map((offer) => offer.messageId), + replyId, + ], + ); + assert.deepEqual( + settled.turnItems + .filter((item) => item.type === "user_message" && item.runId === first.runId) + .toSorted((left, right) => left.ordinal - right.ordinal) + .map((item) => (item.type === "user_message" ? item.messageId : null)), + [MessageId.make("message:first"), ...offers.map((offer) => offer.messageId)], + ); + for (const original of expected) { + const delivered = settled.messages.find((message) => message.id === original.id)!; + assert.equal(delivered.runId, first.runId); + assert.equal(delivered.createdBy, original.createdBy); + assert.equal(delivered.creationSource, original.creationSource); + assert.equal(delivered.text, original.text); + assert.deepEqual(delivered.attachments, original.attachments); + } + assert.deepEqual(steered, []); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect( + "a late native receipt uses the saved attempt after the run changes provider and node", + () => + Effect.scoped( + Effect.gen(function* () { + const { started, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "native-boundary-restarted-identity", + false, + true, + Effect.void, + true, + true, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const sent = yield* sendUpdate(threadId, "agent", "auto"); + const original = yield* orchestrator.getThreadProjection(threadId); + const originalRun = original.runs.find((run) => run.id === first.runId)!; + const originalTurn = original.providerTurns[0]!; + const originalAttempt = original.attempts.find( + (attempt) => attempt.id === originalRun.activeAttemptId, + )!; + const originalNode = original.nodes.find((node) => node.id === originalRun.rootNodeId)!; + const originalThread = original.providerThreads.find( + (thread) => thread.id === originalRun.providerThreadId, + )!; + const offers = yield* requests.claimAgentMessages!({ + threadId, + runId: first.runId, + providerTurnId: originalTurn.id, + }); + assert.lengthOf(offers, 1); + const providerInstanceId = ProviderInstanceId.make("replacement-provider"); + const providerThreadId = ProviderThreadId.make("replacement-thread"); + const providerTurnId = ProviderTurnId.make("replacement-turn"); + const rootNodeId = NodeId.make("replacement-root"); + const activeAttemptId = RunAttemptId.make("replacement-attempt"); + const now = yield* DateTime.now; + yield* sink.write({ + events: [ + { + id: EventId.make("replacement-thread"), + type: "provider-thread.updated", + threadId, + providerInstanceId, + occurredAt: now, + payload: { + ...originalThread, + id: providerThreadId, + providerInstanceId, + providerSessionId: null, + nativeThreadRef: { driver, nativeId: "replacement-native", strength: "strong" }, + }, + }, + { + id: EventId.make("replacement-node"), + type: "node.updated", + threadId, + runId: first.runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: now, + payload: { + ...originalNode, + id: rootNodeId, + rootNodeId, + providerThreadId, + providerTurnId, + }, + }, + { + id: EventId.make("replacement-attempt"), + type: "run-attempt.updated", + threadId, + runId: first.runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: now, + payload: { + ...originalAttempt, + id: activeAttemptId, + attemptOrdinal: originalAttempt.attemptOrdinal + 1, + rootNodeId, + providerThreadId, + providerTurnId, + providerInstanceId, + }, + }, + { + id: EventId.make("replacement-turn"), + type: "provider-turn.updated", + threadId, + runId: first.runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: now, + payload: { + ...originalTurn, + id: providerTurnId, + nodeId: rootNodeId, + providerThreadId, + runAttemptId: activeAttemptId, + }, + }, + { + id: EventId.make("replacement-run"), + type: "run.updated", + threadId, + runId: first.runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: now, + payload: { + ...originalRun, + rootNodeId, + activeAttemptId, + providerThreadId, + providerInstanceId, + modelSelection: { ...originalRun.modelSelection, instanceId: providerInstanceId }, + }, + }, + ], + }); + const receipt = yield* orchestrator.dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make("late-native-receipt"), + threadId, + queuedRunId: sent.run.id, + targetRunId: first.runId, + boundaryOffer: { id: offers[0]!.id, step: "receipt" }, + }); + const settled = yield* orchestrator.getThreadProjection(threadId); + const message = settled.messages.find((message) => message.id === "message:update")!; + const item = settled.turnItems.find( + (item) => item.type === "user_message" && item.messageId === message.id, + )!; + assert.equal(message.nodeId, originalNode.id); + assert.equal(item.nodeId, originalNode.id); + assert.equal(item.providerTurnId, originalTurn.id); + assert.equal(item.providerThreadId, originalThread.id); + const savedMessage = receipt.storedEvents.find( + (event) => event.event.type === "message.updated", + ); + assert.equal(savedMessage?.event.providerInstanceId, originalAttempt.providerInstanceId); + }).pipe(Effect.provide(layer)); + }), + ), +); + +const sendUpdate = Effect.fn("sendUpdate")(function* ( + threadId: ThreadId, + createdBy: "agent" | "user", + mode: "auto" | "steer", + attachments: ReadonlyArray = [], +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const result = yield* threads.sendToThread({ + projectId: (yield* orchestrator.getThreadProjection(threadId)).thread.projectId, + commandId: CommandId.make("update"), + threadId, + messageId: MessageId.make("message:update"), + text: "progress update", + attachments, + mode, + createdBy, + creationSource: createdBy === "agent" ? "mcp" : "web", + }); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + return result; +}); + +const imageAttachment: ChatAttachment = { + type: "image", + id: "attachment-1", + name: "screen.png", + mimeType: "image/png", + sizeBytes: 1, +}; + +it.effect.each([ + { mode: "auto", attachments: [] }, + { mode: "steer", attachments: [] }, + // Steering an image natively would still cancel the running tool. + { mode: "auto", attachments: [imageAttachment] }, + { mode: "steer", attachments: [imageAttachment] }, +] as const)( + "queues an agent $mode update behind a turn whose steering cancels tools (attachments=$attachments.length)", + ({ mode, attachments }) => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `agent-update-queued-${mode}-${attachments.length}`, + false, + true, + Effect.void, + true, + true, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const result = yield* sendUpdate(threadId, "agent", mode, attachments); + assert.equal(result.delivery, "queued"); + assert.deepEqual(steered, []); + const pending = yield* orchestrator.getThreadProjection(threadId); + assert.equal(pending.runs.find((run) => run.id === first.runId)?.status, "running"); + + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "waiting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const turn = pending.providerTurns[0]!; + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "completed", completedAt: yield* DateTime.now }, + }); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(completed); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + assert.deepEqual( + started.map((input) => input.message.messageId), + [MessageId.make("message:first"), MessageId.make("message:update")], + ); + assert.deepEqual(steered, []); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect.each([ + { createdBy: "user", activeSteeringInterruptsTools: false }, + { createdBy: "agent", activeSteeringInterruptsTools: false }, +] as const)( + "steers an update from $createdBy when steering is allowed (interruptsTools=$activeSteeringInterruptsTools)", + ({ createdBy, activeSteeringInterruptsTools }) => + Effect.scoped( + Effect.gen(function* () { + const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `update-steered-${createdBy}-${activeSteeringInterruptsTools}`, + false, + true, + Effect.void, + true, + activeSteeringInterruptsTools, + ); + yield* Effect.gen(function* () { + const threadId = yield* startFirstTurn; + const result = yield* sendUpdate( + threadId, + createdBy, + createdBy === "agent" ? "auto" : "steer", + ); + assert.equal(result.delivery, "steered"); + assert.deepEqual(steered, ["progress update"]); + assert.equal(started.length, 1); + }).pipe(Effect.provide(layer)); + }), + ), +); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 8a0d2d5df6001..243ab6e95ce5d 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -959,13 +959,21 @@ const make = Effect.gen(function* () { // so a cut run is not the answer while its continuation is pending. // Runs per matching event and is already covered by the waitForThread span. const outcome = Effect.fnUntraced(function* (runs: ReadonlyArray) { - const current = runs.find((candidate) => candidate.id === selectedRun.id); - if (current === undefined) { + const selected = runs.find((candidate) => candidate.id === selectedRun.id); + if (selected === undefined) { return yield* new ThreadManagementRunNotFoundError({ threadId: input.threadId, runId: selectedRun.id, }); } + // A boundary receipt moves the message into its receiving run before cancelling the queue row. + const takenBy = + selected.status === "cancelled" && selected.startedAt === null + ? (yield* getProjectThreadRecords(input, ["messages"], { + messageIds: [selected.userMessageId], + })).messages[0]?.runId + : undefined; + const current = runs.find((candidate) => candidate.id === takenBy) ?? selected; const run = latestRestartContinuation(runs, current); const pending = isTerminalRunStatus(run.status) && (yield* orchestrator.restartContinuationPending(run)); @@ -1040,6 +1048,7 @@ const make = Effect.gen(function* () { untilDelegatedWorkSettled || stored.event.type !== "run.updated" || stored.event.payload.id === selectedRun.id || + isTerminalRunStatus(stored.event.payload.status) || stored.event.payload.restartContinuationOfRunId !== undefined, ), ), diff --git a/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts b/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts index ea3a70bdf22af..48f6875f58d67 100644 --- a/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts +++ b/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts @@ -1,5 +1,6 @@ import { assert, it } from "@effect/vitest"; -import { RunId, ThreadId, TurnItemId } from "@t3tools/contracts"; +import { MessageId, ProviderTurnId, RunId, ThreadId, TurnItemId } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as SqlClient from "effect/sql/SqlClient"; @@ -57,6 +58,114 @@ it.effect("reuses an item's position without attempting another allocation inser }).pipe(Effect.provide(layerTest)), ); +it.effect("fences boundary reservations and restores an explicitly retried message's own run", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const positions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; + const threadId = ThreadId.make("thread:boundary-position"); + const turnItemId = TurnItemId.make("item:boundary-message"); + const queuedRunId = RunId.make("run:queued"); + const runId = RunId.make("run:receiving"); + const providerTurnId = ProviderTurnId.make("turn:receiving"); + const offerId = "boundary-offer"; + const now = DateTime.makeUnsafe("2026-11-08T15:00:00Z"); + yield* sql` + INSERT INTO orchestration_v2_projection_runs ( + run_id, thread_id, ordinal, provider, provider_thread_id, status, + requested_at, completed_at, payload_json + ) VALUES ( + ${queuedRunId}, ${threadId}, 5, 'claudeAgent', NULL, 'queued', + ${DateTime.formatIso(now)}, NULL, + ${JSON.stringify({ boundaryOffer: { id: offerId, targetRunId: runId, providerTurnId } })} + ) + `; + const original = yield* positions.allocate({ + threadId, + turnItemId, + runId: queuedRunId, + runOrdinal: 5, + }); + const reserve = { + threadId, + turnItemId, + runId, + runOrdinal: 1, + queuedRunId, + offerId, + providerTurnId, + }; + for (const stale of [ + { ...reserve, offerId: "stale-offer" }, + { ...reserve, providerTurnId: ProviderTurnId.make("turn:replacement") }, + { ...reserve, runId: RunId.make("run:replacement") }, + ]) { + yield* positions.reserveBoundary(stale); + assert.equal(yield* positions.allocate({ threadId, turnItemId, runId }), original); + } + const toolPosition = yield* positions.allocate({ + threadId, + turnItemId: TurnItemId.make("item:preceding-tool"), + runId, + runOrdinal: 1, + }); + yield* positions.reserveBoundary(reserve); + const reserved = yield* positions.allocate({ threadId, turnItemId, runId }); + assert.equal(reserved, toolPosition + 2); + assert.equal( + yield* positions.allocate({ + threadId, + turnItemId: TurnItemId.make("item:following-assistant"), + runId, + runOrdinal: 1, + }), + reserved + 1, + ); + yield* positions.reserveBoundary(reserve); + assert.equal(yield* positions.allocate({ threadId, turnItemId, runId }), reserved); + const replacementOfferId = "replacement-boundary-offer"; + yield* sql` + UPDATE orchestration_v2_projection_runs + SET payload_json = json_set(payload_json, '$.boundaryOffer.id', ${replacementOfferId}) + WHERE run_id = ${queuedRunId} + `; + yield* positions.reserveBoundary({ ...reserve, offerId: replacementOfferId }); + const replacementPosition = yield* positions.allocate({ threadId, turnItemId, runId }); + assert.isAbove(replacementPosition, reserved + 1); + yield* positions.reserveBoundary({ ...reserve, offerId: replacementOfferId }); + yield* positions.reserveBoundary(reserve); + assert.equal(yield* positions.allocate({ threadId, turnItemId, runId }), replacementPosition); + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'starting' WHERE run_id = ${queuedRunId}`; + const retried = yield* positions.normalize( + { + id: turnItemId, + threadId, + runId: queuedRunId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: reserved, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + messageId: MessageId.make("message:retried"), + inputIntent: "queued_turn", + text: "Try again", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + 6, + ); + assert.equal(retried.ordinal, 6_000_001); + yield* positions.reserveBoundary(reserve); + assert.equal(yield* positions.allocate({ threadId, turnItemId, runId }), retried.ordinal); + }).pipe(Effect.provide(layerTest)), +); it.effect("places a run-start item ahead of the run's items, then appends once taken", () => Effect.gen(function* () { const positions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; diff --git a/apps/server/src/orchestration-v2/TurnItemPositionStore.ts b/apps/server/src/orchestration-v2/TurnItemPositionStore.ts index 95eb9cdc2415f..355b9d0bae8c0 100644 --- a/apps/server/src/orchestration-v2/TurnItemPositionStore.ts +++ b/apps/server/src/orchestration-v2/TurnItemPositionStore.ts @@ -1,4 +1,10 @@ -import { OrchestrationV2TurnItem, RunId, ThreadId, TurnItemId } from "@t3tools/contracts"; +import { + OrchestrationV2TurnItem, + ProviderTurnId, + RunId, + ThreadId, + TurnItemId, +} from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -25,6 +31,15 @@ export interface TurnItemPositionStoreV2Shape { // `run_start` takes the run's unused first slot, ahead of its items, and appends once it is taken. readonly placement?: "append" | "run_start"; }) => Effect.Effect; + readonly reserveBoundary: (input: { + readonly threadId: ThreadId; + readonly turnItemId: TurnItemId; + readonly runId: RunId; + readonly runOrdinal?: number; + readonly queuedRunId: RunId; + readonly offerId: string; + readonly providerTurnId: ProviderTurnId; + }) => Effect.Effect; readonly normalize: ( item: OrchestrationV2TurnItem, runOrdinal?: number, @@ -41,7 +56,7 @@ export const layer: Layer.Layer[0] & { + readonly moveToRun?: boolean | "always"; }) { const existing = yield* sql<{ readonly ordinal: number }>` SELECT ordinal @@ -57,7 +75,7 @@ export const layer: Layer.Layer= lowerBound && + existing[0].ordinal <= upperBound + ) { + return existing[0].ordinal; + } + yield* sql` + UPDATE orchestration_v2_turn_item_positions + SET ordinal = ( + SELECT COALESCE(MAX(ordinal), ${lowerBound}) + 1 + FROM orchestration_v2_turn_item_positions + WHERE thread_id = ${threadId} + AND ordinal >= ${lowerBound} + AND ordinal <= ${upperBound} + ) + WHERE thread_id = ${threadId} AND turn_item_id = ${turnItemId} + AND (${moveToRun === "always" ? 1 : 0} = 1 OR ordinal < ${lowerBound} OR ordinal > ${upperBound}) + `; + } else { + yield* sql` INSERT INTO orchestration_v2_turn_item_positions (thread_id, turn_item_id, ordinal) SELECT ${threadId}, @@ -87,6 +126,7 @@ export const layer: Layer.Layer` SELECT ordinal FROM orchestration_v2_turn_item_positions @@ -115,6 +155,37 @@ export const layer: Layer.Layer` + SELECT run_id FROM orchestration_v2_projection_runs + WHERE thread_id = ${input.threadId} AND run_id = ${input.queuedRunId} + AND status = 'queued' + AND json_extract(payload_json, '$.boundaryOffer.id') = ${input.offerId} + AND json_extract(payload_json, '$.boundaryOffer.targetRunId') = ${input.runId} + AND json_extract(payload_json, '$.boundaryOffer.providerTurnId') = ${input.providerTurnId} + `; + if (offered.length === 0) return; + const tokenId = TurnItemId.make(`turn-item:boundary-offer:${input.offerId}`); + const reserved = yield* sql<{ readonly ordinal: number }>` + SELECT ordinal FROM orchestration_v2_turn_item_positions + WHERE thread_id = ${input.threadId} AND turn_item_id = ${tokenId} + `; + if (reserved.length > 0) return; + // The invisible token distinguishes a retry in the same run from a duplicate receipt. + yield* allocate({ ...input, turnItemId: tokenId }); + yield* allocate({ ...input, moveToRun: "always" }); + }, + (effect, { threadId, turnItemId }) => + effect.pipe( + sql.withTransaction, + Effect.mapError((cause) => + isTurnItemPositionStoreError(cause) + ? cause + : new TurnItemPositionStoreError({ threadId, turnItemId, cause }), + ), + ), + ), // Per-event normalization shares the enclosing orchestration span. normalize: Effect.fnUntraced(function* (item, runOrdinal) { return yield* allocate({ @@ -122,6 +193,8 @@ export const layer: Layer.Layer (item.ordinal === ordinal ? item : { ...item, ordinal }))); }), }); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index d83f0603d7f7c..8809a2b0bd76a 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -3,6 +3,7 @@ import * as GuestShareService from "../guest/GuestShareService.ts"; import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; import * as Layer from "effect/Layer"; +import * as Effect from "effect/Effect"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as OrchestrationCommandReceipts from "../persistence/OrchestrationCommandReceipts.ts"; import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts"; @@ -44,6 +45,7 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as CommandOutputHub from "./CommandOutputHub.ts"; import * as ProviderRuntimeRecoveryService from "./ProviderRuntimeRecoveryService.ts"; +import { makeClaudeBoundaryReceiptRecovery } from "./ClaudeBoundaryReceipts.ts"; import * as ProviderSwitchService from "./ProviderSwitchService.ts"; import * as ProviderTurnControlService from "./ProviderTurnControlService.ts"; import * as ProviderTurnStartService from "./ProviderTurnStartService.ts"; @@ -160,6 +162,7 @@ const layerRunExecutionServiceProvided = RunExecutionService.layer.pipe( IdAllocator.layer, layerProviderEventIngestorProvided, TurnItemPositionStore.layer, + ProviderContinuationRequests.layer, ThreadCommandExecutor.layer, ), ), @@ -347,7 +350,29 @@ const layerEffectExecutorProvided = EffectWorker.layerExecutor.pipe( const layerEffectWorkerProvided = EffectWorker.layer.pipe( Layer.provide(Layer.merge(layerStores, layerEffectExecutorProvided)), ); -const layerProviderRuntimeRecoveryProvided = ProviderRuntimeRecoveryService.layer.pipe( +const layerProviderRuntimeRecoveryProvided = Layer.effect( + ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService, + Effect.gen(function* () { + const recovery = yield* ProviderRuntimeRecoveryService.make; + const recoverReceipts = yield* makeClaudeBoundaryReceiptRecovery; + return { + ...recovery, + recover: recovery.recover.pipe( + Effect.tap(() => + recoverReceipts.pipe( + Effect.mapError( + (cause) => + new ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryError({ + operation: "reconcile", + cause, + }), + ), + ), + ), + ), + }; + }), +).pipe( Layer.provide( Layer.mergeAll( layerEffectWorkerProvided, @@ -355,6 +380,8 @@ const layerProviderRuntimeRecoveryProvided = ProviderRuntimeRecoveryService.laye layerEventSink, IdAllocator.layer, ProjectionStore.layer, + layerThreadManagementProvided, + TurnItemPositionStore.layer, ), ), ); diff --git a/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts index 8886daad3e017..9a4299afc5821 100644 --- a/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts @@ -8,6 +8,7 @@ import { type ProviderReplayEntry, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import { ClaudeOrchestratorReplayHarness } from "../Adapters/ClaudeAdapterV2.testkit.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; @@ -67,6 +68,7 @@ describe("Claude Agent SDK replay fixtures", () => { commands: materialized.commands, steps: materialized.steps, projectionThreadIds: materialized.projectionThreadIds, + runtimePolicyOverride: { cwd: yield* checkpointWorkspace("subagent-read-only-child") }, }; yield* Effect.gen(function* () { const result = yield* runOrchestratorV2Scenario(scenario); @@ -106,7 +108,7 @@ describe("Claude Agent SDK replay fixtures", () => { provideDeterministicTestRuntime, Effect.scoped, ); - }), + }).pipe(Effect.scoped), ); const afterRootFixture = findReplayFixture("claude_background_subagent_after_root"); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index c742f83d37586..9605913b1e951 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -428,6 +428,7 @@ export function layerWithRegistry( IdAllocator.layer, layerProviderEventIngestorProvided, layerServerSettings, + layerContinuationRequests, layerStores, ThreadCommandExecutor.layer, ), @@ -562,6 +563,7 @@ export function layerWithRegistry( layerEffectWorkerProvided, layerEventSinkProvided, layerContinuationWorkerProvided, + layerContinuationRequests, layerCommandOutputHubProvided, ).pipe( Layer.provide(ProviderTurnStartServiceTestkit.layer), diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts index 8ee31f2b14460..13faf3828179c 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts @@ -70,7 +70,6 @@ import { grokPromptErrorInput } from "./grok_prompt_error/input.ts"; import { assertGrokPromptErrorOutput } from "./grok_prompt_error/output.ts"; import { grokSubagentLineageInput } from "./grok_subagent_lineage/input.ts"; import { assertGrokSubagentLineageOutput } from "./grok_subagent_lineage/output.ts"; -import { assertClaudeMessageSteeringOutput } from "./message_steering/claude_output.ts"; import { assertMessageSteeringOutput } from "./message_steering/codex_output.ts"; import { assertCursorMessageSteeringOutput } from "./message_steering/cursor_output.ts"; import { assertGrokMessageSteeringOutput } from "./message_steering/grok_output.ts"; @@ -1774,12 +1773,6 @@ export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray ({ run: { id: runId, userMessageId: `${runId}:message` }, + delivery: "queued", + deliveryLabel: "Queued", + automatic: false, + editBlockedReason: null, text, attachments: [], message: message(`${runId}:message`, { text, ...(fields as object) }), @@ -211,6 +223,10 @@ it("stays inside the timeline scroller instead of growing the chat column", asyn queuedRuns: [ { run: { id: "run:queued", userMessageId: "message:queued" }, + delivery: "queued", + deliveryLabel: "Queued", + automatic: false, + editBlockedReason: null, text: "Later", attachments: [], }, diff --git a/apps/web/src/components/chat/QueuedRunsControl.tsx b/apps/web/src/components/chat/QueuedRunsControl.tsx index 66e023a756f6b..2e5cfd833e218 100644 --- a/apps/web/src/components/chat/QueuedRunsControl.tsx +++ b/apps/web/src/components/chat/QueuedRunsControl.tsx @@ -1,5 +1,8 @@ import { scopeThreadRef } from "@t3tools/client-runtime/environment"; -import { deriveThreadQueueWorkflowState } from "@t3tools/client-runtime/state/thread-workflows"; +import { + deriveThreadQueueWorkflowState, + UNCONFIRMED_DELIVERY_HINT, +} from "@t3tools/client-runtime/state/thread-workflows"; import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; import { isAttachmentOnlyBootstrapPrompt, @@ -115,9 +118,10 @@ export function QueuedRunsControl({ const queued = workflow?.queuedRuns ?? []; const activeRun = workflow?.activeRun ?? null; // A fresh message starts its own turn: it may skip compaction, but not steer into a turn. - const canSendRun = (freshContext: boolean) => + const canSendRun = (freshContext: boolean, automatic: boolean) => workflow?.canSendNow === true || - (workflow?.canPromoteToSteer === true && + (!automatic && + workflow?.canPromoteToSteer === true && (!freshContext || workflow.promoteSkipsCompaction === true)); const queuedImageUrlById = new Map( queuedImageAttachmentUrls.flatMap((url, index) => { @@ -129,18 +133,22 @@ export function QueuedRunsControl({ (message) => message.inputIntent === "queued_turn" && !acknowledgedMessageIds.has(message.id), ); const items = [ - ...queued.map(({ run, text, attachments, context, message }, serverIndex) => ({ - key: run.id, - message, - runId: run.id, - messageId: run.userMessageId, - freshContext: run.freshContext === true, + ...queued.map((entry, serverIndex) => ({ + key: entry.run.id, + message: entry.message, + runId: entry.run.id, + messageId: entry.run.userMessageId, + freshContext: entry.run.freshContext === true, + delivery: entry.delivery, + deliveryLabel: entry.deliveryLabel, + automatic: entry.automatic, + editBlockedReason: entry.editBlockedReason, serverIndex, - text, + text: entry.text, // A guest's message keeps the guest's words; the owner can cancel it, not rewrite it. - editable: context?.guest === undefined, - attachments, - thumbnails: attachments + editable: entry.context?.guest === undefined, + attachments: entry.attachments, + thumbnails: entry.attachments .filter((attachment) => attachment.type === "image") .map((attachment) => ({ key: attachment.id, @@ -155,6 +163,10 @@ export function QueuedRunsControl({ runId: null, messageId: null, freshContext: false, + delivery: "queued" as const, + deliveryLabel: "Queued" as const, + automatic: false, + editBlockedReason: null, serverIndex: null, text: message.text, editable: true, @@ -171,10 +183,13 @@ export function QueuedRunsControl({ ]; const sendQueuedMessage = async (queuedRunId: RunId) => { - const freshContext = queued.some( - ({ run }) => run.id === queuedRunId && run.freshContext === true, - ); - if (!canSendRun(freshContext) || steerInFlightRef.current) return; + const entry = queued.find(({ run }) => run.id === queuedRunId); + if ( + !entry || + !canSendRun(entry.run.freshContext === true, entry.automatic) || + steerInFlightRef.current + ) + return; steerInFlightRef.current = true; setBusyRunId(queuedRunId); if (workflow?.canSendNow) { @@ -194,8 +209,10 @@ export function QueuedRunsControl({ useImperativeHandle(ref, () => ({ steerNext(repeat) { - const next = queued[0]; - if (!next || !canSendRun(next.run.freshContext === true)) return false; + const next = queued.find((entry) => + canSendRun(entry.run.freshContext === true, entry.automatic), + ); + if (!next) return false; if (!repeat && busyRunId === null) void sendQueuedMessage(next.run.id); return true; }, @@ -205,6 +222,7 @@ export function QueuedRunsControl({ if ( !latest || latest.context?.guest !== undefined || + latest.editBlockedReason !== null || props.editingRunId !== null || props.editBlockedReason !== null || busyRunId !== null @@ -261,7 +279,7 @@ export function QueuedRunsControl({ return (
  • - {canSendRun(item.freshContext) && !isEditing ? ( + {canSendRun(item.freshContext, item.automatic) && !isEditing ? ( <> }> @@ -306,6 +324,20 @@ export function QueuedRunsControl({ Saving ) : null} + {item.delivery !== "queued" ? ( + + }> + {item.deliveryLabel} + + + {item.delivery === "unconfirmed" + ? UNCONFIRMED_DELIVERY_HINT + : item.deliveryLabel === "Delivering" + ? "Waiting for the agent to save this message" + : "Waiting for the next tool boundary or turn"} + + + ) : null} {isEditing ? ( <> Editing in the composer @@ -330,6 +362,7 @@ export function QueuedRunsControl({ disabled={ item.runId === null || !item.editable || + item.editBlockedReason !== null || busyRunId !== null || props.editBlockedReason !== null } @@ -349,6 +382,7 @@ export function QueuedRunsControl({ {props.editBlockedReason ?? + item.editBlockedReason ?? `Edit in the composer${item.serverIndex === queued.length - 1 && props.editShortcutLabel ? ` (${props.editShortcutLabel})` : ""}`} diff --git a/packages/client-runtime/src/state/threadWorkflows.test.ts b/packages/client-runtime/src/state/threadWorkflows.test.ts index c47b5ed28a738..1bb8289039225 100644 --- a/packages/client-runtime/src/state/threadWorkflows.test.ts +++ b/packages/client-runtime/src/state/threadWorkflows.test.ts @@ -211,7 +211,7 @@ describe("thread workflows", () => { } }); - it("hides automatic completion delivery from the visible queue", () => { + it("keeps offered automatic deliveries visible while hiding unoffered completions", () => { const state = deriveThreadQueueWorkflowState({ thread: { id: "thread", activeProviderThreadId: null }, runs: [ @@ -221,6 +221,7 @@ describe("thread workflows", () => { userMessageId: "provider-wake", ordinal: 4, queuePosition: 3, + boundaryOffer: { id: "offer-wake", targetRunId: "previous", providerTurnId: "turn-old" }, }, { id: "automatic", @@ -265,9 +266,105 @@ describe("thread workflows", () => { expect(state.queuedRuns.map(({ run, text }) => [run.id, text])).toEqual([ ["visible", "Visible queued message"], + ["provider-wake", "Model-facing text"], ]); + expect(state.queuedRuns[1]).toMatchObject({ + delivery: "unconfirmed", + deliveryLabel: "Unconfirmed", + automatic: true, + editBlockedReason: expect.any(String), + }); }); + it.each([ + { change: "active", delivery: "pending", label: "Delivering" }, + { change: "active-automatic", delivery: "pending", label: "Delivering" }, + { change: "ended", delivery: "unconfirmed", label: "Unconfirmed" }, + { change: "stopped-session", delivery: "unconfirmed", label: "Unconfirmed" }, + { change: "replaced-attempt", delivery: "unconfirmed", label: "Unconfirmed" }, + { change: "replaced-turn", delivery: "unconfirmed", label: "Unconfirmed" }, + { change: "foreign-thread", delivery: "unconfirmed", label: "Unconfirmed" }, + { change: "detached-session", delivery: "unconfirmed", label: "Unconfirmed" }, + { change: "held", delivery: "unconfirmed", label: "Unconfirmed" }, + ])( + "shows $change boundary offers with their actual delivery state", + ({ change, delivery, label }) => { + const state = deriveThreadQueueWorkflowState({ + thread: { id: "thread", activeProviderThreadId: "provider-thread" }, + runs: [ + { + id: "active", + status: change === "ended" ? "interrupted" : "running", + providerThreadId: "provider-thread", + activeAttemptId: change === "replaced-attempt" ? "attempt-new" : "attempt-active", + ordinal: 1, + }, + { + id: "offered", + status: "queued", + userMessageId: "message-offered", + ordinal: 2, + queueHeld: change === "held", + boundaryOffer: { id: "offer", targetRunId: "active", providerTurnId: "provider-turn" }, + }, + { id: "waiting", status: "queued", userMessageId: "message-waiting", ordinal: 3 }, + ], + messages: [ + { + id: "message-offered", + text: "Offered", + atToolBoundary: true, + ...(change === "active-automatic" + ? { + notification: { + source: { kind: "monitor" }, + outcome: "updated", + summary: "Updated", + }, + } + : {}), + }, + { id: "message-waiting", text: "Waiting", atToolBoundary: true }, + ], + providerTurns: [ + { + id: "provider-turn", + providerThreadId: change === "foreign-thread" ? "other-thread" : "provider-thread", + runAttemptId: "attempt-active", + status: change === "replaced-turn" ? "completed" : "running", + }, + ], + providerThreads: [ + { + id: "provider-thread", + appThreadId: "thread", + providerSessionId: change === "detached-session" ? null : "provider-session", + }, + ], + providerSessions: [ + { + id: "provider-session", + status: change === "stopped-session" ? "stopped" : "running", + capabilities: capabilities({ queued: true, steer: true }), + }, + ], + } as never); + + expect(state.queuedRuns[0]).toMatchObject({ + delivery, + deliveryLabel: label, + automatic: change === "active-automatic", + editBlockedReason: expect.any(String), + }); + expect(state.queuedRuns[1]).toMatchObject({ + delivery: "pending", + deliveryLabel: "Pending", + automatic: false, + editBlockedReason: null, + }); + }, + ); + it("removes only the promoted head from the visible queue", () => { const state = deriveThreadQueueWorkflowState({ thread: { id: "thread", activeProviderThreadId: "provider-thread" }, diff --git a/packages/client-runtime/src/state/threadWorkflows.ts b/packages/client-runtime/src/state/threadWorkflows.ts index c96090427c4fd..4e4a00356b1e8 100644 --- a/packages/client-runtime/src/state/threadWorkflows.ts +++ b/packages/client-runtime/src/state/threadWorkflows.ts @@ -23,6 +23,10 @@ const MERGE_BACK_BLOCKING_RUN_STATUSES = new Set([ export interface QueuedThreadRun { readonly run: Run; + readonly delivery: "pending" | "queued" | "unconfirmed"; + readonly deliveryLabel: "Pending" | "Queued" | "Delivering" | "Unconfirmed"; + readonly automatic: boolean; + readonly editBlockedReason: string | null; readonly text: string; readonly attachments: ReadonlyArray; readonly messageId: Message["id"]; @@ -31,6 +35,9 @@ export interface QueuedThreadRun { readonly message?: Message; } +export const UNCONFIRMED_DELIVERY_HINT = + "This message may already have reached the agent. Sending it again may duplicate it. Send it again or remove it; automatic updates can be sent again once the turn ends."; + export interface ThreadQueueWorkflowState { readonly activeRun: Run | null; readonly queuedRuns: ReadonlyArray; @@ -132,7 +139,9 @@ export function getUserQueuedThreadRuns( .map((message) => message.id), ); return projection.runs.filter( - (run) => run.status === "queued" && !automaticCompletionMessageIds.has(run.userMessageId), + (run) => + run.status === "queued" && + (run.boundaryOffer !== undefined || !automaticCompletionMessageIds.has(run.userMessageId)), ); } @@ -158,15 +167,56 @@ export function deriveThreadQueueWorkflowState(projection: Projection): ThreadQu left.ordinal - right.ordinal, ).map((run) => { const message = projection.messages.find((candidate) => candidate.id === run.userMessageId); + const offer = run.boundaryOffer; + const offeredTurn = + offer === undefined + ? undefined + : projection.providerTurns.find((turn) => turn.id === offer.providerTurnId); + const offeredThread = projection.providerThreads.find( + (thread) => thread.id === offeredTurn?.providerThreadId, + ); + const delivering = + offer !== undefined && + run.queueHeld !== true && + activeRun?.id === offer.targetRunId && + activeRun.status === "running" && + activeRun.activeAttemptId !== null && + offeredTurn?.runAttemptId === activeRun.activeAttemptId && + offeredTurn?.providerThreadId === activeRun.providerThreadId && + offeredTurn?.status === "running" && + offeredThread?.appThreadId === projection.thread.id && + session !== null && + offeredThread?.providerSessionId === session.id && + session.status !== "stopped" && + session.status !== "error"; + const delivery = + offer !== undefined + ? delivering + ? "pending" + : "unconfirmed" + : message?.atToolBoundary === true + ? "pending" + : "queued"; return { run, + delivery, + deliveryLabel: delivering + ? "Delivering" + : delivery === "pending" + ? "Pending" + : delivery === "unconfirmed" + ? "Unconfirmed" + : "Queued", + automatic: message?.notification !== undefined || message?.delegatedCompletion !== undefined, + editBlockedReason: + offer === undefined ? null : "This message may already have reached the agent", // A queued guest message shows the guest's words; the agent still gets the attributed text. text: message?.context?.guest?.text ?? message?.text ?? "Queued message", attachments: message?.attachments ?? [], messageId: run.userMessageId, ...(message?.context ? { context: message.context } : {}), ...(message ? { message } : {}), - }; + } satisfies QueuedThreadRun; }); return { diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 8d8cc42c235b2..ad3cee487f184 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -298,6 +298,8 @@ export const OrchestrationV2TurnCapabilities = Schema.Struct({ supportsActiveSteering: Schema.Boolean, // Some native steering mechanisms cancel pending tools before consuming the message. activeSteeringInterruptsTools: Schema.optional(Schema.Boolean), + // The adapter hands queued updates to the provider between tool batches, so a steer queues instead. + deliversUpdatesAtToolBoundary: Schema.optional(Schema.Boolean), // Some take text only, so a steer with attachments restarts the turn instead. activeSteeringAcceptsAttachments: Schema.optional(Schema.Boolean), // Only where a steer leaves a running automatic compaction intact; otherwise clients queue. @@ -708,6 +710,14 @@ export const OrchestrationV2Run = Schema.Struct({ status: OrchestrationV2RunStatus, queuePosition: Schema.optional(Schema.NullOr(PositiveInt)), queueHeld: Schema.optional(Schema.Boolean), + /** Native input may already be saved; an outstanding offer must never automatically execute. */ + boundaryOffer: Schema.optional( + Schema.Struct({ + id: TrimmedNonEmptyString, + targetRunId: RunId, + providerTurnId: ProviderTurnId, + }), + ), steerWhenReadyRunId: Schema.optional(RunId), requestedAt: Schema.DateTimeUtc, startedAt: Schema.NullOr(Schema.DateTimeUtc), @@ -1334,6 +1344,8 @@ export const OrchestrationV2ConversationMessage = Schema.Struct({ * the workflow's opening instead and keep the script behind a disclosure. */ workflowScript: Schema.optional(Schema.Boolean), + /** Eligible for native delivery at a root tool boundary instead of interrupting tools. */ + atToolBoundary: Schema.optional(Schema.Literal(true)), id: MessageId, threadId: ThreadId, runId: Schema.NullOr(RunId), @@ -3304,6 +3316,20 @@ export const OrchestrationV2Command = Schema.Union([ threadId: ThreadId, queuedRunId: RunId, targetRunId: RunId, + boundaryOffer: Schema.optional( + Schema.Union([ + Schema.Struct({ + id: TrimmedNonEmptyString, + step: Schema.Literal("offer"), + providerTurnId: ProviderTurnId, + }), + Schema.Struct({ + id: TrimmedNonEmptyString, + step: Schema.Literals(["receipt", "withdraw"]), + }), + ]), + ), + requestedBy: Schema.optional(Schema.Literal("agent")), }), Schema.Struct({ type: Schema.Literal("queue.resume"), @@ -3315,6 +3341,7 @@ export const OrchestrationV2Command = Schema.Union([ commandId: CommandId, threadId: ThreadId, runId: RunId, + requestedBy: Schema.optional(Schema.Literal("agent")), }), Schema.Struct({ type: Schema.Literal("queued-run.reorder"), diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 2a2537c8bf866..bd82c0ddbed1a 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -139,6 +139,17 @@ export const ProviderAdapterV2Event = Schema.Union([ messageId: MessageId, status: Schema.Literals(["pending", "completed", "cancelled"]), }), + Schema.Struct({ + type: Schema.Literal("boundary_messages.updated"), + driver: ProviderDriverKind, + threadId: ThreadId, + runId: RunId, + providerTurnId: ProviderTurnId, + step: Schema.Literals(["reserve", "receipt"]), + offers: Schema.Array( + Schema.Struct({ id: Schema.String, queuedRunId: RunId, messageId: MessageId }), + ), + }), // Start before the root terminal; clear only after the final covered projection. // Consumer lifetime cannot depend on a producer probe that runs ahead of its queue. Schema.Struct({ diff --git a/packages/provider-core/src/server/ProviderContinuationRequests.ts b/packages/provider-core/src/server/ProviderContinuationRequests.ts index 576edcc221a11..4586626e63b36 100644 --- a/packages/provider-core/src/server/ProviderContinuationRequests.ts +++ b/packages/provider-core/src/server/ProviderContinuationRequests.ts @@ -1,8 +1,10 @@ import { type OrchestrationV2Notification, + type OrchestrationV2ConversationMessage, MessageId, ProviderDriverKind, ProviderThreadId, + ProviderTurnId, RunId, ThreadId, } from "@t3tools/contracts"; @@ -11,6 +13,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; export interface ProviderContinuationRequest { readonly threadId: ThreadId; @@ -34,10 +37,48 @@ export interface ProviderContinuationRequest { readonly onRunEndedWithoutStarting?: () => Effect.Effect; } +export interface AgentMessageOffer { + readonly id: `${string}-${string}-${string}-${string}-${string}`; + readonly queuedRunId: RunId; + readonly messageId: MessageId; + readonly text: string; + readonly attachments: OrchestrationV2ConversationMessage["attachments"]; + readonly createdBy: OrchestrationV2ConversationMessage["createdBy"]; + readonly creationSource: OrchestrationV2ConversationMessage["creationSource"]; +} + +export interface AgentMessageOfferRef { + readonly id: string; + readonly queuedRunId: RunId; +} + +export type ClaimAgentMessages = (input: { + readonly threadId: ThreadId; + readonly runId: RunId; + readonly providerTurnId: ProviderTurnId; +}) => Effect.Effect>; + +export type AcknowledgeAgentMessages = (input: { + readonly threadId: ThreadId; + readonly runId: RunId; + readonly offers: ReadonlyArray; +}) => Effect.Effect; + +interface AgentMessageDelivery { + readonly claim: ClaimAgentMessages; + readonly acknowledge: AcknowledgeAgentMessages; + /** Caller has fenced the producer and proved these inputs were never handed off. */ + readonly withdraw: AcknowledgeAgentMessages; +} + /** The default drops requests; share the live layer with the worker that drains them. */ export class ProviderContinuationRequests extends Context.Reference<{ readonly offer: (request: ProviderContinuationRequest) => Effect.Effect; readonly take: Effect.Effect; + readonly claimAgentMessages?: ClaimAgentMessages; + readonly acknowledgeAgentMessages?: AcknowledgeAgentMessages; + readonly withdrawAgentMessages?: AcknowledgeAgentMessages; + readonly serveAgentMessages?: (delivery: AgentMessageDelivery) => Effect.Effect; }>("@t3tools/provider-core/server/ProviderContinuationRequests", { defaultValue: () => ({ offer: () => Effect.void, take: Effect.never }), }) {} @@ -46,6 +87,11 @@ export const layer = Layer.effect( ProviderContinuationRequests, Effect.gen(function* () { const queue = yield* Queue.unbounded(); + const delivery = yield* Ref.make({ + claim: () => Effect.succeed([]), + acknowledge: () => Effect.void, + withdraw: () => Effect.void, + }); return { offer: Effect.fn("ProviderContinuationRequests.offer")(function ( request: ProviderContinuationRequest, @@ -53,6 +99,18 @@ export const layer = Layer.effect( return Queue.offer(queue, request).pipe(Effect.asVoid); }), take: Queue.take(queue), + claimAgentMessages: Effect.fn("ProviderContinuationRequests.claimAgentMessages")((input) => + Ref.get(delivery).pipe(Effect.flatMap((served) => served.claim(input))), + ), + acknowledgeAgentMessages: Effect.fn("ProviderContinuationRequests.acknowledgeAgentMessages")( + (input) => Ref.get(delivery).pipe(Effect.flatMap((served) => served.acknowledge(input))), + ), + withdrawAgentMessages: Effect.fn("ProviderContinuationRequests.withdrawAgentMessages")( + (input) => Ref.get(delivery).pipe(Effect.flatMap((served) => served.withdraw(input))), + ), + serveAgentMessages: Effect.fn("ProviderContinuationRequests.serveAgentMessages")((served) => + Ref.set(delivery, served), + ), }; }), ); diff --git a/packages/provider-core/src/server/randomUuid.ts b/packages/provider-core/src/server/randomUuid.ts index 2f29fc5e399d4..5745902bfedde 100644 --- a/packages/provider-core/src/server/randomUuid.ts +++ b/packages/provider-core/src/server/randomUuid.ts @@ -8,6 +8,6 @@ export const randomUuidV4 = Effect.all( bytes[6] = (bytes[6]! & 0x0f) | 0x40; bytes[8] = (bytes[8]! & 0x3f) | 0x80; const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0")).join(""); - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` as const; }), );