From 954bc76c264426e7e54b2113be2df4ebd202ade4 Mon Sep 17 00:00:00 2001 From: murayy <22128993+murayy@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:38:25 +0000 Subject: [PATCH 01/14] fix(server): a follow-up promoted to Steer no longer reads as stopped Fixes #19351 --- .../DelegatedCompletionDelivery.test.ts | 165 ++++++++++++++++++ .../src/orchestration-v2/Orchestrator.ts | 37 +++- 2 files changed, 196 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 003844efa7206..c893d25cf951d 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1837,3 +1837,168 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { }), ); }); + +it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { + it.effect("settles a promoted follow-up with the turn it steered, not as cancelled", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:steer-parent"); + const projectId = ProjectId.make("project:steer-parent"); + const runId = RunId.make("run:steer-parent"); + const rootNodeId = NodeId.make("node:steer-parent-root"); + yield* seedParentWithTerminalTask({ + threadId, + projectId, + runId, + rootNodeId, + taskId: NodeId.make("node:steer-parent-settled"), + deliveryState: "delivered", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: "steer-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + const steeredRunId = RunId.make("run:steer-child:2"); + const followUpTaskId = NodeId.make("node:steer-child-follow-up"); + const followUpMessageId = MessageId.make("message:steer-child-follow-up"); + const followUpRun = runEvent({ + threadId: child.childThreadId, + runId: RunId.make("run:steer-child:3"), + ordinal: 3, + status: "cancelled", + userMessageId: followUpMessageId, + now, + }); + // The follow-up was queued as its own task, then promoted into the running turn. + yield* eventSink.write({ + commandId: CommandId.make("command:steer-child:promoted"), + events: [ + runEvent({ + threadId: child.childThreadId, + runId: steeredRunId, + ordinal: 2, + status: "running", + now, + }), + { + ...followUpRun, + payload: { ...followUpRun.payload, startedAt: null, delegatedTaskId: followUpTaskId }, + }, + { + id: EventId.make("event:steer-child:follow-up-message"), + type: "message.updated", + threadId: child.childThreadId, + runId: steeredRunId, + occurredAt: now, + payload: { + id: followUpMessageId, + threadId: child.childThreadId, + runId: steeredRunId, + nodeId: null, + role: "user", + text: "Also merge the fixture branch.", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "mcp", + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("event:steer-child:follow-up-task"), + type: "subagent.updated", + threadId, + runId, + nodeId: followUpTaskId, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + id: followUpTaskId, + threadId, + runId, + parentNodeId: rootNodeId, + origin: "app_owned", + createdBy: "agent", + driver, + providerInstanceId: modelSelection.instanceId, + providerThreadId: null, + childThreadId: child.childThreadId, + nativeTaskRef: null, + prompt: "Also merge the fixture branch.", + title: null, + model: null, + completionWake: "always", + status: "pending", + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }, + ], + }); + + yield* orchestrator.recoverDelegatedTask(child.childThreadId, followUpRun.payload.id); + + const steering = yield* orchestrator.getThreadProjection(threadId); + assert.equal(steering.subagents.find((row) => row.id === followUpTaskId)?.status, "pending"); + assert.isTrue( + yield* orchestrator.delegatedTaskResultPending(child.childThreadId, followUpTaskId), + ); + + yield* eventSink.write({ + commandId: CommandId.make("command:steer-child:completed"), + events: [ + { + id: EventId.make("event:steer-child:result"), + type: "message.updated", + threadId: child.childThreadId, + runId: steeredRunId, + occurredAt: now, + payload: { + id: MessageId.make("message:steer-child:result"), + threadId: child.childThreadId, + runId: steeredRunId, + nodeId: null, + role: "assistant", + text: "Merged both branches.", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, + }, + runEvent({ + threadId: child.childThreadId, + runId: steeredRunId, + ordinal: 2, + status: "completed", + now, + }), + ], + }); + yield* orchestrator.recoverDelegatedTask(child.childThreadId, steeredRunId); + + const settled = yield* orchestrator.getThreadProjection(threadId); + const original = settled.subagents.find((row) => row.id === child.taskId); + const followUp = settled.subagents.find((row) => row.id === followUpTaskId); + assert.equal(original?.result, "Merged both branches."); + assert.equal(followUp?.status, "completed"); + assert.equal(followUp?.result, "Merged both branches."); + }), + ); +}); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 35a2ecf02c691..25f7297a0343a 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -387,6 +387,25 @@ export class OrchestratorV2 extends Context.Service, + taskId: NodeId, +): OrchestrationV2Run | undefined { + const run = child.runs + .filter((candidate) => candidate.delegatedTaskId === taskId) + .toSorted((a, b) => b.ordinal - a.ordinal)[0]; + if (run?.status !== "cancelled") return run; + const steeredRunId = child.messages.find((message) => message.id === run.userMessageId)?.runId; + return ( + child.runs.find((candidate) => candidate.id === steeredRunId && candidate.id !== run.id) ?? run + ); +} + function nextRunOrdinal(projection: Pick): number { return projection.runs.length + 1; } @@ -10154,8 +10173,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? options.taskId === forkedFrom.nodeId ? options.resultRun : undefined - : taskRuns.toSorted((a, b) => b.ordinal - a.ordinal)[0]; + : delegatedTaskResultRun(childControls, options.taskId); if (childRun === undefined) return; + const steered = taskRuns.length > 0 && !taskRuns.includes(childRun); const outcome = runOutcome(childRun); const terminalStatus = delegatedTaskTerminalStatus(outcome); if (terminalStatus === null) { @@ -10163,7 +10183,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } if ( yield* childAwaitsRestartContinuation( - taskRuns.length === 0 ? childControls.runs : taskRuns, + taskRuns.length === 0 || steered ? childControls.runs : taskRuns, childRun, options?.settledContinuationOf, ) @@ -10218,7 +10238,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (transfer.sourcePoint.runId === childRun.id || (options.taskId === forkedFrom.nodeId && transfer.sourcePoint.runId === undefined)), ); - if (existingResultTransfer !== undefined) { + // The steered run's own task may already have published that run's result. + if (steered ? task.result !== null : existingResultTransfer !== undefined) { return; } @@ -11428,10 +11449,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio { messageRoles: ["user"] }, ); if (taskId !== undefined) { - const runs = child.runs.filter((run) => run.delegatedTaskId === taskId); - const run = runs.toSorted((a, b) => b.ordinal - a.ordinal)[0]; + const run = delegatedTaskResultRun(child, taskId); if (run === undefined || delegatedTaskTerminalStatus(run.status) === null) return true; - return yield* childAwaitsRestartContinuation(runs, run); + return yield* childAwaitsRestartContinuation( + run.delegatedTaskId === taskId + ? child.runs.filter((candidate) => candidate.delegatedTaskId === taskId) + : child.runs, + run, + ); } const progress = delegatedTaskProgress(child); // A caller's older read saw a result; newer work since then means it is not final. From b6ca13453c8ffc5d2cefd4eb8bdf92a404dcbf2c Mon Sep 17 00:00:00 2001 From: murayy <22128993+murayy@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:14:00 +0000 Subject: [PATCH 02/14] fix(server): settle steered follow-ups without blocking the steered run's task A promoted follow-up shares its steered run with that run's own task. It no longer publishes a second result transfer for the run, which made the run's own task skip settlement. Restart holds use the steered run's task runs, and task_status reads the steered run while the finalizer has not settled yet. --- apps/server/src/mcp/OrchestratorMcpService.ts | 5 +- .../DelegatedCompletionDelivery.test.ts | 166 +++++++++++------- .../src/orchestration-v2/Orchestrator.ts | 59 +++---- .../SubagentProjection.test.ts | 30 ++++ .../src/server/subagentProjection.ts | 22 +++ 5 files changed, 182 insertions(+), 100 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 4055544a39473..e3b860267a134 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -76,6 +76,7 @@ import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterReg import { subagentResultForRun, delegatedTaskProgress, + delegatedTaskResultRun, } from "@t3tools/provider-core/server/subagentProjection"; import { DispatchModeLimit, @@ -1360,9 +1361,7 @@ const make = Effect.gen(function* () { const progress = delegatedTaskProgress(childControls); const taskResultRun = childRun?.delegatedTaskId === task.id - ? childControls.runs - .filter((run) => run.delegatedTaskId === task.id) - .toSorted((a, b) => b.ordinal - a.ordinal)[0] + ? delegatedTaskResultRun(childControls, task.id) : progress.resultRun; const taskWorkState = childRun?.delegatedTaskId === task.id diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index c893d25cf951d..2f2484d7e54af 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1868,45 +1868,81 @@ it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { runStatus: "completed", now, }); + const followUpTask = (taskId: NodeId, prompt: string) => ({ + id: EventId.make(`event:${taskId}`), + type: "subagent.updated" as const, + threadId, + runId, + nodeId: taskId, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + id: taskId, + threadId, + runId, + parentNodeId: rootNodeId, + origin: "app_owned" as const, + createdBy: "agent" as const, + driver, + providerInstanceId: modelSelection.instanceId, + providerThreadId: null, + childThreadId: child.childThreadId, + nativeTaskRef: null, + prompt, + title: null, + model: null, + completionWake: "always" as const, + status: "pending" as const, + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }); const steeredRunId = RunId.make("run:steer-child:2"); - const followUpTaskId = NodeId.make("node:steer-child-follow-up"); - const followUpMessageId = MessageId.make("message:steer-child-follow-up"); - const followUpRun = runEvent({ + const steeredTaskId = NodeId.make("node:steer-child-steered"); + const promotedTaskId = NodeId.make("node:steer-child-promoted"); + const promotedMessageId = MessageId.make("message:steer-child-promoted"); + const steeredRun = runEvent({ + threadId: child.childThreadId, + runId: steeredRunId, + ordinal: 2, + status: "running", + now, + }); + const promotedRun = runEvent({ threadId: child.childThreadId, runId: RunId.make("run:steer-child:3"), ordinal: 3, status: "cancelled", - userMessageId: followUpMessageId, + userMessageId: promotedMessageId, now, }); - // The follow-up was queued as its own task, then promoted into the running turn. + // Both follow-ups were queued as their own tasks. The second was promoted into the first's turn. yield* eventSink.write({ commandId: CommandId.make("command:steer-child:promoted"), events: [ - runEvent({ - threadId: child.childThreadId, - runId: steeredRunId, - ordinal: 2, - status: "running", - now, - }), + followUpTask(steeredTaskId, "Merge the fixture branch."), + followUpTask(promotedTaskId, "Also merge the pricing branch."), + { ...steeredRun, payload: { ...steeredRun.payload, delegatedTaskId: steeredTaskId } }, { - ...followUpRun, - payload: { ...followUpRun.payload, startedAt: null, delegatedTaskId: followUpTaskId }, + ...promotedRun, + payload: { ...promotedRun.payload, startedAt: null, delegatedTaskId: promotedTaskId }, }, { - id: EventId.make("event:steer-child:follow-up-message"), + id: EventId.make("event:steer-child:promoted-message"), type: "message.updated", threadId: child.childThreadId, runId: steeredRunId, occurredAt: now, payload: { - id: followUpMessageId, + id: promotedMessageId, threadId: child.childThreadId, runId: steeredRunId, nodeId: null, role: "user", - text: "Also merge the fixture branch.", + text: "Also merge the pricing branch.", attachments: [], streaming: false, createdBy: "agent", @@ -1915,49 +1951,46 @@ it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { updatedAt: now, }, }, - { - id: EventId.make("event:steer-child:follow-up-task"), - type: "subagent.updated", - threadId, - runId, - nodeId: followUpTaskId, - driver, - providerInstanceId: modelSelection.instanceId, - occurredAt: now, - payload: { - id: followUpTaskId, - threadId, - runId, - parentNodeId: rootNodeId, - origin: "app_owned", - createdBy: "agent", - driver, - providerInstanceId: modelSelection.instanceId, - providerThreadId: null, - childThreadId: child.childThreadId, - nativeTaskRef: null, - prompt: "Also merge the fixture branch.", - title: null, - model: null, - completionWake: "always", - status: "pending", - result: null, - startedAt: now, - completedAt: null, - updatedAt: now, - }, - }, ], }); - yield* orchestrator.recoverDelegatedTask(child.childThreadId, followUpRun.payload.id); + yield* orchestrator.recoverDelegatedTask(child.childThreadId, promotedRun.payload.id); const steering = yield* orchestrator.getThreadProjection(threadId); - assert.equal(steering.subagents.find((row) => row.id === followUpTaskId)?.status, "pending"); + assert.equal(steering.subagents.find((row) => row.id === promotedTaskId)?.status, "pending"); assert.isTrue( - yield* orchestrator.delegatedTaskResultPending(child.childThreadId, followUpTaskId), + yield* orchestrator.delegatedTaskResultPending(child.childThreadId, promotedTaskId), ); + // A later run of an unrelated task still awaits its restart continuation. + const unrelatedRunId = RunId.make("run:steer-child:4"); + const unrelatedRun = runEvent({ + threadId: child.childThreadId, + runId: unrelatedRunId, + ordinal: 4, + status: "cancelled", + now, + }); + yield* eventSink.writeWithEffects({ + commandId: reconcileCommandId("steer-child-unrelated"), + events: [ + { + ...unrelatedRun, + payload: { + ...unrelatedRun.payload, + delegatedTaskId: NodeId.make("node:steer-child-unrelated"), + }, + }, + ], + effects: [ + { + id: `effect:restart-continuation:${unrelatedRunId}`, + commandId: reconcileCommandId("steer-child-unrelated"), + threadId: child.childThreadId, + request: { type: "provider-runtime.continue", sourceRunId: unrelatedRunId }, + }, + ], + }); yield* eventSink.write({ commandId: CommandId.make("command:steer-child:completed"), events: [ @@ -1982,23 +2015,28 @@ it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { updatedAt: now, }, }, - runEvent({ - threadId: child.childThreadId, - runId: steeredRunId, - ordinal: 2, - status: "completed", - now, - }), + { + ...steeredRun, + id: EventId.make("event:steer-child:steered-completed"), + payload: { + ...steeredRun.payload, + delegatedTaskId: steeredTaskId, + status: "completed" as const, + completedAt: now, + }, + }, ], }); yield* orchestrator.recoverDelegatedTask(child.childThreadId, steeredRunId); const settled = yield* orchestrator.getThreadProjection(threadId); - const original = settled.subagents.find((row) => row.id === child.taskId); - const followUp = settled.subagents.find((row) => row.id === followUpTaskId); - assert.equal(original?.result, "Merged both branches."); - assert.equal(followUp?.status, "completed"); - assert.equal(followUp?.result, "Merged both branches."); + const task = (id: NodeId) => settled.subagents.find((row) => row.id === id); + assert.equal(task(steeredTaskId)?.result, "Merged both branches."); + assert.equal(task(promotedTaskId)?.status, "completed"); + assert.equal(task(promotedTaskId)?.result, "Merged both branches."); + assert.isFalse( + yield* orchestrator.delegatedTaskResultPending(child.childThreadId, promotedTaskId), + ); }), ); }); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 25f7297a0343a..bd14928664344 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -126,6 +126,7 @@ import { makeSubagentChildThread, subagentResultForRun, delegatedTaskProgress, + delegatedTaskResultRun, subagentThreadTitle, } from "@t3tools/provider-core/server/subagentProjection"; import { @@ -387,23 +388,14 @@ export class OrchestratorV2 extends Context.Service, - taskId: NodeId, -): OrchestrationV2Run | undefined { - const run = child.runs - .filter((candidate) => candidate.delegatedTaskId === taskId) - .toSorted((a, b) => b.ordinal - a.ordinal)[0]; - if (run?.status !== "cancelled") return run; - const steeredRunId = child.messages.find((message) => message.id === run.userMessageId)?.runId; - return ( - child.runs.find((candidate) => candidate.id === steeredRunId && candidate.id !== run.id) ?? run - ); +/** The runs whose restart continuation can still change `run`'s result: its task's runs, or all runs for the original task. */ +function runsOfTask( + runs: ReadonlyArray, + run: OrchestrationV2Run, +): ReadonlyArray { + return run.delegatedTaskId === undefined + ? runs + : runs.filter((candidate) => candidate.delegatedTaskId === run.delegatedTaskId); } function nextRunOrdinal(projection: Pick): number { @@ -10183,7 +10175,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } if ( yield* childAwaitsRestartContinuation( - taskRuns.length === 0 || steered ? childControls.runs : taskRuns, + taskRuns.length === 0 ? childControls.runs : runsOfTask(childControls.runs, childRun), childRun, options?.settledContinuationOf, ) @@ -10238,7 +10230,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (transfer.sourcePoint.runId === childRun.id || (options.taskId === forkedFrom.nodeId && transfer.sourcePoint.runId === undefined)), ); - // The steered run's own task may already have published that run's result. + // A steered task shares its run's result transfer, so only its own result marks it settled. if (steered ? task.result !== null : existingResultTransfer !== undefined) { return; } @@ -10285,7 +10277,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : parentProjection.providerThreads.find( (candidate) => candidate.id === parentRun.providerThreadId, ); + // The steered run's own task publishes that run's result to the parent's context. const resultHandoff: OrchestrationV2ContextHandoff | null = + steered || parentRun === undefined || childProviderThread === undefined || parentProviderThread === undefined @@ -10435,14 +10429,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: resultHandoff, }, ]), - { - type: "context-transfer.created", - threadId: parentThreadId, - ...(parentRun === undefined ? {} : { runId: parentRun.id }), - providerInstanceId: childRun.providerInstanceId, - occurredAt: now, - payload: resultTransfer, - }, + ...(steered + ? [] + : [ + { + type: "context-transfer.created" as const, + threadId: parentThreadId, + ...(parentRun === undefined ? {} : { runId: parentRun.id }), + providerInstanceId: childRun.providerInstanceId, + occurredAt: now, + payload: resultTransfer, + }, + ]), ]); if (completionPlan.offer && completionPlan.parentRun !== undefined) { @@ -11451,12 +11449,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (taskId !== undefined) { const run = delegatedTaskResultRun(child, taskId); if (run === undefined || delegatedTaskTerminalStatus(run.status) === null) return true; - return yield* childAwaitsRestartContinuation( - run.delegatedTaskId === taskId - ? child.runs.filter((candidate) => candidate.delegatedTaskId === taskId) - : child.runs, - run, - ); + return yield* childAwaitsRestartContinuation(runsOfTask(child.runs, run), run); } const progress = delegatedTaskProgress(child); // A caller's older read saw a result; newer work since then means it is not final. diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index 65b64000b4709..c5a8d45e0e462 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -20,6 +20,7 @@ import * as Effect from "effect/Effect"; import { makeSubagentChildThread, delegatedTaskProgress, + delegatedTaskResultRun, subagentResultForRun, makeSubagentConversationArtifacts, endOrphanedNativeSubagent, @@ -191,6 +192,35 @@ it("reports the run that ended last, not the highest ordinal", () => { assert.equal(progress.resultRun?.ordinal, 3); }); +it("settles a follow-up promoted to Steer with the run its message joined", () => { + const { run } = taskFixture(); + const taskId = NodeId.make("node:follow-up"); + const messageId = MessageId.make("message:follow-up"); + const steered = { ...run, status: "running" as const }; + const queued: OrchestrationV2Run = { + ...run, + id: RunId.make("run:follow-up"), + ordinal: 2, + status: "cancelled", + userMessageId: messageId, + delegatedTaskId: taskId, + }; + const message = { id: messageId, runId: queued.id }; + // Stopped while queued: the message still belongs to its own run. + assert.equal( + delegatedTaskResultRun({ runs: [steered, queued], messages: [message] }, taskId)?.id, + queued.id, + ); + // Promoted to Steer: the message moved into the running turn. + assert.equal( + delegatedTaskResultRun( + { runs: [steered, queued], messages: [{ ...message, runId: steered.id }] }, + taskId, + )?.id, + steered.id, + ); +}); + it("waits for nested work and retains the report across monitor acknowledgements", () => { const { projection, run } = taskFixture(); assert.equal( diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index a3a00c5005e2a..22d57c0101a4c 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -209,6 +209,28 @@ export function subagentResultForRun( }; } +/** + * The run whose outcome settles a delegated task. Promoting a queued follow-up + * to Steer cancels its run and moves its message into the running turn, so the + * task settles with that turn instead of reading as stopped. + */ +export function delegatedTaskResultRun( + child: { + readonly runs: OrchestrationV2ThreadProjection["runs"]; + readonly messages: ReadonlyArray>; + }, + taskId: NodeId, +): OrchestrationV2Run | undefined { + const run = child.runs + .filter((candidate) => candidate.delegatedTaskId === taskId) + .toSorted((a, b) => b.ordinal - a.ordinal)[0]; + if (run?.status !== "cancelled") return run; + const steeredRunId = child.messages.find((message) => message.id === run.userMessageId)?.runId; + return ( + child.runs.find((candidate) => candidate.id === steeredRunId && candidate.id !== run.id) ?? run + ); +} + /** A finished turn can still own live children or queued completion follow-ups. */ export function delegatedTaskProgress(projection: { readonly runs: OrchestrationV2ThreadProjection["runs"]; From b8a2eeaf86abbd41af75e381ea31382fe11dce28 Mon Sep 17 00:00:00 2001 From: murayy <22128993+murayy@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:05:58 +0000 Subject: [PATCH 03/14] fix(server): a promoted follow-up follows its turn's restart continuation A restart can cut the turn a follow-up was steered into and resume it in a continuation run. The promoted task now resolves to that continuation, so it stays open while the turn resumes and settles with the continuation's result. --- .../DelegatedCompletionDelivery.test.ts | 325 +++++++++++------- .../SubagentProjection.test.ts | 17 + .../src/server/subagentProjection.ts | 14 +- 3 files changed, 223 insertions(+), 133 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 2f2484d7e54af..dc92e13b885ff 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1838,123 +1838,182 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { ); }); -it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { - it.effect("settles a promoted follow-up with the turn it steered, not as cancelled", () => - Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const now = yield* DateTime.now; - const threadId = ThreadId.make("thread:steer-parent"); - const projectId = ProjectId.make("project:steer-parent"); - const runId = RunId.make("run:steer-parent"); - const rootNodeId = NodeId.make("node:steer-parent-root"); - yield* seedParentWithTerminalTask({ - threadId, - projectId, - runId, - rootNodeId, - taskId: NodeId.make("node:steer-parent-settled"), - deliveryState: "delivered", - now, - }); - const child = yield* seedRestartCancelledChild({ - parentThreadId: threadId, - projectId, - parentRunId: runId, - rootNodeId, - name: "steer-child", - completionWake: "always", - continuationPending: false, - runStatus: "completed", - now, - }); - const followUpTask = (taskId: NodeId, prompt: string) => ({ - id: EventId.make(`event:${taskId}`), - type: "subagent.updated" as const, +/** + * A child whose second follow-up was queued as its own task, then promoted to + * Steer into the first follow-up's running turn. + */ +const seedPromotedFollowUp = (name: string) => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread:${name}-parent`); + const runId = RunId.make(`run:${name}-parent`); + const rootNodeId = NodeId.make(`node:${name}-parent-root`); + const projectId = ProjectId.make(`project:${name}-parent`); + yield* seedParentWithTerminalTask({ + threadId, + projectId, + runId, + rootNodeId, + taskId: NodeId.make(`node:${name}-parent-settled`), + deliveryState: "delivered", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name, + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + const followUpTask = (taskId: NodeId, prompt: string) => ({ + id: EventId.make(`event:${taskId}`), + type: "subagent.updated" as const, + threadId, + runId, + nodeId: taskId, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + id: taskId, threadId, runId, - nodeId: taskId, + parentNodeId: rootNodeId, + origin: "app_owned" as const, + createdBy: "agent" as const, driver, providerInstanceId: modelSelection.instanceId, - occurredAt: now, + providerThreadId: null, + childThreadId: child.childThreadId, + nativeTaskRef: null, + prompt, + title: null, + model: null, + completionWake: "always" as const, + status: "pending" as const, + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }); + const steeredRunId = RunId.make(`run:${name}:2`); + const steeredTaskId = NodeId.make(`node:${name}-steered`); + const promotedTaskId = NodeId.make(`node:${name}-promoted`); + const promotedMessageId = MessageId.make(`message:${name}-promoted`); + const promotedRun = runEvent({ + threadId: child.childThreadId, + runId: RunId.make(`run:${name}:3`), + ordinal: 3, + status: "cancelled", + userMessageId: promotedMessageId, + now, + }); + const steeredRun = ( + status: OrchestrationV2Run["status"], + id = steeredRunId, + ordinal = 2, + restartContinuationOfRunId?: RunId, + ) => { + const event = runEvent({ threadId: child.childThreadId, runId: id, ordinal, status, now }); + return { + ...event, payload: { - id: taskId, - threadId, - runId, - parentNodeId: rootNodeId, - origin: "app_owned" as const, - createdBy: "agent" as const, - driver, - providerInstanceId: modelSelection.instanceId, - providerThreadId: null, - childThreadId: child.childThreadId, - nativeTaskRef: null, - prompt, - title: null, - model: null, - completionWake: "always" as const, - status: "pending" as const, - result: null, - startedAt: now, - completedAt: null, - updatedAt: now, + ...event.payload, + delegatedTaskId: steeredTaskId, + ...(restartContinuationOfRunId === undefined ? {} : { restartContinuationOfRunId }), }, - }); - const steeredRunId = RunId.make("run:steer-child:2"); - const steeredTaskId = NodeId.make("node:steer-child-steered"); - const promotedTaskId = NodeId.make("node:steer-child-promoted"); - const promotedMessageId = MessageId.make("message:steer-child-promoted"); - const steeredRun = runEvent({ - threadId: child.childThreadId, - runId: steeredRunId, - ordinal: 2, - status: "running", - now, - }); - const promotedRun = runEvent({ - threadId: child.childThreadId, - runId: RunId.make("run:steer-child:3"), - ordinal: 3, - status: "cancelled", - userMessageId: promotedMessageId, - now, - }); - // Both follow-ups were queued as their own tasks. The second was promoted into the first's turn. - yield* eventSink.write({ - commandId: CommandId.make("command:steer-child:promoted"), - events: [ - followUpTask(steeredTaskId, "Merge the fixture branch."), - followUpTask(promotedTaskId, "Also merge the pricing branch."), - { ...steeredRun, payload: { ...steeredRun.payload, delegatedTaskId: steeredTaskId } }, - { - ...promotedRun, - payload: { ...promotedRun.payload, startedAt: null, delegatedTaskId: promotedTaskId }, + }; + }; + yield* eventSink.write({ + commandId: CommandId.make(`command:${name}:promoted`), + events: [ + followUpTask(steeredTaskId, "Merge the fixture branch."), + followUpTask(promotedTaskId, "Also merge the pricing branch."), + steeredRun("running"), + { + ...promotedRun, + payload: { ...promotedRun.payload, startedAt: null, delegatedTaskId: promotedTaskId }, + }, + { + id: EventId.make(`event:${name}:promoted-message`), + type: "message.updated", + threadId: child.childThreadId, + runId: steeredRunId, + occurredAt: now, + payload: { + id: promotedMessageId, + threadId: child.childThreadId, + runId: steeredRunId, + nodeId: null, + role: "user", + text: "Also merge the pricing branch.", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "mcp", + createdAt: now, + updatedAt: now, }, + }, + ], + }); + /** Ends `resultRunId` with an assistant reply; `run` is that run's terminal event. */ + const finish = (resultRunId: RunId, run: ReturnType) => + eventSink.write({ + commandId: CommandId.make(`command:${resultRunId}:completed`), + events: [ { - id: EventId.make("event:steer-child:promoted-message"), + id: EventId.make(`event:${resultRunId}:result`), type: "message.updated", threadId: child.childThreadId, - runId: steeredRunId, + runId: resultRunId, occurredAt: now, payload: { - id: promotedMessageId, + id: MessageId.make(`message:${resultRunId}:result`), threadId: child.childThreadId, - runId: steeredRunId, + runId: resultRunId, nodeId: null, - role: "user", - text: "Also merge the pricing branch.", + role: "assistant", + text: "Merged both branches.", attachments: [], streaming: false, createdBy: "agent", - creationSource: "mcp", + creationSource: "server", createdAt: now, updatedAt: now, }, }, + { ...run, id: EventId.make(`event:${resultRunId}:finished`) }, ], }); + return { + threadId, + child, + steeredRunId, + steeredRun, + steeredTaskId, + promotedTaskId, + promotedRunId: promotedRun.payload.id, + finish, + }; + }); + +it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { + it.effect("settles a promoted follow-up with the turn it steered, not as cancelled", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const seeded = yield* seedPromotedFollowUp("steer-child"); + const { threadId, child, promotedTaskId, steeredTaskId } = seeded; - yield* orchestrator.recoverDelegatedTask(child.childThreadId, promotedRun.payload.id); + yield* orchestrator.recoverDelegatedTask(child.childThreadId, seeded.promotedRunId); const steering = yield* orchestrator.getThreadProjection(threadId); assert.equal(steering.subagents.find((row) => row.id === promotedTaskId)?.status, "pending"); @@ -1991,43 +2050,8 @@ it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { }, ], }); - yield* eventSink.write({ - commandId: CommandId.make("command:steer-child:completed"), - events: [ - { - id: EventId.make("event:steer-child:result"), - type: "message.updated", - threadId: child.childThreadId, - runId: steeredRunId, - occurredAt: now, - payload: { - id: MessageId.make("message:steer-child:result"), - threadId: child.childThreadId, - runId: steeredRunId, - nodeId: null, - role: "assistant", - text: "Merged both branches.", - attachments: [], - streaming: false, - createdBy: "agent", - creationSource: "server", - createdAt: now, - updatedAt: now, - }, - }, - { - ...steeredRun, - id: EventId.make("event:steer-child:steered-completed"), - payload: { - ...steeredRun.payload, - delegatedTaskId: steeredTaskId, - status: "completed" as const, - completedAt: now, - }, - }, - ], - }); - yield* orchestrator.recoverDelegatedTask(child.childThreadId, steeredRunId); + yield* seeded.finish(seeded.steeredRunId, seeded.steeredRun("completed")); + yield* orchestrator.recoverDelegatedTask(child.childThreadId, seeded.steeredRunId); const settled = yield* orchestrator.getThreadProjection(threadId); const task = (id: NodeId) => settled.subagents.find((row) => row.id === id); @@ -2039,4 +2063,45 @@ it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { ); }), ); + + it.effect("keeps a promoted follow-up open while a restart continues its turn", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const seeded = yield* seedPromotedFollowUp("steer-restart-child"); + const { threadId, child, promotedTaskId, steeredRunId } = seeded; + const continuationRunId = RunId.make("run:steer-restart-child:4"); + // A restart cut the steered turn, and its continuation resumed it. + yield* eventSink.write({ + commandId: reconcileCommandId("steer-restart-child-cut"), + events: [seeded.steeredRun("cancelled")], + }); + yield* eventSink.write({ + commandId: CommandId.make("command:steer-restart-child:continued"), + events: [seeded.steeredRun("running", continuationRunId, 4, steeredRunId)], + }); + + yield* orchestrator.recoverDelegatedTask(child.childThreadId, steeredRunId); + + const continuing = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + continuing.subagents.find((row) => row.id === promotedTaskId)?.status, + "pending", + ); + assert.isTrue( + yield* orchestrator.delegatedTaskResultPending(child.childThreadId, promotedTaskId), + ); + + yield* seeded.finish( + continuationRunId, + seeded.steeredRun("completed", continuationRunId, 4, steeredRunId), + ); + yield* orchestrator.recoverDelegatedTask(child.childThreadId, continuationRunId); + + const settled = yield* orchestrator.getThreadProjection(threadId); + const promoted = settled.subagents.find((row) => row.id === promotedTaskId); + assert.equal(promoted?.status, "completed"); + assert.equal(promoted?.result, "Merged both branches."); + }), + ); }); diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index c5a8d45e0e462..781c15116dcf7 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -219,6 +219,23 @@ it("settles a follow-up promoted to Steer with the run its message joined", () = )?.id, steered.id, ); + // A restart cut the steered turn; its continuation now carries the result. + const continuation: OrchestrationV2Run = { + ...run, + id: RunId.make("run:continuation"), + ordinal: 3, + restartContinuationOfRunId: steered.id, + }; + assert.equal( + delegatedTaskResultRun( + { + runs: [{ ...steered, status: "cancelled" }, queued, continuation], + messages: [{ ...message, runId: steered.id }], + }, + taskId, + )?.id, + continuation.id, + ); }); it("waits for nested work and retains the report across monitor acknowledgements", () => { diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index 22d57c0101a4c..0fb07acc74cd2 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -212,7 +212,8 @@ export function subagentResultForRun( /** * The run whose outcome settles a delegated task. Promoting a queued follow-up * to Steer cancels its run and moves its message into the running turn, so the - * task settles with that turn instead of reading as stopped. + * task settles with that turn, or the restart continuation that resumed it, + * instead of reading as stopped. */ export function delegatedTaskResultRun( child: { @@ -226,9 +227,16 @@ export function delegatedTaskResultRun( .toSorted((a, b) => b.ordinal - a.ordinal)[0]; if (run?.status !== "cancelled") return run; const steeredRunId = child.messages.find((message) => message.id === run.userMessageId)?.runId; - return ( - child.runs.find((candidate) => candidate.id === steeredRunId && candidate.id !== run.id) ?? run + let resolved = child.runs.find( + (candidate) => candidate.id === steeredRunId && candidate.id !== run.id, ); + if (resolved === undefined) return run; + const continuationOf = (from: OrchestrationV2Run) => + child.runs.find((candidate) => candidate.restartContinuationOfRunId === from.id); + for (let next = continuationOf(resolved); next !== undefined; next = continuationOf(next)) { + resolved = next; + } + return resolved; } /** A finished turn can still own live children or queued completion follow-ups. */ From fb4b22f7adac4fbdcdb26f7d7cb655e8267a6ed0 Mon Sep 17 00:00:00 2001 From: murayy <22128993+murayy@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:51:34 +0000 Subject: [PATCH 04/14] fix(server): a promoted follow-up reports the steered turn's result transfer --- .../src/mcp/OrchestratorMcpService.test.ts | 91 +++++++++++++++++++ apps/server/src/mcp/OrchestratorMcpService.ts | 12 ++- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index b0688fd217bb8..69f81043d7498 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -278,6 +278,97 @@ describe("OrchestratorMcpService", () => { }), ); + it.effect("reports the steered turn's result transfer for a follow-up promoted to Steer", () => + Effect.gen(function* () { + const parentThreadId = ThreadId.make("thread:mcp-steer-parent"); + const childThreadId = ThreadId.make("thread:mcp-steer-child"); + const steeredTaskId = NodeId.make("node:mcp-steer-steered"); + const promotedTaskId = NodeId.make("node:mcp-steer-promoted"); + const steeredRunId = RunId.make("run:mcp-steer-steered"); + const promotedMessageId = "message:mcp-steer-promoted"; + const task = (id: NodeId) => ({ + id, + threadId: parentThreadId, + origin: "app_owned", + childThreadId, + driver: "codex", + model: "gpt-5.6-terra", + status: "completed", + result: "Merged both branches.", + completionDelivery: { state: "delivered" }, + }); + const parentProjection = { + thread: { id: parentThreadId }, + runs: [], + contextTransfers: [ + { + id: "context-transfer:mcp-steer-result", + type: "subagent_result", + sourceThreadId: childThreadId, + targetThreadId: parentThreadId, + sourcePoint: { runId: steeredRunId }, + }, + ], + subagents: [task(steeredTaskId), task(promotedTaskId)], + } as unknown as OrchestrationV2ThreadProjection; + const childProjection = { + thread: { id: childThreadId }, + runs: [ + { id: steeredRunId, ordinal: 1, status: "completed", delegatedTaskId: steeredTaskId }, + { + id: RunId.make("run:mcp-steer-promoted"), + ordinal: 2, + status: "cancelled", + delegatedTaskId: promotedTaskId, + userMessageId: promotedMessageId, + }, + ], + contextTransfers: [], + // Promoting the follow-up moved its message into the steered run. + messages: [{ id: promotedMessageId, role: "user", runId: steeredRunId }], + subagents: [], + providerThreads: [], + turnItems: [], + } as unknown as OrchestrationV2ThreadProjection; + const layerDependencies = Layer.mergeAll( + NodeServices.layer, + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: (threadId) => + Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection), + delegatedTaskResultPending: () => Effect.succeed(false), + dispatch: () => Effect.succeed({} as never), + }), + ServerSettings.layerTest(), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ + list: () => Effect.succeed([]), + }), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ); + const scope: McpInvocationScope = { + environmentId: EnvironmentId.make("environment:mcp-steer"), + requestNamespace: "provider-session:mcp-steer", + thread: { + threadId: parentThreadId, + providerSessionId: "provider-session:mcp-steer", + providerInstanceId: ProviderInstanceId.make("codex"), + }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }; + + yield* Effect.gen(function* () { + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const promoted = yield* service.taskStatus(scope, promotedTaskId); + assert.equal(promoted.status, "completed"); + assert.equal(promoted.resultContextTransferId, "context-transfer:mcp-steer-result"); + }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(layerDependencies)))); + }), + ); + it.effect("does not dispose delivery when a nonterminal task has no active child run", () => Effect.gen(function* () { const parentThreadId = ThreadId.make("thread:mcp-cancel-parent"); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index e3b860267a134..ff1ae4f9b790e 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -435,10 +435,12 @@ function directAppOwnedChildTasks( function taskResultTransfer( parent: Pick, - child: Pick, + child: Pick, task: OrchestrationV2Subagent, ) { const originalRun = delegatedTaskRun(child, task); + // A follow-up promoted to Steer shares the transfer of the run it joined. + const resultRun = delegatedTaskResultRun(child, task.id); const resultRunIds = new Set( child.runs .filter( @@ -449,6 +451,7 @@ function taskResultTransfer( .map((run) => run.id), ); if (originalRun !== undefined) resultRunIds.add(originalRun.id); + if (resultRun !== undefined) resultRunIds.add(resultRun.id); return parent.contextTransfers.find( (transfer) => transfer.type === "subagent_result" && @@ -2524,8 +2527,13 @@ const make = Effect.gen(function* () { tasks.length > 0 && (input.textOffset ?? 0) === 0 ) { + const targetMessages = messagesByThreadId.get(target.thread.id) ?? []; for (const task of tasks) { - const transfer = taskResultTransfer(parent, target, task); + const transfer = taskResultTransfer( + parent, + { ...target, messages: targetMessages }, + task, + ); const resultRunId = transfer?.sourcePoint.runId ?? delegatedTaskRun(target, task)?.id; const resultRunIds = resultRunId === undefined ? [] : [resultRunId]; const resultRecords = yield* threadManagement From f07216678eb8b836c2422dfb675aca96b37845ce Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:04:03 +0000 Subject: [PATCH 05/14] test(server): cover steered delegated task settlement --- .../src/mcp/OrchestratorMcpService.test.ts | 91 --- .../DelegatedCompletionDelivery.test.ts | 578 +++++++++++------- .../SubagentProjection.test.ts | 47 -- 3 files changed, 342 insertions(+), 374 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 0cfde6a79c2e6..b524921f02a66 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -278,97 +278,6 @@ describe("OrchestratorMcpService", () => { }), ); - it.effect("reports the steered turn's result transfer for a follow-up promoted to Steer", () => - Effect.gen(function* () { - const parentThreadId = ThreadId.make("thread:mcp-steer-parent"); - const childThreadId = ThreadId.make("thread:mcp-steer-child"); - const steeredTaskId = NodeId.make("node:mcp-steer-steered"); - const promotedTaskId = NodeId.make("node:mcp-steer-promoted"); - const steeredRunId = RunId.make("run:mcp-steer-steered"); - const promotedMessageId = "message:mcp-steer-promoted"; - const task = (id: NodeId) => ({ - id, - threadId: parentThreadId, - origin: "app_owned", - childThreadId, - driver: "codex", - model: "gpt-5.6-terra", - status: "completed", - result: "Merged both branches.", - completionDelivery: { state: "delivered" }, - }); - const parentProjection = { - thread: { id: parentThreadId }, - runs: [], - contextTransfers: [ - { - id: "context-transfer:mcp-steer-result", - type: "subagent_result", - sourceThreadId: childThreadId, - targetThreadId: parentThreadId, - sourcePoint: { runId: steeredRunId }, - }, - ], - subagents: [task(steeredTaskId), task(promotedTaskId)], - } as unknown as OrchestrationV2ThreadProjection; - const childProjection = { - thread: { id: childThreadId }, - runs: [ - { id: steeredRunId, ordinal: 1, status: "completed", delegatedTaskId: steeredTaskId }, - { - id: RunId.make("run:mcp-steer-promoted"), - ordinal: 2, - status: "cancelled", - delegatedTaskId: promotedTaskId, - userMessageId: promotedMessageId, - }, - ], - contextTransfers: [], - // Promoting the follow-up moved its message into the steered run. - messages: [{ id: promotedMessageId, role: "user", runId: steeredRunId }], - subagents: [], - providerThreads: [], - turnItems: [], - } as unknown as OrchestrationV2ThreadProjection; - const layerDependencies = Layer.mergeAll( - NodeServices.layer, - Layer.mock(ThreadManagementService.ThreadManagementService)({ - getThreadRecords: (threadId) => - Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection), - delegatedTaskResultPending: () => Effect.succeed(false), - dispatch: () => Effect.succeed({} as never), - }), - ServerSettings.layerTest(), - Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), - Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ - list: () => Effect.succeed([]), - }), - Layer.mock(ProjectService.ProjectService)({}), - Layer.mock(SecretRequests.SecretRequests)({}), - Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), - ); - const scope: McpInvocationScope = { - environmentId: EnvironmentId.make("environment:mcp-steer"), - requestNamespace: "provider-session:mcp-steer", - thread: { - threadId: parentThreadId, - providerSessionId: "provider-session:mcp-steer", - providerInstanceId: ProviderInstanceId.make("codex"), - }, - client: undefined, - capabilities: new Set(["orchestration"]), - issuedAt: 1, - }; - - yield* Effect.gen(function* () { - const service = yield* OrchestratorMcpService.OrchestratorMcpService; - const promoted = yield* service.taskStatus(scope, promotedTaskId); - assert.equal(promoted.status, "completed"); - assert.equal(promoted.resultContextTransferId, "context-transfer:mcp-steer-result"); - }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(layerDependencies)))); - }), - ); - it.effect("does not dispose delivery when a nonterminal task has no active child run", () => Effect.gen(function* () { const parentThreadId = ThreadId.make("thread:mcp-cancel-parent"); diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 53b00d084e8f4..473dcc929760f 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -5,6 +5,7 @@ import { assert, it } from "@effect/vitest"; import { CommandId, EventId, + EnvironmentId, MessageId, type ModelSelection, NodeId, @@ -44,6 +45,16 @@ import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.tes import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; +import { makeSubagentConversationArtifacts } from "@t3tools/provider-core/server/subagentProjection"; +import * as OrchestratorMcpService from "../mcp/OrchestratorMcpService.ts"; +import type { McpInvocationScope } from "../mcp/McpInvocationContext.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ThreadLaunchService from "./ThreadLaunchService.ts"; +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; +import * as SecretRequests from "../secrets/SecretRequests.ts"; +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; + const layerPlatformTest = Layer.merge( NodeServices.layer, Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ @@ -2056,270 +2067,365 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { ); }); -/** - * A child whose second follow-up was queued as its own task, then promoted to - * Steer into the first follow-up's running turn. - */ -const seedPromotedFollowUp = (name: string) => - Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const now = yield* DateTime.now; - const threadId = ThreadId.make(`thread:${name}-parent`); - const runId = RunId.make(`run:${name}-parent`); - const rootNodeId = NodeId.make(`node:${name}-parent-root`); - const projectId = ProjectId.make(`project:${name}-parent`); - yield* seedParentWithTerminalTask({ - threadId, - projectId, - runId, - rootNodeId, - taskId: NodeId.make(`node:${name}-parent-settled`), - deliveryState: "delivered", - now, - }); - const child = yield* seedRestartCancelledChild({ - parentThreadId: threadId, - projectId, - parentRunId: runId, - rootNodeId, - name, - completionWake: "always", - continuationPending: false, - runStatus: "completed", +const layerSteeredTaskTest = OrchestratorMcpService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(GitVcsDriver.GitVcsDriver)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ + list: () => Effect.succeed([]), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ServerSettings.layerTest(), + NodeServices.layer, + ), + ), + Layer.provideMerge(layerTest), +); + +const seedSteeredTasks = Effect.fnUntraced(function* (name: string) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const parentThreadId = ThreadId.make(`${name}:parent`); + const parentRunId = RunId.make(`${name}:parent`); + const rootNodeId = NodeId.make(`${name}:parent`); + const projectId = ProjectId.make(`${name}:project`); + yield* seedParentWithTerminalTask({ + threadId: parentThreadId, + runId: parentRunId, + rootNodeId, + projectId, + taskId: NodeId.make(`${name}:seed`), + deliveryState: "delivered", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId, + parentRunId, + projectId, + rootNodeId, + name, + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.recoverDelegatedTask(child.childThreadId, child.childRunId); + const original = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (task) => task.id === child.taskId, + )!; + const steeredTaskId = NodeId.make(`${name}:steered`); + const promotedTaskId = NodeId.make(`${name}:promoted`); + const promotedMessageId = MessageId.make(`${name}:promoted`); + const steeredRun = { + ...runEvent({ + threadId: child.childThreadId, + runId: RunId.make(`${name}:steered`), + ordinal: 2, + status: "running", now, - }); - const followUpTask = (taskId: NodeId, prompt: string) => ({ - id: EventId.make(`event:${taskId}`), - type: "subagent.updated" as const, - threadId, - runId, - nodeId: taskId, - driver, - providerInstanceId: modelSelection.instanceId, - occurredAt: now, - payload: { - id: taskId, - threadId, - runId, - parentNodeId: rootNodeId, - origin: "app_owned" as const, - createdBy: "agent" as const, - driver, - providerInstanceId: modelSelection.instanceId, - providerThreadId: null, - childThreadId: child.childThreadId, - nativeTaskRef: null, - prompt, - title: null, - model: null, - completionWake: "always" as const, - status: "pending" as const, - result: null, - startedAt: now, - completedAt: null, - updatedAt: now, - }, - }); - const steeredRunId = RunId.make(`run:${name}:2`); - const steeredTaskId = NodeId.make(`node:${name}-steered`); - const promotedTaskId = NodeId.make(`node:${name}-promoted`); - const promotedMessageId = MessageId.make(`message:${name}-promoted`); - const promotedRun = runEvent({ + }).payload, + delegatedTaskId: steeredTaskId, + }; + const promotedRun = { + ...runEvent({ threadId: child.childThreadId, - runId: RunId.make(`run:${name}:3`), + runId: RunId.make(`${name}:promoted`), ordinal: 3, status: "cancelled", userMessageId: promotedMessageId, now, + }).payload, + delegatedTaskId: promotedTaskId, + startedAt: null, + }; + const conversationEvents = (role: "user" | "assistant", runId: RunId, messageId: MessageId) => { + const { message, turnItem } = makeSubagentConversationArtifacts({ + threadId: child.childThreadId, + rootNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + messageId, + turnItemId: TurnItemId.make(`${messageId}:item`), + role, + text: role === "user" ? "Also check the queued follow-up." : "Shared result.", + ordinal: role === "user" ? 1 : 2, + now, }); - const steeredRun = ( - status: OrchestrationV2Run["status"], - id = steeredRunId, - ordinal = 2, - restartContinuationOfRunId?: RunId, - ) => { - const event = runEvent({ threadId: child.childThreadId, runId: id, ordinal, status, now }); - return { - ...event, + return [ + { + id: EventId.make(`${messageId}:message`), + type: "message.updated" as const, + threadId: child.childThreadId, + runId, + occurredAt: now, + payload: { ...message, runId }, + }, + { + id: EventId.make(`${messageId}:item`), + type: "turn-item.updated" as const, + threadId: child.childThreadId, + runId, + occurredAt: now, + payload: { ...turnItem, runId }, + }, + ]; + }; + yield* sink.write({ + events: [ + ...[steeredTaskId, promotedTaskId].map((id) => ({ + id: EventId.make(`${id}:task-seed`), + type: "subagent.updated" as const, + threadId: parentThreadId, + runId: parentRunId, + occurredAt: now, payload: { - ...event.payload, - delegatedTaskId: steeredTaskId, - ...(restartContinuationOfRunId === undefined ? {} : { restartContinuationOfRunId }), + ...original, + id, + status: "running" as const, + result: null, + completedAt: null, + completionDelivery: undefined, }, - }; - }; - yield* eventSink.write({ - commandId: CommandId.make(`command:${name}:promoted`), + })), + ...[steeredRun, promotedRun].map((run) => ({ + id: EventId.make(`${run.id}:run-seed`), + type: "run.updated" as const, + threadId: child.childThreadId, + runId: run.id, + occurredAt: now, + payload: run, + })), + ...conversationEvents("user", steeredRun.id, promotedMessageId), + ], + }); + const finish = Effect.fnUntraced(function* (run: OrchestrationV2Run) { + const messageId = MessageId.make(`${run.id}:result`); + yield* sink.write({ events: [ - followUpTask(steeredTaskId, "Merge the fixture branch."), - followUpTask(promotedTaskId, "Also merge the pricing branch."), - steeredRun("running"), - { - ...promotedRun, - payload: { ...promotedRun.payload, startedAt: null, delegatedTaskId: promotedTaskId }, - }, + ...conversationEvents("assistant", run.id, messageId), { - id: EventId.make(`event:${name}:promoted-message`), - type: "message.updated", + id: EventId.make(`${run.id}:finished`), + type: "run.updated", threadId: child.childThreadId, - runId: steeredRunId, + runId: run.id, occurredAt: now, - payload: { - id: promotedMessageId, - threadId: child.childThreadId, - runId: steeredRunId, - nodeId: null, - role: "user", - text: "Also merge the pricing branch.", - attachments: [], - streaming: false, - createdBy: "agent", - creationSource: "mcp", - createdAt: now, - updatedAt: now, - }, + payload: { ...run, completedAt: now }, }, ], }); - /** Ends `resultRunId` with an assistant reply; `run` is that run's terminal event. */ - const finish = (resultRunId: RunId, run: ReturnType) => - eventSink.write({ - commandId: CommandId.make(`command:${resultRunId}:completed`), - events: [ - { - id: EventId.make(`event:${resultRunId}:result`), - type: "message.updated", - threadId: child.childThreadId, - runId: resultRunId, - occurredAt: now, - payload: { - id: MessageId.make(`message:${resultRunId}:result`), - threadId: child.childThreadId, - runId: resultRunId, - nodeId: null, - role: "assistant", - text: "Merged both branches.", - attachments: [], - streaming: false, - createdBy: "agent", - creationSource: "server", - createdAt: now, - updatedAt: now, - }, - }, - { ...run, id: EventId.make(`event:${resultRunId}:finished`) }, - ], - }); - return { - threadId, - child, - steeredRunId, - steeredRun, - steeredTaskId, - promotedTaskId, - promotedRunId: promotedRun.payload.id, - finish, - }; + yield* orchestrator.recoverDelegatedTask(child.childThreadId, run.id); + return TurnItemId.make(`${messageId}:item`); }); + const scope: McpInvocationScope = { + environmentId: EnvironmentId.make("steered-test"), + requestNamespace: name, + thread: { + threadId: parentThreadId, + providerSessionId: name, + providerInstanceId: modelSelection.instanceId, + }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }; + return { + ...child, + parentThreadId, + steeredRun, + promotedRun, + promotedTaskId, + steeredTaskId, + now, + finish, + scope, + }; +}); -it.layer(layerTest)("delegated follow-ups promoted to Steer", (it) => { - it.effect("settles a promoted follow-up with the turn it steered, not as cancelled", () => - Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const now = yield* DateTime.now; - const seeded = yield* seedPromotedFollowUp("steer-child"); - const { threadId, child, promotedTaskId, steeredTaskId } = seeded; - - yield* orchestrator.recoverDelegatedTask(child.childThreadId, seeded.promotedRunId); - - const steering = yield* orchestrator.getThreadProjection(threadId); - assert.equal(steering.subagents.find((row) => row.id === promotedTaskId)?.status, "pending"); - assert.isTrue( - yield* orchestrator.delegatedTaskResultPending(child.childThreadId, promotedTaskId), - ); +it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { + it.effect.each(["completed", "failed"] as const)( + "shares the running turn outcome and delivers each task once (%s)", + (status) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const seeded = yield* seedSteeredTasks(`steered-${status}`); + yield* orchestrator.recoverDelegatedTask(seeded.childThreadId, seeded.promotedRun.id); + const working = yield* service.taskStatus(seeded.scope, seeded.promotedTaskId); + assert.equal(working.status, "running"); + assert.equal(working.workState, "working"); + assert.isNull(working.summary); + assert.isTrue( + yield* orchestrator.delegatedTaskResultPending( + seeded.childThreadId, + seeded.promotedTaskId, + ), + ); - // A later run of an unrelated task still awaits its restart continuation. - const unrelatedRunId = RunId.make("run:steer-child:4"); - const unrelatedRun = runEvent({ - threadId: child.childThreadId, - runId: unrelatedRunId, - ordinal: 4, - status: "cancelled", - now, - }); - yield* eventSink.writeWithEffects({ - commandId: reconcileCommandId("steer-child-unrelated"), - events: [ - { - ...unrelatedRun, - payload: { - ...unrelatedRun.payload, - delegatedTaskId: NodeId.make("node:steer-child-unrelated"), + const sink = yield* EventSink.EventSinkV2; + const unrelated = runEvent({ + threadId: seeded.childThreadId, + runId: RunId.make(`${status}:unrelated`), + ordinal: 4, + status: "cancelled", + now: seeded.now, + }); + yield* sink.writeWithEffects({ + commandId: reconcileCommandId(`${status}:unrelated`), + events: [ + { + ...unrelated, + payload: { + ...unrelated.payload, + delegatedTaskId: NodeId.make(`${status}:unrelated`), + }, }, - }, - ], - effects: [ - { - id: `effect:restart-continuation:${unrelatedRunId}`, - commandId: reconcileCommandId("steer-child-unrelated"), - threadId: child.childThreadId, - request: { type: "provider-runtime.continue", sourceRunId: unrelatedRunId }, - }, - ], - }); - yield* seeded.finish(seeded.steeredRunId, seeded.steeredRun("completed")); - yield* orchestrator.recoverDelegatedTask(child.childThreadId, seeded.steeredRunId); + ], + effects: [ + { + id: `effect:restart-continuation:${unrelated.runId}`, + commandId: reconcileCommandId(`${status}:unrelated`), + threadId: seeded.childThreadId, + request: { type: "provider-runtime.continue", sourceRunId: unrelated.runId }, + }, + ], + }); + yield* seeded.finish({ ...seeded.steeredRun, status }); + const settled = yield* orchestrator.getThreadProjection(seeded.parentThreadId); + const shared = settled.contextTransfers.filter( + (transfer) => transfer.sourcePoint.runId === seeded.steeredRun.id, + ); + assert.lengthOf(shared, 1); + yield* orchestrator.recoverDelegatedTasks; + yield* orchestrator.recoverDelegatedTasks; + const again = yield* orchestrator.getThreadProjection(seeded.parentThreadId); + assert.deepEqual(again.subagents, settled.subagents); + assert.deepEqual(again.contextTransfers, settled.contextTransfers); + assert.deepEqual(again.runs, settled.runs); + for (const id of [seeded.steeredTaskId, seeded.promotedTaskId]) { + const result = yield* service.taskStatus(seeded.scope, id); + assert.equal(result.status, status); + assert.equal(result.summary, "Shared result."); + assert.equal(result.resultContextTransferId, shared[0]!.id); + } + assert.isFalse( + yield* orchestrator.delegatedTaskResultPending( + seeded.childThreadId, + seeded.promotedTaskId, + ), + ); + }), + ); - const settled = yield* orchestrator.getThreadProjection(threadId); - const task = (id: NodeId) => settled.subagents.find((row) => row.id === id); - assert.equal(task(steeredTaskId)?.result, "Merged both branches."); - assert.equal(task(promotedTaskId)?.status, "completed"); - assert.equal(task(promotedTaskId)?.result, "Merged both branches."); - assert.isFalse( - yield* orchestrator.delegatedTaskResultPending(child.childThreadId, promotedTaskId), - ); - }), + it.effect( + "holds a promoted task through two restart cuts and shares the continuation result", + () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const sink = yield* EventSink.EventSinkV2; + const seeded = yield* seedSteeredTasks("steered-restarts"); + let run: OrchestrationV2Run = seeded.steeredRun; + for (const ordinal of [4, 5]) { + yield* sink.writeWithEffects({ + commandId: reconcileCommandId(`steered-restarts-${ordinal}`), + events: [ + { + id: EventId.make(`${run.id}:cut`), + type: "run.updated", + threadId: seeded.childThreadId, + runId: run.id, + occurredAt: seeded.now, + payload: { ...run, status: "cancelled", completedAt: seeded.now }, + }, + ], + effects: [ + { + id: `effect:restart-continuation:${run.id}`, + commandId: reconcileCommandId(`steered-restarts-${ordinal}`), + threadId: seeded.childThreadId, + request: { type: "provider-runtime.continue", sourceRunId: run.id }, + }, + ], + }); + yield* orchestrator.recoverDelegatedTasks; + assert.isTrue( + yield* orchestrator.delegatedTaskResultPending( + seeded.childThreadId, + seeded.promotedTaskId, + ), + ); + assert.equal( + (yield* service.taskStatus(seeded.scope, seeded.promotedTaskId)).workState, + "working", + ); + run = { + ...seeded.steeredRun, + id: RunId.make(`steered-restarts:${ordinal}`), + ordinal, + restartContinuationOfRunId: run.id, + status: "starting", + startedAt: null, + }; + yield* sink.write({ + events: [ + { + id: EventId.make(`${run.id}:continued`), + type: "run.updated", + threadId: seeded.childThreadId, + runId: run.id, + occurredAt: seeded.now, + payload: run, + }, + ], + }); + } + yield* seeded.finish({ ...run, status: "completed", startedAt: seeded.now }); + const result = yield* service.taskStatus(seeded.scope, seeded.promotedTaskId); + assert.equal(result.status, "completed"); + assert.equal(result.summary, "Shared result."); + const parent = yield* orchestrator.getThreadProjection(seeded.parentThreadId); + const transfers = parent.contextTransfers.filter( + (transfer) => transfer.sourcePoint.runId === run.id, + ); + assert.lengthOf(transfers, 1); + assert.equal(result.resultContextTransferId, transfers[0]!.id); + }), ); - it.effect("keeps a promoted follow-up open while a restart continues its turn", () => + it.effect("acknowledges both tasks when a result page excludes the promoted user message", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const seeded = yield* seedPromotedFollowUp("steer-restart-child"); - const { threadId, child, promotedTaskId, steeredRunId } = seeded; - const continuationRunId = RunId.make("run:steer-restart-child:4"); - // A restart cut the steered turn, and its continuation resumed it. - yield* eventSink.write({ - commandId: reconcileCommandId("steer-restart-child-cut"), - events: [seeded.steeredRun("cancelled")], - }); - yield* eventSink.write({ - commandId: CommandId.make("command:steer-restart-child:continued"), - events: [seeded.steeredRun("running", continuationRunId, 4, steeredRunId)], + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const seeded = yield* seedSteeredTasks("steered-page"); + const itemId = yield* seeded.finish({ ...seeded.steeredRun, status: "completed" }); + const page = yield* service.readThread(seeded.scope, { + threadId: seeded.childThreadId, + itemId, + limit: 1, }); - - yield* orchestrator.recoverDelegatedTask(child.childThreadId, steeredRunId); - - const continuing = yield* orchestrator.getThreadProjection(threadId); + assert.lengthOf(page.items, 1); + assert.equal(page.items[0]?.type, "assistant_message"); + const parent = yield* orchestrator.getThreadProjection(seeded.parentThreadId); + for (const id of [seeded.steeredTaskId, seeded.promotedTaskId]) { + assert.equal( + parent.subagents.find((task) => task.id === id)?.completionDelivery?.state, + "acknowledged", + id, + ); + } + const result = yield* service.taskStatus(seeded.scope, seeded.promotedTaskId); + assert.equal(result.summary, "Shared result."); assert.equal( - continuing.subagents.find((row) => row.id === promotedTaskId)?.status, - "pending", - ); - assert.isTrue( - yield* orchestrator.delegatedTaskResultPending(child.childThreadId, promotedTaskId), - ); - - yield* seeded.finish( - continuationRunId, - seeded.steeredRun("completed", continuationRunId, 4, steeredRunId), + result.resultContextTransferId, + parent.contextTransfers.find( + (transfer) => transfer.sourcePoint.runId === seeded.steeredRun.id, + )?.id, ); - yield* orchestrator.recoverDelegatedTask(child.childThreadId, continuationRunId); - - const settled = yield* orchestrator.getThreadProjection(threadId); - const promoted = settled.subagents.find((row) => row.id === promotedTaskId); - assert.equal(promoted?.status, "completed"); - assert.equal(promoted?.result, "Merged both branches."); }), ); }); diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index d9a5474246bc1..961add539fbea 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -20,7 +20,6 @@ import * as Effect from "effect/Effect"; import { makeSubagentChildThread, delegatedTaskProgress, - delegatedTaskResultRun, subagentResultForRun, makeSubagentConversationArtifacts, endOrphanedNativeSubagent, @@ -191,52 +190,6 @@ it("reports the run that ended last, not the highest ordinal", () => { assert.equal(progress.resultRun?.ordinal, 3); }); -it("settles a follow-up promoted to Steer with the run its message joined", () => { - const { run } = taskFixture(); - const taskId = NodeId.make("node:follow-up"); - const messageId = MessageId.make("message:follow-up"); - const steered = { ...run, status: "running" as const }; - const queued: OrchestrationV2Run = { - ...run, - id: RunId.make("run:follow-up"), - ordinal: 2, - status: "cancelled", - userMessageId: messageId, - delegatedTaskId: taskId, - }; - const message = { id: messageId, runId: queued.id }; - // Stopped while queued: the message still belongs to its own run. - assert.equal( - delegatedTaskResultRun({ runs: [steered, queued], messages: [message] }, taskId)?.id, - queued.id, - ); - // Promoted to Steer: the message moved into the running turn. - assert.equal( - delegatedTaskResultRun( - { runs: [steered, queued], messages: [{ ...message, runId: steered.id }] }, - taskId, - )?.id, - steered.id, - ); - // A restart cut the steered turn; its continuation now carries the result. - const continuation: OrchestrationV2Run = { - ...run, - id: RunId.make("run:continuation"), - ordinal: 3, - restartContinuationOfRunId: steered.id, - }; - assert.equal( - delegatedTaskResultRun( - { - runs: [{ ...steered, status: "cancelled" }, queued, continuation], - messages: [{ ...message, runId: steered.id }], - }, - taskId, - )?.id, - continuation.id, - ); -}); - it("waits for nested work and retains the report across monitor acknowledgements", () => { const { projection, run } = taskFixture(); assert.equal( From fc21bef2473355eeda84adda82e859d28c5a6268 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:09:49 +0000 Subject: [PATCH 06/14] fix(server): settle steered delegates with their shared turn --- apps/server/src/mcp/OrchestratorMcpService.ts | 70 +++++++------------ .../src/orchestration-v2/Orchestrator.ts | 10 ++- .../src/server/subagentProjection.ts | 7 +- 3 files changed, 32 insertions(+), 55 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 7a1cfdd381146..64bfb67b924fd 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -419,8 +419,7 @@ function isTerminalTaskStatus( function directAppOwnedChildTasks( parent: Pick, - target: Pick, - returnedRunIds: ReadonlySet, + target: Pick, ): ReadonlyArray { if ( target.thread.lineage.parentThreadId !== parent.thread.id || @@ -428,23 +427,10 @@ function directAppOwnedChildTasks( ) { return []; } - const original = parent.subagents.find( - (task) => - task.origin === "app_owned" && - task.threadId === parent.thread.id && - task.id === - (target.thread.forkedFrom?.type === "node" ? target.thread.forkedFrom.nodeId : undefined) && - task.childThreadId === target.thread.id, - ); - const taskIds = new Set( - target.runs - .filter((run) => returnedRunIds.has(run.id)) - .map((run) => run.delegatedTaskId ?? original?.id), - ); return parent.subagents.filter( (task) => - taskIds.has(task.id) && task.origin === "app_owned" && + task.threadId === parent.thread.id && task.childThreadId === target.thread.id, ); } @@ -479,24 +465,14 @@ function taskResultTransfer( } function pageIncludesTerminalTaskResult(input: { - readonly parent: Pick; readonly page: ReadonlyArray; - readonly task: OrchestrationV2Subagent; - readonly target: Pick< - OrchestrationV2ThreadProjection, - "thread" | "runs" | "contextTransfers" | "messages" | "turnItems" - >; + readonly run: OrchestrationV2Run; + readonly target: Pick; readonly maxChars: number; }): boolean { - const transfer = taskResultTransfer(input.parent, input.target, input.task); - if (transfer === undefined) return false; - const run = - transfer.sourcePoint.runId === undefined - ? delegatedTaskRun(input.target, input.task) - : input.target.runs.find((run) => run.id === transfer.sourcePoint.runId); - if (run === undefined || !isTerminalTaskStatus(taskStatusForRun(run))) return false; - - const result = subagentResultForRun(input.target, run); + if (!isTerminalTaskStatus(taskStatusForRun(input.run))) return false; + + const result = subagentResultForRun(input.target, input.run); if (result.messageId === null && result.turnItemId === null) return false; return input.page.some((row) => { @@ -2553,23 +2529,32 @@ const make = Effect.gen(function* () { .filter((row) => row.sourceThreadId === target.thread.id) .map((row) => row.item.runId), ); - const tasks = - parent === undefined ? [] : directAppOwnedChildTasks(parent, target, returnedRunIds); + const tasks = parent === undefined ? [] : directAppOwnedChildTasks(parent, target); if ( parent !== undefined && scope.thread !== undefined && tasks.length > 0 && (input.textOffset ?? 0) === 0 ) { - const targetMessages = messagesByThreadId.get(target.thread.id) ?? []; + // Promotion moves the queued user message to its joined run, possibly outside this page. + const taskMessages = yield* threadManagement + .getThreadRecords(target.thread.id, ["messages"], { + messageIds: target.runs + .filter((run) => run.status === "cancelled" && run.delegatedTaskId !== undefined) + .map((run) => run.userMessageId) + .filter((id): id is MessageId => id !== null), + }) + .pipe(Effect.mapError(threadManagementFailure)); + const taskTarget = { ...target, messages: taskMessages.messages }; for (const task of tasks) { - const transfer = taskResultTransfer( - parent, - { ...target, messages: targetMessages }, - task, - ); - const resultRunId = transfer?.sourcePoint.runId ?? delegatedTaskRun(target, task)?.id; - const resultRunIds = resultRunId === undefined ? [] : [resultRunId]; + const transfer = taskResultTransfer(parent, taskTarget, task); + if (transfer === undefined) continue; + const run = + transfer.sourcePoint.runId === undefined + ? delegatedTaskRun(target, task) + : target.runs.find((candidate) => candidate.id === transfer.sourcePoint.runId); + if (run === undefined || !returnedRunIds.has(run.id)) continue; + const resultRunIds = [run.id]; const resultRecords = yield* threadManagement .getThreadRecords(target.thread.id, ["messages", "turnItems"], { messageRoles: ["assistant"], @@ -2580,9 +2565,8 @@ const make = Effect.gen(function* () { .pipe(Effect.mapError(threadManagementFailure)); if ( pageIncludesTerminalTaskResult({ - parent, page, - task, + run, target: { ...target, ...resultRecords }, maxChars, }) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index e42428c76c3f8..762a2729b7446 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -392,14 +392,12 @@ export class OrchestratorV2 extends Context.Service, run: OrchestrationV2Run, ): ReadonlyArray { - return run.delegatedTaskId === undefined - ? runs - : runs.filter((candidate) => candidate.delegatedTaskId === run.delegatedTaskId); + return runs.filter((candidate) => candidate.delegatedTaskId === run.delegatedTaskId); } /** Run ordinals can skip values: a queued run moves past newer runs when it starts. */ @@ -10266,9 +10264,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? options.task.id === forkedFrom.nodeId ? options.resultRun : undefined - : delegatedTaskResultRun(childControls, options.taskId); + : delegatedTaskResultRun(childControls, options.task.id); if (childRun === undefined) return; - const steered = taskRuns.length > 0 && !taskRuns.includes(childRun); + const steered = taskRuns.length > 0 && childRun.delegatedTaskId !== options.task.id; const outcome = runOutcome(childRun); const terminalStatus = delegatedTaskTerminalStatus(outcome); if (terminalStatus === null) { diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index 60d897fe6b45d..0c3e7f8fb630e 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -219,12 +219,7 @@ export function subagentResultForRun( }; } -/** - * The run whose outcome settles a delegated task. Promoting a queued follow-up - * to Steer cancels its run and moves its message into the running turn, so the - * task settles with that turn, or the restart continuation that resumed it, - * instead of reading as stopped. - */ +/** Promotion cancels the queued run; its moved message identifies the turn that now owns the work. */ export function delegatedTaskResultRun( child: { readonly runs: OrchestrationV2ThreadProjection["runs"]; From 589b34e2bada363733901973a51f61eae1ac612e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:27:44 +0000 Subject: [PATCH 07/14] fix(server): sync promoted follow-up state and result transfers --- .../DelegatedCompletionDelivery.test.ts | 160 +++++++++++++----- .../src/orchestration-v2/Orchestrator.ts | 16 +- 2 files changed, 131 insertions(+), 45 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 473dcc929760f..fbce52d9ffaab 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -2085,7 +2085,10 @@ const layerSteeredTaskTest = OrchestratorMcpService.layer.pipe( Layer.provideMerge(layerTest), ); -const seedSteeredTasks = Effect.fnUntraced(function* (name: string) { +const seedSteeredTasks = Effect.fnUntraced(function* ( + name: string, + { queueHeld = false, monitor = false } = {}, +) { const orchestrator = yield* Orchestrator.OrchestratorV2; const sink = yield* EventSink.EventSinkV2; const now = yield* DateTime.now; @@ -2120,15 +2123,48 @@ const seedSteeredTasks = Effect.fnUntraced(function* (name: string) { const steeredTaskId = NodeId.make(`${name}:steered`); const promotedTaskId = NodeId.make(`${name}:promoted`); const promotedMessageId = MessageId.make(`${name}:promoted`); - const steeredRun = { - ...runEvent({ + let monitorRun: OrchestrationV2Run | undefined; + if (monitor) { + const messageId = MessageId.make(`${name}:monitor`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "server", + commandId: CommandId.make(`${name}:monitor`), threadId: child.childThreadId, - runId: RunId.make(`${name}:steered`), - ordinal: 2, - status: "running", - now, - }).payload, - delegatedTaskId: steeredTaskId, + messageId, + text: "A monitor reported new output.", + attachments: [], + notification: { + source: { kind: "monitor" }, + outcome: "updated", + summary: "Monitor updated", + }, + dispatchMode: { type: "queue_after_active" }, + }); + monitorRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( + (run) => run.userMessageId === messageId, + )!; + assert.isUndefined(monitorRun.delegatedTaskId); + assert.lengthOf( + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.filter( + (task) => task.childThreadId === child.childThreadId, + ), + 1, + ); + } + const steeredRun: OrchestrationV2Run = { + ...(monitorRun ?? { + ...runEvent({ + threadId: child.childThreadId, + runId: RunId.make(`${name}:steered`), + ordinal: 2, + status: "running", + now, + }).payload, + delegatedTaskId: steeredTaskId, + }), + status: "running", }; const promotedRun = { ...runEvent({ @@ -2175,9 +2211,10 @@ const seedSteeredTasks = Effect.fnUntraced(function* (name: string) { }, ]; }; + const beforePromotion = yield* sink.latestSequence(); yield* sink.write({ events: [ - ...[steeredTaskId, promotedTaskId].map((id) => ({ + ...(monitor ? [promotedTaskId] : [steeredTaskId, promotedTaskId]).map((id) => ({ id: EventId.make(`${id}:task-seed`), type: "subagent.updated" as const, threadId: parentThreadId, @@ -2186,7 +2223,12 @@ const seedSteeredTasks = Effect.fnUntraced(function* (name: string) { payload: { ...original, id, - status: "running" as const, + status: + id === promotedTaskId + ? queueHeld + ? ("idle" as const) + : ("pending" as const) + : ("running" as const), result: null, completedAt: null, completionDelivery: undefined, @@ -2236,6 +2278,7 @@ const seedSteeredTasks = Effect.fnUntraced(function* (name: string) { return { ...child, parentThreadId, + beforePromotion, steeredRun, promotedRun, promotedTaskId, @@ -2253,8 +2296,27 @@ it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const service = yield* OrchestratorMcpService.OrchestratorMcpService; - const seeded = yield* seedSteeredTasks(`steered-${status}`); + const seeded = yield* seedSteeredTasks(`steered-${status}`, { + queueHeld: status === "failed", + }); yield* orchestrator.recoverDelegatedTask(seeded.childThreadId, seeded.promotedRun.id); + yield* (yield* EventSink.EventSinkV2) + .stream({ afterSequence: seeded.beforePromotion, eventType: "subagent.updated" }) + .pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === seeded.promotedTaskId && + stored.event.payload.status === "running", + ), + Stream.take(1), + Stream.runHead, + ); + const activeParent = yield* orchestrator.getThreadProjection(seeded.parentThreadId); + assert.equal( + activeParent.subagents.find((task) => task.id === seeded.promotedTaskId)?.status, + "running", + ); const working = yield* service.taskStatus(seeded.scope, seeded.promotedTaskId); assert.equal(working.status, "running"); assert.equal(working.workState, "working"); @@ -2397,35 +2459,53 @@ it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { }), ); - it.effect("acknowledges both tasks when a result page excludes the promoted user message", () => - Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const service = yield* OrchestratorMcpService.OrchestratorMcpService; - const seeded = yield* seedSteeredTasks("steered-page"); - const itemId = yield* seeded.finish({ ...seeded.steeredRun, status: "completed" }); - const page = yield* service.readThread(seeded.scope, { - threadId: seeded.childThreadId, - itemId, - limit: 1, - }); - assert.lengthOf(page.items, 1); - assert.equal(page.items[0]?.type, "assistant_message"); - const parent = yield* orchestrator.getThreadProjection(seeded.parentThreadId); - for (const id of [seeded.steeredTaskId, seeded.promotedTaskId]) { + it.effect.each(["delegated", "monitor"] as const)( + "acknowledges tasks when a result page excludes the promoted user message (%s)", + (owner) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const seeded = yield* seedSteeredTasks(`steered-page-${owner}`, { + monitor: owner === "monitor", + }); + const itemId = yield* seeded.finish({ ...seeded.steeredRun, status: "completed" }); + const settled = yield* orchestrator.getThreadProjection(seeded.parentThreadId); + assert.lengthOf( + settled.contextTransfers.filter( + (transfer) => transfer.sourcePoint.runId === seeded.steeredRun.id, + ), + 1, + ); + yield* orchestrator.recoverDelegatedTasks; + assert.deepEqual( + (yield* orchestrator.getThreadProjection(seeded.parentThreadId)).contextTransfers, + settled.contextTransfers, + ); + const page = yield* service.readThread(seeded.scope, { + threadId: seeded.childThreadId, + itemId, + limit: 1, + }); + assert.lengthOf(page.items, 1); + assert.equal(page.items[0]?.type, "assistant_message"); + const parent = yield* orchestrator.getThreadProjection(seeded.parentThreadId); + for (const id of owner === "monitor" + ? [seeded.promotedTaskId] + : [seeded.steeredTaskId, seeded.promotedTaskId]) { + assert.equal( + parent.subagents.find((task) => task.id === id)?.completionDelivery?.state, + "acknowledged", + id, + ); + } + const result = yield* service.taskStatus(seeded.scope, seeded.promotedTaskId); + assert.equal(result.summary, "Shared result."); assert.equal( - parent.subagents.find((task) => task.id === id)?.completionDelivery?.state, - "acknowledged", - id, + result.resultContextTransferId, + parent.contextTransfers.find( + (transfer) => transfer.sourcePoint.runId === seeded.steeredRun.id, + )?.id, ); - } - const result = yield* service.taskStatus(seeded.scope, seeded.promotedTaskId); - assert.equal(result.summary, "Shared result."); - assert.equal( - result.resultContextTransferId, - parent.contextTransfers.find( - (transfer) => transfer.sourcePoint.runId === seeded.steeredRun.id, - )?.id, - ); - }), + }), ); }); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 762a2729b7446..825cf21e8435f 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -10270,7 +10270,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const outcome = runOutcome(childRun); const terminalStatus = delegatedTaskTerminalStatus(outcome); if (terminalStatus === null) { - if (childRun.delegatedTaskId === undefined || childRun.id !== options.syncRunId) return; + if ( + (childRun.delegatedTaskId === undefined || childRun.id !== options.syncRunId) && + !(steered && taskRuns.some((run) => run.id === options.syncRunId)) + ) + return; const task = options.task; const status: OrchestrationV2Subagent["status"] = childRun.status === "queued" @@ -10382,11 +10386,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (transfer.sourcePoint.runId === childRun.id || (options.task.id === forkedFrom.nodeId && transfer.sourcePoint.runId === undefined)), ); - // A steered task shares its run's result transfer, so only its own result marks it settled. + // A promoted task may share a transfer before its own result is settled. if (steered ? task.result !== null : existingResultTransfer !== undefined) { return; } + const sharesResultTransfer = + steered && (childRun.delegatedTaskId !== undefined || existingResultTransfer !== undefined); const now = yield* DateTime.now; const result = subagentResultForRun(childProjection, { ...childRun, status: outcome }); const parentRun = @@ -10429,9 +10435,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : parentProjection.providerThreads.find( (candidate) => candidate.id === parentRun.providerThreadId, ); - // The steered run's own task publishes that run's result to the parent's context. + // A joined run without a task needs its first promoted task to publish the result. const resultHandoff: OrchestrationV2ContextHandoff | null = - steered || + sharesResultTransfer || parentRun === undefined || childProviderThread === undefined || parentProviderThread === undefined @@ -10581,7 +10587,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: resultHandoff, }, ]), - ...(steered + ...(sharesResultTransfer ? [] : [ { From a50cd989aee06ed0762e26027eadf35933eb52ac Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 21 Oct 2026 18:40:54 -0700 Subject: [PATCH 08/14] feat(server): webhook filters fan out over arrays with a [] path wildcard A filter path segment ending in [] resolves to the list of every element's value, so a task can run only when, say, an issue carries a given label: {path: 'body.issue.labels[].name', op: 'equals', value: 'needs more info'}. equals, in and startsWith match when any element does, notEquals only when none equals the value, exists when the list is not empty. Placeholders accept the same syntax and render the list. Before this, filters could only compare scalars and arrays satisfied exists alone, so a GitHub issue_comment hook had to wake the bound thread on every comment in the repository and let the agent discard the rest. Co-Authored-By: Claude Fable 5.1 --- .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../src/scheduledTasks/webhookFilter.test.ts | 95 +++++++++++++++++++ .../src/scheduledTasks/webhookFilter.ts | 26 ++++- .../scheduledTasks/webhookTemplate.test.ts | 17 ++++ .../src/scheduledTasks/webhookTemplate.ts | 24 ++++- packages/contracts/src/scheduledTask.ts | 4 +- 6 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 apps/server/src/scheduledTasks/webhookFilter.test.ts diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 111870be9bbda..567cb92727326 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -118,7 +118,7 @@ const WorkflowStopTool = Tool.make("t3_workflow_stop", { const ScheduleTaskTool = Tool.make("schedule_task", { description: - "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhook dedupe accepts replayKey and coalesceKey templates with coalesceWindowSeconds (10–3600, default 120); replay keys expire after 7 days and coalescing keeps the latest payload in a window measured from its first delivery. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.", + "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhook dedupe accepts replayKey and coalesceKey templates with coalesceWindowSeconds (10–3600, default 120); replay keys expire after 7 days and coalescing keeps the latest payload in a window measured from its first delivery. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. A path segment ending in [] fans out over an array: {path:'body.issue.labels[].name', op:'equals', value:'needs more info'} matches when any label has that name, and notEquals only when none does. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.", parameters: OrchestratorMcpScheduleTaskInput, success: OrchestratorMcpScheduleTaskResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/scheduledTasks/webhookFilter.test.ts b/apps/server/src/scheduledTasks/webhookFilter.test.ts new file mode 100644 index 0000000000000..1194a1aed5177 --- /dev/null +++ b/apps/server/src/scheduledTasks/webhookFilter.test.ts @@ -0,0 +1,95 @@ +import { assert, describe, it } from "@effect/vitest"; + +import { findFailedWebhookFilter } from "./webhookFilter.ts"; +import type { WebhookRequest } from "./webhookTemplate.ts"; + +const issueComment: WebhookRequest = { + method: "POST", + path: "/api/hooks/task-1", + query: "", + headers: { "content-type": "application/json", "x-github-event": "issue_comment" }, + bodyText: JSON.stringify({ + action: "created", + issue: { number: 7, labels: [{ name: "bug" }, { name: "needs more info" }], assignees: [] }, + comment: { user: { login: "reporter", type: "User" } }, + }), +}; + +describe("findFailedWebhookFilter", () => { + it("matches a wildcard path when any element matches", () => { + assert.isNull( + findFailedWebhookFilter( + [ + { path: "body.issue.labels[].name", op: "equals", value: "needs more info" }, + { path: "body.issue.labels[].name", op: "in", value: ["needs more info", "triage"] }, + { path: "body.issue.labels[].name", op: "startsWith", value: "needs" }, + { path: "body.issue.labels[].name", op: "exists" }, + ], + issueComment, + ), + ); + assert.deepEqual( + findFailedWebhookFilter( + [{ path: "body.issue.labels[].name", op: "equals", value: "wontfix" }], + issueComment, + ), + { index: 0, path: "body.issue.labels[].name" }, + ); + }); + + it("fails notEquals on a wildcard path when any element equals the value", () => { + assert.deepEqual( + findFailedWebhookFilter( + [{ path: "body.issue.labels[].name", op: "notEquals", value: "bug" }], + issueComment, + ), + { index: 0, path: "body.issue.labels[].name" }, + ); + assert.isNull( + findFailedWebhookFilter( + [{ path: "body.issue.labels[].name", op: "notEquals", value: "wontfix" }], + issueComment, + ), + ); + }); + + it("treats an empty or missing wildcard list as nothing to match", () => { + assert.deepEqual( + findFailedWebhookFilter( + [{ path: "body.issue.assignees[].login", op: "exists" }], + issueComment, + ), + { index: 0, path: "body.issue.assignees[].login" }, + ); + assert.isNull( + findFailedWebhookFilter( + [{ path: "body.issue.assignees[].login", op: "notEquals", value: "anyone" }], + issueComment, + ), + ); + assert.deepEqual( + findFailedWebhookFilter([{ path: "body.issue.number[].x", op: "exists" }], issueComment), + { index: 0, path: "body.issue.number[].x" }, + ); + }); + + it("still compares scalars without a wildcard", () => { + assert.isNull( + findFailedWebhookFilter( + [ + { path: "body.action", op: "equals", value: "created" }, + { path: "body.comment.user.type", op: "notEquals", value: "Bot" }, + ], + issueComment, + ), + ); + // A list without a wildcard in the path only satisfies exists. + assert.deepEqual( + findFailedWebhookFilter( + [{ path: "body.issue.labels", op: "equals", value: "bug" }], + issueComment, + ), + { index: 0, path: "body.issue.labels" }, + ); + }); +}); diff --git a/apps/server/src/scheduledTasks/webhookFilter.ts b/apps/server/src/scheduledTasks/webhookFilter.ts index 6f880ef7cebe6..882f300364288 100644 --- a/apps/server/src/scheduledTasks/webhookFilter.ts +++ b/apps/server/src/scheduledTasks/webhookFilter.ts @@ -13,11 +13,7 @@ function scalarText(value: unknown): string | undefined { return undefined; } -/** Placeholders render null as missing, so filters treat it the same way. */ -function matchesWebhookFilter(filter: ScheduledTaskWebhookFilter, value: unknown): boolean { - if (filter.op === "exists") return value !== undefined && value !== null; - const text = scalarText(value); - if (text === undefined) return false; +function matchesScalar(filter: ScheduledTaskWebhookFilter, text: string): boolean { switch (filter.op) { case "equals": return text === filter.value; @@ -27,9 +23,29 @@ function matchesWebhookFilter(filter: ScheduledTaskWebhookFilter, value: unknown return filter.value.includes(text); case "startsWith": return text.startsWith(filter.value); + case "exists": + return true; } } +/** + * Placeholders render null as missing, so filters treat it the same way. A + * path with a `[]` wildcard resolves to a list: `equals`, `in` and + * `startsWith` match when any element does, `notEquals` only when none equals + * the value, and `exists` when the list has an element. + */ +function matchesWebhookFilter(filter: ScheduledTaskWebhookFilter, value: unknown): boolean { + if (Array.isArray(value) && /\[\]/.test(filter.path)) { + const texts = value.map(scalarText).filter((text): text is string => text !== undefined); + if (filter.op === "notEquals") return texts.every((text) => text !== filter.value); + return texts.some((text) => matchesScalar(filter, text)); + } + if (filter.op === "exists") return value !== undefined && value !== null; + const text = scalarText(value); + if (text === undefined) return false; + return matchesScalar(filter, text); +} + /** The first filter the request fails, or null when every filter matches. */ export function findFailedWebhookFilter( filters: ReadonlyArray, diff --git a/apps/server/src/scheduledTasks/webhookTemplate.test.ts b/apps/server/src/scheduledTasks/webhookTemplate.test.ts index b5ac2e69aa38f..6aa23e12b0363 100644 --- a/apps/server/src/scheduledTasks/webhookTemplate.test.ts +++ b/apps/server/src/scheduledTasks/webhookTemplate.test.ts @@ -165,6 +165,23 @@ describe("renderWebhookPrompt", () => { ); }); + it("renders a [] wildcard as the list of matching values", () => { + const rendered = renderWebhookPrompt( + "Labels: {{body.pull_request.labels[]}} Names: {{body.reviewers[].login}}", + { + ...githubPullRequest, + bodyText: JSON.stringify({ + pull_request: { labels: ["bug", "ui"] }, + reviewers: [{ login: "ada" }, { name: "no login" }, { login: "bob" }], + }), + }, + "id", + true, + ); + assert.equal(rendered.prompt, 'Labels: [\n "bug",\n "ui"\n] Names: [\n "ada",\n "bob"\n]'); + assert.deepEqual(rendered.missing, []); + }); + it("does not resolve inherited object properties", () => { const rendered = renderWebhookPrompt( "{{body.constructor}}{{body.__proto__}}{{headers.constructor}}{{headers.__proto__}}", diff --git a/apps/server/src/scheduledTasks/webhookTemplate.ts b/apps/server/src/scheduledTasks/webhookTemplate.ts index 063d41a7ec462..84e2599dc0d75 100644 --- a/apps/server/src/scheduledTasks/webhookTemplate.ts +++ b/apps/server/src/scheduledTasks/webhookTemplate.ts @@ -83,12 +83,30 @@ function parseBody(request: WebhookRequest): unknown { } } +const WILDCARD = /^(.*)\[\]$/; + +/** + * Walks `segments` from `root`. A segment ending in `[]` names an array and + * the rest of the path applies to each element, so `labels[].name` resolves + * to the list of every label's name. Elements without the rest of the path + * are dropped; a wildcard on a non-array resolves to undefined. + */ function lookup(root: unknown, segments: ReadonlyArray): unknown { let current = root; - for (const segment of segments) { + for (const [index, segment] of segments.entries()) { if (current === null || typeof current !== "object") return undefined; - if (!Object.hasOwn(current, segment)) return undefined; - current = (current as Record)[segment]; + const wildcard = WILDCARD.exec(segment); + const key = wildcard ? wildcard[1]! : segment; + if (!Object.hasOwn(current, key)) return undefined; + current = (current as Record)[key]; + if (!wildcard) continue; + if (!Array.isArray(current)) return undefined; + const rest = segments.slice(index + 1); + return current.flatMap((element) => { + const value = rest.length === 0 ? element : lookup(element, rest); + if (value === undefined) return []; + return rest.some((part) => WILDCARD.test(part)) && Array.isArray(value) ? value : [value]; + }); } return current; } diff --git a/packages/contracts/src/scheduledTask.ts b/packages/contracts/src/scheduledTask.ts index 67e2314a8b010..3e1ad5fbdf350 100644 --- a/packages/contracts/src/scheduledTask.ts +++ b/packages/contracts/src/scheduledTask.ts @@ -109,7 +109,7 @@ const WebhookFilterPath = TrimmedNonEmptyString.check( Schema.isPattern(/^(body|headers|query)(\.|$)/), ).annotate({ description: - "Request value to test, in placeholder syntax without braces: headers.x-github-event, body.action, query.source.", + "Request value to test, in placeholder syntax without braces: headers.x-github-event, body.action, query.source. A segment ending in [] fans out over an array, so body.issue.labels[].name tests every label's name.", }); /** @@ -134,7 +134,7 @@ export const ScheduledTaskWebhookFilter = Schema.Union([ Schema.Struct({ path: WebhookFilterPath, op: Schema.Literal("exists") }), ]).annotate({ description: - "Scalars compare as strings; objects and arrays only satisfy 'exists'. Every op but 'exists' fails when the path is missing.", + "Scalars compare as strings; objects and arrays only satisfy 'exists'. Every op but 'exists' fails when the path is missing. A [] wildcard path yields a list: equals, in and startsWith match when any element does, notEquals when no element equals the value, exists when the list is not empty.", }); export type ScheduledTaskWebhookFilter = typeof ScheduledTaskWebhookFilter.Type; export type ScheduledTaskWebhookFilterOp = ScheduledTaskWebhookFilter["op"]; From 7beebeac9b8545d73371c1beab7b996e4b3b6a51 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 01:54:49 +0000 Subject: [PATCH 09/14] test(server): ownerless monitor run settles despite an unrelated ownerless restart cut Co-Authored-By: Claude Opus 5.5 --- .../DelegatedCompletionDelivery.test.ts | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index b4c3d2b4e2aff..c80fe12f14e4a 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -2858,14 +2858,19 @@ const seedSteeredTasks = Effect.fnUntraced(function* ( }); it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { - it.effect.each(["completed", "failed"] as const)( - "shares the running turn outcome and delivers each task once (%s)", - (status) => + it.effect.each([ + { status: "completed", owner: "delegated" }, + { status: "failed", owner: "delegated" }, + { status: "completed", owner: "monitor" }, + ] as const)( + "shares the running turn outcome and delivers each task once ($status, $owner)", + ({ status, owner }) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const service = yield* OrchestratorMcpService.OrchestratorMcpService; - const seeded = yield* seedSteeredTasks(`steered-${status}`, { + const seeded = yield* seedSteeredTasks(`steered-${status}-${owner}`, { queueHeld: status === "failed", + monitor: owner === "monitor", }); yield* orchestrator.recoverDelegatedTask(seeded.childThreadId, seeded.promotedRun.id); yield* (yield* EventSink.EventSinkV2) @@ -2899,26 +2904,28 @@ it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { const sink = yield* EventSink.EventSinkV2; const unrelated = runEvent({ threadId: seeded.childThreadId, - runId: RunId.make(`${status}:unrelated`), + runId: RunId.make(`${status}-${owner}:unrelated`), ordinal: 4, status: "cancelled", now: seeded.now, }); yield* sink.writeWithEffects({ - commandId: reconcileCommandId(`${status}:unrelated`), + commandId: reconcileCommandId(`${status}-${owner}:unrelated`), events: [ - { - ...unrelated, - payload: { - ...unrelated.payload, - delegatedTaskId: NodeId.make(`${status}:unrelated`), - }, - }, + owner === "monitor" + ? unrelated + : { + ...unrelated, + payload: { + ...unrelated.payload, + delegatedTaskId: NodeId.make(`${status}:unrelated`), + }, + }, ], effects: [ { id: `effect:restart-continuation:${unrelated.runId}`, - commandId: reconcileCommandId(`${status}:unrelated`), + commandId: reconcileCommandId(`${status}-${owner}:unrelated`), threadId: seeded.childThreadId, request: { type: "provider-runtime.continue", sourceRunId: unrelated.runId }, }, @@ -2929,6 +2936,10 @@ it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { const shared = settled.contextTransfers.filter( (transfer) => transfer.sourcePoint.runId === seeded.steeredRun.id, ); + assert.equal( + settled.subagents.find((task) => task.id === seeded.promotedTaskId)?.status, + status, + ); assert.lengthOf(shared, 1); yield* orchestrator.recoverDelegatedTasks; yield* orchestrator.recoverDelegatedTasks; @@ -2936,7 +2947,9 @@ it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { assert.deepEqual(again.subagents, settled.subagents); assert.deepEqual(again.contextTransfers, settled.contextTransfers); assert.deepEqual(again.runs, settled.runs); - for (const id of [seeded.steeredTaskId, seeded.promotedTaskId]) { + for (const id of owner === "monitor" + ? [seeded.promotedTaskId] + : [seeded.steeredTaskId, seeded.promotedTaskId]) { const result = yield* service.taskStatus(seeded.scope, id); assert.equal(result.status, status); assert.equal(result.summary, "Shared result."); From fa914efc1b8af9ba4438102f60e08a4a7d242d77 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 02:09:56 +0000 Subject: [PATCH 10/14] fix(server): an unrelated ownerless restart cut no longer holds a joined monitor task open Co-Authored-By: Claude Opus 5.5 --- .../server/src/orchestration-v2/Orchestrator.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 8417a784fa2d5..fb84eb0eaa315 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -402,7 +402,22 @@ function runsOfTask( runs: ReadonlyArray, run: OrchestrationV2Run, ): ReadonlyArray { - return runs.filter((candidate) => candidate.delegatedTaskId === run.delegatedTaskId); + if (run.delegatedTaskId !== undefined) { + return runs.filter((candidate) => candidate.delegatedTaskId === run.delegatedTaskId); + } + // Ownerless runs (a monitor turn a task joined) share no task id; only the + // run's own restart continuations can still change its result. + const chain = [run]; + const seen = new Set([run.id]); + for (let index = 0; index < chain.length; index += 1) { + for (const candidate of runs) { + if (candidate.restartContinuationOfRunId === chain[index]!.id && !seen.has(candidate.id)) { + seen.add(candidate.id); + chain.push(candidate); + } + } + } + return chain; } /** Run ordinals can skip values: a queued run moves past newer runs when it starts. */ From 321edbb8bdf1dca19f779cfeb984ef1d7988b285 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 02:25:46 +0000 Subject: [PATCH 11/14] test(server): make promoted delegation runtime evidence reproducible --- .../scripts/verify-promoted-delegation.ts | 638 ++++++++++++++++++ 1 file changed, 638 insertions(+) create mode 100644 apps/server/scripts/verify-promoted-delegation.ts diff --git a/apps/server/scripts/verify-promoted-delegation.ts b/apps/server/scripts/verify-promoted-delegation.ts new file mode 100644 index 0000000000000..d97fd0f748512 --- /dev/null +++ b/apps/server/scripts/verify-promoted-delegation.ts @@ -0,0 +1,638 @@ +// @effect-diagnostics nodeBuiltinImport:off globalConsole:off globalDate:off preferSchemaOverJson:off - Operational evidence runner owns its artifact output. +// Run from the repository root: node apps/server/scripts/verify-promoted-delegation.ts +// Headless server-service evidence only: no listening server, UI, live inference, or process restart. +import * as NodeAssert from "node:assert/strict"; +import * as NodeFS from "node:fs"; +import * as NodeCrypto from "node:crypto"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + CommandId, + EnvironmentId, + EventId, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ProviderThreadId, + ProviderTurnId, + ThreadId, + TurnItemId, + type OrchestrationV2DomainEvent, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Deferred from "effect/Deferred"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { makeSubagentConversationArtifacts } from "@t3tools/provider-core/server/subagentProjection"; +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; +import * as Harness from "../src/orchestration-v2/testkit/ProviderReplayHarness.ts"; +import * as Orchestrator from "../src/orchestration-v2/Orchestrator.ts"; +import * as EffectWorker from "../src/orchestration-v2/EffectWorker.ts"; +import * as EventSink from "../src/orchestration-v2/EventSink.ts"; +import * as ProviderAdapterRegistry from "../src/orchestration-v2/ProviderAdapterRegistry.ts"; +import { CodexProviderCapabilitiesV2 } from "../src/orchestration-v2/Adapters/CodexAdapterV2.ts"; +import * as ThreadManagement from "../src/orchestration-v2/ThreadManagementService.ts"; +import * as ThreadLaunch from "../src/orchestration-v2/ThreadLaunchService.ts"; +import * as Mcp from "../src/mcp/OrchestratorMcpService.ts"; +import type { McpInvocationScope } from "../src/mcp/McpInvocationContext.ts"; +import * as GitDriver from "../src/vcs/GitVcsDriver.ts"; +import * as Providers from "../src/provider/ProviderRegistry.ts"; +import * as Projects from "../src/project/ProjectService.ts"; +import * as Secrets from "../src/secrets/SecretRequests.ts"; +import * as Scheduled from "../src/scheduledTasks/ScheduledTaskService.ts"; +import * as Settings from "../src/serverSettings.ts"; + +const output = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-promoted-delegation-")); +const root = NodePath.resolve(import.meta.dirname, "../../.."); +const revision = NodeChildProcess.execFileSync("git", ["rev-parse", "HEAD"], { + cwd: root, + encoding: "utf8", +}).trim(); +const changed = NodeChildProcess.execFileSync("git", ["diff", "--name-only", "HEAD"], { + cwd: root, + encoding: "utf8", +}).trim(); +const trace = (value: unknown) => + NodeFS.appendFileSync(NodePath.join(output, "trace.ndjson"), JSON.stringify(value) + "\n"); +NodeFS.writeFileSync( + NodePath.join(output, "manifest.json"), + JSON.stringify( + { + revision, + productionRevision: NodeChildProcess.execFileSync( + "git", + ["log", "-1", "--format=%H", "--", "apps/server/src/orchestration-v2/Orchestrator.ts"], + { cwd: root, encoding: "utf8" }, + ).trim(), + fixtureSha256: NodeCrypto.createHash("sha256") + .update(NodeFS.readFileSync(import.meta.filename)) + .digest("hex"), + changed, + evidence: "headless actual server services / deterministic adapter / in-memory SQLite", + command: "node apps/server/scripts/verify-promoted-delegation.ts", + missing: [ + "UI", + "live provider", + "real process restart", + "native clients", + "task_wait (not exposed by this revision; mode=wait cannot address a follow-up task)", + "queue-held/idle opening", + "synthetic restart chain", + "unpromoted cancellation and interrupted/waiting boundary controls", + ], + }, + null, + 2, + ), +); +console.log(`Evidence: ${output}`); +const driver = ProviderDriverKind.make("codex"); +const instanceId = ProviderInstanceId.make("codex"); +const modelSelection = { instanceId, model: "test-model" }; + +const run = (owner: "delegated" | "monitor", outcome: "completed" | "failed") => + Effect.scoped( + Effect.gen(function* () { + const name = `${owner}-${outcome}`; + const cwd = yield* checkpointWorkspace(`promoted-${name}`); + const started = new Map(); + const queues = new Map>(); + const steered = yield* Deferred.make(); + const capabilities = CodexProviderCapabilitiesV2; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId, + driver, + getCapabilities: () => Effect.succeed(capabilities), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: Effect.fn("PromotedFixture.openSession")(function* ( + input: ProviderAdapter.ProviderAdapterV2OpenSessionInput, + ): Effect.fn.Return { + const now = yield* DateTime.now; + const events = yield* Queue.unbounded(); + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd, + model: modelSelection.model, + capabilities, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId }) => + Effect.sync(() => { + queues.set(threadId, events); + return { + id: ProviderThreadId.make(`provider-thread:${threadId}`), + driver, + providerInstanceId: instanceId, + providerSessionId: input.providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { + driver, + nativeId: `native:${threadId}`, + strength: "strong" as const, + }, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: Effect.fn("PromotedFixture.startTurn")(function* ( + turn: ProviderAdapter.ProviderAdapterV2TurnInput, + ) { + started.set(turn.threadId, [...(started.get(turn.threadId) ?? []), turn]); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: { + driver, + nativeId: `native:${turn.attemptId}`, + strength: "strong", + }, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + }), + steerTurn: () => Deferred.succeed(steered, undefined).pipe(Effect.asVoid), + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused snapshot"), + rollbackThread: () => Effect.die("unused rollback"), + forkThread: () => Effect.die("unused fork"), + }; + }), + }; + const registry = Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistryV2, { + get: () => Effect.succeed(adapter), + list: () => Effect.succeed([instanceId]), + }); + const runtime = Harness.layerWithRegistry({ name: `promoted-${name}` }, registry, { + runEffectWorker: false, + }); + // The replay harness owns the real runtime. This facade forwards only reads/commands; + // unrelated launch, secrets, scheduling and provider discovery must never be exercised. + const threads = Layer.unwrap( + Effect.gen(function* () { + const o = yield* Orchestrator.OrchestratorV2; + return Layer.mock(ThreadManagement.ThreadManagementService)({ + dispatch: o.dispatch, + getThreadProjection: o.getThreadProjection, + getThreadShell: o.getThreadShell, + getThreadRecords: o.getThreadRecords, + getTimelinePage: o.getTimelinePage, + getProjectThreadRecords: (input, fields, filter) => + o.getThreadRecords(input.threadId, fields, filter), + recoverDelegatedTask: o.recoverDelegatedTask, + delegatedTaskResultPending: o.delegatedTaskResultPending, + streamStoredEvents: o.streamStoredEvents, + streamStoredEventsFrom: o.streamStoredEventsFrom, + streamDomainEvents: o.streamDomainEvents, + }); + }), + ).pipe(Layer.provide(runtime)); + const mcp = Mcp.layer.pipe( + Layer.provide( + Layer.mergeAll( + threads, + registry, + Layer.mock(ThreadLaunch.ThreadLaunchService)({}), + Layer.mock(GitDriver.GitVcsDriver)({}), + Layer.mock(Providers.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(Projects.ProjectService)({}), + Layer.mock(Secrets.SecretRequests)({}), + Layer.mock(Scheduled.ScheduledTaskService)({}), + Settings.layerTest(), + NodeServices.layer, + ), + ), + ); + yield* Effect.gen(function* () { + const o = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sink = yield* EventSink.EventSinkV2; + const service = yield* Mcp.OrchestratorMcpService; + const dispatch: typeof o.dispatch = (command) => { + trace({ name, stage: "command", command }); + return o.dispatch(command); + }; + const parent = ThreadId.make(`${name}:parent`); + const projectId = ProjectId.make(`${name}:project`); + let sequence = 0; + const commandId = () => CommandId.make(`${name}:command:${++sequence}`); + const watch = Effect.fn("PromotedFixture.watch")(function* ( + predicate: (event: OrchestrationV2DomainEvent) => boolean, + ) { + const afterSequence = yield* sink.latestSequence(); + yield* o.streamStoredEventsFrom({ afterSequence }).pipe( + Stream.filter((stored) => predicate(stored.event)), + Stream.take(1), + Stream.runDrain, + ); + }); + const waitRunning = (threadId: ThreadId) => + watch( + (e) => + e.threadId === threadId && + e.type === "provider-turn.updated" && + e.payload.status === "running", + ); + const start = ( + threadId: ThreadId, + text: string, + senderThreadId?: ThreadId, + notification?: { source: { kind: "monitor" }; outcome: "updated"; summary: string }, + ) => + dispatch({ + type: "message.dispatch", + commandId: commandId(), + threadId, + messageId: MessageId.make(`${name}:message:${sequence}`), + text, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: senderThreadId || notification ? "agent" : "user", + creationSource: senderThreadId ? "mcp" : "server", + ...(senderThreadId ? { senderThreadId } : {}), + ...(notification ? { notification } : {}), + }); + const finish = Effect.fn("PromotedFixture.finish")(function* ( + threadId: ThreadId, + status: "completed" | "failed", + text: string, + ) { + const projection = yield* o.getThreadProjection(threadId); + const turn = projection.providerTurns.find((t) => t.status === "running"); + NodeAssert.ok(turn); + const input = started.get(threadId)!.at(-1)!; + const run = projection.runs.find((r) => r.id === input.runId)!; + const now = yield* DateTime.now; + const itemId = TurnItemId.make(`${name}:result:${sequence++}`); + const { message, turnItem } = makeSubagentConversationArtifacts({ + threadId, + rootNodeId: input.rootNodeId, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + nativeItemRef: null, + messageId: MessageId.make(`${itemId}:message`), + turnItemId: itemId, + role: "assistant", + text, + ordinal: 100, + now, + }); + yield* sink.write({ + events: [ + { + id: EventId.make(`${itemId}:message`), + type: "message.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...message, runId: run.id }, + }, + { + id: EventId.make(`${itemId}:item`), + type: "turn-item.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...turnItem, runId: run.id }, + }, + ], + }); + const waiting = yield* watch( + (e) => + e.type === "run.updated" && + e.payload.id === run.id && + ["waiting", "failed"].includes(e.payload.status), + ).pipe(Effect.forkScoped); + const queue = queues.get(threadId)!; + yield* Queue.offer(queue, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status, completedAt: now }, + }); + yield* Queue.offer(queue, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: input.runOrdinal, + ...(status === "failed" + ? { + status, + failureItemOrdinal: 101, + failure: { + class: "provider_error" as const, + message: text, + code: "fixture_failure", + retryable: false, + }, + } + : { status, failure: null }), + threadDisposition: "reusable", + }); + trace({ name, stage: "adapter-terminal-offered", status, runId: run.id }); + yield* Fiber.join(waiting); + yield* worker.drain(); + const final = yield* o.getThreadProjection(threadId); + const resultItemId = + status === "failed" + ? final.turnItems.find((item) => item.type === "error" && item.runId === run.id)!.id + : itemId; + return { run, itemId: resultItemId }; + }); + yield* dispatch({ + type: "thread.create", + commandId: commandId(), + threadId: parent, + projectId, + title: name, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + yield* start(parent, "PARENT_HELD"); + const parentRunning = yield* waitRunning(parent).pipe(Effect.forkScoped); + yield* worker.drain(); + yield* Fiber.join(parentRunning); + const parentRun = (yield* o.getThreadProjection(parent)).runs[0]!; + const delegation = yield* dispatch({ + type: "delegated_task.request", + commandId: commandId(), + parentThreadId: parent, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId!, + task: "INITIAL_HELD", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + completionWake: "settled_only", + createdBy: "agent", + creationSource: "mcp", + }); + const taskEvent = delegation.storedEvents.find((e) => e.event.type === "subagent.updated")!; + NodeAssert.equal(taskEvent.event.type, "subagent.updated"); + if (taskEvent.event.type !== "subagent.updated") throw new Error("missing task"); + const originalTask = taskEvent.event.payload; + const child = originalTask.childThreadId!; + const childRunning = yield* waitRunning(child).pipe(Effect.forkScoped); + yield* worker.drain(); + yield* Fiber.join(childRunning); + yield* finish(child, "completed", "INITIAL_DONE"); + if (owner === "monitor") + yield* start(child, "MONITOR_HELD", undefined, { + source: { kind: "monitor" }, + outcome: "updated", + summary: "Fixture monitor output", + }); + else yield* start(child, "DELEGATED_HELD", parent); + const nextRunning = yield* waitRunning(child).pipe(Effect.forkScoped); + yield* worker.drain(); + yield* Fiber.join(nextRunning); + const active = (yield* o.getThreadProjection(child)).runs.find( + (r) => r.status === "running", + )!; + NodeAssert.ok(active); + NodeAssert.equal(active.delegatedTaskId === undefined, owner === "monitor"); + yield* start(child, "PROMOTED_HELD", parent); + const queued = (yield* o.getThreadProjection(child)).runs.find( + (r) => r.status === "queued", + )!; + NodeAssert.ok(queued?.delegatedTaskId); + const taskId = queued.delegatedTaskId; + const scope: McpInvocationScope = { + environmentId: EnvironmentId.make("fixture"), + capabilities: new Set(["orchestration"]), + issuedAt: 0, + requestNamespace: name, + thread: { + threadId: parent, + providerSessionId: "fixture", + providerInstanceId: instanceId, + }, + client: undefined, + }; + trace({ + name, + stage: "queued", + task: yield* service.taskStatus(scope, taskId), + activeRunId: active.id, + queuedRunId: queued.id, + }); + const beforePromotion = yield* sink.latestSequence(); + yield* dispatch({ + type: "queued-message.promote-to-steer", + commandId: commandId(), + threadId: child, + queuedRunId: queued.id, + targetRunId: active.id, + }); + yield* worker.drain(); + yield* Deferred.await(steered); + yield* o.recoverDelegatedTask(child, queued.id); + yield* sink.stream({ afterSequence: beforePromotion, eventType: "subagent.updated" }).pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === taskId && + stored.event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + ); + const working = yield* service.taskStatus(scope, taskId); + const promotedChild = yield* o.getThreadProjection(child); + NodeAssert.equal( + promotedChild.runs.find((run) => run.id === queued.id)?.status, + "cancelled", + ); + NodeAssert.equal( + promotedChild.messages.find((message) => message.id === queued.userMessageId)?.runId, + active.id, + ); + NodeAssert.ok( + promotedChild.turnItems.some( + (item) => + item.type === "user_message" && + item.messageId === queued.userMessageId && + item.inputIntent === "promoted_queued_to_steer", + ), + ); + const p = yield* o.getThreadProjection(parent); + const task = p.subagents.find((t) => t.id === taskId)!; + const node = p.nodes.find((n) => n.id === taskId)!; + const item = p.turnItems.find((i) => i.type === "subagent" && i.subagentId === taskId); + trace({ + name, + stage: "promoted-held", + task: working, + storedTask: task, + storedNode: node, + storedItem: item, + child: promotedChild, + }); + NodeAssert.equal(working.status, "running"); + NodeAssert.equal(working.workState, "working"); + NodeAssert.equal(working.summary, null); + NodeAssert.equal(task.status, "running"); + NodeAssert.equal(task.result, null); + NodeAssert.equal(node.status, "running"); + NodeAssert.equal(item?.status, "running"); + NodeAssert.equal(yield* o.delegatedTaskResultPending(child, taskId), true); + yield* o.recoverDelegatedTasks; + yield* o.recoverDelegatedTasks; + NodeAssert.equal((yield* service.taskStatus(scope, taskId)).status, "running"); + const resultText = `JOINED_${name}_RESULT`; + const beforeFinish = yield* sink.latestSequence(); + const result = yield* finish(child, outcome, resultText); + yield* o.recoverDelegatedTask(child, active.id); + yield* sink.stream({ afterSequence: beforeFinish, eventType: "subagent.updated" }).pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === taskId && + stored.event.payload.status === outcome, + ), + Stream.take(1), + Stream.runDrain, + ); + const settled = yield* o.getThreadProjection(parent); + const shared = settled.contextTransfers.filter( + (t) => t.sourcePoint.runId === result.run.id, + ); + const ids = owner === "delegated" ? [active.delegatedTaskId!, taskId] : [taskId]; + NodeAssert.equal(shared.length, 1); + for (const id of ids) { + const value = settled.subagents.find((task) => task.id === id)!; + NodeAssert.equal(value.status, outcome); + NodeAssert.equal(value.result, resultText); + NodeAssert.notEqual(value.completionDelivery?.state, "acknowledged"); + NodeAssert.equal(settled.nodes.find((node) => node.id === id)?.status, outcome); + NodeAssert.equal( + settled.turnItems.find((item) => item.type === "subagent" && item.subagentId === id) + ?.status, + outcome, + ); + } + const acknowledgement = () => + o + .getThreadProjection(parent) + .pipe( + Effect.map((p) => + ids.map((id) => p.subagents.find((t) => t.id === id)!.completionDelivery?.state), + ), + ); + const beforeRead = yield* acknowledgement(); + NodeAssert.ok(beforeRead.every((state) => state !== "acknowledged")); + const childProjection = yield* o.getThreadProjection(child); + const nonResult = childProjection.turnItems.find((i) => i.type === "user_message")!; + for (const input of [ + { itemId: nonResult.id }, + { itemId: result.itemId, maxCharsPerItem: 3 }, + { itemId: result.itemId, textOffset: 1 }, + ]) { + const page = yield* service.readThread(scope, { threadId: child, limit: 1, ...input }); + NodeAssert.deepEqual(yield* acknowledgement(), beforeRead); + trace({ name, stage: "negative-page", input, page, delivery: yield* acknowledgement() }); + } + const page = yield* service.readThread(scope, { + threadId: child, + itemId: result.itemId, + limit: 1, + }); + NodeAssert.equal(page.items.length, 1); + NodeAssert.equal( + page.items[0]!.type, + outcome === "completed" ? "assistant_message" : "error", + ); + NodeAssert.deepEqual( + yield* acknowledgement(), + ids.map(() => "acknowledged"), + ); + for (const id of ids) { + const resolved = yield* service.taskStatus(scope, id); + NodeAssert.equal(resolved.status, outcome); + NodeAssert.equal(resolved.summary, resultText); + NodeAssert.equal(resolved.resultContextTransferId, shared[0]!.id); + trace({ name, stage: "resolved-task", task: resolved }); + } + const afterRead = yield* o.getThreadProjection(parent); + yield* service.readThread(scope, { threadId: child, itemId: result.itemId, limit: 1 }); + yield* o.recoverDelegatedTasks; + yield* o.recoverDelegatedTasks; + const again = yield* o.getThreadProjection(parent); + NodeAssert.deepEqual(again.contextTransfers, settled.contextTransfers); + NodeAssert.deepEqual(again.subagents, afterRead.subagents); + NodeAssert.deepEqual(again.runs, afterRead.runs); + trace({ + name, + stage: "settled", + result: yield* service.taskStatus(scope, taskId), + sharedTransferId: shared[0]!.id, + resultPage: page, + acknowledgedTasks: ids, + parent: again, + }); + console.log( + JSON.stringify({ + name, + status: "passed", + outcome, + sharedTransfers: shared.length, + acknowledgedTasks: ids.length, + }), + ); + }).pipe(Effect.provide(Layer.merge(runtime, mcp))); + }), + ); + +try { + for (const owner of ["delegated", "monitor"] as const) + for (const outcome of ["completed", "failed"] as const) + await Effect.runPromise(run(owner, outcome).pipe(Effect.timeout("5 minutes"))); + NodeFS.writeFileSync( + NodePath.join(output, "verdict.json"), + JSON.stringify({ status: "passed", revision }), + ); +} catch (error) { + NodeFS.writeFileSync( + NodePath.join(output, "verdict.json"), + JSON.stringify({ status: "failed", revision, error: String(error) }), + ); + throw error; +} From 18e90fc0a24119b7141d8e6496922c8f13b12a45 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 03:32:00 +0000 Subject: [PATCH 12/14] fix(server): a promoted delegated task reports approvals pending in its joined run A follow-up promoted to Steer cancels its own run and continues in the run it joined, but pendingTaskApproval only looked at the task's own non-terminal runs, so task_status and wait never surfaced an approval the joined run was blocked on. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/OrchestratorMcpService.ts | 5 +- .../DelegatedCompletionDelivery.test.ts | 73 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index bad214f7e830e..f986c4d3ab2e2 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -510,16 +510,19 @@ function endsTaskWait(result: OrchestratorMcpDelegateTaskResult): boolean { const isApprovalRequestKind = Schema.is(ProviderRequestKind); function pendingTaskApproval( - child: Pick, + child: Pick, taskId: NodeId, originalRun: OrchestrationV2Run | undefined, ) { + // A follow-up promoted to Steer waits on approvals in the run it joined. + const joinedRunId = delegatedTaskResultRun(child, taskId)?.id; const taskRunIds = new Set( child.runs .filter( (run) => !ThreadManagementService.isTerminalRunStatus(run.status) && (run.delegatedTaskId === taskId || + run.id === joinedRunId || // Legacy tasks own only their original run, not later turns in the child thread. (run.delegatedTaskId === undefined && run.id === originalRun?.id)), ) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index c80fe12f14e4a..ac8343164237c 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -14,8 +14,10 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, + ProviderSessionId, ProviderThreadId, RunId, + RuntimeRequestId, ThreadId, TurnItemId, } from "@t3tools/contracts"; @@ -3040,6 +3042,77 @@ it.layer(layerSteeredTaskTest)("steered delegated follow-ups", (it) => { }), ); + it.effect.each(["delegated", "monitor"] as const)( + "reports an approval pending in the joined run to the promoted task (%s)", + (owner) => + Effect.gen(function* () { + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const sink = yield* EventSink.EventSinkV2; + const seeded = yield* seedSteeredTasks(`steered-approval-${owner}`, { + monitor: owner === "monitor", + }); + const nodeId = NodeId.make(`steered-approval-${owner}:node`); + const requestId = RuntimeRequestId.make(`steered-approval-${owner}:request`); + yield* sink.write({ + events: [ + { + id: EventId.make(`${nodeId}:seed`), + type: "node.updated", + threadId: seeded.childThreadId, + runId: seeded.steeredRun.id, + nodeId, + occurredAt: seeded.now, + payload: { + id: nodeId, + threadId: seeded.childThreadId, + runId: seeded.steeredRun.id, + rootNodeId: nodeId, + parentNodeId: null, + kind: "root_turn", + status: "running", + countsForRun: true, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: seeded.now, + completedAt: null, + }, + }, + { + id: EventId.make(`${requestId}:seed`), + type: "runtime-request.updated", + threadId: seeded.childThreadId, + runId: seeded.steeredRun.id, + nodeId, + occurredAt: seeded.now, + payload: { + id: requestId, + nodeId, + providerTurnId: null, + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: ProviderSessionId.make(`steered-approval-${owner}`), + }, + createdAt: seeded.now, + resolvedAt: null, + }, + }, + ], + }); + const result = yield* service.taskStatus(seeded.scope, seeded.promotedTaskId); + assert.deepEqual(result.pendingApproval, { + requestId, + kind: "command", + runId: seeded.steeredRun.id, + }); + }), + ); + it.effect.each(["delegated", "monitor"] as const)( "acknowledges tasks when a result page excludes the promoted user message (%s)", (owner) => From f37400565325917b9703b7e8996ddd1f868db85e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 04:22:18 +0000 Subject: [PATCH 13/14] fix(ci): register promoted delegation evidence runner --- knip.jsonc | 1 + 1 file changed, 1 insertion(+) diff --git a/knip.jsonc b/knip.jsonc index 7fbfe38259951..5fa70b3624022 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -44,6 +44,7 @@ "scripts/probe-claude-fork-local-rollback-replay.ts", "scripts/update-test-shard-weights.ts", "scripts/verify-background-live.ts", + "scripts/verify-promoted-delegation.ts", "src/provider/testFixtures/*.mjs", // graphqlgen reads its config. "graphql.config.ts", From a49a734210a3669c604abfe061728a58f02bb732 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 01:43:07 +0000 Subject: [PATCH 14/14] chore(server): drop the promoted delegation evidence runner and fix the steered test layer The one-off evidence script is not part of the fix. The steered follow-up suite now provides the services the MCP layer gained on main. AI-assisted; verified with the tests and tools listed above. --- .../scripts/verify-promoted-delegation.ts | 638 ------------------ .../DelegatedCompletionDelivery.test.ts | 3 + knip.jsonc | 1 - 3 files changed, 3 insertions(+), 639 deletions(-) delete mode 100644 apps/server/scripts/verify-promoted-delegation.ts diff --git a/apps/server/scripts/verify-promoted-delegation.ts b/apps/server/scripts/verify-promoted-delegation.ts deleted file mode 100644 index d97fd0f748512..0000000000000 --- a/apps/server/scripts/verify-promoted-delegation.ts +++ /dev/null @@ -1,638 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off globalConsole:off globalDate:off preferSchemaOverJson:off - Operational evidence runner owns its artifact output. -// Run from the repository root: node apps/server/scripts/verify-promoted-delegation.ts -// Headless server-service evidence only: no listening server, UI, live inference, or process restart. -import * as NodeAssert from "node:assert/strict"; -import * as NodeFS from "node:fs"; -import * as NodeCrypto from "node:crypto"; -import * as NodeOS from "node:os"; -import * as NodePath from "node:path"; -import * as NodeChildProcess from "node:child_process"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { - CommandId, - EnvironmentId, - EventId, - MessageId, - ProjectId, - ProviderDriverKind, - ProviderInstanceId, - ProviderThreadId, - ProviderTurnId, - ThreadId, - TurnItemId, - type OrchestrationV2DomainEvent, -} from "@t3tools/contracts"; -import * as DateTime from "effect/DateTime"; -import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; -import * as Deferred from "effect/Deferred"; -import * as Layer from "effect/Layer"; -import * as Queue from "effect/Queue"; -import * as Stream from "effect/Stream"; -import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; -import { makeSubagentConversationArtifacts } from "@t3tools/provider-core/server/subagentProjection"; -import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; -import * as Harness from "../src/orchestration-v2/testkit/ProviderReplayHarness.ts"; -import * as Orchestrator from "../src/orchestration-v2/Orchestrator.ts"; -import * as EffectWorker from "../src/orchestration-v2/EffectWorker.ts"; -import * as EventSink from "../src/orchestration-v2/EventSink.ts"; -import * as ProviderAdapterRegistry from "../src/orchestration-v2/ProviderAdapterRegistry.ts"; -import { CodexProviderCapabilitiesV2 } from "../src/orchestration-v2/Adapters/CodexAdapterV2.ts"; -import * as ThreadManagement from "../src/orchestration-v2/ThreadManagementService.ts"; -import * as ThreadLaunch from "../src/orchestration-v2/ThreadLaunchService.ts"; -import * as Mcp from "../src/mcp/OrchestratorMcpService.ts"; -import type { McpInvocationScope } from "../src/mcp/McpInvocationContext.ts"; -import * as GitDriver from "../src/vcs/GitVcsDriver.ts"; -import * as Providers from "../src/provider/ProviderRegistry.ts"; -import * as Projects from "../src/project/ProjectService.ts"; -import * as Secrets from "../src/secrets/SecretRequests.ts"; -import * as Scheduled from "../src/scheduledTasks/ScheduledTaskService.ts"; -import * as Settings from "../src/serverSettings.ts"; - -const output = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-promoted-delegation-")); -const root = NodePath.resolve(import.meta.dirname, "../../.."); -const revision = NodeChildProcess.execFileSync("git", ["rev-parse", "HEAD"], { - cwd: root, - encoding: "utf8", -}).trim(); -const changed = NodeChildProcess.execFileSync("git", ["diff", "--name-only", "HEAD"], { - cwd: root, - encoding: "utf8", -}).trim(); -const trace = (value: unknown) => - NodeFS.appendFileSync(NodePath.join(output, "trace.ndjson"), JSON.stringify(value) + "\n"); -NodeFS.writeFileSync( - NodePath.join(output, "manifest.json"), - JSON.stringify( - { - revision, - productionRevision: NodeChildProcess.execFileSync( - "git", - ["log", "-1", "--format=%H", "--", "apps/server/src/orchestration-v2/Orchestrator.ts"], - { cwd: root, encoding: "utf8" }, - ).trim(), - fixtureSha256: NodeCrypto.createHash("sha256") - .update(NodeFS.readFileSync(import.meta.filename)) - .digest("hex"), - changed, - evidence: "headless actual server services / deterministic adapter / in-memory SQLite", - command: "node apps/server/scripts/verify-promoted-delegation.ts", - missing: [ - "UI", - "live provider", - "real process restart", - "native clients", - "task_wait (not exposed by this revision; mode=wait cannot address a follow-up task)", - "queue-held/idle opening", - "synthetic restart chain", - "unpromoted cancellation and interrupted/waiting boundary controls", - ], - }, - null, - 2, - ), -); -console.log(`Evidence: ${output}`); -const driver = ProviderDriverKind.make("codex"); -const instanceId = ProviderInstanceId.make("codex"); -const modelSelection = { instanceId, model: "test-model" }; - -const run = (owner: "delegated" | "monitor", outcome: "completed" | "failed") => - Effect.scoped( - Effect.gen(function* () { - const name = `${owner}-${outcome}`; - const cwd = yield* checkpointWorkspace(`promoted-${name}`); - const started = new Map(); - const queues = new Map>(); - const steered = yield* Deferred.make(); - const capabilities = CodexProviderCapabilitiesV2; - const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { - instanceId, - driver, - getCapabilities: () => Effect.succeed(capabilities), - planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), - openSession: Effect.fn("PromotedFixture.openSession")(function* ( - input: ProviderAdapter.ProviderAdapterV2OpenSessionInput, - ): Effect.fn.Return { - const now = yield* DateTime.now; - const events = yield* Queue.unbounded(); - return { - instanceId, - driver, - providerSessionId: input.providerSessionId, - providerSession: { - id: input.providerSessionId, - driver, - providerInstanceId: instanceId, - status: "ready", - cwd, - model: modelSelection.model, - capabilities, - createdAt: now, - updatedAt: now, - lastError: null, - }, - events: Stream.fromQueue(events), - ensureThread: ({ threadId }) => - Effect.sync(() => { - queues.set(threadId, events); - return { - id: ProviderThreadId.make(`provider-thread:${threadId}`), - driver, - providerInstanceId: instanceId, - providerSessionId: input.providerSessionId, - appThreadId: threadId, - ownerNodeId: null, - nativeThreadRef: { - driver, - nativeId: `native:${threadId}`, - strength: "strong" as const, - }, - nativeConversationHeadRef: null, - status: "idle" as const, - firstRunOrdinal: null, - lastRunOrdinal: null, - handoffIds: [], - forkedFrom: null, - createdAt: now, - updatedAt: now, - }; - }), - resumeThread: ({ providerThread }) => Effect.succeed(providerThread), - startTurn: Effect.fn("PromotedFixture.startTurn")(function* ( - turn: ProviderAdapter.ProviderAdapterV2TurnInput, - ) { - started.set(turn.threadId, [...(started.get(turn.threadId) ?? []), turn]); - yield* Queue.offer(events, { - type: "provider_turn.updated", - driver, - providerTurn: { - id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), - providerThreadId: turn.providerThread.id, - nodeId: turn.rootNodeId, - runAttemptId: turn.attemptId, - nativeTurnRef: { - driver, - nativeId: `native:${turn.attemptId}`, - strength: "strong", - }, - ordinal: turn.providerTurnOrdinal, - status: "running", - startedAt: now, - completedAt: null, - }, - }); - }), - steerTurn: () => Deferred.succeed(steered, undefined).pipe(Effect.asVoid), - interruptTurn: () => Effect.void, - respondToRuntimeRequest: () => Effect.void, - readThreadSnapshot: () => Effect.die("unused snapshot"), - rollbackThread: () => Effect.die("unused rollback"), - forkThread: () => Effect.die("unused fork"), - }; - }), - }; - const registry = Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistryV2, { - get: () => Effect.succeed(adapter), - list: () => Effect.succeed([instanceId]), - }); - const runtime = Harness.layerWithRegistry({ name: `promoted-${name}` }, registry, { - runEffectWorker: false, - }); - // The replay harness owns the real runtime. This facade forwards only reads/commands; - // unrelated launch, secrets, scheduling and provider discovery must never be exercised. - const threads = Layer.unwrap( - Effect.gen(function* () { - const o = yield* Orchestrator.OrchestratorV2; - return Layer.mock(ThreadManagement.ThreadManagementService)({ - dispatch: o.dispatch, - getThreadProjection: o.getThreadProjection, - getThreadShell: o.getThreadShell, - getThreadRecords: o.getThreadRecords, - getTimelinePage: o.getTimelinePage, - getProjectThreadRecords: (input, fields, filter) => - o.getThreadRecords(input.threadId, fields, filter), - recoverDelegatedTask: o.recoverDelegatedTask, - delegatedTaskResultPending: o.delegatedTaskResultPending, - streamStoredEvents: o.streamStoredEvents, - streamStoredEventsFrom: o.streamStoredEventsFrom, - streamDomainEvents: o.streamDomainEvents, - }); - }), - ).pipe(Layer.provide(runtime)); - const mcp = Mcp.layer.pipe( - Layer.provide( - Layer.mergeAll( - threads, - registry, - Layer.mock(ThreadLaunch.ThreadLaunchService)({}), - Layer.mock(GitDriver.GitVcsDriver)({}), - Layer.mock(Providers.ProviderRegistry)({ getProviders: Effect.succeed([]) }), - Layer.mock(Projects.ProjectService)({}), - Layer.mock(Secrets.SecretRequests)({}), - Layer.mock(Scheduled.ScheduledTaskService)({}), - Settings.layerTest(), - NodeServices.layer, - ), - ), - ); - yield* Effect.gen(function* () { - const o = yield* Orchestrator.OrchestratorV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const sink = yield* EventSink.EventSinkV2; - const service = yield* Mcp.OrchestratorMcpService; - const dispatch: typeof o.dispatch = (command) => { - trace({ name, stage: "command", command }); - return o.dispatch(command); - }; - const parent = ThreadId.make(`${name}:parent`); - const projectId = ProjectId.make(`${name}:project`); - let sequence = 0; - const commandId = () => CommandId.make(`${name}:command:${++sequence}`); - const watch = Effect.fn("PromotedFixture.watch")(function* ( - predicate: (event: OrchestrationV2DomainEvent) => boolean, - ) { - const afterSequence = yield* sink.latestSequence(); - yield* o.streamStoredEventsFrom({ afterSequence }).pipe( - Stream.filter((stored) => predicate(stored.event)), - Stream.take(1), - Stream.runDrain, - ); - }); - const waitRunning = (threadId: ThreadId) => - watch( - (e) => - e.threadId === threadId && - e.type === "provider-turn.updated" && - e.payload.status === "running", - ); - const start = ( - threadId: ThreadId, - text: string, - senderThreadId?: ThreadId, - notification?: { source: { kind: "monitor" }; outcome: "updated"; summary: string }, - ) => - dispatch({ - type: "message.dispatch", - commandId: commandId(), - threadId, - messageId: MessageId.make(`${name}:message:${sequence}`), - text, - attachments: [], - dispatchMode: { type: "queue_after_active" }, - createdBy: senderThreadId || notification ? "agent" : "user", - creationSource: senderThreadId ? "mcp" : "server", - ...(senderThreadId ? { senderThreadId } : {}), - ...(notification ? { notification } : {}), - }); - const finish = Effect.fn("PromotedFixture.finish")(function* ( - threadId: ThreadId, - status: "completed" | "failed", - text: string, - ) { - const projection = yield* o.getThreadProjection(threadId); - const turn = projection.providerTurns.find((t) => t.status === "running"); - NodeAssert.ok(turn); - const input = started.get(threadId)!.at(-1)!; - const run = projection.runs.find((r) => r.id === input.runId)!; - const now = yield* DateTime.now; - const itemId = TurnItemId.make(`${name}:result:${sequence++}`); - const { message, turnItem } = makeSubagentConversationArtifacts({ - threadId, - rootNodeId: input.rootNodeId, - providerThreadId: turn.providerThreadId, - providerTurnId: turn.id, - nativeItemRef: null, - messageId: MessageId.make(`${itemId}:message`), - turnItemId: itemId, - role: "assistant", - text, - ordinal: 100, - now, - }); - yield* sink.write({ - events: [ - { - id: EventId.make(`${itemId}:message`), - type: "message.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...message, runId: run.id }, - }, - { - id: EventId.make(`${itemId}:item`), - type: "turn-item.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...turnItem, runId: run.id }, - }, - ], - }); - const waiting = yield* watch( - (e) => - e.type === "run.updated" && - e.payload.id === run.id && - ["waiting", "failed"].includes(e.payload.status), - ).pipe(Effect.forkScoped); - const queue = queues.get(threadId)!; - yield* Queue.offer(queue, { - type: "provider_turn.updated", - driver, - providerTurn: { ...turn, status, completedAt: now }, - }); - yield* Queue.offer(queue, { - type: "turn.terminal", - driver, - providerThreadId: turn.providerThreadId, - providerTurnId: turn.id, - runOrdinal: input.runOrdinal, - ...(status === "failed" - ? { - status, - failureItemOrdinal: 101, - failure: { - class: "provider_error" as const, - message: text, - code: "fixture_failure", - retryable: false, - }, - } - : { status, failure: null }), - threadDisposition: "reusable", - }); - trace({ name, stage: "adapter-terminal-offered", status, runId: run.id }); - yield* Fiber.join(waiting); - yield* worker.drain(); - const final = yield* o.getThreadProjection(threadId); - const resultItemId = - status === "failed" - ? final.turnItems.find((item) => item.type === "error" && item.runId === run.id)!.id - : itemId; - return { run, itemId: resultItemId }; - }); - yield* dispatch({ - type: "thread.create", - commandId: commandId(), - threadId: parent, - projectId, - title: name, - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: cwd, - createdBy: "user", - creationSource: "web", - }); - yield* start(parent, "PARENT_HELD"); - const parentRunning = yield* waitRunning(parent).pipe(Effect.forkScoped); - yield* worker.drain(); - yield* Fiber.join(parentRunning); - const parentRun = (yield* o.getThreadProjection(parent)).runs[0]!; - const delegation = yield* dispatch({ - type: "delegated_task.request", - commandId: commandId(), - parentThreadId: parent, - parentRunId: parentRun.id, - parentNodeId: parentRun.rootNodeId!, - task: "INITIAL_HELD", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - completionWake: "settled_only", - createdBy: "agent", - creationSource: "mcp", - }); - const taskEvent = delegation.storedEvents.find((e) => e.event.type === "subagent.updated")!; - NodeAssert.equal(taskEvent.event.type, "subagent.updated"); - if (taskEvent.event.type !== "subagent.updated") throw new Error("missing task"); - const originalTask = taskEvent.event.payload; - const child = originalTask.childThreadId!; - const childRunning = yield* waitRunning(child).pipe(Effect.forkScoped); - yield* worker.drain(); - yield* Fiber.join(childRunning); - yield* finish(child, "completed", "INITIAL_DONE"); - if (owner === "monitor") - yield* start(child, "MONITOR_HELD", undefined, { - source: { kind: "monitor" }, - outcome: "updated", - summary: "Fixture monitor output", - }); - else yield* start(child, "DELEGATED_HELD", parent); - const nextRunning = yield* waitRunning(child).pipe(Effect.forkScoped); - yield* worker.drain(); - yield* Fiber.join(nextRunning); - const active = (yield* o.getThreadProjection(child)).runs.find( - (r) => r.status === "running", - )!; - NodeAssert.ok(active); - NodeAssert.equal(active.delegatedTaskId === undefined, owner === "monitor"); - yield* start(child, "PROMOTED_HELD", parent); - const queued = (yield* o.getThreadProjection(child)).runs.find( - (r) => r.status === "queued", - )!; - NodeAssert.ok(queued?.delegatedTaskId); - const taskId = queued.delegatedTaskId; - const scope: McpInvocationScope = { - environmentId: EnvironmentId.make("fixture"), - capabilities: new Set(["orchestration"]), - issuedAt: 0, - requestNamespace: name, - thread: { - threadId: parent, - providerSessionId: "fixture", - providerInstanceId: instanceId, - }, - client: undefined, - }; - trace({ - name, - stage: "queued", - task: yield* service.taskStatus(scope, taskId), - activeRunId: active.id, - queuedRunId: queued.id, - }); - const beforePromotion = yield* sink.latestSequence(); - yield* dispatch({ - type: "queued-message.promote-to-steer", - commandId: commandId(), - threadId: child, - queuedRunId: queued.id, - targetRunId: active.id, - }); - yield* worker.drain(); - yield* Deferred.await(steered); - yield* o.recoverDelegatedTask(child, queued.id); - yield* sink.stream({ afterSequence: beforePromotion, eventType: "subagent.updated" }).pipe( - Stream.filter( - (stored) => - stored.event.type === "subagent.updated" && - stored.event.payload.id === taskId && - stored.event.payload.status === "running", - ), - Stream.take(1), - Stream.runDrain, - ); - const working = yield* service.taskStatus(scope, taskId); - const promotedChild = yield* o.getThreadProjection(child); - NodeAssert.equal( - promotedChild.runs.find((run) => run.id === queued.id)?.status, - "cancelled", - ); - NodeAssert.equal( - promotedChild.messages.find((message) => message.id === queued.userMessageId)?.runId, - active.id, - ); - NodeAssert.ok( - promotedChild.turnItems.some( - (item) => - item.type === "user_message" && - item.messageId === queued.userMessageId && - item.inputIntent === "promoted_queued_to_steer", - ), - ); - const p = yield* o.getThreadProjection(parent); - const task = p.subagents.find((t) => t.id === taskId)!; - const node = p.nodes.find((n) => n.id === taskId)!; - const item = p.turnItems.find((i) => i.type === "subagent" && i.subagentId === taskId); - trace({ - name, - stage: "promoted-held", - task: working, - storedTask: task, - storedNode: node, - storedItem: item, - child: promotedChild, - }); - NodeAssert.equal(working.status, "running"); - NodeAssert.equal(working.workState, "working"); - NodeAssert.equal(working.summary, null); - NodeAssert.equal(task.status, "running"); - NodeAssert.equal(task.result, null); - NodeAssert.equal(node.status, "running"); - NodeAssert.equal(item?.status, "running"); - NodeAssert.equal(yield* o.delegatedTaskResultPending(child, taskId), true); - yield* o.recoverDelegatedTasks; - yield* o.recoverDelegatedTasks; - NodeAssert.equal((yield* service.taskStatus(scope, taskId)).status, "running"); - const resultText = `JOINED_${name}_RESULT`; - const beforeFinish = yield* sink.latestSequence(); - const result = yield* finish(child, outcome, resultText); - yield* o.recoverDelegatedTask(child, active.id); - yield* sink.stream({ afterSequence: beforeFinish, eventType: "subagent.updated" }).pipe( - Stream.filter( - (stored) => - stored.event.type === "subagent.updated" && - stored.event.payload.id === taskId && - stored.event.payload.status === outcome, - ), - Stream.take(1), - Stream.runDrain, - ); - const settled = yield* o.getThreadProjection(parent); - const shared = settled.contextTransfers.filter( - (t) => t.sourcePoint.runId === result.run.id, - ); - const ids = owner === "delegated" ? [active.delegatedTaskId!, taskId] : [taskId]; - NodeAssert.equal(shared.length, 1); - for (const id of ids) { - const value = settled.subagents.find((task) => task.id === id)!; - NodeAssert.equal(value.status, outcome); - NodeAssert.equal(value.result, resultText); - NodeAssert.notEqual(value.completionDelivery?.state, "acknowledged"); - NodeAssert.equal(settled.nodes.find((node) => node.id === id)?.status, outcome); - NodeAssert.equal( - settled.turnItems.find((item) => item.type === "subagent" && item.subagentId === id) - ?.status, - outcome, - ); - } - const acknowledgement = () => - o - .getThreadProjection(parent) - .pipe( - Effect.map((p) => - ids.map((id) => p.subagents.find((t) => t.id === id)!.completionDelivery?.state), - ), - ); - const beforeRead = yield* acknowledgement(); - NodeAssert.ok(beforeRead.every((state) => state !== "acknowledged")); - const childProjection = yield* o.getThreadProjection(child); - const nonResult = childProjection.turnItems.find((i) => i.type === "user_message")!; - for (const input of [ - { itemId: nonResult.id }, - { itemId: result.itemId, maxCharsPerItem: 3 }, - { itemId: result.itemId, textOffset: 1 }, - ]) { - const page = yield* service.readThread(scope, { threadId: child, limit: 1, ...input }); - NodeAssert.deepEqual(yield* acknowledgement(), beforeRead); - trace({ name, stage: "negative-page", input, page, delivery: yield* acknowledgement() }); - } - const page = yield* service.readThread(scope, { - threadId: child, - itemId: result.itemId, - limit: 1, - }); - NodeAssert.equal(page.items.length, 1); - NodeAssert.equal( - page.items[0]!.type, - outcome === "completed" ? "assistant_message" : "error", - ); - NodeAssert.deepEqual( - yield* acknowledgement(), - ids.map(() => "acknowledged"), - ); - for (const id of ids) { - const resolved = yield* service.taskStatus(scope, id); - NodeAssert.equal(resolved.status, outcome); - NodeAssert.equal(resolved.summary, resultText); - NodeAssert.equal(resolved.resultContextTransferId, shared[0]!.id); - trace({ name, stage: "resolved-task", task: resolved }); - } - const afterRead = yield* o.getThreadProjection(parent); - yield* service.readThread(scope, { threadId: child, itemId: result.itemId, limit: 1 }); - yield* o.recoverDelegatedTasks; - yield* o.recoverDelegatedTasks; - const again = yield* o.getThreadProjection(parent); - NodeAssert.deepEqual(again.contextTransfers, settled.contextTransfers); - NodeAssert.deepEqual(again.subagents, afterRead.subagents); - NodeAssert.deepEqual(again.runs, afterRead.runs); - trace({ - name, - stage: "settled", - result: yield* service.taskStatus(scope, taskId), - sharedTransferId: shared[0]!.id, - resultPage: page, - acknowledgedTasks: ids, - parent: again, - }); - console.log( - JSON.stringify({ - name, - status: "passed", - outcome, - sharedTransfers: shared.length, - acknowledgedTasks: ids.length, - }), - ); - }).pipe(Effect.provide(Layer.merge(runtime, mcp))); - }), - ); - -try { - for (const owner of ["delegated", "monitor"] as const) - for (const outcome of ["completed", "failed"] as const) - await Effect.runPromise(run(owner, outcome).pipe(Effect.timeout("5 minutes"))); - NodeFS.writeFileSync( - NodePath.join(output, "verdict.json"), - JSON.stringify({ status: "passed", revision }), - ); -} catch (error) { - NodeFS.writeFileSync( - NodePath.join(output, "verdict.json"), - JSON.stringify({ status: "failed", revision, error: String(error) }), - ); - throw error; -} diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index f6f8a4e697b36..597959fa61874 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -65,6 +65,7 @@ import * as ThreadLaunchService from "./ThreadLaunchService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; +import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; const layerPlatformTest = Layer.merge( Layer.mergeAll(NodeServices.layer, NodeCrypto.layer), @@ -4149,9 +4150,11 @@ const layerSteeredTaskTest = OrchestratorMcpService.layer.pipe( list: () => Effect.succeed([]), }), Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({}), Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), ServerSettings.layerTest(), NodeServices.layer, + NodeCrypto.layer, ), ), Layer.provideMerge(layerTest), diff --git a/knip.jsonc b/knip.jsonc index db91427d01412..068c8cf84ab69 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -57,7 +57,6 @@ "scripts/probe-claude-fork-local-rollback-replay.ts", "scripts/update-test-shard-weights.ts", "scripts/verify-background-live.ts", - "scripts/verify-promoted-delegation.ts", "src/provider/testFixtures/*.mjs", // graphqlgen reads its config. "graphql.config.ts",