From 79d753663a8bac58fbd6051d087045cca11ccd38 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 22 Oct 2026 02:59:17 -0400 Subject: [PATCH 01/11] fix(chat): requeued steers no longer look delivered Hide the retained timeline item when a steer falls back to the durable queue. Restore it when the run leaves the queue, regardless of startup event order. Republish retained inputs before queued startup failures so bounded clients show the failed prompt too, matching server snapshots. Fixes #23403. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: Claude Opus 5.5 at high effort, diagnosis and implementation subagents Co-Authored-By: GPT-6.1 Sol at high effort, diagnosis subagent Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagents --- .../src/orchestration-v2/Orchestrator.ts | 17 +++- .../orchestration-v2/ProjectionStore.test.ts | 50 ++++++----- .../src/orchestration-v2/ProjectionStore.ts | 2 +- .../SteeringCompletion.integration.test.ts | 26 +++++- .../ProviderSwitch.integration.test.ts | 30 +++++++ .../state/orchestrationV2Projection.test.ts | 85 +++++++++++++++++++ .../src/state/orchestrationV2Projection.ts | 31 ++++++- .../src/orchestrationV2Timeline.test.ts | 4 +- .../shared/src/orchestrationV2Timeline.ts | 11 +-- 9 files changed, 223 insertions(+), 33 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 35e706b9775cb..4d9e6bea4baf6 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1283,13 +1283,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const projection = yield* projectionStore.getThreadRecords( threadId, ["runs", "nodes", "attempts", "providerThreads", "turnItems", "messages"], - { turnItemTypes: [], messageRoles: ["user"] }, + { turnItemTypes: ["user_message"], messageRoles: ["user"] }, ); const queuedRun = queuedRunId === undefined ? nextQueuedRun(projection) : projection.runs.find((run) => run.id === queuedRunId && run.status === "queued"); if (queuedRun === undefined) return; + // A steer that fell back to the queue keeps its user item, hidden while the + // run is queued. Republish it before the failure so clients whose bounded + // snapshot omitted it can show the prompt beside the error. + const retainedUserItems = projection.turnItems.filter( + (item) => item.type === "user_message" && item.runId === queuedRun.id, + ); const now = yield* DateTime.now; const rootNode = projection.nodes.find((node) => node.id === queuedRun.rootNodeId); const attempt = projection.attempts.find((entry) => entry.id === queuedRun.activeAttemptId); @@ -1315,6 +1321,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio class: handoffUnsupported ? "validation_error" : "unknown", }); yield* writeSystemEvents([ + ...retainedUserItems.map((item) => ({ + type: "turn-item.updated" as const, + threadId, + runId: queuedRun.id, + ...(rootNode === undefined ? {} : { nodeId: rootNode.id }), + providerInstanceId: queuedRun.providerInstanceId, + occurredAt: now, + payload: item, + })), ...(attempt !== undefined && rootNode !== undefined ? [ { diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 6f87afb2eb233..1cd34f71fdc7f 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -1824,26 +1824,36 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { })} ) `; - const cancelledSuffixSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, { - rowLimit: sqlPageLimit, - }); - const cancelledSuffixPage = buildBoundedThreadProjection({ - policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined }, - projection: cancelledSuffixSnapshot.projection, - snapshotSequence: cancelledSuffixSnapshot.snapshotSequence, - }); - assert.strictEqual(cancelledSuffixPage.latestLocalTurnOrdinal, 1_202); - assert.lengthOf(cancelledSuffixPage.projection.visibleTurnItems, 75); - assert.isFalse( - cancelledSuffixPage.projection.visibleTurnItems.some((row) => - String(row.sourceItemId).includes(":cancelled:"), - ), - ); - assert.isTrue( - cancelledSuffixPage.projection.visibleTurnItems.some( - (row) => row.sourceItemId === runlessQueuedId, - ), - ); + // A steer that fell back to the queue keeps its queued-turn item while its + // run waits. End on cancelled for the checks below. + for (const status of ["queued", "cancelled"] as const) { + yield* sql` + UPDATE orchestration_v2_projection_runs + SET status = ${status}, payload_json = json_set(payload_json, '$.status', ${status}) + WHERE run_id = ${cancelledRunId} + `; + const cancelledSuffixSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + }); + const cancelledSuffixPage = buildBoundedThreadProjection({ + policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined }, + projection: cancelledSuffixSnapshot.projection, + snapshotSequence: cancelledSuffixSnapshot.snapshotSequence, + }); + assert.strictEqual(cancelledSuffixPage.latestLocalTurnOrdinal, 1_202); + assert.lengthOf(cancelledSuffixPage.projection.visibleTurnItems, 75); + assert.isFalse( + cancelledSuffixPage.projection.visibleTurnItems.some((row) => + String(row.sourceItemId).includes(":cancelled:"), + ), + status, + ); + assert.isTrue( + cancelledSuffixPage.projection.visibleTurnItems.some( + (row) => row.sourceItemId === runlessQueuedId, + ), + ); + } const interruptRunId = RunId.make("run:bounded-sql-history:interrupt"); const interruptNodeId = NodeId.make("node:bounded-sql-history:interrupt"); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 92ef73ebf1ba4..b3f32d8054428 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -2846,7 +2846,7 @@ export const layer: Layer.Layer = AND NOT ( item.type = 'user_message' AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn' - AND run.status IS 'cancelled' + AND COALESCE(run.status IN ('queued', 'cancelled'), 0) ) ) ) diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 208934624d9e1..20a88a997d208 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -31,26 +31,30 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" const driver = ProviderDriverKind.make("codex"); const instanceId = ProviderInstanceId.make("codex"); -const modelSelection = { instanceId, model: "test-model" }; it.effect.each( ( [ [false, "during delivery"], + [false, "during delivery", "claudeAgent"], [false, "before dispatch"], [true, "after delivery"], [true, "without native steering"], [true, "with interrupting native steering"], [true, "settled only"], ] as const - ).map(([mailbox, timing]) => ({ + ).map(([mailbox, timing, provider = "codex"]) => ({ mailbox, timing, + provider, label: mailbox ? "mailbox notification" : "steering", })), -)("delivers $label when completion wins $timing", ({ mailbox, timing }) => +)("delivers $label when completion wins $timing ($provider)", ({ mailbox, timing, provider }) => Effect.scoped( Effect.gen(function* () { + const driver = ProviderDriverKind.make(provider); + const instanceId = ProviderInstanceId.make(provider); + const modelSelection = { instanceId, model: "test-model" }; const cwd = yield* checkpointWorkspace(`steering-completion-${timing.replaceAll(" ", "-")}`); const events = yield* Queue.unbounded(); const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; @@ -315,6 +319,14 @@ it.effect.each( if (delivery !== null) { yield* Deferred.succeed(rejectSteer, undefined); yield* Fiber.join(delivery); + const requeued = yield* orchestrator.getThreadProjection(threadId); + assert.isTrue(requeued.runs.some((run) => run.status === "queued")); + assert.isFalse( + requeued.visibleTurnItems.some( + ({ item }) => item.type === "user_message" && item.messageId === messageId, + ), + "a steer that fell back to the queue must not appear delivered", + ); } if (timing === "before dispatch") yield* dispatchSteer; yield* worker.drain(); @@ -380,6 +392,14 @@ it.effect.each( ).length, 1, ); + if (!mailbox) { + assert.equal( + final.visibleTurnItems.filter( + ({ item }) => item.type === "user_message" && item.messageId === messageId, + ).length, + 1, + ); + } yield* worker.drain(); assert.equal(started.length, 2); }).pipe( diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 72aae60c0d075..049ea77f55c26 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -2591,6 +2591,7 @@ describe("orchestration v2 provider switching", () => { ); assert.equal(queuedItem?.runId, queued.runs[1]?.id); assert.equal(queuedItem?.providerThreadId, queued.runs[1]?.providerThreadId); + const queuedSequence = yield* eventSink.latestSequence({ threadId: rejectedThreadId }); yield* eventSink.write({ events: [ { @@ -2615,6 +2616,30 @@ describe("orchestration v2 provider switching", () => { Stream.runHead, ); yield* worker.drain(); + // Clients whose bounded snapshot omitted the hidden queued item must + // receive it while the run is still queued, before it fails. + const throughSequence = yield* eventSink.latestSequence({ threadId: rejectedThreadId }); + const failureEvents = yield* eventSink + .stream({ threadId: rejectedThreadId, afterSequence: queuedSequence }) + .pipe( + Stream.takeUntil((stored) => stored.sequence === throughSequence), + Stream.map(({ event }) => event), + Stream.filter((event) => event.runId === queued.runs[1]!.id), + Stream.runCollect, + ); + const republished = failureEvents.findIndex( + (event) => + event.type === "turn-item.updated" && + event.payload.type === "user_message" && + event.payload.messageId === rejectedMessageId, + ); + assert.isAtLeast(republished, 0); + assert.isBelow( + republished, + failureEvents.findIndex( + (event) => event.type === "run.updated" && event.payload.status === "failed", + ), + ); return yield* orchestrator.getThreadProjection(rejectedThreadId); }).pipe( Effect.provide( @@ -2658,6 +2683,11 @@ describe("orchestration v2 provider switching", () => { failureItem?.type === "error" ? failureItem.failure.code : null, "context_handoff_unsupported", ); + assert.isTrue( + projection.visibleTurnItems.some( + ({ item }) => item.type === "user_message" && item.messageId === rejectedMessageId, + ), + ); assert.deepEqual( (yield* Ref.get(capturedTurns)).map((turn) => turn.driver), [CODEX_DRIVER, CODEX_DRIVER], diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 9d96d8001430d..c5553f452d8ae 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -481,6 +481,91 @@ describe("applyOrchestrationV2ProjectionEvent", () => { }); }); + it("hides a steer that fell back to the queue until its run starts", () => { + const queuedRunId = RunId.make("run-requeued-steer"); + const queuedRun = { ...run, id: queuedRunId, ordinal: 2, status: "queued" } as const; + const reply = commandItem("item-reply", "reply", 102); + const steer: OrchestrationV2TurnItem = { + id: TurnItemId.make("item-steer"), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 101, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + messageId: MessageId.make("message-steer"), + inputIntent: "steer", + text: "steer", + attachments: [], + createdBy: "user", + creationSource: "web", + }; + const event = (type: "run.updated" | "turn-item.updated", payload: unknown) => + ({ + id: `event-${type}`, + type, + threadId, + occurredAt: now, + payload, + }) as OrchestrationV2DomainEvent; + const apply = ( + projection: OrchestrationV2ThreadProjection | null, + events: ReadonlyArray, + options?: Parameters[2], + ) => + events.reduce( + (next, item) => applyOrchestrationV2ProjectionEvent(next, item, options), + projection, + ); + const visibleIds = (projection: OrchestrationV2ThreadProjection | null) => + projection?.visibleTurnItems.map((row) => row.item.id); + + const accepted = apply(emptyProjection, [ + event("run.updated", { ...run, status: "running" }), + event("turn-item.updated", steer), + event("turn-item.updated", reply), + ]); + expect(visibleIds(accepted)).toEqual([steer.id, reply.id]); + + const queuedSteer = event("turn-item.updated", { + ...steer, + runId: queuedRunId, + inputIntent: "queued_turn", + }); + const requeued = apply(accepted, [event("run.updated", queuedRun), queuedSteer]); + expect(visibleIds(requeued)).toEqual([reply.id]); + + // Servers publish the start's user item and run update in either order. + const starting = event("run.updated", { ...queuedRun, status: "starting" }); + expect(visibleIds(apply(requeued, [queuedSteer, starting]))).toEqual([steer.id, reply.id]); + expect(visibleIds(apply(requeued, [starting, queuedSteer]))).toEqual([steer.id, reply.id]); + const cancelled = event("run.updated", { ...queuedRun, status: "cancelled" }); + expect(visibleIds(apply(requeued, [cancelled]))).toEqual([reply.id]); + + // A bounded snapshot taken while queued omits the hidden item, and the + // newer reply puts it below the watermark. + const bounded = { + ...requeued!, + turnItems: requeued!.turnItems.filter((item) => item.id !== steer.id), + }; + const partial = { partialTimeline: true, latestLocalTurnOrdinal: reply.ordinal }; + const failed = event("run.updated", { ...queuedRun, status: "failed" }); + expect(visibleIds(apply(bounded, [queuedSteer, failed], partial))).toEqual([ + steer.id, + reply.id, + ]); + // Leaving the queue never invents an item the client did not receive. + expect(visibleIds(apply(bounded, [failed], partial))).toEqual([reply.id]); + }); + it("removes only hidden local items while preserving inherited rows", () => { const inherited = commandItem("item-inherited"); const local = commandItem("item-local"); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index d954e8a1eda33..0e85375be3d8d 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -210,6 +210,29 @@ function upsertVisibleTurnItem( return updated; } +// A user item retained for a queued run stays hidden until the run leaves the +// queue, and its update can arrive before or after that run update. Items in +// `turnItems` were loaded or streamed, so they skip the partial-window check; +// items never received stay absent. +function restoreQueuedUserItems( + projection: OrchestrationV2ThreadProjection, + runId: OrchestrationV2Run["id"], +): OrchestrationV2ThreadProjection["visibleTurnItems"] { + let visibleTurnItems = projection.visibleTurnItems; + for (const item of projection.turnItems) { + if (item.runId !== runId || item.type !== "user_message" || item.inputIntent !== "queued_turn") + continue; + if (!shouldShowLocalTurnItem(projection, item)) continue; + visibleTurnItems = upsertVisibleTurnItem( + { ...projection, visibleTurnItems }, + item, + false, + null, + ); + } + return visibleTurnItems; +} + export function applyOrchestrationV2ProjectionEvent( projection: OrchestrationV2ThreadProjection | null, event: OrchestrationV2DomainEvent, @@ -253,7 +276,13 @@ export function applyOrchestrationV2ProjectionEvent( case "run.created": case "run.updated": { const next = { ...base, runs: upsertEntity(base.runs, event.payload) }; - return { ...next, visibleTurnItems: activeVisibleTurnItems(next) }; + const visible = { ...next, visibleTurnItems: activeVisibleTurnItems(next) }; + const wasQueued = base.runs.some( + (run) => run.id === event.payload.id && run.status === "queued", + ); + return wasQueued && event.payload.status !== "queued" + ? { ...visible, visibleTurnItems: restoreQueuedUserItems(visible, event.payload.id) } + : visible; } case "run.background-work-cancelled": return { diff --git a/packages/shared/src/orchestrationV2Timeline.test.ts b/packages/shared/src/orchestrationV2Timeline.test.ts index 8d38d1db59b30..23c79641fdbe5 100644 --- a/packages/shared/src/orchestrationV2Timeline.test.ts +++ b/packages/shared/src/orchestrationV2Timeline.test.ts @@ -53,11 +53,11 @@ describe.each([ ).toBe(true); }); - it("hides queued user messages once their run is cancelled", () => { + it.each(["queued", "cancelled"] as const)("hides queued user messages on %s runs", (status) => { expect( isVisible({ item: { type: "user_message", inputIntent: "queued_turn", runId, nodeId }, - runs: [{ id: runId, status: "cancelled" }], + runs: [{ id: runId, status }], attempts: [], items: [{ type: "user_message", inputIntent: "queued_turn", runId, nodeId }], }), diff --git a/packages/shared/src/orchestrationV2Timeline.ts b/packages/shared/src/orchestrationV2Timeline.ts index d160b318a3e0d..fdc90dc4dc2a3 100644 --- a/packages/shared/src/orchestrationV2Timeline.ts +++ b/packages/shared/src/orchestrationV2Timeline.ts @@ -53,14 +53,15 @@ export function isOrchestrationV2TurnItemVisible(input: { return false; } - // A queued message is composer state, not conversation history. When its run - // is cancelled the input never reached the provider, so it must not surface - // as a transcript row. + // A steer can return to the queue before delivery. Keep its retained item out + // of conversation history until its run starts; cancelled queued inputs stay hidden. if ( item.type === "user_message" && item.inputIntent === "queued_turn" && item.runId !== null && - input.runs.some((run) => run.id === item.runId && run.status === "cancelled") + input.runs.some( + (run) => run.id === item.runId && (run.status === "queued" || run.status === "cancelled"), + ) ) { return false; } @@ -95,7 +96,7 @@ export function createOrchestrationV2TurnItemVisibility(input: { const status = item.runId === null ? undefined : statuses.get(item.runId); if (status === "rolled_back") return false; if ( - status === "cancelled" && + (status === "queued" || status === "cancelled") && item.type === "user_message" && item.inputIntent === "queued_turn" ) From b6fbef756b0151585098969872798bf9224fc3fc Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 22 Oct 2026 03:46:47 -0400 Subject: [PATCH 02/11] fix(server): forks exclude undelivered queued inputs Apply queued-input visibility to inherited fork prefixes, SQL windows and shell counts while preserving source history after rollback. Cover projections, pages, counts and filled-window positions for queued and cancelled inputs. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: Claude Opus 5.5 at high effort, implementation subagent Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent --- .../orchestration-v2/ProjectionStore.test.ts | 186 ++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 104 ++++++---- .../shared/src/orchestrationV2Timeline.ts | 2 +- 3 files changed, 252 insertions(+), 40 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 1cd34f71fdc7f..adbced124d571 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5552,6 +5552,192 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }), ); + it.effect("keeps undelivered queued input out of forks taken after a later run", () => + Effect.gen(function* () { + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = ProjectId.make("project:projection-fork-held-queue"); + + for (const heldStatus of ["queued", "cancelled"] as const) { + const prefix = `projection-fork-held-queue:${heldStatus}`; + const sourceThreadId = ThreadId.make(`thread:${prefix}:source`); + const targetThreadId = ThreadId.make(`thread:${prefix}:target`); + const heldRunId = RunId.make(`run:${prefix}:held`); + const answeredRunId = RunId.make(`run:${prefix}:answered`); + + for (const [threadId, forkedFrom] of [ + [sourceThreadId, null], + [targetThreadId, { type: "run", threadId: sourceThreadId, runId: answeredRunId }], + ] as const) { + yield* projectionStore.apply({ + id: EventId.make(`event:${threadId}:created`), + type: "thread.created", + threadId, + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: threadId, + projectId, + title: threadId, + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { + parentThreadId: forkedFrom === null ? null : sourceThreadId, + relationshipToParent: forkedFrom === null ? null : "fork", + rootThreadId: sourceThreadId, + }, + forkedFrom, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + }); + } + + const applyRun = ( + runId: RunId, + ordinal: number, + status: typeof heldStatus | "completed" | "rolled_back", + ) => + projectionStore.apply({ + id: EventId.make(`event:${runId}:${status}`), + type: "run.updated", + threadId: sourceThreadId, + runId, + nodeId: NodeId.make(`node:${runId}`), + driver, + occurredAt: now, + payload: { + id: runId, + threadId: sourceThreadId, + ordinal, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`message:${runId}`), + rootNodeId: NodeId.make(`node:${runId}`), + activeAttemptId: null, + status, + requestedAt: now, + startedAt: status === "queued" ? null : now, + completedAt: status === "queued" ? null : now, + checkpointId: null, + contextHandoffId: null, + }, + }); + const applyUserItem = (input: { + readonly threadId: ThreadId; + readonly runId: RunId | null; + readonly ordinal: number; + readonly text: string; + readonly inputIntent: "turn_start" | "queued_turn"; + }) => + projectionStore.apply({ + id: EventId.make(`event:${input.threadId}:item:${input.ordinal}`), + type: "turn-item.updated", + threadId: input.threadId, + ...(input.runId === null ? {} : { runId: input.runId }), + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: TurnItemId.make(`turn-item:${input.threadId}:${input.ordinal}`), + threadId: input.threadId, + runId: input.runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: input.ordinal, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + messageId: MessageId.make(`message:${input.threadId}:${input.ordinal}`), + inputIntent: input.inputIntent, + text: input.text, + attachments: [], + }, + }); + + yield* applyRun(heldRunId, 1, heldStatus); + yield* applyRun(answeredRunId, 2, "completed"); + yield* applyUserItem({ + threadId: sourceThreadId, + runId: heldRunId, + ordinal: 100, + text: "held", + inputIntent: "queued_turn", + }); + yield* applyUserItem({ + threadId: sourceThreadId, + runId: answeredRunId, + ordinal: 200, + text: "answered", + inputIntent: "turn_start", + }); + // Enough local turns for a turn-limited window to count inherited rows in SQL. + for (let index = 0; index < THREAD_HISTORY_PAGE_POLICY.maxUserTurns; index += 1) { + yield* applyUserItem({ + threadId: targetThreadId, + runId: null, + ordinal: index, + text: `local ${index}`, + inputIntent: "turn_start", + }); + } + + const assertForkHidesHeldInput = Effect.fn("assertForkHidesHeldInput")(function* () { + const target = yield* projectionStore.getThreadProjection(targetThreadId); + assert.deepEqual( + target.visibleTurnItems + .slice(0, 2) + .map((row) => (row.item.type === "user_message" ? row.item.text : row.item.type)), + ["answered", "fork"], + ); + const page = yield* projectionStore.getTimelinePage(targetThreadId, { + view: "activity", + limit: target.visibleTurnItems.length + 1, + }); + assert.deepEqual(page.items, target.visibleTurnItems); + const shellSnapshot = yield* projectionStore.getShellSnapshot(); + for (const shell of [ + yield* projectionStore.getThreadShell(targetThreadId), + shellSnapshot.threads.find((thread) => thread.id === targetThreadId), + ]) { + assert.equal(shell?.visibleItemCount, target.visibleTurnItems.length); + } + const filledLocalPage = yield* projectionStore.getThreadSnapshotWindow(targetThreadId, { + rowLimit: target.visibleTurnItems.length, + userTurnLimit: THREAD_HISTORY_PAGE_POLICY.maxUserTurns, + }); + assert.deepEqual( + filledLocalPage.projection.visibleTurnItems.filter((row) => row.visibility === "local"), + target.visibleTurnItems.filter((row) => row.visibility === "local"), + ); + }); + + yield* assertForkHidesHeldInput(); + // Forks keep rolled-back source history, but still not undelivered input. + yield* applyRun(answeredRunId, 2, "rolled_back"); + yield* assertForkHidesHeldInput(); + } + }), + ); + it.effect("a pull request watch keeps a finished thread working until it ends", () => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index b3f32d8054428..8319d183e6a25 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -68,10 +68,7 @@ import { NodeId, ProviderInstanceId, } from "@t3tools/contracts"; -import { - createOrchestrationV2TurnItemVisibility, - isOrchestrationV2SupersededInterrupt, -} from "@t3tools/shared/orchestrationV2Timeline"; +import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline"; import { deriveBlockedOnThreadWait, derivePendingBackgroundWork, @@ -1393,6 +1390,17 @@ function isTurnItemAtOrBeforeRun(input: { return ordinal !== undefined && ordinal <= input.sourceRunOrdinal; } +// Forks keep source history their cutoff covered even after the source rolls it +// back, so only the remaining visibility rules apply to inherited items. +function createInheritedTurnItemVisibility( + source: Parameters[0], +) { + return createOrchestrationV2TurnItemVisibility({ + ...source, + runs: source.runs.filter((run) => run.status !== "rolled_back"), + }); +} + function visibleTurnItemsThroughRun(input: { readonly sourceProjection: OrchestrationV2ThreadProjection; readonly sourceRunId: NonNullable; @@ -1403,6 +1411,11 @@ function visibleTurnItemsThroughRun(input: { } const runOrdinalById = new Map(input.sourceProjection.runs.map((run) => [run.id, run.ordinal])); + const isVisible = createInheritedTurnItemVisibility({ + runs: input.sourceProjection.runs, + attempts: input.sourceProjection.attempts, + items: input.sourceProjection.turnItems, + }); const inheritedPrefix = input.sourceProjection.visibleTurnItems .filter( (row) => row.item.threadId !== input.sourceProjection.thread.id || row.item.type === "fork", @@ -1414,23 +1427,16 @@ function visibleTurnItemsThroughRun(input: { item: row.item, })); const localPrefix = inheritedVisibleTurnItemsFromLocalItems( - input.sourceProjection.turnItems.filter((item) => { - if ( - isOrchestrationV2SupersededInterrupt({ - item, - attempts: input.sourceProjection.attempts, - items: input.sourceProjection.turnItems, - }) - ) { - return false; - } - return isTurnItemAtOrBeforeRun({ - historyOrigin: input.sourceProjection.thread.historyOrigin, - itemRunId: item.runId, - runOrdinalById, - sourceRunOrdinal: sourceRun.ordinal, - }); - }), + input.sourceProjection.turnItems.filter( + (item) => + isVisible(item) && + isTurnItemAtOrBeforeRun({ + historyOrigin: input.sourceProjection.thread.historyOrigin, + itemRunId: item.runId, + runOrdinalById, + sourceRunOrdinal: sourceRun.ordinal, + }), + ), ); return [...inheritedPrefix, ...localPrefix]; @@ -2839,16 +2845,16 @@ export const layer: Layer.Layer = ), 9223372036854775807 ) + -- Fork ancestry keeps rolled-back runs; undelivered queued input stays hidden. AND ( ${window.requiredRunId ?? null} IS NOT NULL - OR ( - (run.status IS NULL OR run.status <> 'rolled_back') - AND NOT ( - item.type = 'user_message' - AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn' - AND COALESCE(run.status IN ('queued', 'cancelled'), 0) - ) - ) + OR run.status IS NULL + OR run.status <> 'rolled_back' + ) + AND NOT ( + item.type = 'user_message' + AND COALESCE(run.status IN ('queued', 'cancelled'), 0) + AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn' ) AND ( ${window.requiredRunId ?? null} IS NULL @@ -5360,18 +5366,18 @@ export const layer: Layer.Layer = let inherited: Array = []; if (sourceRun !== undefined) { const ordinals = new Map(source.records.runs.map((run) => [run.id, run.ordinal])); - const sourceItems = source.local.map((row) => row.item); + const isInheritedVisible = createInheritedTurnItemVisibility({ + runs: source.records.runs, + attempts: source.records.attempts, + items: source.local.map((row) => row.item), + }); inherited = [ ...source.visible.filter( (row) => row.item.threadId !== fork.threadId || row.item.type === "fork", ), ...source.local.filter( (row) => - !isOrchestrationV2SupersededInterrupt({ - item: row.item, - attempts: source.records.attempts, - items: sourceItems, - }) && + isInheritedVisible(row.item) && isTurnItemAtOrBeforeRun({ historyOrigin: source.records.thread.historyOrigin, itemRunId: row.item.runId, @@ -5831,12 +5837,32 @@ export const layer: Layer.Layer = WHERE thread_id IN ${sql.in(threadIds)} `; + // Forks do not inherit undelivered queued input. Checking the run first keeps + // payload reads to the few queued or cancelled runs. const selectShellRunItemCounts = (threadIds: ReadonlyArray) => sql` - SELECT thread_id, run_id, COUNT(*) AS item_count - FROM orchestration_v2_projection_turn_items - WHERE run_id IS NOT NULL AND thread_id IN ${sql.in(threadIds)} - GROUP BY thread_id, run_id + SELECT per_run.thread_id, per_run.run_id, + per_run.item_count - CASE + WHEN EXISTS ( + SELECT 1 + FROM orchestration_v2_projection_runs AS run + WHERE run.run_id = per_run.run_id + AND run.status IN ('queued', 'cancelled') + ) THEN ( + SELECT COUNT(*) + FROM orchestration_v2_projection_turn_items AS held + WHERE held.run_id = per_run.run_id + AND held.type = 'user_message' + AND json_extract(held.payload_json, '$.inputIntent') = 'queued_turn' + ) + ELSE 0 + END AS item_count + FROM ( + SELECT thread_id, run_id, COUNT(*) AS item_count + FROM orchestration_v2_projection_turn_items + WHERE run_id IS NOT NULL AND thread_id IN ${sql.in(threadIds)} + GROUP BY thread_id, run_id + ) AS per_run `; // Count all selected threads in one index walk; fork sources still need per-run counts. diff --git a/packages/shared/src/orchestrationV2Timeline.ts b/packages/shared/src/orchestrationV2Timeline.ts index fdc90dc4dc2a3..7add05b5200ac 100644 --- a/packages/shared/src/orchestrationV2Timeline.ts +++ b/packages/shared/src/orchestrationV2Timeline.ts @@ -11,7 +11,7 @@ type TimelineTurnItem = Pick; readonly items: ReadonlyArray; From c7546134faa24f4e78b273dec361081ffefd54b9 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 22 Oct 2026 04:44:34 -0400 Subject: [PATCH 03/11] fix(server): shell counts exclude undelivered queued inputs Keep local SQL shell counts aligned with queued-input timeline visibility. Extend the existing regression to check source and fork shell APIs before and after source rollback. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent --- .../src/orchestration-v2/ProjectionStore.test.ts | 13 +++++++++---- apps/server/src/orchestration-v2/ProjectionStore.ts | 12 +++++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index adbced124d571..f25d4c38c3a32 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5714,11 +5714,16 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }); assert.deepEqual(page.items, target.visibleTurnItems); const shellSnapshot = yield* projectionStore.getShellSnapshot(); - for (const shell of [ - yield* projectionStore.getThreadShell(targetThreadId), - shellSnapshot.threads.find((thread) => thread.id === targetThreadId), + for (const projection of [ + yield* projectionStore.getThreadProjection(sourceThreadId), + target, ]) { - assert.equal(shell?.visibleItemCount, target.visibleTurnItems.length); + for (const shell of [ + yield* projectionStore.getThreadShell(projection.thread.id), + shellSnapshot.threads.find((thread) => thread.id === projection.thread.id), + ]) { + assert.equal(shell?.visibleItemCount, projection.visibleTurnItems.length); + } } const filledLocalPage = yield* projectionStore.getThreadSnapshotWindow(targetThreadId, { rowLimit: target.visibleTurnItems.length, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8319d183e6a25..f80dac99f46f5 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -5870,7 +5870,17 @@ export const layer: Layer.Layer = sql` SELECT counts.thread_id, SUM(CASE WHEN counts.run_id IS NULL OR r.status <> 'rolled_back' - THEN counts.item_count ELSE 0 END) AS item_count, + THEN counts.item_count - CASE + WHEN r.status IN ('queued', 'cancelled') THEN ( + SELECT COUNT(*) + FROM orchestration_v2_projection_turn_items held + WHERE held.run_id = counts.run_id + AND held.type = 'user_message' + AND json_extract(held.payload_json, '$.inputIntent') = 'queued_turn' + ) + ELSE 0 + END + ELSE 0 END) AS item_count, SUM(CASE WHEN counts.run_id IS NULL THEN counts.item_count ELSE 0 END) AS runless_item_count FROM ( From a51b33cfa5ea6a08bd493042ccf5cd38e46333c8 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 22 Oct 2026 20:09:11 -0400 Subject: [PATCH 04/11] test(server): preserve delivered queued inputs in fork history Include a completed queued-turn input in the inherited-history regression so intent-only exclusions cannot hide delivered prompts or undercount rows. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent for delivered queued-input regression --- .../src/orchestration-v2/ProjectionStore.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index f25d4c38c3a32..3df68e2886a2c 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5689,6 +5689,13 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { text: "answered", inputIntent: "turn_start", }); + yield* applyUserItem({ + threadId: sourceThreadId, + runId: answeredRunId, + ordinal: 201, + text: "delivered steer", + inputIntent: "queued_turn", + }); // Enough local turns for a turn-limited window to count inherited rows in SQL. for (let index = 0; index < THREAD_HISTORY_PAGE_POLICY.maxUserTurns; index += 1) { yield* applyUserItem({ @@ -5704,9 +5711,9 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const target = yield* projectionStore.getThreadProjection(targetThreadId); assert.deepEqual( target.visibleTurnItems - .slice(0, 2) + .slice(0, 3) .map((row) => (row.item.type === "user_message" ? row.item.text : row.item.type)), - ["answered", "fork"], + ["answered", "delivered steer", "fork"], ); const page = yield* projectionStore.getTimelinePage(targetThreadId, { view: "activity", From 3ebddb5160f0e94ca7d94d09e911431ffb9a40e9 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 22 Oct 2026 21:15:47 -0400 Subject: [PATCH 05/11] fix(server): keep cancelled queued inputs hidden after source rollback Rollback replaces a cancelled run's status but preserves its cancelled attempt. Use that evidence in inherited visibility, history windows and shell counts, while keeping delivered source history visible after rollback. Extend the fork-history regression to roll back the cancelled held run and check projections, activity pages, both shell APIs and loaded row positions. Co-Authored-By: GPT-6.1 Sol at high effort Co-Authored-By: Claude Opus 5.5 at high effort, implementation subagent for cancelled-input rollback Co-Authored-By: GPT-6.1 Sol at xhigh effort, review subagent for cancelled-input rollback --- .../orchestration-v2/ProjectionStore.test.ts | 32 ++++++++++++- .../src/orchestration-v2/ProjectionStore.ts | 47 ++++++++++++++++--- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 3df68e2886a2c..451a46692ab6f 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5563,6 +5563,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const sourceThreadId = ThreadId.make(`thread:${prefix}:source`); const targetThreadId = ThreadId.make(`thread:${prefix}:target`); const heldRunId = RunId.make(`run:${prefix}:held`); + const heldAttemptId = RunAttemptId.make(`attempt:${prefix}:held`); const answeredRunId = RunId.make(`run:${prefix}:answered`); for (const [threadId, forkedFrom] of [ @@ -5626,10 +5627,10 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { providerThreadId: null, userMessageId: MessageId.make(`message:${runId}`), rootNodeId: NodeId.make(`node:${runId}`), - activeAttemptId: null, + activeAttemptId: runId === heldRunId ? heldAttemptId : null, status, requestedAt: now, - startedAt: status === "queued" ? null : now, + startedAt: runId === heldRunId ? null : now, completedAt: status === "queued" ? null : now, checkpointId: null, contextHandoffId: null, @@ -5674,6 +5675,28 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }); yield* applyRun(heldRunId, 1, heldStatus); + yield* projectionStore.apply({ + id: EventId.make(`event:${heldAttemptId}:created`), + type: "run-attempt.created", + threadId: sourceThreadId, + runId: heldRunId, + nodeId: NodeId.make(`node:${heldRunId}`), + driver, + occurredAt: now, + payload: { + id: heldAttemptId, + runId: heldRunId, + attemptOrdinal: 1, + rootNodeId: NodeId.make(`node:${heldRunId}`), + providerInstanceId, + providerThreadId: ProviderThreadId.make(`provider-thread:${prefix}`), + providerTurnId: null, + reason: "initial", + status: heldStatus === "queued" ? "pending" : "cancelled", + startedAt: null, + completedAt: heldStatus === "queued" ? null : now, + }, + }); yield* applyRun(answeredRunId, 2, "completed"); yield* applyUserItem({ threadId: sourceThreadId, @@ -5746,6 +5769,11 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { // Forks keep rolled-back source history, but still not undelivered input. yield* applyRun(answeredRunId, 2, "rolled_back"); yield* assertForkHidesHeldInput(); + // Rollback also takes cancelled runs, and overwrites the status that hid their input. + if (heldStatus === "cancelled") { + yield* applyRun(heldRunId, 1, "rolled_back"); + yield* assertForkHidesHeldInput(); + } } }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index f80dac99f46f5..85f3520434797 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1391,13 +1391,26 @@ function isTurnItemAtOrBeforeRun(input: { } // Forks keep source history their cutoff covered even after the source rolls it -// back, so only the remaining visibility rules apply to inherited items. +// back, so only the remaining visibility rules apply to inherited items. Rollback +// overwrites a cancelled run's status; its unchanged attempt preserves the +// cancelled-input visibility rule. function createInheritedTurnItemVisibility( source: Parameters[0], ) { + const cancelledRunIds = new Set( + source.attempts + .filter((attempt) => attempt.status === "cancelled") + .map((attempt) => attempt.runId), + ); return createOrchestrationV2TurnItemVisibility({ ...source, - runs: source.runs.filter((run) => run.status !== "rolled_back"), + runs: source.runs.flatMap((run) => + run.status !== "rolled_back" + ? [run] + : cancelledRunIds.has(run.id) + ? [{ ...run, status: "cancelled" as const }] + : [], + ), }); } @@ -2853,7 +2866,18 @@ export const layer: Layer.Layer = ) AND NOT ( item.type = 'user_message' - AND COALESCE(run.status IN ('queued', 'cancelled'), 0) + AND ( + COALESCE(run.status IN ('queued', 'cancelled'), 0) + OR ( + run.status = 'rolled_back' + AND EXISTS ( + SELECT 1 + FROM orchestration_v2_projection_run_attempts AS attempt + WHERE attempt.run_id = run.run_id + AND attempt.status = 'cancelled' + ) + ) + ) AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn' ) AND ( @@ -5837,8 +5861,8 @@ export const layer: Layer.Layer = WHERE thread_id IN ${sql.in(threadIds)} `; - // Forks do not inherit undelivered queued input. Checking the run first keeps - // payload reads to the few queued or cancelled runs. + // Forks do not inherit undelivered queued input, including from cancelled runs + // rolled back later. Checking the run first keeps payload reads to those few runs. const selectShellRunItemCounts = (threadIds: ReadonlyArray) => sql` SELECT per_run.thread_id, per_run.run_id, @@ -5847,7 +5871,18 @@ export const layer: Layer.Layer = SELECT 1 FROM orchestration_v2_projection_runs AS run WHERE run.run_id = per_run.run_id - AND run.status IN ('queued', 'cancelled') + AND ( + run.status IN ('queued', 'cancelled') + OR ( + run.status = 'rolled_back' + AND EXISTS ( + SELECT 1 + FROM orchestration_v2_projection_run_attempts AS attempt + WHERE attempt.run_id = run.run_id + AND attempt.status = 'cancelled' + ) + ) + ) ) THEN ( SELECT COUNT(*) FROM orchestration_v2_projection_turn_items AS held From 241ac99713da0b133edb6c7806b1094e322e8831 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 17:40:28 +0000 Subject: [PATCH 06/11] test(chat): cover bounded inherited queued input visibility --- apps/server/src/orchestration-v2/ProjectionStore.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index f4f362c50e5f2..c1067b159d64b 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -6258,6 +6258,10 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { limit: target.visibleTurnItems.length + 1, }); assert.deepEqual(page.items, target.visibleTurnItems); + const bounded = yield* projectionStore.getThreadSnapshotWindow(targetThreadId, { + rowLimit: target.visibleTurnItems.length + 1, + }); + assert.deepEqual(bounded.projection.visibleTurnItems, target.visibleTurnItems); const shellSnapshot = yield* projectionStore.getShellSnapshot(); for (const projection of [ yield* projectionStore.getThreadProjection(sourceThreadId), From 1b47d3995d44a5915e0332e9ba99f4ae853fa226 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Wed, 28 Oct 2026 08:51:25 +0000 Subject: [PATCH 07/11] fix(chat): restore retained prompts after queued startup failure Mark the retained prompt failed when queued startup fails so web and mobile stop suppressing it as pending. Preserve its identity, intent and ordinal, and keep it hidden until the queued run fails. Co-Authored-By: GPT-6.1 Sol at high effort --- apps/mobile/src/lib/threadActivity.test.ts | 24 +++++++++++++++++ .../src/orchestration-v2/Orchestrator.ts | 2 +- .../ProviderSwitch.integration.test.ts | 14 ++++++++++ apps/web/src/session-logic.test.ts | 26 ++++++++++++++++++- .../state/orchestrationV2Projection.test.ts | 26 ++++++++++++++----- 5 files changed, 84 insertions(+), 8 deletions(-) diff --git a/apps/mobile/src/lib/threadActivity.test.ts b/apps/mobile/src/lib/threadActivity.test.ts index 6582b8a55d91e..59e21658ccb0b 100644 --- a/apps/mobile/src/lib/threadActivity.test.ts +++ b/apps/mobile/src/lib/threadActivity.test.ts @@ -333,6 +333,30 @@ function userMessage(updatedAt = "2026-06-20T00:00:01.000Z") { }; } +it("restores a retained queued prompt after startup fails", () => { + const pending = { + ...userMessage(), + inputIntent: "queued_turn" as const, + status: "pending" as const, + startedAt: null, + completedAt: null, + }; + for (const visibility of ["local", "inherited"] as const) { + const cache = createThreadFeedCache(); + expect(buildThreadFeed([projected(pending, 0, visibility)], { cache })).toEqual([]); + const failedAt = DateTime.makeUnsafe("2026-06-20T00:00:02.000Z"); + const failed = { + ...pending, + status: "failed" as const, + completedAt: failedAt, + updatedAt: failedAt, + }; + expect(buildThreadFeed([projected(failed, 0, visibility)], { cache })).toMatchObject([ + { type: "message", message: { text: pending.text, inputIntent: "queued_turn" } }, + ]); + } +}); + function command(updatedAt = "2026-06-20T00:00:02.000Z") { return { ...base("item-command", updatedAt, 1), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 86373485781a8..beea18bcf1b78 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1393,7 +1393,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...(rootNode === undefined ? {} : { nodeId: rootNode.id }), providerInstanceId: queuedRun.providerInstanceId, occurredAt: now, - payload: item, + payload: { ...item, status: "failed" as const, completedAt: now, updatedAt: now }, })), ...(attempt !== undefined && rootNode !== undefined ? [ diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 158619dd31572..fcdf7dcee3af3 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -2886,6 +2886,10 @@ describe("orchestration v2 provider switching", () => { ); assert.equal(queuedItem?.runId, queued.runs[1]?.id); assert.equal(queuedItem?.providerThreadId, queued.runs[1]?.providerThreadId); + assert.equal(queuedItem?.status, "pending"); + assert.isNull(queuedItem?.startedAt); + assert.isNull(queuedItem?.completedAt); + assert.isFalse(queued.visibleTurnItems.some((row) => row.item.id === queuedItem?.id)); const queuedSequence = yield* eventSink.latestSequence({ threadId: rejectedThreadId }); yield* eventSink.write({ events: [ @@ -2929,6 +2933,16 @@ describe("orchestration v2 provider switching", () => { event.payload.messageId === rejectedMessageId, ); assert.isAtLeast(republished, 0); + const restored = failureEvents[republished]; + assert.equal(restored?.type, "turn-item.updated"); + if (restored?.type === "turn-item.updated") { + assert.deepEqual(restored.payload, { + ...queuedItem, + status: "failed", + completedAt: restored.occurredAt, + updatedAt: restored.occurredAt, + }); + } assert.isBelow( republished, failureEvents.findIndex( diff --git a/apps/web/src/session-logic.test.ts b/apps/web/src/session-logic.test.ts index de213de2f3c1c..ea30cd0606aa3 100644 --- a/apps/web/src/session-logic.test.ts +++ b/apps/web/src/session-logic.test.ts @@ -624,7 +624,7 @@ describe("V2 session presentation", () => { }, ); - it("waits for a dispatched turn item before adding queued input to the timeline", () => { + it("keeps queued input out of the timeline until dispatch or startup failure", () => { const projection = makeThreadProjectionFixture(); const now = DateTime.makeUnsafe("2026-06-20T00:00:00.000Z"); const runId = RunId.make("run-dispatched-queued"); @@ -693,6 +693,30 @@ describe("V2 session presentation", () => { }), ).toHaveLength(inputIntent === "queued_turn" ? 0 : 1); } + for (const status of ["completed", "failed"] as const) { + const entries = deriveTimelineEntriesFromVisibleTurnItems({ + visibleTurnItems: [ + { + position: 0, + visibility: "local", + sourceThreadId: projection.thread.id, + sourceItemId: dispatchedItem.id, + item: { + ...dispatchedItem, + inputIntent: "queued_turn", + status, + startedAt: status === "failed" ? null : now, + }, + }, + ], + optimisticMessages: [optimisticMessage], + }); + expect(entries.map((entry) => entry.id)).toEqual([messageId]); + expect(entries[0]).toMatchObject({ + kind: "message", + message: { text: "Queued input", inputIntent: "queued_turn" }, + }); + } const promotedEntries = deriveTimelineEntriesFromVisibleTurnItems({ visibleTurnItems: [ { diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index c972ca97903e0..3d3db1478df93 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -539,14 +539,22 @@ describe("applyOrchestrationV2ProjectionEvent", () => { ...steer, runId: queuedRunId, inputIntent: "queued_turn", + status: "pending", + startedAt: null, + completedAt: null, }); const requeued = apply(accepted, [event("run.updated", queuedRun), queuedSteer]); expect(visibleIds(requeued)).toEqual([reply.id]); // Servers publish the start's user item and run update in either order. const starting = event("run.updated", { ...queuedRun, status: "starting" }); - expect(visibleIds(apply(requeued, [queuedSteer, starting]))).toEqual([steer.id, reply.id]); - expect(visibleIds(apply(requeued, [starting, queuedSteer]))).toEqual([steer.id, reply.id]); + const dispatchedSteer = event("turn-item.updated", { + ...steer, + runId: queuedRunId, + inputIntent: "queued_turn", + }); + expect(visibleIds(apply(requeued, [dispatchedSteer, starting]))).toEqual([steer.id, reply.id]); + expect(visibleIds(apply(requeued, [starting, dispatchedSteer]))).toEqual([steer.id, reply.id]); const cancelled = event("run.updated", { ...queuedRun, status: "cancelled" }); expect(visibleIds(apply(requeued, [cancelled]))).toEqual([reply.id]); @@ -558,10 +566,16 @@ describe("applyOrchestrationV2ProjectionEvent", () => { }; const partial = { partialTimeline: true, latestLocalTurnOrdinal: reply.ordinal }; const failed = event("run.updated", { ...queuedRun, status: "failed" }); - expect(visibleIds(apply(bounded, [queuedSteer, failed], partial))).toEqual([ - steer.id, - reply.id, - ]); + const failedSteer = event("turn-item.updated", { + ...steer, + runId: queuedRunId, + inputIntent: "queued_turn", + status: "failed", + startedAt: null, + }); + const retained = apply(bounded, [failedSteer], partial); + expect(visibleIds(retained)).toEqual([reply.id]); + expect(visibleIds(apply(retained, [failed], partial))).toEqual([steer.id, reply.id]); // Leaving the queue never invents an item the client did not receive. expect(visibleIds(apply(bounded, [failed], partial))).toEqual([reply.id]); }); From 9a6a0e824929b16678cfcf9ef6b8624e322252eb Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 29 Oct 2026 18:34:37 +0000 Subject: [PATCH 08/11] test(mobile): isolate git branches sheet occupancy test Co-Authored-By: GPT-6 Luna at low effort, primary PR maintenance Co-Authored-By: GPT-6.1 Sol at high effort, implementation subagent for mobile test isolation --- .../threads/git/GitBranchesSheet.occupancy.test.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx b/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx index 7667b0c793fc4..6bd8652899ad7 100644 --- a/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx +++ b/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx @@ -36,8 +36,10 @@ vi.mock("../../../state/use-thread-selection", () => ({ selectedThreadProject: { workspaceRoot: checkout.projectRoot }, }), })); -vi.mock("../../../state/use-thread-detail", () => ({ - useSelectedThreadWorktreePath: () => checkout.worktreePath, +vi.mock("../../../state/use-selected-thread-worktree", () => ({ + useSelectedThreadWorktree: () => ({ + selectedThreadCwd: checkout.worktreePath ?? checkout.projectRoot, + }), })); vi.mock("../../../state/use-selected-thread-git-state", () => ({ useSelectedThreadGitState: () => ({ From e2a62fa416c040e8c1337a81c338d0efc8f6ee95 Mon Sep 17 00:00:00 2001 From: Alb11747 Date: Thu, 12 Nov 2026 12:42:25 +0000 Subject: [PATCH 09/11] test(server): include queued input index in preview migration expectations Co-Authored-By: GPT-6.1 Sol (high; implementation sub-agent) --- apps/server/src/persistence/reconcileV2PreviewMigration.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 3c2cb075cfab4..98a79e7a7370d 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -65,6 +65,7 @@ describe("V2 preview upgrade", () => { [83, "CommandReadIndexes"], [84, "LastExecutedRunIndex"], [85, "TurnItemCounts"], + [86, "QueuedTurnInputIndex"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -173,6 +174,7 @@ describe("V2 preview upgrade", () => { [83, "CommandReadIndexes"], [84, "LastExecutedRunIndex"], [85, "TurnItemCounts"], + [86, "QueuedTurnInputIndex"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); From 9616c6619cb6094b84bcab9fa7c29d3c78e01c54 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 21:57:36 +0000 Subject: [PATCH 10/11] test: keep queued-steer coverage to the layers the fix changes Drop renderer tests that pass on main without this change, the redundant provider row in the steering matrix, and an unrelated mobile test mock edit. AI-assisted; verified with the tests and tools listed above. --- .../git/GitBranchesSheet.occupancy.test.tsx | 6 ++--- apps/mobile/src/lib/threadActivity.test.ts | 24 ----------------- .../SteeringCompletion.integration.test.ts | 10 +++---- apps/web/src/session-logic.test.ts | 26 +------------------ 4 files changed, 6 insertions(+), 60 deletions(-) diff --git a/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx b/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx index 2660f584dd0e4..b2d29db8c8c72 100644 --- a/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx +++ b/apps/mobile/src/features/threads/git/GitBranchesSheet.occupancy.test.tsx @@ -37,10 +37,8 @@ vi.mock("../../../state/use-thread-selection", () => ({ selectedThreadProject: { workspaceRoot: checkout.projectRoot }, }), })); -vi.mock("../../../state/use-selected-thread-worktree", () => ({ - useSelectedThreadWorktree: () => ({ - selectedThreadCwd: checkout.worktreePath ?? checkout.projectRoot, - }), +vi.mock("../../../state/use-thread-detail", () => ({ + useSelectedThreadWorktreePath: () => checkout.worktreePath, })); vi.mock("../../../state/use-selected-thread-git-state", () => ({ useSelectedThreadGitState: () => ({ diff --git a/apps/mobile/src/lib/threadActivity.test.ts b/apps/mobile/src/lib/threadActivity.test.ts index 7bcd8d677504e..86e7f3f7182ac 100644 --- a/apps/mobile/src/lib/threadActivity.test.ts +++ b/apps/mobile/src/lib/threadActivity.test.ts @@ -386,30 +386,6 @@ function userMessage(updatedAt = "2026-06-20T00:00:01.000Z") { }; } -it("restores a retained queued prompt after startup fails", () => { - const pending = { - ...userMessage(), - inputIntent: "queued_turn" as const, - status: "pending" as const, - startedAt: null, - completedAt: null, - }; - for (const visibility of ["local", "inherited"] as const) { - const cache = createThreadFeedCache(); - expect(buildThreadFeed([projected(pending, 0, visibility)], { cache })).toEqual([]); - const failedAt = DateTime.makeUnsafe("2026-06-20T00:00:02.000Z"); - const failed = { - ...pending, - status: "failed" as const, - completedAt: failedAt, - updatedAt: failedAt, - }; - expect(buildThreadFeed([projected(failed, 0, visibility)], { cache })).toMatchObject([ - { type: "message", message: { text: pending.text, inputIntent: "queued_turn" } }, - ]); - } -}); - function command(updatedAt = "2026-06-20T00:00:02.000Z") { return { ...base("item-command", updatedAt, 1), diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 40538f9923ee7..f4f2e09d24c28 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -40,12 +40,12 @@ import * as ProviderContinuationRequests from "@t3tools/provider-core/server/Pro const driver = ProviderDriverKind.make("codex"); const instanceId = ProviderInstanceId.make("codex"); +const modelSelection = { instanceId, model: "test-model" }; it.effect.each( ( [ [false, "during delivery"], - [false, "during delivery", "claudeAgent"], [false, "before dispatch"], [true, "after delivery"], [true, "without native steering"], @@ -54,20 +54,16 @@ it.effect.each( [false, "during continuation"], ["question", "during delivery"], ] as const - ).map(([kind, timing, provider = "codex"]) => ({ + ).map(([kind, timing]) => ({ mailbox: kind === true, question: kind === "question", timing, - provider, label: kind === "question" ? "delegated task question" : kind ? "mailbox notification" : "steering", })), -)("delivers $label on completion $timing ($provider)", ({ mailbox, question, timing, provider }) => +)("delivers $label when completion wins $timing", ({ mailbox, question, timing }) => Effect.scoped( Effect.gen(function* () { - const driver = ProviderDriverKind.make(provider); - const instanceId = ProviderInstanceId.make(provider); - const modelSelection = { instanceId, model: "test-model" }; const cwd = yield* checkpointWorkspace(`steering-completion-${timing.replaceAll(" ", "-")}`); const events = yield* Queue.unbounded(); const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; diff --git a/apps/web/src/session-logic.test.ts b/apps/web/src/session-logic.test.ts index c544a7fb9e1f3..013e38c701ca1 100644 --- a/apps/web/src/session-logic.test.ts +++ b/apps/web/src/session-logic.test.ts @@ -643,7 +643,7 @@ describe("V2 session presentation", () => { }, ); - it("keeps queued input out of the timeline until dispatch or startup failure", () => { + it("waits for a dispatched turn item before adding queued input to the timeline", () => { const projection = makeThreadProjectionFixture(); const now = DateTime.makeUnsafe("2026-06-20T00:00:00.000Z"); const runId = RunId.make("run-dispatched-queued"); @@ -712,30 +712,6 @@ describe("V2 session presentation", () => { }), ).toHaveLength(inputIntent === "queued_turn" ? 0 : 1); } - for (const status of ["completed", "failed"] as const) { - const entries = deriveTimelineEntriesFromVisibleTurnItems({ - visibleTurnItems: [ - { - position: 0, - visibility: "local", - sourceThreadId: projection.thread.id, - sourceItemId: dispatchedItem.id, - item: { - ...dispatchedItem, - inputIntent: "queued_turn", - status, - startedAt: status === "failed" ? null : now, - }, - }, - ], - optimisticMessages: [optimisticMessage], - }); - expect(entries.map((entry) => entry.id)).toEqual([messageId]); - expect(entries[0]).toMatchObject({ - kind: "message", - message: { text: "Queued input", inputIntent: "queued_turn" }, - }); - } const promotedEntries = deriveTimelineEntriesFromVisibleTurnItems({ visibleTurnItems: [ { From 6c8969f0ecd63393bbb8138773daddd41e27c5f2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 22:12:42 +0000 Subject: [PATCH 11/11] fix(server): timeline pages count input held by queued runs as hidden The windowed timeline read only looked for hidden rows in rolled-back and cancelled runs, so a queued_turn item of a queued run vanished from pages without being subtracted from counts and offsets. AI-assisted; verified with the tests and tools listed above. --- .../src/orchestration-v2/ProjectionStore.test.ts | 16 ++++++++++++---- .../src/orchestration-v2/ProjectionStore.ts | 16 ++++++++-------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 94837ecbf080a..68862a6db3916 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5840,6 +5840,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { : [ "rolled back", "cancelled queued turn", + "held queued turn", "replaced attempt", "stopped", "stopped", @@ -5870,9 +5871,11 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { ? "rolled_back" : kind === "cancelled queued turn" ? "cancelled" - : kind === "short" && chance(0.5) - ? "interrupted" - : "completed"; + : kind === "held queued turn" + ? "queued" + : kind === "short" && chance(0.5) + ? "interrupted" + : "completed"; runs.push({ id: runId, threadId, @@ -5951,7 +5954,12 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { ? run % 10 !== 0 : kind !== "late wake" && (agentDriven ? run > 1 : chance(0.2)); if (wakes) notification(); - else userMessage(kind === "cancelled queued turn" ? "queued_turn" : "turn_start"); + else + userMessage( + kind === "cancelled queued turn" || kind === "held queued turn" + ? "queued_turn" + : "turn_start", + ); // A stopped run never folds, so it is long enough for page ends to fall inside it. const work = kind === "late wake" diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8cae70d4673a1..3aeac72d533a1 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -6603,13 +6603,13 @@ export const layer: Layer.Layer = })), }); // Only these runs can hide rows: every row of a rolled-back run, the queued turn of a - // cancelled run, and the stop result of a replaced attempt. So the hidden rows are found - // without reading the thread. + // queued or cancelled run, and the stop result of a replaced attempt. So the hidden rows + // are found without reading the thread. const stoppedRuns = new Set(stopRequests.map((row) => row.run_id)); - const runsWithStatus = (status: "rolled_back" | "cancelled") => - records.runs.flatMap((run) => (run.status === status ? [String(run.id)] : [])); + const runsWithStatus = (...statuses: ReadonlyArray) => + records.runs.flatMap((run) => (statuses.includes(run.status) ? [String(run.id)] : [])); const rolledBack = runsWithStatus("rolled_back"); - const cancelled = runsWithStatus("cancelled"); + const heldInput = runsWithStatus("queued", "cancelled"); const steered = [...new Set(superseded.map((attempt) => String(attempt.runId)))].filter( (runId) => !stoppedRuns.has(runId) && !rolledBack.includes(runId), ); @@ -6634,8 +6634,8 @@ export const layer: Layer.Layer = INDEXED BY orchestration_v2_projection_turn_items_thread_run_idx ON item.thread_id = ${threadId} AND item.run_id = wanted.value`, // The conversation index holds the type and run of these rows, so only the matches are - // fetched. Through the run index, every row of each cancelled run would be. - cancelled.length + steered.length === 0 + // fetched. Through the run index, every row of each such run would be. + heldInput.length + steered.length === 0 ? Effect.succeed([]) : sql` SELECT ${candidateColumns} @@ -6644,7 +6644,7 @@ export const layer: Layer.Layer = WHERE item.thread_id = ${threadId} AND item.type NOT IN ${CONVERSATION_INDEX_EXCLUDED_TYPES} AND ((item.type = 'user_message' - AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(cancelled)}))) + AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(heldInput)}))) OR (item.type = 'run_interrupt_result' AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(steered)}))))`, ]);