From 4171833be76c29d74d0a1742ae7266dcdcabf517 Mon Sep 17 00:00:00 2001 From: Matthew Riddell Date: Sun, 1 Nov 2026 10:03:41 -0500 Subject: [PATCH 1/5] fix(server): renew idle native context without replacing thread (#27890) --- .../src/orchestration-v2/Orchestrator.ts | 133 ++++++++---- .../ProviderTurnStartService.test.ts | 66 ++++++ .../ProviderTurnStartService.ts | 29 ++- .../freshProviderContext.test.ts | 200 ++++++++++++++++++ .../orchestration-v2/freshProviderContext.ts | 94 ++++++++ .../testkit/OrchestratorScenario.ts | 1 + packages/contracts/src/orchestrationV2.ts | 20 ++ 7 files changed, 494 insertions(+), 49 deletions(-) create mode 100644 apps/server/src/orchestration-v2/freshProviderContext.test.ts create mode 100644 apps/server/src/orchestration-v2/freshProviderContext.ts diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index d0edd5dacf8f..a80387e88e05 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -77,6 +77,7 @@ import { derivePendingBackgroundWork, pendingBackgroundTurnItems, } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; +import { prepareFreshProviderThread } from "./freshProviderContext.ts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -507,6 +508,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId { case "thread.runtime-mode.set": case "thread.interaction-mode.set": case "thread.model-selection.set": + case "provider-session.renew": case "provider-session.detach": case "message.dispatch": case "notification.delivery.accept": @@ -3948,58 +3950,102 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } }); - const dispatchProviderSessionDetach = Effect.fn("orchestrationV2.dispatch.providerSessionDetach")( - function* ( - command: Extract, - events: Ref.Ref>, - effects: Ref.Ref>, - ) { - const projection = yield* projectionStore - .getThreadRecords(command.threadId, ["providerSessions"]) + const dispatchProviderSessionLifecycle = Effect.fn( + "orchestrationV2.dispatch.providerSessionLifecycle", + )(function* ( + command: Extract< + OrchestrationV2Command, + { readonly type: "provider-session.detach" | "provider-session.renew" } + >, + events: Ref.Ref>, + effects: Ref.Ref>, + ) { + const projection = yield* projectionStore + .getThreadRecords(command.threadId, ["providerSessions"]) + .pipe( + Effect.mapError( + (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }), + ), + ); + const session = projection.providerSessions.find( + (candidate) => candidate.id === command.providerSessionId, + ); + if (session === undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Provider session ${command.providerSessionId} does not belong to thread ${command.threadId}.`, + }); + } + const now = yield* DateTime.now; + if (command.type === "provider-session.renew") { + const records = yield* projectionStore + .getThreadRecords(command.threadId, [ + "providerThreads", + "runs", + "providerTurns", + "runtimeRequests", + "subagents", + "turnItems", + ]) .pipe( Effect.mapError( (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }), ), ); - const session = projection.providerSessions.find( - (candidate) => candidate.id === command.providerSessionId, - ); - if (session === undefined) { + const prepared = prepareFreshProviderThread(records, command, now); + if (typeof prepared === "string") { return yield* new OrchestratorDispatchError({ commandId: command.commandId, commandType: command.type, - cause: `Provider session ${command.providerSessionId} does not belong to thread ${command.threadId}.`, + cause: prepared, }); } - const now = yield* DateTime.now; - yield* emit( - events, - command, - )({ - type: "provider-session.detached", - threadId: command.threadId, - driver: session.driver, - providerInstanceId: session.providerInstanceId, - occurredAt: now, - payload: { - providerSessionId: session.id, - detachedAt: now, - ...(command.reason === undefined ? {} : { reason: command.reason }), - }, - }); - const pendingEffect = { - id: `effect:${command.commandId}:provider-session.detach:${command.providerSessionId}`, - commandId: command.commandId, - threadId: command.threadId, - request: { - type: "provider-session.detach", - providerSessionId: command.providerSessionId, - ...(command.reason === undefined ? {} : { detail: command.reason }), - }, - } satisfies PendingOrchestrationEffectV2; - yield* Ref.update(effects, (existing) => [...existing, pendingEffect]); - }, - ); + for (const providerThread of [prepared.previous, prepared.fresh]) { + yield* emit( + events, + command, + )({ + type: "provider-thread.updated", + threadId: command.threadId, + driver: session.driver, + providerInstanceId: session.providerInstanceId, + occurredAt: now, + payload: providerThread, + }); + } + // Keep the transport attached. A delayed detach effect could otherwise + // interrupt input that arrives after this atomic renewal. The next run + // starts a new native thread from the new row's null native reference. + return; + } + yield* emit( + events, + command, + )({ + type: "provider-session.detached", + threadId: command.threadId, + driver: session.driver, + providerInstanceId: session.providerInstanceId, + occurredAt: now, + payload: { + providerSessionId: session.id, + detachedAt: now, + ...(command.reason === undefined ? {} : { reason: command.reason }), + }, + }); + const pendingEffect = { + id: `effect:${command.commandId}:provider-session.detach:${command.providerSessionId}`, + commandId: command.commandId, + threadId: command.threadId, + request: { + type: "provider-session.detach", + providerSessionId: command.providerSessionId, + ...(command.reason === undefined ? {} : { detail: command.reason }), + }, + } satisfies PendingOrchestrationEffectV2; + yield* Ref.update(effects, (existing) => [...existing, pendingEffect]); + }); const dispatchThreadProjectAdoptFolder = Effect.fn( "orchestrationV2.dispatch.threadProjectAdoptFolder", @@ -12552,8 +12598,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "thread.guest-message.dismiss": yield* dispatchGuestShare(command, events, effects); break; + case "provider-session.renew": case "provider-session.detach": - yield* dispatchProviderSessionDetach(command, events, effects); + yield* dispatchProviderSessionLifecycle(command, events, effects); break; case "thread.project.adopt-folder": yield* dispatchThreadProjectAdoptFolder(command, events); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 7a922d79c731..484d971561b9 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -263,6 +263,7 @@ it.each([ function makeLocalCommandHarness(input: { readonly text: string; + readonly freshBootstrap?: string; readonly worktreePath?: string; readonly worktreesDir?: string; readonly workspaceRoot?: string; @@ -329,6 +330,9 @@ function makeLocalCommandHarness(input: { firstRunOrdinal: 2, lastRunOrdinal: 2, handoffIds: [], + ...(input.freshBootstrap === undefined + ? {} + : { freshContext: { bootstrap: input.freshBootstrap, throughRunOrdinal: 1 } }), forkedFrom: null, createdAt: now, updatedAt: now, @@ -1417,3 +1421,65 @@ for (const previousMessages of [[], ["/compact", " /COMPACT "]]) { }), ); } + +effectIt.effect( + "starts a renewed native thread from its seed without reading the old transcript", + () => + Effect.gen(function* () { + const seed = "Role: matt-console. Handoff: pending ask. Lessons: read the bounded role file."; + const harness = makeLocalCommandHarness({ text: "New ask", freshBootstrap: seed }); + const startTurn = vi.fn((_input: unknown) => Effect.void); + const old = harness.projection().providerThreads[0]!; + const ensureThread = vi.fn(() => + Effect.succeed({ + ...old, + nativeThreadRef: { + driver: old.driver, + nativeId: "fresh-native", + strength: "strong" as const, + }, + }), + ); + const resumeThread = vi.fn(() => Effect.die("Must not resume the previous native thread")); + harness.open.mockImplementation(() => + Effect.succeed({ + driver: old.driver, + providerSession: { + id: old.providerSessionId, + driver: old.driver, + providerInstanceId: old.providerInstanceId, + status: "ready", + cwd: "/tmp/native-account-command", + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: old.createdAt, + updatedAt: old.updatedAt, + lastError: null, + }, + ensureThread, + resumeThread, + startTurn, + } as never), + ); + harness.startRootRun.mockImplementation((input) => + input.session + .startTurn({ + threadId: input.appThread.id, + runId: input.run.id, + message: input.message, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + providerThread: input.providerThread, + } as never) + .pipe(Effect.orDie), + ); + yield* harness.start; + expect(ensureThread).toHaveBeenCalledOnce(); + expect(resumeThread).not.toHaveBeenCalled(); + expect(startTurn).toHaveBeenCalledOnce(); + expect(startTurn.mock.calls[0]?.[0]).toMatchObject({ + message: { text: `${seed}\n\nUser message:\nNew ask` }, + }); + expect(harness.projection().providerThreads[0]?.freshContext?.bootstrap).toBe(seed); + }), +); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 94ba406b106f..c344315a177d 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -793,6 +793,11 @@ export const layer: Layer.Layer< const tokenCap = yield* handoffTokenCapConfig.pipe( Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), ); + const freshBootstrapForThread = (loaded: OrchestrationV2ProviderThread) => + providerThread.freshContext !== undefined && + (run.ordinal === providerThread.firstRunOrdinal || + loaded.nativeThreadRef?.nativeId !== providerThread.nativeThreadRef?.nativeId) + ? providerThread.freshContext.bootstrap : ""; const handoffContextForThread = (loadedProviderThread: OrchestrationV2ProviderThread) => { const measuredContext = latestNativeContextUsage(projection, providerThread); const previousSelection = @@ -935,7 +940,7 @@ export const layer: Layer.Layer< return handoffBudget({ tokenCap, modelContextWindow, - userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, + userText: [freshBootstrapForThread(loadedProviderThread), restartNote, userText].filter(Boolean).join("\n\n"), attachments: message.attachments, providerThread: budgetProviderThread, nativeContextEstimate: @@ -1002,14 +1007,20 @@ export const layer: Layer.Layer< toProviderThreadId: providerThread.id, fromProviderInstanceId: providerThread.providerInstanceId, toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, + coveredRunOrdinals: { + from: (providerThread.freshContext?.throughRunOrdinal ?? 0) + 1, + to: Math.max(1, run.ordinal - 1), + }, strategy: "full_thread_summary", - runs: projection.runs, + runs: projection.runs.filter( + (source) => source.ordinal > (providerThread.freshContext?.throughRunOrdinal ?? 0), + ), items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter( (item) => - item.runId === null || + (item.runId === null && providerThread.freshContext === undefined) || projection.runs.some( - (source) => source.id === item.runId && source.ordinal < run.ordinal, + (source) => source.id === item.runId && source.ordinal < run.ordinal && + source.ordinal > (providerThread.freshContext?.throughRunOrdinal ?? 0), ), ), createdAt, @@ -1186,6 +1197,10 @@ export const layer: Layer.Layer< }), ); if (replacement === undefined) return undefined; + if ( + providerThread.freshContext !== undefined && + run.ordinal <= providerThread.freshContext.throughRunOrdinal + 1 + ) return replacement; const { handoff, transferId, createdAt } = yield* recoveryHandoff; effectiveHandoffs = withRecoveryHandoff(handoff); resumeFellBack = true; @@ -1255,6 +1270,7 @@ export const layer: Layer.Layer< providerSessionId, appThreadId: projection.thread.id, ownerNodeId: providerThread.ownerNodeId, + freshContext: providerThread.freshContext, firstRunOrdinal: providerThread.firstRunOrdinal ?? run.ordinal, lastRunOrdinal: run.ordinal, handoffIds: providerThread.handoffIds, @@ -1411,7 +1427,7 @@ export const layer: Layer.Layer< }); if (!(yield* isCurrentAttemptInStatus("running"))) return; const start = compact ? session.compactThread! : session.startTurn; - const context = [delivery.context, restartNote] + const context = [freshBootstrapForThread(runningProviderThread), delivery.context, restartNote] .filter((part) => part !== "") .join("\n\n"); const { restartContinuationOfRunId: _resumedRunId, ...promptedInput } = turnInput; @@ -1461,6 +1477,7 @@ export const layer: Layer.Layer< ), ); const deliverySession = + providerThread.freshContext === undefined && effectiveHandoffs.length === 0 && retryHandoff.length === 0 && restartNote === "" && diff --git a/apps/server/src/orchestration-v2/freshProviderContext.test.ts b/apps/server/src/orchestration-v2/freshProviderContext.test.ts new file mode 100644 index 000000000000..361ed0afdc04 --- /dev/null +++ b/apps/server/src/orchestration-v2/freshProviderContext.test.ts @@ -0,0 +1,200 @@ +import { assert, it } from "@effect/vitest"; +import { + CommandId, + EventId, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ProviderSessionId, + ProviderThreadId, + RunId, + ThreadId, + type OrchestrationV2DomainEvent, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { prepareFreshProviderThread } from "./freshProviderContext.ts"; + +const instanceId = ProviderInstanceId.make("codex"); +const adapter = { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }), + openSession: () => Effect.die("Renewal must not start a provider"), +} as ProviderAdapter.ProviderAdapterV2["Service"]; +const database = SqlitePersistence.layerMemory; +const layer = Layer.mergeAll( + database, + ProjectionStore.layer.pipe(Layer.provide(database)), + ProviderReplayHarness.layerWithRegistry( + { name: "fresh-context" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { databaseLayer: database, runEffectWorker: false }, + ), +); + +it.effect( + "renews native context while retaining visible identity, messages, runs and old refs; refuses unsafe twins", + () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("console"); + const sessionId = ProviderSessionId.make("console-session"); + const providerThreadId = ProviderThreadId.make("old-console-native"); + const now = yield* DateTime.now; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create"), + threadId, + projectId: ProjectId.make("project"), + title: "Work With Matt", + modelSelection: { instanceId, model: "gpt-5.1-codex" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + const apply = (type: OrchestrationV2DomainEvent["type"], payload: unknown) => + store.apply({ + id: EventId.make(`seed:${type}`), + type, + threadId, + occurredAt: now, + payload, + } as OrchestrationV2DomainEvent); + yield* apply("provider-session.attached", { + id: sessionId, + driver: adapter.driver, + providerInstanceId: instanceId, + status: "ready", + cwd: "/repo", + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }); + yield* apply("provider-thread.updated", { + id: providerThreadId, + driver: adapter.driver, + providerInstanceId: instanceId, + providerSessionId: sessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver: adapter.driver, nativeId: "keep-history", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }); + yield* apply("run.updated", { + id: RunId.make("old-run"), + threadId, + ordinal: 1, + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "gpt-5.1-codex" }, + providerThreadId, + userMessageId: MessageId.make("old-message"), + rootNodeId: null, + activeAttemptId: null, + status: "completed", + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }); + yield* apply("message.updated", { + id: MessageId.make("old-message"), + threadId, + runId: RunId.make("old-run"), + nodeId: null, + role: "user", + createdBy: "user", + creationSource: "web", + text: "Keep this full original ask in the visible history.", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }); + const before = yield* store.getThreadProjection(threadId); + const command = { + type: "provider-session.renew" as const, + commandId: CommandId.make("renew"), + threadId, + providerSessionId: sessionId, + freshContext: { + providerThreadId, + expectedUpdatedAt: before.thread.updatedAt, + bootstrap: "Role; complete handoff; lessons pointer.", + }, + }; + assert.equal( + typeof prepareFreshProviderThread(before, command, now), + "string", + "NO: less than 30 minutes", + ); + yield* TestClock.adjust("31 minutes"); + const later = yield* DateTime.now; + const refuse = (records: typeof before, cmd = command) => + assert.equal(typeof prepareFreshProviderThread(records, cmd, later), "string"); + refuse(before, { + ...command, + freshContext: { ...command.freshContext, bootstrap: "😀".repeat(1250) }, + }); + refuse(before, { ...command, freshContext: { ...command.freshContext, bootstrap: " " } }); + refuse(before, { + ...command, + freshContext: { ...command.freshContext, expectedUpdatedAt: later }, + }); + refuse(before, { ...command, providerSessionId: ProviderSessionId.make("wrong") }); + for (const status of ["queued", "starting", "running", "waiting"] as const) { + refuse({ ...before, runs: before.runs.map((row) => ({ ...row, status })) }); + } + refuse({ + ...before, + providerThreads: before.providerThreads.map((row) => ({ + ...row, + pendingBackgroundTasks: [{ taskId: "active-background", kind: "monitor" as const }], + })), + }); + yield* orchestrator.dispatch(command); + const after = yield* store.getThreadProjection(threadId); + assert.equal(after.thread.id, before.thread.id); + assert.equal(after.thread.title, before.thread.title); + assert.deepEqual(after.messages, before.messages); + assert.deepEqual(after.runs, before.runs); + assert.deepEqual(after.providerSessions, before.providerSessions); + // No delayed detach may interrupt a new turn after the renewal commits. + assert.deepEqual(after.providerTurns, before.providerTurns); + const old = after.providerThreads.find((row) => row.id === providerThreadId)!; + assert.equal(old.nativeThreadRef?.nativeId, "keep-history"); + assert.equal(old.status, "archived"); + const fresh = after.providerThreads.find( + (row) => row.id === after.thread.activeProviderThreadId, + )!; + assert.notEqual(fresh.id, providerThreadId); + assert.equal(fresh.nativeThreadRef, null); + assert.equal(fresh.freshContext?.bootstrap, command.freshContext.bootstrap); + refuse(after); // NO: repeated/stale renewal must not reset again. + }).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/orchestration-v2/freshProviderContext.ts b/apps/server/src/orchestration-v2/freshProviderContext.ts new file mode 100644 index 000000000000..86f9694863cf --- /dev/null +++ b/apps/server/src/orchestration-v2/freshProviderContext.ts @@ -0,0 +1,94 @@ +import { + ProviderThreadId, + isOrchestrationV2WorkActive, + type OrchestrationV2Command, + type OrchestrationV2ProviderThread, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import type { ProjectionRecords } from "./ProjectionStore.ts"; + +// Refuse before emitting anything. The command's timestamp is a compare-and-swap +// against the saved handoff's snapshot, including input queued since that snapshot. +export function prepareFreshProviderThread( + records: ProjectionRecords< + "providerThreads" | "runs" | "providerTurns" | "runtimeRequests" | "subagents" | "turnItems" + >, + command: Extract, + now: DateTime.Utc, +): string | { previous: OrchestrationV2ProviderThread; fresh: OrchestrationV2ProviderThread } { + const seed = command.freshContext; + if (seed === undefined) return "Fresh context requires an explicit seed."; + const previous = records.providerThreads.find((row) => row.id === seed.providerThreadId); + if ( + previous === undefined || + previous.id !== records.thread.activeProviderThreadId || + previous.appThreadId !== command.threadId || + previous.ownerNodeId !== null || + previous.providerSessionId !== command.providerSessionId || + previous.nativeThreadRef === null || + previous.status === "archived" || + previous.status === "closed" + ) { + return "Fresh context requires the current root provider thread and native session."; + } + if ( + DateTime.toEpochMillis(records.thread.updatedAt) !== + DateTime.toEpochMillis(seed.expectedUpdatedAt) + ) { + return "Thread changed since the handoff snapshot."; + } + if ( + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(records.thread.updatedAt) < + 30 * 60_000 + ) { + return "Thread has not been idle for 30 minutes."; + } + if (!seed.bootstrap.trim() || new TextEncoder().encode(seed.bootstrap).length >= 5_000) { + return "Bootstrap must contain fewer than 5000 UTF-8 bytes."; + } + const latest = records.runs.reduce<(typeof records.runs)[number] | undefined>( + (last, run) => (last === undefined || run.ordinal > last.ordinal ? run : last), + undefined, + ); + if ( + latest === undefined || + latest.status !== "completed" || + latest.completedAt === null || + latest.providerThreadId !== previous.id || + !Number.isSafeInteger(latest.ordinal + 1) || + records.runs.some( + (run) => + !["completed", "failed", "cancelled", "interrupted", "rolled_back"].includes(run.status), + ) || + records.providerTurns.some((turn) => turn.status === "pending" || turn.status === "running") || + records.runtimeRequests.some((request) => request.status === "pending") || + records.subagents.some((task) => isOrchestrationV2WorkActive(task.status)) || + records.turnItems.some((item) => isOrchestrationV2WorkActive(item.status)) || + records.providerThreads.some((row) => (row.pendingBackgroundTasks?.length ?? 0) > 0) + ) { + return "Thread has active, queued, waiting or uncompleted work."; + } + return { + previous: { ...previous, status: "archived", updatedAt: now }, + fresh: { + ...previous, + id: ProviderThreadId.make(`fresh:${command.commandId}`), + providerSessionId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "not_loaded", + // Non-null distinguishes this deliberate renewal from a queued placeholder. + firstRunOrdinal: latest.ordinal + 1, + lastRunOrdinal: latest.ordinal + 1, + handoffIds: [], + forkedFrom: null, + contextUsage: null, + nativeMetadata: null, + goal: null, + pendingBackgroundTasks: [], + freshContext: { bootstrap: seed.bootstrap, throughRunOrdinal: latest.ordinal }, + createdAt: now, + updatedAt: now, + }, + }; +} diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts index 77e8bae8f085..f77c228ceae7 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts @@ -159,6 +159,7 @@ function commandThreadIds(command: OrchestrationV2Command): ReadonlyArray Date: Sun, 1 Nov 2026 12:45:38 -0500 Subject: [PATCH 2/5] fix(server): retain pending renewal seed until accepted (#27890) --- .../src/orchestration-v2/ProjectionStore.ts | 31 ++- .../ProviderTurnStartService.test.ts | 233 +++++++++++++----- .../ProviderTurnStartService.ts | 57 ++++- .../freshProviderContext.test.ts | 55 +++++ .../orchestration-v2/freshProviderContext.ts | 20 ++ packages/contracts/src/orchestrationV2.ts | 1 + 6 files changed, 323 insertions(+), 74 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2a21bfae4bd3..dd61a47c8c33 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1,6 +1,7 @@ import { pendingGuestMessageCount } from "@t3tools/contracts"; import { applyThreadPullRequestLinkUpdate } from "../sourceControl/threadPullRequests.ts"; import { projectTurnItemForWire } from "./WireProjection.ts"; +import { preserveFreshContext } from "./freshProviderContext.ts"; import * as Stream from "effect/Stream"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import { makeThreadFind } from "./ThreadFind.ts"; @@ -855,7 +856,13 @@ export function applyToProjection( activeProviderThreadId: event.payload.id, } : base.thread, - providerThreads: upsertById(base.providerThreads, event.payload), + providerThreads: upsertById( + base.providerThreads, + preserveFreshContext( + base.providerThreads.find((thread) => thread.id === event.payload.id), + event.payload, + ), + ), }; case "provider-turn.updated": return { @@ -2190,12 +2197,22 @@ export const layer: Layer.Layer = WHERE provider_session_id = ${event.payload.providerSessionId} AND thread_id = ${event.threadId} `; - break; - } - case "provider-thread.updated": { - const payloadJson = yield* encodeProviderThreadPayload(event.payload); - const payload = parseEncodedPayload(payloadJson); - yield* sql` + break; + } + case "provider-thread.updated": { + const existing = yield* sql` + SELECT payload_json FROM orchestration_v2_projection_provider_threads + WHERE provider_thread_id = ${event.payload.id} + `; + const previous = + existing[0] === undefined + ? undefined + : yield* decodeProviderThreadPayload(existing[0].payload_json); + const payloadJson = yield* encodeProviderThreadPayload( + preserveFreshContext(previous, event.payload), + ); + const payload = parseEncodedPayload(payloadJson); + yield* sql` INSERT INTO orchestration_v2_projection_provider_threads ( provider_thread_id, thread_id, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 484d971561b9..07e77ad90464 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -275,6 +275,7 @@ function makeLocalCommandHarness(input: { readonly openFailure?: unknown; readonly ensureThreadFailure?: unknown; readonly historyReadFailureAfterFallback?: unknown; + readonly emptyRetryHistory?: boolean; readonly interruptOpen?: boolean; readonly interruptRunBeforeOpenFailure?: boolean; readonly writeFailure?: unknown; @@ -650,29 +651,30 @@ function makeLocalCommandHarness(input: { }), ), ); - const writeIfRunCurrent = vi.fn(({ events: incoming, activeAttemptId, expectedStatus }) => - "writeFailure" in input - ? Effect.fail( - new EventSink.EventSinkWriteError({ - eventCount: incoming.length, - cause: input.writeFailure, - }), - ) - : Effect.sync(() => { - const current = projection.runs.find((candidate) => candidate.id === runId); - const committed = - current !== undefined && - current.activeAttemptId === activeAttemptId && - current.status === expectedStatus; - if (committed) { - for (const event of incoming) { - expect(isDomainEvent(event)).toBe(true); - events.push(event); - projection = ProjectionStore.applyToProjection(projection, event); + const writeIfRunCurrent = vi.fn( + ({ events: incoming, activeAttemptId, expectedStatus, runId: currentRunId }) => + "writeFailure" in input + ? Effect.fail( + new EventSink.EventSinkWriteError({ + eventCount: incoming.length, + cause: input.writeFailure, + }), + ) + : Effect.sync(() => { + const current = projection.runs.find((candidate) => candidate.id === currentRunId); + const committed = + current !== undefined && + current.activeAttemptId === activeAttemptId && + current.status === expectedStatus; + if (committed) { + for (const event of incoming) { + expect(isDomainEvent(event)).toBe(true); + events.push(event); + projection = ProjectionStore.applyToProjection(projection, event); + } } - } - return { committed, storedEvents: [] }; - }), + return { committed, storedEvents: [] }; + }), ); const resolveRuntimePolicy = vi.fn( ({ thread }: Parameters[0]) => @@ -700,7 +702,20 @@ function makeLocalCommandHarness(input: { Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({ prepareProviderHandoff: () => Effect.die("history read must fail first"), }), - Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }), + Layer.mock(EventSink.EventSinkV2)({ + writeIfRunCurrent, + writeIfProviderThreadUnchanged: ({ expected, events: incoming }) => + Effect.sync(() => { + const current = projection.providerThreads.find((row) => row.id === expected.id); + if (current !== expected) return { committed: false, storedEvents: [] }; + for (const event of incoming) { + expect(isDomainEvent(event)).toBe(true); + events.push(event); + projection = ProjectionStore.applyToProjection(projection, event); + } + return { committed: true, storedEvents: [] }; + }), + }), IdAllocator.layer, layerWorktreeLocation(input.worktreesDir), input.fileSystem === undefined @@ -731,13 +746,15 @@ function makeLocalCommandHarness(input: { (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0), ), }), - getTurnStartHistory: () => - Effect.fail( - new ProjectionStore.ProjectionStoreReadError({ - threadId, - cause: input.historyReadFailureAfterFallback, - }), - ), + getTurnStartHistory: (_threadId, runIds) => + input.emptyRetryHistory && runIds !== undefined + ? Effect.succeed([]) + : Effect.fail( + new ProjectionStore.ProjectionStoreReadError({ + threadId, + cause: input.historyReadFailureAfterFallback, + }), + ), }), Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open, @@ -779,7 +796,10 @@ function makeLocalCommandHarness(input: { attemptId, projection: () => projection, start: Effect.gen(function* () { - yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ threadId, runId }); + yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ + threadId, + runId: projection.runs.at(-1)!.id, + }); }).pipe(Effect.provide(layer)), startWithRetry: Effect.gen(function* () { yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ @@ -1422,25 +1442,41 @@ for (const previousMessages of [[], ["/compact", " /COMPACT "]]) { ); } -effectIt.effect( - "starts a renewed native thread from its seed without reading the old transcript", - () => +for (const first of ["success", "rejected", "compact"] as const) { + effectIt.effect(`keeps bootstrap delivery durable after a first ${first} turn`, () => Effect.gen(function* () { - const seed = "Role: matt-console. Handoff: pending ask. Lessons: read the bounded role file."; - const harness = makeLocalCommandHarness({ text: "New ask", freshBootstrap: seed }); - const startTurn = vi.fn((_input: unknown) => Effect.void); + const seed = "Role: matt-console. Handoff: pending ask. Lessons: bounded role file."; + const harness = makeLocalCommandHarness({ + text: first === "compact" ? "/compact" : "New ask", + freshBootstrap: seed, + emptyRetryHistory: true, + previousMessages: ["Pre-renewal transcript must stay out of native context"], + }); + harness.tryHandlePromptCommand.mockImplementation(() => Effect.succeed(false)); const old = harness.projection().providerThreads[0]!; - const ensureThread = vi.fn(() => - Effect.succeed({ - ...old, - nativeThreadRef: { - driver: old.driver, - nativeId: "fresh-native", - strength: "strong" as const, - }, - }), + let nativeId = "fresh-native"; + const withNative = (thread: typeof old) => ({ + ...thread, + nativeThreadRef: { driver: old.driver, nativeId, strength: "strong" as const }, + }); + const startTurn = vi.fn((_input: unknown) => + first === "rejected" && startTurn.mock.calls.length === 1 + ? Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: old.driver, + threadId: old.appThreadId!, + providerThreadId: old.id, + runId: harness.projection().runs.at(-1)!.id, + cause: "Provider refused before accepting the prompt", + }), + ) + : Effect.void, + ); + const compactThread = vi.fn((_input: unknown) => Effect.void); + const ensureThread = vi.fn(() => Effect.succeed(withNative(old))); + const resumeThread = vi.fn(({ providerThread }: { providerThread: typeof old }) => + Effect.succeed(withNative(providerThread)), ); - const resumeThread = vi.fn(() => Effect.die("Must not resume the previous native thread")); harness.open.mockImplementation(() => Effect.succeed({ driver: old.driver, @@ -1459,27 +1495,96 @@ effectIt.effect( ensureThread, resumeThread, startTurn, + compactThread, } as never), ); - harness.startRootRun.mockImplementation((input) => - input.session - .startTurn({ - threadId: input.appThread.id, - runId: input.run.id, - message: input.message, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, - providerThread: input.providerThread, - } as never) - .pipe(Effect.orDie), - ); + harness.startRootRun.mockImplementation((input) => { + const start = + input.message.text === "/compact" + ? input.session.compactThread! + : input.session.startTurn; + return start({ + threadId: input.appThread.id, + runId: input.run.id, + message: input.message, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + providerThread: input.providerThread, + } as never).pipe(Effect.catch(() => Effect.void)); + }); + const next = (text: string) => { + const projection = harness.projection(); + const prior = projection.runs.at(-1)!; + const runId = RunId.make(`next:${prior.ordinal}`); + harness.setProjection({ + ...projection, + runs: [ + ...projection.runs.map((run) => ({ + ...run, + status: + first === "rejected" && run.ordinal === old.firstRunOrdinal + ? ("failed" as const) + : ("completed" as const), + })), + { ...prior, id: runId, ordinal: prior.ordinal + 1, status: "starting" }, + ], + attempts: projection.attempts.map((attempt) => ({ + ...attempt, + runId, + status: "pending", + })), + messages: projection.messages.map((message) => + message.id === prior.userMessageId ? { ...message, runId, text } : message, + ), + }); + }; yield* harness.start; expect(ensureThread).toHaveBeenCalledOnce(); expect(resumeThread).not.toHaveBeenCalled(); - expect(startTurn).toHaveBeenCalledOnce(); - expect(startTurn.mock.calls[0]?.[0]).toMatchObject({ - message: { text: `${seed}\n\nUser message:\nNew ask` }, + expect(harness.projection().providerThreads[0]?.freshContext?.acceptedNativeThreadId).toBe( + first === "success" ? nativeId : undefined, + ); + if (first === "compact") { + expect(compactThread).toHaveBeenCalledOnce(); + expect(compactThread.mock.calls[0]?.[0]).toMatchObject({ message: { text: "/compact" } }); + } else { + expect(startTurn.mock.calls[0]?.[0]).toMatchObject({ + message: { text: `${seed}\n\nUser message:\nNew ask` }, + }); + } + next("Next ordinary ask"); + yield* harness.start; + expect(resumeThread).toHaveBeenCalledOnce(); + expect(startTurn.mock.calls.at(-1)?.[0]).toMatchObject({ + message: { + text: + first === "success" + ? "Next ordinary ask" + : `${seed}\n\nUser message:\nNext ordinary ask`, + }, + }); + expect(harness.projection().providerThreads[0]?.freshContext?.acceptedNativeThreadId).toBe( + nativeId, + ); + next("After acceptance"); + yield* harness.start; + expect(startTurn.mock.calls.at(-1)?.[0]).toMatchObject({ + message: { text: "After acceptance" }, + }); + nativeId = "replacement-native"; + next("After native replacement"); + yield* harness.start; + expect(startTurn.mock.calls.at(-1)?.[0]).toMatchObject({ + message: { text: `${seed}\n\nUser message:\nAfter native replacement` }, }); - expect(harness.projection().providerThreads[0]?.freshContext?.bootstrap).toBe(seed); + expect(harness.projection().providerThreads[0]?.freshContext?.acceptedNativeThreadId).toBe( + nativeId, + ); + for (const [call] of startTurn.mock.calls) { + expect((call as { message: { text: string } }).message.text).not.toContain( + "Pre-renewal transcript", + ); + } }), -); + ); +} diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index c344315a177d..ab6544ecfbc9 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -795,8 +795,8 @@ export const layer: Layer.Layer< ); const freshBootstrapForThread = (loaded: OrchestrationV2ProviderThread) => providerThread.freshContext !== undefined && - (run.ordinal === providerThread.firstRunOrdinal || - loaded.nativeThreadRef?.nativeId !== providerThread.nativeThreadRef?.nativeId) + (providerThread.freshContext.acceptedNativeThreadId === undefined || + providerThread.freshContext.acceptedNativeThreadId !== loaded.nativeThreadRef?.nativeId) ? providerThread.freshContext.bootstrap : ""; const handoffContextForThread = (loadedProviderThread: OrchestrationV2ProviderThread) => { const measuredContext = latestNativeContextUsage(projection, providerThread); @@ -1391,6 +1391,7 @@ export const layer: Layer.Layer< ); const startWithHandoffs = Effect.fn("ProviderTurnStartService.startWithHandoffs")( function* (turnInput: Parameters[0], compact = false) { + const freshBootstrap = compact ? "" : freshBootstrapForThread(runningProviderThread); const delivery = yield* deliverContextHandoffs({ handoffs: [...effectiveHandoffs, ...retryHandoff], deferInline: compact || goalCommand, @@ -1427,7 +1428,7 @@ export const layer: Layer.Layer< }); if (!(yield* isCurrentAttemptInStatus("running"))) return; const start = compact ? session.compactThread! : session.startTurn; - const context = [freshBootstrapForThread(runningProviderThread), delivery.context, restartNote] + const context = [freshBootstrap, delivery.context, restartNote] .filter((part) => part !== "") .join("\n\n"); const { restartContinuationOfRunId: _resumedRunId, ...promptedInput } = turnInput; @@ -1454,6 +1455,56 @@ export const layer: Layer.Layer< : Effect.void, ), ); + if (freshBootstrap !== "") { + yield* Effect.gen(function* () { + const nativeId = runningProviderThread.nativeThreadRef?.nativeId; + if (nativeId === undefined || nativeId === null) return; + for (let retry = 0; retry < 3; retry += 1) { + const records = yield* projectionStore.getThreadRecords(input.threadId, [ + "providerThreads", + ]); + const current = records.providerThreads.find((row) => row.id === providerThread.id); + if ( + current?.nativeThreadRef?.nativeId !== nativeId || + current.freshContext?.bootstrap !== freshBootstrap || + current.freshContext.throughRunOrdinal !== + providerThread.freshContext?.throughRunOrdinal || + current.freshContext.acceptedNativeThreadId === nativeId + ) + return; + const acceptedAt = yield* DateTime.now; + const result = yield* eventSink.writeIfProviderThreadUnchanged({ + expected: current, + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: input.threadId }), + type: "provider-thread.updated", + threadId: input.threadId, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: acceptedAt, + payload: { + ...current, + freshContext: { ...current.freshContext, acceptedNativeThreadId: nativeId }, + updatedAt: acceptedAt, + }, + }, + ], + }); + if (result.committed) return; + } + yield* Effect.logWarning("Bootstrap receipt lost concurrent update races", { + runId: run.id, + }); + }).pipe( + Effect.catchCause(() => + Effect.logWarning("Failed to record accepted bootstrap", { + runId: run.id, + deliveryStatus: "pending", + }), + ), + ); + } // After provider acceptance, failed delivery bookkeeping must not stop live ingestion. yield* delivery.delivered.pipe( Effect.catchCause(() => diff --git a/apps/server/src/orchestration-v2/freshProviderContext.test.ts b/apps/server/src/orchestration-v2/freshProviderContext.test.ts index 361ed0afdc04..4ab7224f0951 100644 --- a/apps/server/src/orchestration-v2/freshProviderContext.test.ts +++ b/apps/server/src/orchestration-v2/freshProviderContext.test.ts @@ -196,5 +196,60 @@ it.effect( assert.equal(fresh.nativeThreadRef, null); assert.equal(fresh.freshContext?.bootstrap, command.freshContext.bootstrap); refuse(after); // NO: repeated/stale renewal must not reset again. + + // Both reducers must keep a durable receipt when adapters/finalization + // write snapshots captured before successful bootstrap delivery. + let memory = after; + const update = (payload: typeof fresh) => + Effect.gen(function* () { + const event = { + id: EventId.make(`receipt:${payload.nativeThreadRef?.nativeId}:${payload.updatedAt}`), + type: "provider-thread.updated" as const, + threadId, + occurredAt: later, + payload, + }; + memory = ProjectionStore.applyToProjection(memory, event); + yield* store.apply(event); + const persisted = yield* store.getThreadProjection(threadId); + assert.deepEqual( + persisted.providerThreads.find((row) => row.id === fresh.id)?.freshContext, + memory.providerThreads.find((row) => row.id === fresh.id)?.freshContext, + ); + return persisted.providerThreads.find((row) => row.id === fresh.id)!; + }); + const pending = { + ...fresh, + nativeThreadRef: { + driver: adapter.driver, + nativeId: "seed-native", + strength: "strong" as const, + }, + }; + yield* update({ + ...pending, + freshContext: { ...fresh.freshContext!, acceptedNativeThreadId: "seed-native" }, + }); + for (const stale of [pending, { ...pending, freshContext: undefined }]) { + const kept = yield* update(stale); + assert.equal(kept.freshContext?.acceptedNativeThreadId, "seed-native"); + } + const replacement = { + ...pending, + nativeThreadRef: { ...pending.nativeThreadRef, nativeId: "replacement" }, + }; + const unaccepted = yield* update(replacement); + assert.notEqual(unaccepted.freshContext?.acceptedNativeThreadId, "replacement"); + yield* update({ + ...replacement, + freshContext: { ...fresh.freshContext!, acceptedNativeThreadId: "replacement" }, + }); + const retained = yield* update(replacement); + assert.equal(retained.freshContext?.acceptedNativeThreadId, "replacement"); + const renewed = yield* update({ + ...replacement, + freshContext: { bootstrap: "New generation", throughRunOrdinal: 2 }, + }); + assert.equal(renewed.freshContext?.acceptedNativeThreadId, undefined); }).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/orchestration-v2/freshProviderContext.ts b/apps/server/src/orchestration-v2/freshProviderContext.ts index 86f9694863cf..779f815143f0 100644 --- a/apps/server/src/orchestration-v2/freshProviderContext.ts +++ b/apps/server/src/orchestration-v2/freshProviderContext.ts @@ -7,6 +7,26 @@ import { import * as DateTime from "effect/DateTime"; import type { ProjectionRecords } from "./ProjectionStore.ts"; +// Adapter snapshots and finalization can predate acceptance of the bootstrap. +// Only a new renewal generation or native context invalidates that receipt. +export function preserveFreshContext( + previous: OrchestrationV2ProviderThread | undefined, + incoming: OrchestrationV2ProviderThread, +): OrchestrationV2ProviderThread { + const saved = previous?.freshContext; + if (saved === undefined) return incoming; + const freshContext = incoming.freshContext ?? saved; + if ( + freshContext.bootstrap === saved.bootstrap && + freshContext.throughRunOrdinal === saved.throughRunOrdinal && + saved.acceptedNativeThreadId !== undefined && + saved.acceptedNativeThreadId === incoming.nativeThreadRef?.nativeId + ) { + return { ...incoming, freshContext: saved }; + } + return { ...incoming, freshContext }; +} + // Refuse before emitting anything. The command's timestamp is a compare-and-swap // against the saved handoff's snapshot, including input queued since that snapshot. export function prepareFreshProviderThread( diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 75c4e7f44358..b0052d5001a2 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1068,6 +1068,7 @@ export const OrchestrationV2ProviderThread = Schema.Struct({ Schema.Struct({ bootstrap: Schema.String, throughRunOrdinal: PositiveInt, + acceptedNativeThreadId: Schema.optional(Schema.String), }), ), pendingBackgroundTasks: Schema.optional(Schema.Array(OrchestrationV2PendingBackgroundTask)).pipe( From 43ac11daa5c7d004078779d3ab3e29dad4008026 Mon Sep 17 00:00:00 2001 From: Matthew Riddell Date: Sun, 1 Nov 2026 21:32:19 -0500 Subject: [PATCH 3/5] fix(server): persist renewal acceptance transactionally (#27890) --- apps/server/src/orchestration-v2/EventSink.ts | 58 +++++++++++ .../FoundationPersistence.test.ts | 96 +++++++++++++++++++ .../ProviderTurnStartService.test.ts | 39 +++++++- .../ProviderTurnStartService.ts | 54 ++++------- 4 files changed, 206 insertions(+), 41 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index b210d3c9840a..164eee6a12da 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -67,6 +67,15 @@ export const EventSinkV2Error = Schema.Union([EventSinkWriteError, EventSinkStre export type EventSinkV2Error = typeof EventSinkV2Error.Type; export interface EventSinkV2Shape { + readonly acceptFreshContext: ( + event: Extract, + ) => Effect.Effect< + { + readonly committed: boolean; + readonly storedEvents: ReadonlyArray; + }, + EventSinkV2Error + >; readonly write: (input: { /** Return no events if another thread owns this native session at commit time. */ readonly unlessNativeThreadOwned?: Pick< @@ -616,6 +625,51 @@ const layerBase: Layer.Layer< ); }); + const acceptFreshContextEffect = Effect.fn("orchestrationV2.EventSink.acceptFreshContext")( + function* (event: Parameters[0]) { + return yield* commitThenPublish( + Effect.gen(function* () { + const records = yield* projectionStore.getThreadRecords(event.threadId, [ + "providerThreads", + ]); + const current = records.providerThreads.find((row) => row.id === event.payload.id); + const accepted = event.payload.freshContext; + const nativeId = event.payload.nativeThreadRef?.nativeId; + if ( + nativeId == null || + accepted?.acceptedNativeThreadId !== nativeId || + current?.nativeThreadRef?.nativeId !== nativeId || + current.freshContext?.bootstrap !== accepted.bootstrap || + current.freshContext.throughRunOrdinal !== accepted.throughRunOrdinal || + current.freshContext.acceptedNativeThreadId === nativeId + ) { + return { + committed: false as const, + storedEvents: [] as ReadonlyArray, + }; + } + // Read and merge under the transaction so ingestion cannot invalidate + // acceptance by changing unrelated provider-thread fields. + const storedEvents = yield* eventStore.append({ + events: [ + { + ...event, + payload: { + ...current, + freshContext: { ...current.freshContext, acceptedNativeThreadId: nativeId }, + updatedAt: event.occurredAt, + }, + }, + ], + }); + const outcomes = yield* applyStoredEvents(storedEvents); + return { committed: true as const, storedEvents, outcomes }; + }), + (result) => (result.committed ? publishStoredEvents(result.storedEvents, result.outcomes) : Effect.void), + ); + }, + ); + const existingCommandResult = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (commandId: CommandId) { @@ -922,6 +976,10 @@ const layerBase: Layer.Layer< }), ), ), + acceptFreshContext: (event) => + acceptFreshContextEffect(event).pipe( + Effect.mapError((cause) => new EventSinkWriteError({ eventCount: 1, cause })), + ), writeIfProviderThreadUnchanged: (input) => writeIfProviderThreadUnchangedEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 770da1939848..9f3fe0405cb8 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -14,6 +14,7 @@ import { type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2Run, + type OrchestrationV2ProviderThread, type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, ProjectId, @@ -147,6 +148,101 @@ function threadCreatedEvent(input: { }; } +it.effect("persists fresh acceptance despite three provider-thread receipt collisions", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:fresh-acceptance-collisions"); + const providerThread: OrchestrationV2ProviderThread = { + id: ProviderThreadId.make("provider-thread:fresh-acceptance-collisions"), + driver: providerDriver, + providerInstanceId, + providerSessionId: null, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver: providerDriver, nativeId: "renewed-native", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 2, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + freshContext: { bootstrap: "role and lessons", throughRunOrdinal: 1 }, + createdAt: now, + updatedAt: now, + }; + const update = (id: string, payload: OrchestrationV2ProviderThread) => ({ + id: EventId.make(id), + type: "provider-thread.updated" as const, + threadId, + occurredAt: now, + payload, + }); + yield* sink.write({ + events: [ + threadCreatedEvent({ + id: "event:fresh-acceptance:thread", + thread: makeThread(threadId, now), + now, + }), + update("event:fresh-acceptance:initial", providerThread), + ], + }); + const receipt = update("event:fresh-acceptance:receipt", { + ...providerThread, + freshContext: { ...providerThread.freshContext!, acceptedNativeThreadId: "renewed-native" }, + }); + for (let collision = 0; collision < 3; collision += 1) { + const snapshot = (yield* store.getThreadRecords(threadId, ["providerThreads"])) + .providerThreads[0]!; + yield* sink.write({ + events: [ + update(`event:fresh-acceptance:roster-${collision}`, { + ...snapshot, + pendingBackgroundTasks: [ + { taskId: `roster-${collision}`, description: "ingestion update", kind: "command" }, + ], + }), + ], + }); + const staleReceipt = yield* sink.writeIfProviderThreadUnchanged({ + expected: snapshot, + events: [receipt], + }); + assert.isFalse(staleReceipt.committed); + } + const accepted = yield* sink.acceptFreshContext(receipt); + assert.isTrue(accepted.committed); + const durable = (yield* store.getThreadRecords(threadId, ["providerThreads"])) + .providerThreads[0]!; + assert.equal(durable.freshContext?.acceptedNativeThreadId, "renewed-native"); + assert.equal(durable.pendingBackgroundTasks?.[0]?.taskId, "roster-2"); + assert.isFalse((yield* sink.acceptFreshContext(receipt)).committed); + + for (const replacement of [ + { + ...providerThread, + nativeThreadRef: { ...providerThread.nativeThreadRef!, nativeId: "replacement-native" }, + }, + { ...providerThread, freshContext: { bootstrap: "new generation", throughRunOrdinal: 2 } }, + ]) { + yield* sink.write({ + events: [ + update( + `event:fresh-acceptance:replacement-${replacement.freshContext?.throughRunOrdinal}`, + replacement, + ), + ], + }); + assert.isFalse((yield* sink.acceptFreshContext(receipt)).committed); + const current = (yield* store.getThreadRecords(threadId, ["providerThreads"])) + .providerThreads[0]!; + assert.isUndefined(current.freshContext?.acceptedNativeThreadId); + } + }).pipe(Effect.provide(layerTest)), +); + it.effect("rebuilds event history one bounded page at a time", () => Effect.gen(function* () { const eventStore = yield* EventStore.EventStoreV2; diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 07e77ad90464..5a7bcafc47de 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -704,6 +704,19 @@ function makeLocalCommandHarness(input: { }), Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent, + acceptFreshContext: (event) => + Effect.sync(() => { + const current = projection.providerThreads.find( + (row) => row.id === event.payload.id, + )!; + const merged = { + ...event, + payload: { ...current, freshContext: event.payload.freshContext }, + }; + events.push(merged); + projection = ProjectionStore.applyToProjection(projection, merged); + return { committed: true, storedEvents: [] }; + }), writeIfProviderThreadUnchanged: ({ expected, events: incoming }) => Effect.sync(() => { const current = projection.providerThreads.find((row) => row.id === expected.id); @@ -1442,7 +1455,7 @@ for (const previousMessages of [[], ["/compact", " /COMPACT "]]) { ); } -for (const first of ["success", "rejected", "compact"] as const) { +for (const first of ["success", "concurrent-updates", "rejected", "compact"] as const) { effectIt.effect(`keeps bootstrap delivery durable after a first ${first} turn`, () => Effect.gen(function* () { const seed = "Role: matt-console. Handoff: pending ask. Lessons: bounded role file."; @@ -1470,7 +1483,25 @@ for (const first of ["success", "rejected", "compact"] as const) { cause: "Provider refused before accepting the prompt", }), ) - : Effect.void, + : Effect.sync(() => { + if (first !== "concurrent-updates") return; + for (let update = 0; update < 3; update += 1) { + const current = harness.projection(); + harness.setProjection({ + ...current, + providerThreads: current.providerThreads.map((row) => ({ + ...row, + pendingBackgroundTasks: [ + { + taskId: `roster-${update}`, + description: "ingestion update", + kind: "command" as const, + }, + ], + })), + }); + } + }), ); const compactThread = vi.fn((_input: unknown) => Effect.void); const ensureThread = vi.fn(() => Effect.succeed(withNative(old))); @@ -1542,7 +1573,7 @@ for (const first of ["success", "rejected", "compact"] as const) { expect(ensureThread).toHaveBeenCalledOnce(); expect(resumeThread).not.toHaveBeenCalled(); expect(harness.projection().providerThreads[0]?.freshContext?.acceptedNativeThreadId).toBe( - first === "success" ? nativeId : undefined, + first === "success" || first === "concurrent-updates" ? nativeId : undefined, ); if (first === "compact") { expect(compactThread).toHaveBeenCalledOnce(); @@ -1558,7 +1589,7 @@ for (const first of ["success", "rejected", "compact"] as const) { expect(startTurn.mock.calls.at(-1)?.[0]).toMatchObject({ message: { text: - first === "success" + first === "success" || first === "concurrent-updates" ? "Next ordinary ask" : `${seed}\n\nUser message:\nNext ordinary ask`, }, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index ab6544ecfbc9..4631416c9fa2 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1391,7 +1391,7 @@ export const layer: Layer.Layer< ); const startWithHandoffs = Effect.fn("ProviderTurnStartService.startWithHandoffs")( function* (turnInput: Parameters[0], compact = false) { - const freshBootstrap = compact ? "" : freshBootstrapForThread(runningProviderThread); + const freshBootstrap = compact || goalCommand ? "" : freshBootstrapForThread(runningProviderThread); const delivery = yield* deliverContextHandoffs({ handoffs: [...effectiveHandoffs, ...retryHandoff], deferInline: compact || goalCommand, @@ -1458,43 +1458,23 @@ export const layer: Layer.Layer< if (freshBootstrap !== "") { yield* Effect.gen(function* () { const nativeId = runningProviderThread.nativeThreadRef?.nativeId; - if (nativeId === undefined || nativeId === null) return; - for (let retry = 0; retry < 3; retry += 1) { - const records = yield* projectionStore.getThreadRecords(input.threadId, [ - "providerThreads", - ]); - const current = records.providerThreads.find((row) => row.id === providerThread.id); - if ( - current?.nativeThreadRef?.nativeId !== nativeId || - current.freshContext?.bootstrap !== freshBootstrap || - current.freshContext.throughRunOrdinal !== - providerThread.freshContext?.throughRunOrdinal || - current.freshContext.acceptedNativeThreadId === nativeId - ) - return; - const acceptedAt = yield* DateTime.now; - const result = yield* eventSink.writeIfProviderThreadUnchanged({ - expected: current, - events: [ - { - id: yield* idAllocator.allocate.event({ threadId: input.threadId }), - type: "provider-thread.updated", - threadId: input.threadId, - runId: run.id, - providerInstanceId: run.providerInstanceId, - occurredAt: acceptedAt, - payload: { - ...current, - freshContext: { ...current.freshContext, acceptedNativeThreadId: nativeId }, - updatedAt: acceptedAt, - }, - }, - ], - }); - if (result.committed) return; - } - yield* Effect.logWarning("Bootstrap receipt lost concurrent update races", { + if (nativeId == null || runningProviderThread.freshContext === undefined) return; + const acceptedAt = yield* DateTime.now; + yield* eventSink.acceptFreshContext({ + id: yield* idAllocator.allocate.event({ threadId: input.threadId }), + type: "provider-thread.updated", + threadId: input.threadId, runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: acceptedAt, + payload: { + ...runningProviderThread, + freshContext: { + ...runningProviderThread.freshContext, + acceptedNativeThreadId: nativeId, + }, + updatedAt: acceptedAt, + }, }); }).pipe( Effect.catchCause(() => From 50f702e043ed689b962fc1b9da07a830b197527f Mon Sep 17 00:00:00 2001 From: Matthew Riddell Date: Mon, 2 Nov 2026 02:42:42 -0500 Subject: [PATCH 4/5] fix(server): preserve mainline recovery and defer renewal seed for goals (#27890) --- apps/server/src/orchestration-v2/EventSink.ts | 5 +++- .../src/orchestration-v2/ProjectionStore.ts | 26 +++++++++---------- .../ProviderTurnStartService.test.ts | 11 +++++--- .../ProviderTurnStartService.ts | 17 ++++++++---- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 164eee6a12da..e5940ee10a28 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -665,7 +665,10 @@ const layerBase: Layer.Layer< const outcomes = yield* applyStoredEvents(storedEvents); return { committed: true as const, storedEvents, outcomes }; }), - (result) => (result.committed ? publishStoredEvents(result.storedEvents, result.outcomes) : Effect.void), + (result) => + result.committed + ? publishStoredEvents(result.storedEvents, result.outcomes) + : Effect.void, ); }, ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index dd61a47c8c33..b8f716685525 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -2197,22 +2197,22 @@ export const layer: Layer.Layer = WHERE provider_session_id = ${event.payload.providerSessionId} AND thread_id = ${event.threadId} `; - break; - } - case "provider-thread.updated": { - const existing = yield* sql` + break; + } + case "provider-thread.updated": { + const existing = yield* sql` SELECT payload_json FROM orchestration_v2_projection_provider_threads WHERE provider_thread_id = ${event.payload.id} `; - const previous = - existing[0] === undefined - ? undefined - : yield* decodeProviderThreadPayload(existing[0].payload_json); - const payloadJson = yield* encodeProviderThreadPayload( - preserveFreshContext(previous, event.payload), - ); - const payload = parseEncodedPayload(payloadJson); - yield* sql` + const previous = + existing[0] === undefined + ? undefined + : yield* decodeProviderThreadPayload(existing[0].payload_json); + const payloadJson = yield* encodeProviderThreadPayload( + preserveFreshContext(previous, event.payload), + ); + const payload = parseEncodedPayload(payloadJson); + yield* sql` INSERT INTO orchestration_v2_projection_provider_threads ( provider_thread_id, thread_id, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 5a7bcafc47de..ce565963e1ff 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -1455,12 +1455,13 @@ for (const previousMessages of [[], ["/compact", " /COMPACT "]]) { ); } -for (const first of ["success", "concurrent-updates", "rejected", "compact"] as const) { +for (const first of ["success", "concurrent-updates", "rejected", "compact", "goal"] as const) { effectIt.effect(`keeps bootstrap delivery durable after a first ${first} turn`, () => Effect.gen(function* () { const seed = "Role: matt-console. Handoff: pending ask. Lessons: bounded role file."; const harness = makeLocalCommandHarness({ - text: first === "compact" ? "/compact" : "New ask", + text: + first === "compact" ? "/compact" : first === "goal" ? "/goal Keep working" : "New ask", freshBootstrap: seed, emptyRetryHistory: true, previousMessages: ["Pre-renewal transcript must stay out of native context"], @@ -1541,7 +1542,7 @@ for (const first of ["success", "concurrent-updates", "rejected", "compact"] as modelSelection: input.modelSelection, runtimePolicy: input.runtimePolicy, providerThread: input.providerThread, - } as never).pipe(Effect.catch(() => Effect.void)); + } as never).pipe(Effect.ignore); }); const next = (text: string) => { const projection = harness.projection(); @@ -1578,6 +1579,10 @@ for (const first of ["success", "concurrent-updates", "rejected", "compact"] as if (first === "compact") { expect(compactThread).toHaveBeenCalledOnce(); expect(compactThread.mock.calls[0]?.[0]).toMatchObject({ message: { text: "/compact" } }); + } else if (first === "goal") { + expect(startTurn.mock.calls[0]?.[0]).toMatchObject({ + message: { text: "/goal Keep working" }, + }); } else { expect(startTurn.mock.calls[0]?.[0]).toMatchObject({ message: { text: `${seed}\n\nUser message:\nNew ask` }, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 4631416c9fa2..fb631c3a5545 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -797,7 +797,8 @@ export const layer: Layer.Layer< providerThread.freshContext !== undefined && (providerThread.freshContext.acceptedNativeThreadId === undefined || providerThread.freshContext.acceptedNativeThreadId !== loaded.nativeThreadRef?.nativeId) - ? providerThread.freshContext.bootstrap : ""; + ? providerThread.freshContext.bootstrap + : ""; const handoffContextForThread = (loadedProviderThread: OrchestrationV2ProviderThread) => { const measuredContext = latestNativeContextUsage(projection, providerThread); const previousSelection = @@ -940,7 +941,9 @@ export const layer: Layer.Layer< return handoffBudget({ tokenCap, modelContextWindow, - userText: [freshBootstrapForThread(loadedProviderThread), restartNote, userText].filter(Boolean).join("\n\n"), + userText: [freshBootstrapForThread(loadedProviderThread), restartNote, userText] + .filter(Boolean) + .join("\n\n"), attachments: message.attachments, providerThread: budgetProviderThread, nativeContextEstimate: @@ -1019,7 +1022,9 @@ export const layer: Layer.Layer< (item) => (item.runId === null && providerThread.freshContext === undefined) || projection.runs.some( - (source) => source.id === item.runId && source.ordinal < run.ordinal && + (source) => + source.id === item.runId && + source.ordinal < run.ordinal && source.ordinal > (providerThread.freshContext?.throughRunOrdinal ?? 0), ), ), @@ -1200,7 +1205,8 @@ export const layer: Layer.Layer< if ( providerThread.freshContext !== undefined && run.ordinal <= providerThread.freshContext.throughRunOrdinal + 1 - ) return replacement; + ) + return replacement; const { handoff, transferId, createdAt } = yield* recoveryHandoff; effectiveHandoffs = withRecoveryHandoff(handoff); resumeFellBack = true; @@ -1391,7 +1397,8 @@ export const layer: Layer.Layer< ); const startWithHandoffs = Effect.fn("ProviderTurnStartService.startWithHandoffs")( function* (turnInput: Parameters[0], compact = false) { - const freshBootstrap = compact || goalCommand ? "" : freshBootstrapForThread(runningProviderThread); + const freshBootstrap = + compact || goalCommand ? "" : freshBootstrapForThread(runningProviderThread); const delivery = yield* deliverContextHandoffs({ handoffs: [...effectiveHandoffs, ...retryHandoff], deferInline: compact || goalCommand, From d21b7f0517b902d49ead09095ccfa681dfae9526 Mon Sep 17 00:00:00 2001 From: Matthew Riddell Date: Mon, 2 Nov 2026 06:41:37 -0500 Subject: [PATCH 5/5] fix(server): validate renewal inside the event transaction (#27890) --- apps/server/src/orchestration-v2/EventSink.ts | 58 +++++++++- .../src/orchestration-v2/Orchestrator.ts | 1 + .../freshProviderContext.test.ts | 103 +++++++++++++++++- 3 files changed, 157 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index e5940ee10a28..791705261819 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -1,5 +1,6 @@ import { CommandId, + type OrchestrationV2Command, type OrchestrationV2Run, OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, @@ -34,6 +35,7 @@ import * as EventStore from "./EventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +import { prepareFreshProviderThread } from "./freshProviderContext.ts"; export class EventSinkWriteError extends Schema.TaggedError()( "EventSinkWriteError", @@ -151,6 +153,11 @@ export interface EventSinkV2Shape { readonly acceptedAt: DateTime.Utc; readonly events: ReadonlyArray; readonly effects: ReadonlyArray; + /** Recheck renewal safety against ingestion in the same transaction as its events. */ + readonly freshProviderRenewal?: Extract< + OrchestrationV2Command, + { type: "provider-session.renew" } + >; readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; @@ -708,11 +715,58 @@ const layerBase: Layer.Layer< return { ...existing, committed: false as const, cancelledEffectIds: [] }; } + let events = input.events; + if (input.freshProviderRenewal !== undefined) { + const records = yield* projectionStore.getThreadRecords(input.threadId, [ + "providerThreads", + "runs", + "providerTurns", + "runtimeRequests", + "subagents", + "turnItems", + ]); + const prepared = prepareFreshProviderThread( + records, + input.freshProviderRenewal, + input.acceptedAt, + ); + if (typeof prepared === "string") { + const receipt: CommandReceiptStore.CommandReceiptV2 = { + commandId: input.commandId, + threadId: input.threadId, + commandType: input.commandType, + acceptedAt: input.acceptedAt, + resultSequence: yield* eventStore.latestSequence({ threadId: input.threadId }), + status: "rejected", + error: prepared, + }; + yield* commandReceipts.upsert(receipt); + return { + receipt, + storedEvents: [] as ReadonlyArray, + committed: false as const, + cancelledEffectIds: [], + }; + } + // Ingestion does not take the command lock. Rebuild from the current + // row so even an unrelated adapter update keeps all of its fields. + events = events.map((event) => + event.type !== "provider-thread.updated" + ? event + : { + ...event, + payload: + event.payload.id === prepared.previous.id + ? prepared.previous + : prepared.fresh, + }, + ); + } const runId = input.updateDelegatedCompletionOnlyRunId; const normalized = yield* normalizeEvents( runId === undefined - ? input.events - : yield* updateDelegatedCompletionOnly(input.events, (id) => id === runId), + ? events + : yield* updateDelegatedCompletionOnly(events, (id) => id === runId), ); const storedEvents = yield* eventStore.append({ commandId: input.commandId, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index a80387e88e05..b7d8be3ca03a 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -13018,6 +13018,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio acceptedAt, events: plan.events, effects: plan.effects, + ...(command.type === "provider-session.renew" ? { freshProviderRenewal: command } : {}), ...(plan.cancelUnsettledEffects === undefined ? {} : { cancelUnsettledEffects: plan.cancelUnsettledEffects }), diff --git a/apps/server/src/orchestration-v2/freshProviderContext.test.ts b/apps/server/src/orchestration-v2/freshProviderContext.test.ts index 4ab7224f0951..82cc8566a2ec 100644 --- a/apps/server/src/orchestration-v2/freshProviderContext.test.ts +++ b/apps/server/src/orchestration-v2/freshProviderContext.test.ts @@ -3,21 +3,28 @@ import { CommandId, EventId, MessageId, + NodeId, ProjectId, ProviderDriverKind, ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, RunId, ThreadId, type OrchestrationV2DomainEvent, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as TestClock from "effect/testing/TestClock"; +import { vi } from "vite-plus/test"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; @@ -44,9 +51,9 @@ const layer = Layer.mergeAll( ), ); -it.effect( - "renews native context while retaining visible identity, messages, runs and old refs; refuses unsafe twins", - () => +it.effect.each(["none", "background", "turn", "metadata"] as const)( + "renews native context and preserves concurrent ingestion; refuses unsafe twins (%s)", + (race) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const store = yield* ProjectionStore.ProjectionStoreV2; @@ -177,6 +184,96 @@ it.effect( pendingBackgroundTasks: [{ taskId: "active-background", kind: "monitor" as const }], })), }); + if (race !== "none") { + const sink = yield* EventSink.EventSinkV2; + const commit = sink.commitCommand; + const planned = yield* Deferred.make(); + const resume = yield* Deferred.make(); + const previous = before.providerThreads.find((row) => row.id === providerThreadId)!; + const ingested: OrchestrationV2DomainEvent = + race === "turn" + ? { + id: EventId.make("concurrent-native-turn"), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { + id: ProviderTurnId.make("concurrent-turn"), + providerThreadId, + nodeId: NodeId.make("concurrent-node"), + runAttemptId: null, + nativeTurnRef: { + driver: adapter.driver, + nativeId: "concurrent-native-turn", + strength: "strong", + }, + ordinal: 1, + status: "running", + startedAt: later, + completedAt: null, + }, + } + : { + id: EventId.make(`concurrent-${race}`), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: { + ...previous, + // Adapter updates need not advance the row/thread timestamp. + ...(race === "background" + ? { pendingBackgroundTasks: [{ taskId: "concurrent-task", kind: "monitor" }] } + : { nativeMetadata: { title: "Retain this adapter update" } }), + }, + }; + yield* Effect.acquireUseRelease( + Effect.sync(() => + vi + .spyOn(sink, "commitCommand") + .mockImplementationOnce((input) => + Deferred.succeed(planned, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + Effect.andThen(commit(input)), + ), + ), + ), + () => + Effect.gen(function* () { + const renewal = yield* orchestrator + .dispatch(command) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(planned); + // This is the transactional ingestion writer, without the command + // lock. A delayed event may retain the old snapshot timestamp. + yield* sink.write({ events: [ingested] }); + const concurrent = yield* store.getThreadProjection(threadId); + yield* Deferred.succeed(resume, undefined); + const result = yield* Fiber.join(renewal); + const afterRace = yield* store.getThreadProjection(threadId); + if (race === "metadata") { + assert.isTrue(Exit.isSuccess(result)); + assert.deepEqual( + afterRace.providerThreads.find((row) => row.id === providerThreadId) + ?.nativeMetadata, + { title: "Retain this adapter update" }, + ); + assert.notEqual(afterRace.thread.activeProviderThreadId, providerThreadId); + } else { + assert.isTrue(Exit.isFailure(result), "NO: work arrived after renewal planning"); + assert.deepEqual(afterRace, concurrent, "NO: refusal must preserve ingestion"); + assert.isTrue( + Exit.isFailure(yield* orchestrator.dispatch(command).pipe(Effect.exit)), + ); + assert.deepEqual(yield* store.getThreadProjection(threadId), concurrent); + } + }), + (spy) => + Deferred.succeed(resume, undefined).pipe( + Effect.andThen(Effect.sync(() => spy.mockRestore())), + ), + ); + return; + } yield* orchestrator.dispatch(command); const after = yield* store.getThreadProjection(threadId); assert.equal(after.thread.id, before.thread.id);