diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index b210d3c9840a..791705261819 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -1,5 +1,6 @@ import { CommandId, + type OrchestrationV2Command, type OrchestrationV2Run, OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, @@ -34,6 +35,7 @@ import * as EventStore from "./EventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +import { prepareFreshProviderThread } from "./freshProviderContext.ts"; export class EventSinkWriteError extends Schema.TaggedError()( "EventSinkWriteError", @@ -67,6 +69,15 @@ export const EventSinkV2Error = Schema.Union([EventSinkWriteError, EventSinkStre export type EventSinkV2Error = typeof EventSinkV2Error.Type; export interface EventSinkV2Shape { + readonly acceptFreshContext: ( + event: Extract, + ) => Effect.Effect< + { + readonly committed: boolean; + readonly storedEvents: ReadonlyArray; + }, + EventSinkV2Error + >; readonly write: (input: { /** Return no events if another thread owns this native session at commit time. */ readonly unlessNativeThreadOwned?: Pick< @@ -142,6 +153,11 @@ export interface EventSinkV2Shape { readonly acceptedAt: DateTime.Utc; readonly events: ReadonlyArray; readonly effects: ReadonlyArray; + /** Recheck renewal safety against ingestion in the same transaction as its events. */ + readonly freshProviderRenewal?: Extract< + OrchestrationV2Command, + { type: "provider-session.renew" } + >; readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; @@ -616,6 +632,54 @@ const layerBase: Layer.Layer< ); }); + const acceptFreshContextEffect = Effect.fn("orchestrationV2.EventSink.acceptFreshContext")( + function* (event: Parameters[0]) { + return yield* commitThenPublish( + Effect.gen(function* () { + const records = yield* projectionStore.getThreadRecords(event.threadId, [ + "providerThreads", + ]); + const current = records.providerThreads.find((row) => row.id === event.payload.id); + const accepted = event.payload.freshContext; + const nativeId = event.payload.nativeThreadRef?.nativeId; + if ( + nativeId == null || + accepted?.acceptedNativeThreadId !== nativeId || + current?.nativeThreadRef?.nativeId !== nativeId || + current.freshContext?.bootstrap !== accepted.bootstrap || + current.freshContext.throughRunOrdinal !== accepted.throughRunOrdinal || + current.freshContext.acceptedNativeThreadId === nativeId + ) { + return { + committed: false as const, + storedEvents: [] as ReadonlyArray, + }; + } + // Read and merge under the transaction so ingestion cannot invalidate + // acceptance by changing unrelated provider-thread fields. + const storedEvents = yield* eventStore.append({ + events: [ + { + ...event, + payload: { + ...current, + freshContext: { ...current.freshContext, acceptedNativeThreadId: nativeId }, + updatedAt: event.occurredAt, + }, + }, + ], + }); + const outcomes = yield* applyStoredEvents(storedEvents); + return { committed: true as const, storedEvents, outcomes }; + }), + (result) => + result.committed + ? publishStoredEvents(result.storedEvents, result.outcomes) + : Effect.void, + ); + }, + ); + const existingCommandResult = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (commandId: CommandId) { @@ -651,11 +715,58 @@ const layerBase: Layer.Layer< return { ...existing, committed: false as const, cancelledEffectIds: [] }; } + let events = input.events; + if (input.freshProviderRenewal !== undefined) { + const records = yield* projectionStore.getThreadRecords(input.threadId, [ + "providerThreads", + "runs", + "providerTurns", + "runtimeRequests", + "subagents", + "turnItems", + ]); + const prepared = prepareFreshProviderThread( + records, + input.freshProviderRenewal, + input.acceptedAt, + ); + if (typeof prepared === "string") { + const receipt: CommandReceiptStore.CommandReceiptV2 = { + commandId: input.commandId, + threadId: input.threadId, + commandType: input.commandType, + acceptedAt: input.acceptedAt, + resultSequence: yield* eventStore.latestSequence({ threadId: input.threadId }), + status: "rejected", + error: prepared, + }; + yield* commandReceipts.upsert(receipt); + return { + receipt, + storedEvents: [] as ReadonlyArray, + committed: false as const, + cancelledEffectIds: [], + }; + } + // Ingestion does not take the command lock. Rebuild from the current + // row so even an unrelated adapter update keeps all of its fields. + events = events.map((event) => + event.type !== "provider-thread.updated" + ? event + : { + ...event, + payload: + event.payload.id === prepared.previous.id + ? prepared.previous + : prepared.fresh, + }, + ); + } const runId = input.updateDelegatedCompletionOnlyRunId; const normalized = yield* normalizeEvents( runId === undefined - ? input.events - : yield* updateDelegatedCompletionOnly(input.events, (id) => id === runId), + ? events + : yield* updateDelegatedCompletionOnly(events, (id) => id === runId), ); const storedEvents = yield* eventStore.append({ commandId: input.commandId, @@ -922,6 +1033,10 @@ const layerBase: Layer.Layer< }), ), ), + acceptFreshContext: (event) => + acceptFreshContextEffect(event).pipe( + Effect.mapError((cause) => new EventSinkWriteError({ eventCount: 1, cause })), + ), writeIfProviderThreadUnchanged: (input) => writeIfProviderThreadUnchangedEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 770da1939848..9f3fe0405cb8 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -14,6 +14,7 @@ import { type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2Run, + type OrchestrationV2ProviderThread, type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, ProjectId, @@ -147,6 +148,101 @@ function threadCreatedEvent(input: { }; } +it.effect("persists fresh acceptance despite three provider-thread receipt collisions", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:fresh-acceptance-collisions"); + const providerThread: OrchestrationV2ProviderThread = { + id: ProviderThreadId.make("provider-thread:fresh-acceptance-collisions"), + driver: providerDriver, + providerInstanceId, + providerSessionId: null, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver: providerDriver, nativeId: "renewed-native", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 2, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + freshContext: { bootstrap: "role and lessons", throughRunOrdinal: 1 }, + createdAt: now, + updatedAt: now, + }; + const update = (id: string, payload: OrchestrationV2ProviderThread) => ({ + id: EventId.make(id), + type: "provider-thread.updated" as const, + threadId, + occurredAt: now, + payload, + }); + yield* sink.write({ + events: [ + threadCreatedEvent({ + id: "event:fresh-acceptance:thread", + thread: makeThread(threadId, now), + now, + }), + update("event:fresh-acceptance:initial", providerThread), + ], + }); + const receipt = update("event:fresh-acceptance:receipt", { + ...providerThread, + freshContext: { ...providerThread.freshContext!, acceptedNativeThreadId: "renewed-native" }, + }); + for (let collision = 0; collision < 3; collision += 1) { + const snapshot = (yield* store.getThreadRecords(threadId, ["providerThreads"])) + .providerThreads[0]!; + yield* sink.write({ + events: [ + update(`event:fresh-acceptance:roster-${collision}`, { + ...snapshot, + pendingBackgroundTasks: [ + { taskId: `roster-${collision}`, description: "ingestion update", kind: "command" }, + ], + }), + ], + }); + const staleReceipt = yield* sink.writeIfProviderThreadUnchanged({ + expected: snapshot, + events: [receipt], + }); + assert.isFalse(staleReceipt.committed); + } + const accepted = yield* sink.acceptFreshContext(receipt); + assert.isTrue(accepted.committed); + const durable = (yield* store.getThreadRecords(threadId, ["providerThreads"])) + .providerThreads[0]!; + assert.equal(durable.freshContext?.acceptedNativeThreadId, "renewed-native"); + assert.equal(durable.pendingBackgroundTasks?.[0]?.taskId, "roster-2"); + assert.isFalse((yield* sink.acceptFreshContext(receipt)).committed); + + for (const replacement of [ + { + ...providerThread, + nativeThreadRef: { ...providerThread.nativeThreadRef!, nativeId: "replacement-native" }, + }, + { ...providerThread, freshContext: { bootstrap: "new generation", throughRunOrdinal: 2 } }, + ]) { + yield* sink.write({ + events: [ + update( + `event:fresh-acceptance:replacement-${replacement.freshContext?.throughRunOrdinal}`, + replacement, + ), + ], + }); + assert.isFalse((yield* sink.acceptFreshContext(receipt)).committed); + const current = (yield* store.getThreadRecords(threadId, ["providerThreads"])) + .providerThreads[0]!; + assert.isUndefined(current.freshContext?.acceptedNativeThreadId); + } + }).pipe(Effect.provide(layerTest)), +); + it.effect("rebuilds event history one bounded page at a time", () => Effect.gen(function* () { const eventStore = yield* EventStore.EventStoreV2; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 0a5e56d98d63..f08896ba40c3 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -78,6 +78,7 @@ import { derivePendingBackgroundWork, pendingBackgroundTurnItems, } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; +import { prepareFreshProviderThread } from "./freshProviderContext.ts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -523,6 +524,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId { case "thread.runtime-mode.set": case "thread.interaction-mode.set": case "thread.model-selection.set": + case "provider-session.renew": case "provider-session.detach": case "message.dispatch": case "notification.delivery.accept": @@ -4106,58 +4108,102 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } }); - const dispatchProviderSessionDetach = Effect.fn("orchestrationV2.dispatch.providerSessionDetach")( - function* ( - command: Extract, - events: Ref.Ref>, - effects: Ref.Ref>, - ) { - const projection = yield* projectionStore - .getThreadRecords(command.threadId, ["providerSessions"]) + const dispatchProviderSessionLifecycle = Effect.fn( + "orchestrationV2.dispatch.providerSessionLifecycle", + )(function* ( + command: Extract< + OrchestrationV2Command, + { readonly type: "provider-session.detach" | "provider-session.renew" } + >, + events: Ref.Ref>, + effects: Ref.Ref>, + ) { + const projection = yield* projectionStore + .getThreadRecords(command.threadId, ["providerSessions"]) + .pipe( + Effect.mapError( + (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }), + ), + ); + const session = projection.providerSessions.find( + (candidate) => candidate.id === command.providerSessionId, + ); + if (session === undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Provider session ${command.providerSessionId} does not belong to thread ${command.threadId}.`, + }); + } + const now = yield* DateTime.now; + if (command.type === "provider-session.renew") { + const records = yield* projectionStore + .getThreadRecords(command.threadId, [ + "providerThreads", + "runs", + "providerTurns", + "runtimeRequests", + "subagents", + "turnItems", + ]) .pipe( Effect.mapError( (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }), ), ); - const session = projection.providerSessions.find( - (candidate) => candidate.id === command.providerSessionId, - ); - if (session === undefined) { + const prepared = prepareFreshProviderThread(records, command, now); + if (typeof prepared === "string") { return yield* new OrchestratorDispatchError({ commandId: command.commandId, commandType: command.type, - cause: `Provider session ${command.providerSessionId} does not belong to thread ${command.threadId}.`, + cause: prepared, }); } - const now = yield* DateTime.now; - yield* emit( - events, - command, - )({ - type: "provider-session.detached", - threadId: command.threadId, - driver: session.driver, - providerInstanceId: session.providerInstanceId, - occurredAt: now, - payload: { - providerSessionId: session.id, - detachedAt: now, - ...(command.reason === undefined ? {} : { reason: command.reason }), - }, - }); - const pendingEffect = { - id: `effect:${command.commandId}:provider-session.detach:${command.providerSessionId}`, - commandId: command.commandId, - threadId: command.threadId, - request: { - type: "provider-session.detach", - providerSessionId: command.providerSessionId, - ...(command.reason === undefined ? {} : { detail: command.reason }), - }, - } satisfies PendingOrchestrationEffectV2; - yield* Ref.update(effects, (existing) => [...existing, pendingEffect]); - }, - ); + for (const providerThread of [prepared.previous, prepared.fresh]) { + yield* emit( + events, + command, + )({ + type: "provider-thread.updated", + threadId: command.threadId, + driver: session.driver, + providerInstanceId: session.providerInstanceId, + occurredAt: now, + payload: providerThread, + }); + } + // Keep the transport attached. A delayed detach effect could otherwise + // interrupt input that arrives after this atomic renewal. The next run + // starts a new native thread from the new row's null native reference. + return; + } + yield* emit( + events, + command, + )({ + type: "provider-session.detached", + threadId: command.threadId, + driver: session.driver, + providerInstanceId: session.providerInstanceId, + occurredAt: now, + payload: { + providerSessionId: session.id, + detachedAt: now, + ...(command.reason === undefined ? {} : { reason: command.reason }), + }, + }); + const pendingEffect = { + id: `effect:${command.commandId}:provider-session.detach:${command.providerSessionId}`, + commandId: command.commandId, + threadId: command.threadId, + request: { + type: "provider-session.detach", + providerSessionId: command.providerSessionId, + ...(command.reason === undefined ? {} : { detail: command.reason }), + }, + } satisfies PendingOrchestrationEffectV2; + yield* Ref.update(effects, (existing) => [...existing, pendingEffect]); + }); const dispatchThreadProjectAdoptFolder = Effect.fn( "orchestrationV2.dispatch.threadProjectAdoptFolder", @@ -12988,8 +13034,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio case "thread.guest-message.dismiss": yield* dispatchGuestShare(command, events, effects); break; + case "provider-session.renew": case "provider-session.detach": - yield* dispatchProviderSessionDetach(command, events, effects); + yield* dispatchProviderSessionLifecycle(command, events, effects); break; case "thread.project.adopt-folder": yield* dispatchThreadProjectAdoptFolder(command, events); @@ -13414,6 +13461,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio acceptedAt, events: plan.events, effects: plan.effects, + ...(command.type === "provider-session.renew" ? { freshProviderRenewal: command } : {}), ...(plan.cancelUnsettledEffects === undefined ? {} : { cancelUnsettledEffects: plan.cancelUnsettledEffects }), diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2e88fdc11908..bea21e7faba4 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1,6 +1,7 @@ import { pendingGuestMessageCount } from "@t3tools/contracts"; import { applyThreadPullRequestLinkUpdate } from "../sourceControl/threadPullRequests.ts"; import { projectTurnItemForWire } from "./WireProjection.ts"; +import { preserveFreshContext } from "./freshProviderContext.ts"; import * as Stream from "effect/Stream"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import { makeThreadFind } from "./ThreadFind.ts"; @@ -889,7 +890,13 @@ export function applyToProjection( activeProviderThreadId: event.payload.id, } : base.thread, - providerThreads: upsertById(base.providerThreads, event.payload), + providerThreads: upsertById( + base.providerThreads, + preserveFreshContext( + base.providerThreads.find((thread) => thread.id === event.payload.id), + event.payload, + ), + ), }; case "provider-turn.updated": return { @@ -2239,7 +2246,17 @@ export const layer: Layer.Layer = break; } case "provider-thread.updated": { - const payloadJson = yield* encodeProviderThreadPayload(event.payload); + const existing = yield* sql` + SELECT payload_json FROM orchestration_v2_projection_provider_threads + WHERE provider_thread_id = ${event.payload.id} + `; + const previous = + existing[0] === undefined + ? undefined + : yield* decodeProviderThreadPayload(existing[0].payload_json); + const payloadJson = yield* encodeProviderThreadPayload( + preserveFreshContext(previous, event.payload), + ); const payload = parseEncodedPayload(payloadJson); yield* sql` INSERT INTO orchestration_v2_projection_provider_threads ( diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 05fc1b8545bb..ecbdf517f951 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -267,6 +267,7 @@ it.each([ function makeLocalCommandHarness(input: { readonly text: string; + readonly freshBootstrap?: string; readonly worktreePath?: string; readonly worktreesDir?: string; readonly workspaceRoot?: string; @@ -278,6 +279,7 @@ function makeLocalCommandHarness(input: { readonly openFailure?: unknown; readonly ensureThreadFailure?: unknown; readonly historyReadFailureAfterFallback?: unknown; + readonly emptyRetryHistory?: boolean; readonly interruptOpen?: boolean; readonly interruptRunBeforeOpenFailure?: boolean; readonly writeFailure?: unknown; @@ -343,6 +345,9 @@ function makeLocalCommandHarness(input: { firstRunOrdinal: 2, lastRunOrdinal: 2, handoffIds: [], + ...(input.freshBootstrap === undefined + ? {} + : { freshContext: { bootstrap: input.freshBootstrap, throughRunOrdinal: 1 } }), forkedFrom: null, createdAt: now, updatedAt: now, @@ -817,29 +822,30 @@ function makeLocalCommandHarness(input: { }), ), ); - const writeIfRunCurrent = vi.fn(({ events: incoming, activeAttemptId, expectedStatus }) => - "writeFailure" in input - ? Effect.fail( - new EventSink.EventSinkWriteError({ - eventCount: incoming.length, - cause: input.writeFailure, - }), - ) - : Effect.sync(() => { - const current = projection.runs.find((candidate) => candidate.id === runId); - const committed = - current !== undefined && - current.activeAttemptId === activeAttemptId && - current.status === expectedStatus; - if (committed) { - for (const event of incoming) { - expect(isDomainEvent(event)).toBe(true); - events.push(event); - projection = ProjectionStore.applyToProjection(projection, event); + const writeIfRunCurrent = vi.fn( + ({ events: incoming, activeAttemptId, expectedStatus, runId: currentRunId }) => + "writeFailure" in input + ? Effect.fail( + new EventSink.EventSinkWriteError({ + eventCount: incoming.length, + cause: input.writeFailure, + }), + ) + : Effect.sync(() => { + const current = projection.runs.find((candidate) => candidate.id === currentRunId); + const committed = + current !== undefined && + current.activeAttemptId === activeAttemptId && + current.status === expectedStatus; + if (committed) { + for (const event of incoming) { + expect(isDomainEvent(event)).toBe(true); + events.push(event); + projection = ProjectionStore.applyToProjection(projection, event); + } } - } - return { committed, storedEvents: [] }; - }), + return { committed, storedEvents: [] }; + }), ); const resolveRuntimePolicy = vi.fn( ({ thread }: Parameters[0]) => @@ -891,6 +897,30 @@ function makeLocalCommandHarness(input: { } return []; }), + acceptFreshContext: (event) => + Effect.sync(() => { + const current = projection.providerThreads.find( + (row) => row.id === event.payload.id, + )!; + const merged = { + ...event, + payload: { ...current, freshContext: event.payload.freshContext }, + }; + events.push(merged); + projection = ProjectionStore.applyToProjection(projection, merged); + return { committed: true, storedEvents: [] }; + }), + writeIfProviderThreadUnchanged: ({ expected, events: incoming }) => + Effect.sync(() => { + const current = projection.providerThreads.find((row) => row.id === expected.id); + if (current !== expected) return { committed: false, storedEvents: [] }; + for (const event of incoming) { + expect(isDomainEvent(event)).toBe(true); + events.push(event); + projection = ProjectionStore.applyToProjection(projection, event); + } + return { committed: true, storedEvents: [] }; + }), }), IdAllocator.layer, layerWorktreeLocation(input.worktreesDir), @@ -922,8 +952,8 @@ function makeLocalCommandHarness(input: { (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0), ), }), - getTurnStartHistory: () => - input.sideChat !== undefined + getTurnStartHistory: (_threadId, runIds) => + input.sideChat !== undefined || (input.emptyRetryHistory && runIds !== undefined) ? Effect.succeed([]) : Effect.fail( new ProjectionStore.ProjectionStoreReadError({ @@ -974,7 +1004,10 @@ function makeLocalCommandHarness(input: { attemptId, projection: () => projection, start: Effect.gen(function* () { - yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ threadId, runId }); + yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ + threadId, + runId: projection.runs.at(-1)!.id, + }); }).pipe(Effect.provide(layer)), startWithRetry: Effect.gen(function* () { yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ @@ -1749,3 +1782,173 @@ for (const previousMessages of [[], ["/compact", " /COMPACT "]]) { }), ); } + +for (const first of ["success", "concurrent-updates", "rejected", "compact", "goal"] as const) { + effectIt.effect(`keeps bootstrap delivery durable after a first ${first} turn`, () => + Effect.gen(function* () { + const seed = "Role: matt-console. Handoff: pending ask. Lessons: bounded role file."; + const harness = makeLocalCommandHarness({ + text: + first === "compact" ? "/compact" : first === "goal" ? "/goal Keep working" : "New ask", + freshBootstrap: seed, + emptyRetryHistory: true, + previousMessages: ["Pre-renewal transcript must stay out of native context"], + }); + harness.tryHandlePromptCommand.mockImplementation(() => Effect.succeed(false)); + const old = harness.projection().providerThreads[0]!; + let nativeId = "fresh-native"; + const withNative = (thread: typeof old) => ({ + ...thread, + nativeThreadRef: { driver: old.driver, nativeId, strength: "strong" as const }, + }); + const startTurn = vi.fn((_input: unknown) => + first === "rejected" && startTurn.mock.calls.length === 1 + ? Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: old.driver, + threadId: old.appThreadId!, + providerThreadId: old.id, + runId: harness.projection().runs.at(-1)!.id, + cause: "Provider refused before accepting the prompt", + }), + ) + : Effect.sync(() => { + if (first !== "concurrent-updates") return; + for (let update = 0; update < 3; update += 1) { + const current = harness.projection(); + harness.setProjection({ + ...current, + providerThreads: current.providerThreads.map((row) => ({ + ...row, + pendingBackgroundTasks: [ + { + taskId: `roster-${update}`, + description: "ingestion update", + kind: "command" as const, + }, + ], + })), + }); + } + }), + ); + const compactThread = vi.fn((_input: unknown) => Effect.void); + const ensureThread = vi.fn(() => Effect.succeed(withNative(old))); + const resumeThread = vi.fn(({ providerThread }: { providerThread: typeof old }) => + Effect.succeed(withNative(providerThread)), + ); + harness.open.mockImplementation(() => + Effect.succeed({ + driver: old.driver, + providerSession: { + id: old.providerSessionId, + driver: old.driver, + providerInstanceId: old.providerInstanceId, + status: "ready", + cwd: "/tmp/native-account-command", + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: old.createdAt, + updatedAt: old.updatedAt, + lastError: null, + }, + ensureThread, + resumeThread, + startTurn, + compactThread, + } as never), + ); + harness.startRootRun.mockImplementation((input) => { + const start = + input.message.text === "/compact" + ? input.session.compactThread! + : input.session.startTurn; + return start({ + threadId: input.appThread.id, + runId: input.run.id, + message: input.message, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + providerThread: input.providerThread, + } as never).pipe(Effect.ignore); + }); + const next = (text: string) => { + const projection = harness.projection(); + const prior = projection.runs.at(-1)!; + const runId = RunId.make(`next:${prior.ordinal}`); + harness.setProjection({ + ...projection, + runs: [ + ...projection.runs.map((run) => ({ + ...run, + status: + first === "rejected" && run.ordinal === old.firstRunOrdinal + ? ("failed" as const) + : ("completed" as const), + })), + { ...prior, id: runId, ordinal: prior.ordinal + 1, status: "starting" }, + ], + attempts: projection.attempts.map((attempt) => ({ + ...attempt, + runId, + status: "pending", + })), + messages: projection.messages.map((message) => + message.id === prior.userMessageId ? { ...message, runId, text } : message, + ), + }); + }; + yield* harness.start; + expect(ensureThread).toHaveBeenCalledOnce(); + expect(resumeThread).not.toHaveBeenCalled(); + expect(harness.projection().providerThreads[0]?.freshContext?.acceptedNativeThreadId).toBe( + first === "success" || first === "concurrent-updates" ? nativeId : undefined, + ); + if (first === "compact") { + expect(compactThread).toHaveBeenCalledOnce(); + expect(compactThread.mock.calls[0]?.[0]).toMatchObject({ message: { text: "/compact" } }); + } else if (first === "goal") { + expect(startTurn.mock.calls[0]?.[0]).toMatchObject({ + message: { text: "/goal Keep working" }, + }); + } else { + expect(startTurn.mock.calls[0]?.[0]).toMatchObject({ + message: { text: `${seed}\n\nUser message:\nNew ask` }, + }); + } + next("Next ordinary ask"); + yield* harness.start; + expect(resumeThread).toHaveBeenCalledOnce(); + expect(startTurn.mock.calls.at(-1)?.[0]).toMatchObject({ + message: { + text: + first === "success" || first === "concurrent-updates" + ? "Next ordinary ask" + : `${seed}\n\nUser message:\nNext ordinary ask`, + }, + }); + expect(harness.projection().providerThreads[0]?.freshContext?.acceptedNativeThreadId).toBe( + nativeId, + ); + next("After acceptance"); + yield* harness.start; + expect(startTurn.mock.calls.at(-1)?.[0]).toMatchObject({ + message: { text: "After acceptance" }, + }); + nativeId = "replacement-native"; + next("After native replacement"); + yield* harness.start; + expect(startTurn.mock.calls.at(-1)?.[0]).toMatchObject({ + message: { text: `${seed}\n\nUser message:\nAfter native replacement` }, + }); + expect(harness.projection().providerThreads[0]?.freshContext?.acceptedNativeThreadId).toBe( + nativeId, + ); + for (const [call] of startTurn.mock.calls) { + expect((call as { message: { text: string } }).message.text).not.toContain( + "Pre-renewal transcript", + ); + } + }), + ); +} diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 402b2eaf3e04..a45103b66a3f 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -855,6 +855,12 @@ export const layer: Layer.Layer< const tokenCap = yield* handoffTokenCapConfig.pipe( Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), ); + const freshBootstrapForThread = (loaded: OrchestrationV2ProviderThread) => + providerThread.freshContext !== undefined && + (providerThread.freshContext.acceptedNativeThreadId === undefined || + providerThread.freshContext.acceptedNativeThreadId !== loaded.nativeThreadRef?.nativeId) + ? providerThread.freshContext.bootstrap + : ""; const handoffContextForThread = (loadedProviderThread: OrchestrationV2ProviderThread) => { const measuredContext = latestNativeContextUsage(projection, providerThread); const previousSelection = @@ -997,7 +1003,9 @@ export const layer: Layer.Layer< return handoffBudget({ tokenCap, modelContextWindow, - userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, + userText: [freshBootstrapForThread(loadedProviderThread), restartNote, userText] + .filter(Boolean) + .join("\n\n"), attachments: message.attachments, providerThread: budgetProviderThread, nativeContextEstimate: @@ -1064,14 +1072,22 @@ export const layer: Layer.Layer< toProviderThreadId: providerThread.id, fromProviderInstanceId: providerThread.providerInstanceId, toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, + coveredRunOrdinals: { + from: (providerThread.freshContext?.throughRunOrdinal ?? 0) + 1, + to: Math.max(1, run.ordinal - 1), + }, strategy: "full_thread_summary", - runs: projection.runs, + runs: projection.runs.filter( + (source) => source.ordinal > (providerThread.freshContext?.throughRunOrdinal ?? 0), + ), items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter( (item) => - item.runId === null || + (item.runId === null && providerThread.freshContext === undefined) || projection.runs.some( - (source) => source.id === item.runId && source.ordinal < run.ordinal, + (source) => + source.id === item.runId && + source.ordinal < run.ordinal && + source.ordinal > (providerThread.freshContext?.throughRunOrdinal ?? 0), ), ), createdAt, @@ -1322,6 +1338,11 @@ export const layer: Layer.Layer< }), ); if (replacement === undefined) return undefined; + if ( + providerThread.freshContext !== undefined && + run.ordinal <= providerThread.freshContext.throughRunOrdinal + 1 + ) + return replacement; const { handoff, transferId, createdAt } = yield* recoveryHandoff; effectiveHandoffs = withRecoveryHandoff(handoff); resumeFellBack = true; @@ -1391,6 +1412,7 @@ export const layer: Layer.Layer< providerSessionId, appThreadId: projection.thread.id, ownerNodeId: providerThread.ownerNodeId, + freshContext: providerThread.freshContext, firstRunOrdinal: providerThread.firstRunOrdinal ?? run.ordinal, lastRunOrdinal: run.ordinal, handoffIds: @@ -1536,6 +1558,8 @@ export const layer: Layer.Layer< }); const startWithHandoffs = Effect.fn("ProviderTurnStartService.startWithHandoffs")( function* (turnInput: Parameters[0], compact = false) { + const freshBootstrap = + compact || goalCommand ? "" : freshBootstrapForThread(runningProviderThread); const delivery = yield* deliverContextHandoffs({ handoffs: [...effectiveHandoffs, ...retryHandoff], deferInline: compact || goalCommand, @@ -1560,7 +1584,7 @@ export const layer: Layer.Layer< sideChatHandoff === undefined || compact || goalCommand ? "" : sideChatHandoff.summaryText; - const context = [delivery.context, sideChatContext, restartNote] + const context = [freshBootstrap, delivery.context, sideChatContext, restartNote] .filter((part) => part !== "") .join("\n\n"); const { restartContinuationOfRunId: _resumedRunId, ...promptedInput } = turnInput; @@ -1584,6 +1608,36 @@ export const layer: Layer.Layer< : Effect.void, ), ); + if (freshBootstrap !== "") { + yield* Effect.gen(function* () { + const nativeId = runningProviderThread.nativeThreadRef?.nativeId; + if (nativeId == null || runningProviderThread.freshContext === undefined) return; + const acceptedAt = yield* DateTime.now; + yield* eventSink.acceptFreshContext({ + id: yield* idAllocator.allocate.event({ threadId: input.threadId }), + type: "provider-thread.updated", + threadId: input.threadId, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: acceptedAt, + payload: { + ...runningProviderThread, + freshContext: { + ...runningProviderThread.freshContext, + acceptedNativeThreadId: nativeId, + }, + updatedAt: acceptedAt, + }, + }); + }).pipe( + Effect.catchCause(() => + Effect.logWarning("Failed to record accepted bootstrap", { + runId: run.id, + deliveryStatus: "pending", + }), + ), + ); + } // After provider acceptance, failed delivery bookkeeping must not stop live ingestion. const nativeThreadId = runningProviderThread.nativeThreadRef?.nativeId; yield* Effect.forEach( @@ -1624,6 +1678,7 @@ export const layer: Layer.Layer< ), ); const deliverySession = + providerThread.freshContext === undefined && effectiveHandoffs.length === 0 && retryHandoff.length === 0 && sideChatHandoff === undefined && diff --git a/apps/server/src/orchestration-v2/freshProviderContext.test.ts b/apps/server/src/orchestration-v2/freshProviderContext.test.ts new file mode 100644 index 000000000000..82cc8566a2ec --- /dev/null +++ b/apps/server/src/orchestration-v2/freshProviderContext.test.ts @@ -0,0 +1,352 @@ +import { assert, it } from "@effect/vitest"; +import { + CommandId, + EventId, + MessageId, + NodeId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ProviderSessionId, + ProviderThreadId, + ProviderTurnId, + RunId, + ThreadId, + type OrchestrationV2DomainEvent, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; +import { vi } from "vite-plus/test"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as EventSink from "./EventSink.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { prepareFreshProviderThread } from "./freshProviderContext.ts"; + +const instanceId = ProviderInstanceId.make("codex"); +const adapter = { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }), + openSession: () => Effect.die("Renewal must not start a provider"), +} as ProviderAdapter.ProviderAdapterV2["Service"]; +const database = SqlitePersistence.layerMemory; +const layer = Layer.mergeAll( + database, + ProjectionStore.layer.pipe(Layer.provide(database)), + ProviderReplayHarness.layerWithRegistry( + { name: "fresh-context" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { databaseLayer: database, runEffectWorker: false }, + ), +); + +it.effect.each(["none", "background", "turn", "metadata"] as const)( + "renews native context and preserves concurrent ingestion; refuses unsafe twins (%s)", + (race) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("console"); + const sessionId = ProviderSessionId.make("console-session"); + const providerThreadId = ProviderThreadId.make("old-console-native"); + const now = yield* DateTime.now; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create"), + threadId, + projectId: ProjectId.make("project"), + title: "Work With Matt", + modelSelection: { instanceId, model: "gpt-5.1-codex" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + const apply = (type: OrchestrationV2DomainEvent["type"], payload: unknown) => + store.apply({ + id: EventId.make(`seed:${type}`), + type, + threadId, + occurredAt: now, + payload, + } as OrchestrationV2DomainEvent); + yield* apply("provider-session.attached", { + id: sessionId, + driver: adapter.driver, + providerInstanceId: instanceId, + status: "ready", + cwd: "/repo", + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }); + yield* apply("provider-thread.updated", { + id: providerThreadId, + driver: adapter.driver, + providerInstanceId: instanceId, + providerSessionId: sessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver: adapter.driver, nativeId: "keep-history", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }); + yield* apply("run.updated", { + id: RunId.make("old-run"), + threadId, + ordinal: 1, + providerInstanceId: instanceId, + modelSelection: { instanceId, model: "gpt-5.1-codex" }, + providerThreadId, + userMessageId: MessageId.make("old-message"), + rootNodeId: null, + activeAttemptId: null, + status: "completed", + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }); + yield* apply("message.updated", { + id: MessageId.make("old-message"), + threadId, + runId: RunId.make("old-run"), + nodeId: null, + role: "user", + createdBy: "user", + creationSource: "web", + text: "Keep this full original ask in the visible history.", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }); + const before = yield* store.getThreadProjection(threadId); + const command = { + type: "provider-session.renew" as const, + commandId: CommandId.make("renew"), + threadId, + providerSessionId: sessionId, + freshContext: { + providerThreadId, + expectedUpdatedAt: before.thread.updatedAt, + bootstrap: "Role; complete handoff; lessons pointer.", + }, + }; + assert.equal( + typeof prepareFreshProviderThread(before, command, now), + "string", + "NO: less than 30 minutes", + ); + yield* TestClock.adjust("31 minutes"); + const later = yield* DateTime.now; + const refuse = (records: typeof before, cmd = command) => + assert.equal(typeof prepareFreshProviderThread(records, cmd, later), "string"); + refuse(before, { + ...command, + freshContext: { ...command.freshContext, bootstrap: "😀".repeat(1250) }, + }); + refuse(before, { ...command, freshContext: { ...command.freshContext, bootstrap: " " } }); + refuse(before, { + ...command, + freshContext: { ...command.freshContext, expectedUpdatedAt: later }, + }); + refuse(before, { ...command, providerSessionId: ProviderSessionId.make("wrong") }); + for (const status of ["queued", "starting", "running", "waiting"] as const) { + refuse({ ...before, runs: before.runs.map((row) => ({ ...row, status })) }); + } + refuse({ + ...before, + providerThreads: before.providerThreads.map((row) => ({ + ...row, + pendingBackgroundTasks: [{ taskId: "active-background", kind: "monitor" as const }], + })), + }); + if (race !== "none") { + const sink = yield* EventSink.EventSinkV2; + const commit = sink.commitCommand; + const planned = yield* Deferred.make(); + const resume = yield* Deferred.make(); + const previous = before.providerThreads.find((row) => row.id === providerThreadId)!; + const ingested: OrchestrationV2DomainEvent = + race === "turn" + ? { + id: EventId.make("concurrent-native-turn"), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { + id: ProviderTurnId.make("concurrent-turn"), + providerThreadId, + nodeId: NodeId.make("concurrent-node"), + runAttemptId: null, + nativeTurnRef: { + driver: adapter.driver, + nativeId: "concurrent-native-turn", + strength: "strong", + }, + ordinal: 1, + status: "running", + startedAt: later, + completedAt: null, + }, + } + : { + id: EventId.make(`concurrent-${race}`), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: { + ...previous, + // Adapter updates need not advance the row/thread timestamp. + ...(race === "background" + ? { pendingBackgroundTasks: [{ taskId: "concurrent-task", kind: "monitor" }] } + : { nativeMetadata: { title: "Retain this adapter update" } }), + }, + }; + yield* Effect.acquireUseRelease( + Effect.sync(() => + vi + .spyOn(sink, "commitCommand") + .mockImplementationOnce((input) => + Deferred.succeed(planned, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + Effect.andThen(commit(input)), + ), + ), + ), + () => + Effect.gen(function* () { + const renewal = yield* orchestrator + .dispatch(command) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(planned); + // This is the transactional ingestion writer, without the command + // lock. A delayed event may retain the old snapshot timestamp. + yield* sink.write({ events: [ingested] }); + const concurrent = yield* store.getThreadProjection(threadId); + yield* Deferred.succeed(resume, undefined); + const result = yield* Fiber.join(renewal); + const afterRace = yield* store.getThreadProjection(threadId); + if (race === "metadata") { + assert.isTrue(Exit.isSuccess(result)); + assert.deepEqual( + afterRace.providerThreads.find((row) => row.id === providerThreadId) + ?.nativeMetadata, + { title: "Retain this adapter update" }, + ); + assert.notEqual(afterRace.thread.activeProviderThreadId, providerThreadId); + } else { + assert.isTrue(Exit.isFailure(result), "NO: work arrived after renewal planning"); + assert.deepEqual(afterRace, concurrent, "NO: refusal must preserve ingestion"); + assert.isTrue( + Exit.isFailure(yield* orchestrator.dispatch(command).pipe(Effect.exit)), + ); + assert.deepEqual(yield* store.getThreadProjection(threadId), concurrent); + } + }), + (spy) => + Deferred.succeed(resume, undefined).pipe( + Effect.andThen(Effect.sync(() => spy.mockRestore())), + ), + ); + return; + } + yield* orchestrator.dispatch(command); + const after = yield* store.getThreadProjection(threadId); + assert.equal(after.thread.id, before.thread.id); + assert.equal(after.thread.title, before.thread.title); + assert.deepEqual(after.messages, before.messages); + assert.deepEqual(after.runs, before.runs); + assert.deepEqual(after.providerSessions, before.providerSessions); + // No delayed detach may interrupt a new turn after the renewal commits. + assert.deepEqual(after.providerTurns, before.providerTurns); + const old = after.providerThreads.find((row) => row.id === providerThreadId)!; + assert.equal(old.nativeThreadRef?.nativeId, "keep-history"); + assert.equal(old.status, "archived"); + const fresh = after.providerThreads.find( + (row) => row.id === after.thread.activeProviderThreadId, + )!; + assert.notEqual(fresh.id, providerThreadId); + assert.equal(fresh.nativeThreadRef, null); + assert.equal(fresh.freshContext?.bootstrap, command.freshContext.bootstrap); + refuse(after); // NO: repeated/stale renewal must not reset again. + + // Both reducers must keep a durable receipt when adapters/finalization + // write snapshots captured before successful bootstrap delivery. + let memory = after; + const update = (payload: typeof fresh) => + Effect.gen(function* () { + const event = { + id: EventId.make(`receipt:${payload.nativeThreadRef?.nativeId}:${payload.updatedAt}`), + type: "provider-thread.updated" as const, + threadId, + occurredAt: later, + payload, + }; + memory = ProjectionStore.applyToProjection(memory, event); + yield* store.apply(event); + const persisted = yield* store.getThreadProjection(threadId); + assert.deepEqual( + persisted.providerThreads.find((row) => row.id === fresh.id)?.freshContext, + memory.providerThreads.find((row) => row.id === fresh.id)?.freshContext, + ); + return persisted.providerThreads.find((row) => row.id === fresh.id)!; + }); + const pending = { + ...fresh, + nativeThreadRef: { + driver: adapter.driver, + nativeId: "seed-native", + strength: "strong" as const, + }, + }; + yield* update({ + ...pending, + freshContext: { ...fresh.freshContext!, acceptedNativeThreadId: "seed-native" }, + }); + for (const stale of [pending, { ...pending, freshContext: undefined }]) { + const kept = yield* update(stale); + assert.equal(kept.freshContext?.acceptedNativeThreadId, "seed-native"); + } + const replacement = { + ...pending, + nativeThreadRef: { ...pending.nativeThreadRef, nativeId: "replacement" }, + }; + const unaccepted = yield* update(replacement); + assert.notEqual(unaccepted.freshContext?.acceptedNativeThreadId, "replacement"); + yield* update({ + ...replacement, + freshContext: { ...fresh.freshContext!, acceptedNativeThreadId: "replacement" }, + }); + const retained = yield* update(replacement); + assert.equal(retained.freshContext?.acceptedNativeThreadId, "replacement"); + const renewed = yield* update({ + ...replacement, + freshContext: { bootstrap: "New generation", throughRunOrdinal: 2 }, + }); + assert.equal(renewed.freshContext?.acceptedNativeThreadId, undefined); + }).pipe(Effect.provide(layer)), +); diff --git a/apps/server/src/orchestration-v2/freshProviderContext.ts b/apps/server/src/orchestration-v2/freshProviderContext.ts new file mode 100644 index 000000000000..779f815143f0 --- /dev/null +++ b/apps/server/src/orchestration-v2/freshProviderContext.ts @@ -0,0 +1,114 @@ +import { + ProviderThreadId, + isOrchestrationV2WorkActive, + type OrchestrationV2Command, + type OrchestrationV2ProviderThread, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import type { ProjectionRecords } from "./ProjectionStore.ts"; + +// Adapter snapshots and finalization can predate acceptance of the bootstrap. +// Only a new renewal generation or native context invalidates that receipt. +export function preserveFreshContext( + previous: OrchestrationV2ProviderThread | undefined, + incoming: OrchestrationV2ProviderThread, +): OrchestrationV2ProviderThread { + const saved = previous?.freshContext; + if (saved === undefined) return incoming; + const freshContext = incoming.freshContext ?? saved; + if ( + freshContext.bootstrap === saved.bootstrap && + freshContext.throughRunOrdinal === saved.throughRunOrdinal && + saved.acceptedNativeThreadId !== undefined && + saved.acceptedNativeThreadId === incoming.nativeThreadRef?.nativeId + ) { + return { ...incoming, freshContext: saved }; + } + return { ...incoming, freshContext }; +} + +// Refuse before emitting anything. The command's timestamp is a compare-and-swap +// against the saved handoff's snapshot, including input queued since that snapshot. +export function prepareFreshProviderThread( + records: ProjectionRecords< + "providerThreads" | "runs" | "providerTurns" | "runtimeRequests" | "subagents" | "turnItems" + >, + command: Extract, + now: DateTime.Utc, +): string | { previous: OrchestrationV2ProviderThread; fresh: OrchestrationV2ProviderThread } { + const seed = command.freshContext; + if (seed === undefined) return "Fresh context requires an explicit seed."; + const previous = records.providerThreads.find((row) => row.id === seed.providerThreadId); + if ( + previous === undefined || + previous.id !== records.thread.activeProviderThreadId || + previous.appThreadId !== command.threadId || + previous.ownerNodeId !== null || + previous.providerSessionId !== command.providerSessionId || + previous.nativeThreadRef === null || + previous.status === "archived" || + previous.status === "closed" + ) { + return "Fresh context requires the current root provider thread and native session."; + } + if ( + DateTime.toEpochMillis(records.thread.updatedAt) !== + DateTime.toEpochMillis(seed.expectedUpdatedAt) + ) { + return "Thread changed since the handoff snapshot."; + } + if ( + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(records.thread.updatedAt) < + 30 * 60_000 + ) { + return "Thread has not been idle for 30 minutes."; + } + if (!seed.bootstrap.trim() || new TextEncoder().encode(seed.bootstrap).length >= 5_000) { + return "Bootstrap must contain fewer than 5000 UTF-8 bytes."; + } + const latest = records.runs.reduce<(typeof records.runs)[number] | undefined>( + (last, run) => (last === undefined || run.ordinal > last.ordinal ? run : last), + undefined, + ); + if ( + latest === undefined || + latest.status !== "completed" || + latest.completedAt === null || + latest.providerThreadId !== previous.id || + !Number.isSafeInteger(latest.ordinal + 1) || + records.runs.some( + (run) => + !["completed", "failed", "cancelled", "interrupted", "rolled_back"].includes(run.status), + ) || + records.providerTurns.some((turn) => turn.status === "pending" || turn.status === "running") || + records.runtimeRequests.some((request) => request.status === "pending") || + records.subagents.some((task) => isOrchestrationV2WorkActive(task.status)) || + records.turnItems.some((item) => isOrchestrationV2WorkActive(item.status)) || + records.providerThreads.some((row) => (row.pendingBackgroundTasks?.length ?? 0) > 0) + ) { + return "Thread has active, queued, waiting or uncompleted work."; + } + return { + previous: { ...previous, status: "archived", updatedAt: now }, + fresh: { + ...previous, + id: ProviderThreadId.make(`fresh:${command.commandId}`), + providerSessionId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "not_loaded", + // Non-null distinguishes this deliberate renewal from a queued placeholder. + firstRunOrdinal: latest.ordinal + 1, + lastRunOrdinal: latest.ordinal + 1, + handoffIds: [], + forkedFrom: null, + contextUsage: null, + nativeMetadata: null, + goal: null, + pendingBackgroundTasks: [], + freshContext: { bootstrap: seed.bootstrap, throughRunOrdinal: latest.ordinal }, + createdAt: now, + updatedAt: now, + }, + }; +} diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts index 77e8bae8f085..f77c228ceae7 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts @@ -159,6 +159,7 @@ function commandThreadIds(command: OrchestrationV2Command): ReadonlyArray