From aa05723fd425e23648ecbbb7f9bf8dc97d57a421 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Tue, 3 Nov 2026 16:41:13 -0600 Subject: [PATCH 1/9] perf(server): add bounded provider group commits --- apps/server/src/orchestration-v2/EventSink.ts | 77 +++++-- .../FoundationPersistence.test.ts | 108 +++++++++ .../ProviderCommitBatch.test.ts | 207 ++++++++++++++++++ .../orchestration-v2/ProviderCommitBatch.ts | 131 +++++++++++ .../orchestration-v2/ProviderEventIngestor.ts | 1 + 5 files changed, 511 insertions(+), 13 deletions(-) create mode 100644 apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts create mode 100644 apps/server/src/orchestration-v2/ProviderCommitBatch.ts diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index b210d3c9840ae..f99129e0157ff 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -13,6 +13,7 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; +import * as Config from "effect/Config"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -34,6 +35,7 @@ import * as EventStore from "./EventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; export class EventSinkWriteError extends Schema.TaggedError()( "EventSinkWriteError", @@ -68,6 +70,8 @@ export type EventSinkV2Error = typeof EventSinkV2Error.Type; export interface EventSinkV2Shape { readonly write: (input: { + /** Group independent provider snapshots; return only after commit and publication. */ + readonly batch?: boolean; /** Return no events if another thread owns this native session at commit time. */ readonly unlessNativeThreadOwned?: Pick< OrchestrationV2ProviderThread, @@ -256,7 +260,7 @@ const layerBase: Layer.Layer< // Serialize commit and publish; reordered publication makes clients drop earlier sequences. const publishLane = yield* Semaphore.make(1); - const commitThenPublish = ( + const commitThenPublishNow = ( transaction: Effect.Effect, publish: (committed: A) => Effect.Effect, ) => @@ -409,6 +413,7 @@ const layerBase: Layer.Layer< const writeEffect = Effect.fn("orchestrationV2.EventSink.write")(function* ( input: Parameters[0], + grouped = false, ) { yield* Effect.annotateCurrentSpan({ "orchestration_v2.command_id": input.commandId ?? null, @@ -416,7 +421,7 @@ const layerBase: Layer.Layer< "orchestration_v2.thread_id": input.events[0]?.threadId ?? null, }); - const { committed } = yield* commitThenPublish( + const { committed } = yield* (grouped ? commitThenPublishNow : commitThenPublish)( Effect.gen(function* () { // Check ownership in the same transaction that claims the native session. const imported = input.unlessNativeThreadOwned; @@ -877,18 +882,64 @@ const layerBase: Layer.Layer< ); }; - return EventSinkV2.of({ - write: (input) => - writeEffect({ ...input, effects: [] }).pipe( - Effect.mapError( - (cause) => - new EventSinkWriteError({ - eventCount: input.events.length, - ...(input.commandId === undefined ? {} : { commandId: input.commandId }), - cause, - }), - ), + const rawWindow = yield* Config.String("T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS").pipe( + Config.withDefault("0"), + Effect.orDie, + ); + const window = Number(rawWindow); + const batch = + Number.isInteger(window) && window > 0 + ? yield* ProviderCommitBatch.make( + (events) => + writeEffect({ events, effects: [], guardPendingUserInputCancellations: true }, true), + Math.min(window, 25), + ) + : undefined; + + const commitThenPublish: typeof commitThenPublishNow = (transaction, publish) => { + const effect = commitThenPublishNow(transaction, publish); + if (batch === undefined) return effect; + // A caller holding the writer cannot wait for another fiber to acquire it. + return Effect.flatMap(Effect.serviceOption(sql.transactionService), (existing) => + Option.isSome(existing) ? effect : batch.barrier(effect), + ); + }; + + const write: EventSinkV2Shape["write"] = (input) => { + const immediate = writeEffect({ ...input, effects: [] }); + const eligible = + batch !== undefined && + input.batch === true && + input.commandId === undefined && + input.unlessNativeThreadOwned === undefined && + input.events.length > 0 && + input.events.every( + (event) => + event.type === "node.updated" || + event.type === "message.updated" || + event.type === "turn-item.updated" || + event.type === "plan.updated", + ); + return ( + eligible + ? Effect.flatMap(Effect.serviceOption(sql.transactionService), (existing) => + Option.isSome(existing) ? immediate : batch.write(input.events), + ) + : immediate + ).pipe( + Effect.mapError( + (cause) => + new EventSinkWriteError({ + eventCount: input.events.length, + ...(input.commandId === undefined ? {} : { commandId: input.commandId }), + cause, + }), ), + ); + }; + + return EventSinkV2.of({ + write, writeWithEffects: (input) => writeEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 475e0a0e4ae7b..176d873eee834 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -30,6 +30,8 @@ import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Arr from "effect/Array"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Exit from "effect/Exit"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -325,6 +327,112 @@ it.effect("paces scheduled compaction while preserving retention state and fixed }).pipe(Effect.scoped, Effect.provide(layerTest)), ); +it.effect( + "group commits streaming writes atomically, preserves barriers and bypasses existing transactions", + () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const store = yield* EventStore.EventStoreV2; + const sql = yield* SqlClient.SqlClient; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threads = [1, 2].map((id) => + makeThread(ThreadId.make(`thread:group-commit:${id}`), now), + ); + yield* sink.write({ + events: threads.map((thread, id) => + threadCreatedEvent({ id: `group-created:${id}`, thread, now }), + ), + }); + const message = (index: number, id: string): OrchestrationV2DomainEvent => ({ + id: EventId.make(id), + type: "message.updated", + threadId: threads[index]!.id, + occurredAt: now, + payload: { + id: MessageId.make(`group-message:${index}`), + threadId: threads[index]!.id, + runId: null, + nodeId: null, + createdBy: "agent", + creationSource: "web", + role: "assistant", + text: id, + streaming: true, + attachments: [], + createdAt: now, + updatedAt: now, + }, + }); + const highWater = yield* sink.latestSequence(); + const first = yield* sink + .write({ batch: true, events: [message(0, "group-duplicate")] }) + .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); + const second = yield* sink + .write({ batch: true, events: [message(1, "group-duplicate")] }) + .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); + yield* TestClock.adjust(0); + assert.equal(yield* sink.latestSequence(), highWater); + yield* TestClock.adjust(25); + assert.isTrue(Exit.isFailure(yield* Fiber.join(first))); + assert.isTrue(Exit.isFailure(yield* Fiber.join(second))); + assert.equal(yield* sink.latestSequence(), highWater); + assert.equal( + (yield* projections.getThreadRecords(threads[0]!.id, ["messages"])).messages.length, + 0, + ); + assert.equal( + (yield* projections.getThreadRecords(threads[1]!.id, ["messages"])).messages.length, + 0, + ); + + const live = yield* sink + .stream({ afterSequence: highWater }) + .pipe(Stream.take(3), Stream.runCollect, Effect.forkScoped({ startImmediately: true })); + const pending = yield* sink + .write({ batch: true, events: [message(0, "group-first")] }) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* TestClock.adjust(0); + const [control] = yield* sink.write({ + events: [ + { + id: EventId.make("group-control"), + threadId: threads[1]!.id, + occurredAt: now, + payload: threads[1]!, + type: "thread.metadata-updated", + }, + ], + }); + const [committed] = yield* Fiber.join(pending); + assert.isBelow(committed!.sequence, control!.sequence); + // Holding the writer must not wait for a batch worker to acquire that same connection. + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sink.write({ batch: true, events: [message(1, "group-nested")] }); + assert.equal( + (yield* projections.getThreadRecords(threads[1]!.id, ["messages"])).messages[0]?.text, + "group-nested", + ); + }), + ); + const published = yield* Fiber.join(live); + const replayed = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect); + assert.deepEqual( + published.map((stored) => stored.event.id), + ["group-first", "group-control", "group-nested"], + ); + assert.deepEqual(replayed, published); + }).pipe( + Effect.provide(Layer.fresh(layerTest)), + Effect.provideService( + ConfigProvider.ConfigProvider, + ConfigProvider.fromUnknown({ T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: "25" }), + ), + Effect.scoped, + ), +); + it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { it.effect("projects oversized tool bodies before both replay and live RPC retention", () => Effect.scoped( diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts new file mode 100644 index 0000000000000..d6a8e24f1c402 --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -0,0 +1,207 @@ +import { assert, it } from "@effect/vitest"; +import { + EventId, + MessageId, + ThreadId, + type OrchestrationV2DomainEvent, + type OrchestrationV2StoredEvent, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; +import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; + +function event(id: number, text = "delta"): OrchestrationV2DomainEvent { + const threadId = ThreadId.make(`thread:${id}`); + const now = DateTime.makeUnsafe("2026-11-03T00:00:00Z"); + return { + id: EventId.make(`event:${id}`), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + id: MessageId.make(`message:${id}`), + threadId, + runId: null, + nodeId: null, + createdBy: "agent", + creationSource: "web", + role: "assistant", + text, + attachments: [], + streaming: true, + createdAt: now, + updatedAt: now, + }, + }; +} + +it.effect.each([false, true])( + "acknowledges shared commits through publication with cancellation=%s", + (cancel) => + Effect.gen(function* () { + const commits: Array> = []; + const publishing = yield* Deferred.make(); + const published = yield* Deferred.make(); + const batch = yield* ProviderCommitBatch.make( + (events) => + Effect.gen(function* () { + commits.push(events); + yield* Deferred.succeed(publishing, undefined); + yield* Deferred.await(published); + return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); + }), + 25, + ); + const first = yield* batch + .write([event(1)]) + .pipe(Effect.forkScoped({ startImmediately: true })); + const second = yield* batch + .write([event(2)]) + .pipe(Effect.forkScoped({ startImmediately: true })); + assert.deepEqual(commits, []); + yield* TestClock.adjust(25); + yield* Deferred.await(publishing); + assert.isUndefined(yield* Effect.sync(() => first.pollUnsafe())); + const interrupted = cancel + ? yield* Fiber.interrupt(second).pipe(Effect.forkScoped({ startImmediately: true })) + : undefined; + yield* Deferred.succeed(published, undefined); + assert.deepEqual( + (yield* Fiber.join(first)).map((stored) => stored.event.id), + ["event:1"], + ); + if (interrupted !== undefined) { + yield* Fiber.join(interrupted); + assert.isTrue(Exit.isFailure(yield* Fiber.await(second))); + } else { + assert.deepEqual( + (yield* Fiber.join(second)).map((stored) => stored.event.id), + ["event:2"], + ); + } + assert.equal(commits.length, 1); + }).pipe(Effect.scoped), +); + +it.effect("flushes before a control write and does not let an old timer flush a new window", () => + Effect.gen(function* () { + const order: string[] = []; + const batch = yield* ProviderCommitBatch.make( + (events) => + Effect.sync(() => { + order.push(...events.map((event) => event.id)); + return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); + }), + 25, + ); + const first = yield* batch + .write([event(1)]) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* TestClock.adjust(10); + yield* batch.barrier(Effect.sync(() => order.push("stop"))); + yield* Fiber.join(first); + const next = yield* batch.write([event(2)]).pipe(Effect.forkScoped({ startImmediately: true })); + yield* TestClock.adjust(15); + assert.deepEqual(order, ["event:1", "stop"]); + yield* TestClock.adjust(10); + yield* Fiber.join(next); + assert.deepEqual(order, ["event:1", "stop", "event:2"]); + }).pipe(Effect.scoped), +); + +it.effect("fails every affected writer immediately and accepts the next batch", () => + Effect.gen(function* () { + let fail = true; + const batch = yield* ProviderCommitBatch.make( + (events) => + fail + ? Effect.fail("disk full") + : Effect.succeed( + events.map((event, index) => ({ sequence: index + 1, commandId: null, event })), + ), + 25, + ); + const first = yield* batch + .write([event(1)]) + .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); + const second = yield* batch + .write([event(2)]) + .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); + yield* TestClock.adjust(25); + assert.deepEqual(yield* Fiber.join(first), Exit.fail("disk full")); + assert.deepEqual(yield* Fiber.join(second), Exit.fail("disk full")); + fail = false; + const next = yield* batch.write([event(3)]).pipe(Effect.forkScoped({ startImmediately: true })); + yield* batch.barrier(Effect.void); + assert.equal((yield* Fiber.join(next))[0]?.event.id, "event:3"); + }).pipe(Effect.scoped), +); + +it.effect("bounds groups by event count and JSON bytes without waiting for the timer", () => + Effect.gen(function* () { + const commits: Array> = []; + const batch = yield* ProviderCommitBatch.make( + (events) => + Effect.sync(() => { + commits.push(events); + return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); + }), + 25, + ); + const first = yield* batch + .write(Array.from({ length: 63 }, (_, i) => event(i))) + .pipe(Effect.forkScoped({ startImmediately: true })); + const second = yield* batch + .write([event(64)]) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Fiber.join(first); + yield* Fiber.join(second); + assert.equal(commits[0]?.length, 64); + const small = yield* batch + .write([event(65)]) + .pipe(Effect.forkScoped({ startImmediately: true })); + const large = yield* batch.write([event(66, "x".repeat(1024 * 1024))]); + yield* Fiber.join(small); + assert.deepEqual( + commits.map((events) => events.length), + [64, 1, 1], + ); + assert.equal(large.length, 1); + }).pipe(Effect.scoped), +); + +it.effect( + "removes interrupted queued writes and flushes outstanding writes when its scope closes", + () => + Effect.gen(function* () { + const commits: string[] = []; + const write = ( + events: ReadonlyArray, + ): Effect.Effect> => + Effect.sync(() => { + commits.push(...events.map((event) => event.id)); + return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); + }); + const batch = yield* ProviderCommitBatch.make(write, 25); + const cancelled = yield* batch + .write([event(1)]) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Fiber.interrupt(cancelled); + yield* batch.barrier(Effect.void); + assert.deepEqual(commits, []); + const parentScope = yield* Effect.scope; + yield* Effect.scoped( + Effect.gen(function* () { + const closing = yield* ProviderCommitBatch.make(write, 25); + yield* closing + .write([event(2)]) + .pipe(Effect.forkIn(parentScope, { startImmediately: true })); + }), + ); + assert.deepEqual(commits, ["event:2"]); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts new file mode 100644 index 0000000000000..760fcb589848c --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -0,0 +1,131 @@ +import type { OrchestrationV2DomainEvent, OrchestrationV2StoredEvent } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Semaphore from "effect/Semaphore"; +import * as Schema from "effect/Schema"; +import * as Clock from "effect/Clock"; + +const MAX_EVENTS = 64; +const MAX_BYTES = 1024 * 1024; +const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); + +export const make = Effect.fn("ProviderCommitBatch.make")(function* ( + commit: ( + events: ReadonlyArray, + ) => Effect.Effect, E>, + windowMs: number, +) { + const scope = yield* Effect.scope; + const lane = yield* Semaphore.make(1); + type Pending = { + readonly events: ReadonlyArray; + readonly bytes: number; + readonly result: Deferred.Deferred, E>; + }; + let pending: Array = []; + let eventCount = 0; + let byteCount = 0; + let generation = 0; + let closed = false; + let openedAt = 0; + + // The lane stays held through commit, publication and acknowledgement. + const flush = Effect.fn("orchestrationV2.EventSink.flushProviderBatch")(function* ( + reason: "deadline" | "limit" | "barrier" | "shutdown", + ) { + if (pending.length === 0) return; + const batch = pending; + pending = []; + yield* Effect.annotateCurrentSpan({ + "orchestration_v2.batch_requests": batch.length, + "orchestration_v2.event_count": eventCount, + "orchestration_v2.queued_json_bytes": byteCount, + "orchestration_v2.batch_threads": new Set( + batch.flatMap((entry) => entry.events.map((event) => event.threadId)), + ).size, + "orchestration_v2.commit_window_ms": windowMs, + "orchestration_v2.batch_wait_ms": (yield* Clock.currentTimeMillis) - openedAt, + "orchestration_v2.batch_flush_reason": reason, + }); + eventCount = 0; + byteCount = 0; + generation++; + const exit = yield* Effect.exit(commit(batch.flatMap((entry) => entry.events))); + // Unguarded snapshots preserve one stored event per input, in input order. + let offset = 0; + for (const entry of batch) { + yield* Deferred.done( + entry.result, + Exit.isFailure(exit) + ? exit + : Exit.succeed(exit.value.slice(offset, offset + entry.events.length)), + ); + offset += entry.events.length; + } + }, Effect.uninterruptible); + + yield* Effect.addFinalizer(() => + lane.withPermits(1)( + Effect.sync(() => { + closed = true; + }).pipe(Effect.andThen(flush("shutdown"))), + ), + ); + + // Per-write calls share the batch's tracing boundary. + const write = Effect.fnUntraced(function* (events: ReadonlyArray) { + const result = yield* Deferred.make, E>(); + const entry: Pending = { events, result, bytes: Buffer.byteLength(yield* encodeQueue(events)) }; + yield* lane.withPermits(1)( + Effect.gen(function* () { + if (closed) return yield* Effect.interrupt; + if (eventCount + events.length > MAX_EVENTS || byteCount + entry.bytes > MAX_BYTES) { + yield* flush("limit"); + } + const opensWindow = pending.length === 0; + if (opensWindow) openedAt = yield* Clock.currentTimeMillis; + pending.push(entry); + eventCount += events.length; + byteCount += entry.bytes; + if (eventCount >= MAX_EVENTS || byteCount >= MAX_BYTES) { + yield* flush("limit"); + } else if (opensWindow) { + const token = ++generation; + yield* Effect.sleep(windowMs).pipe( + Effect.andThen( + lane.withPermits(1)( + Effect.suspend(() => (token === generation ? flush("deadline") : Effect.void)), + ), + ), + Effect.interruptible, + Effect.forkIn(scope, { startImmediately: true }), + ); + } + }), + ); + return yield* Deferred.await(result).pipe( + Effect.onInterrupt(() => + lane.withPermits(1)( + Effect.sync(() => { + const index = pending.indexOf(entry); + if (index < 0) return; + pending.splice(index, 1); + eventCount -= events.length; + byteCount -= entry.bytes; + if (pending.length === 0) generation++; + }), + ), + ), + ); + }); + + const barrier = (effect: Effect.Effect) => + lane.withPermits(1)( + Effect.suspend(() => + closed ? Effect.interrupt : flush("barrier").pipe(Effect.andThen(effect)), + ), + ); + + return { write, barrier }; +}); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index f27580d909aff..97d7c1680ef00 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -697,6 +697,7 @@ export const layer: Layer.Layer< if (input.writeIfRunCurrent === undefined) { return yield* eventSink .write({ + batch: true, guardPendingUserInputCancellations: true, ...(input.commandId === undefined ? {} : { commandId: input.commandId }), events, From 1064bc94a74fb13f2fb8bfee193c33a282fb129e Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Tue, 3 Nov 2026 23:03:17 -0600 Subject: [PATCH 2/9] perf(server): batch ordered snapshots from one provider stream --- apps/server/src/orchestration-v2/EventSink.ts | 121 ++--- .../FoundationPersistence.test.ts | 297 ++++++------- .../ProviderCommitBatch.test.ts | 261 +++++------ .../orchestration-v2/ProviderCommitBatch.ts | 217 +++++---- .../ProviderEventIngestor.test.ts | 8 +- .../orchestration-v2/ProviderEventIngestor.ts | 1 - .../orchestration-v2/RunExecutionService.ts | 418 +++++++++--------- .../CodexReplayFixtures.integration.test.ts | 49 +- 8 files changed, 681 insertions(+), 691 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index f99129e0157ff..f110ef513a919 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -13,7 +13,6 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; -import * as Config from "effect/Config"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -23,6 +22,7 @@ import * as Semaphore from "effect/Semaphore"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; +import * as SqlError from "effect/sql/SqlError"; import { replayAndBufferProjectedLiveEvents } from "./LiveStreamBudget.ts"; import type { UnsequencedProjectEvent } from "../persistence/OrchestrationEventStore.ts"; @@ -35,7 +35,6 @@ import * as EventStore from "./EventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; -import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; export class EventSinkWriteError extends Schema.TaggedError()( "EventSinkWriteError", @@ -68,10 +67,22 @@ export class EventSinkStreamError extends Schema.TaggedError>; + eventCount: number; +} +const CurrentProviderBatch = Context.Reference( + "t3/orchestration-v2/EventSink/CurrentProviderBatch", + { defaultValue: () => undefined }, +); + export interface EventSinkV2Shape { + /** Normalize and persist an ordered group against one transaction; publish after its commit. */ + readonly withProviderBatch: ( + effect: Effect.Effect, + ) => Effect.Effect; + readonly write: (input: { - /** Group independent provider snapshots; return only after commit and publication. */ - readonly batch?: boolean; /** Return no events if another thread owns this native session at commit time. */ readonly unlessNativeThreadOwned?: Pick< OrchestrationV2ProviderThread, @@ -413,7 +424,6 @@ const layerBase: Layer.Layer< const writeEffect = Effect.fn("orchestrationV2.EventSink.write")(function* ( input: Parameters[0], - grouped = false, ) { yield* Effect.annotateCurrentSpan({ "orchestration_v2.command_id": input.commandId ?? null, @@ -421,7 +431,9 @@ const layerBase: Layer.Layer< "orchestration_v2.thread_id": input.events[0]?.threadId ?? null, }); - const { committed } = yield* (grouped ? commitThenPublishNow : commitThenPublish)( + const batch = yield* CurrentProviderBatch; + if (batch !== undefined) batch.eventCount += input.events.length; + const { committed } = yield* commitThenPublish( Effect.gen(function* () { // Check ownership in the same transaction that claims the native session. const imported = input.unlessNativeThreadOwned; @@ -882,64 +894,61 @@ const layerBase: Layer.Layer< ); }; - const rawWindow = yield* Config.String("T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS").pipe( - Config.withDefault("0"), - Effect.orDie, - ); - const window = Number(rawWindow); - const batch = - Number.isInteger(window) && window > 0 - ? yield* ProviderCommitBatch.make( - (events) => - writeEffect({ events, effects: [], guardPendingUserInputCancellations: true }, true), - Math.min(window, 25), - ) - : undefined; - - const commitThenPublish: typeof commitThenPublishNow = (transaction, publish) => { - const effect = commitThenPublishNow(transaction, publish); - if (batch === undefined) return effect; - // A caller holding the writer cannot wait for another fiber to acquire it. - return Effect.flatMap(Effect.serviceOption(sql.transactionService), (existing) => - Option.isSome(existing) ? effect : batch.barrier(effect), + const commitThenPublish: typeof commitThenPublishNow = (transaction, publish) => + Effect.flatMap(CurrentProviderBatch, (batch) => + batch === undefined + ? commitThenPublishNow(transaction, publish) + : transaction.pipe( + Effect.tap((committed) => + Effect.sync(() => { + batch.publish.push(publish(committed)); + }), + ), + ), ); - }; - const write: EventSinkV2Shape["write"] = (input) => { - const immediate = writeEffect({ ...input, effects: [] }); - const eligible = - batch !== undefined && - input.batch === true && - input.commandId === undefined && - input.unlessNativeThreadOwned === undefined && - input.events.length > 0 && - input.events.every( - (event) => - event.type === "node.updated" || - event.type === "message.updated" || - event.type === "turn-item.updated" || - event.type === "plan.updated", - ); - return ( - eligible - ? Effect.flatMap(Effect.serviceOption(sql.transactionService), (existing) => - Option.isSome(existing) ? immediate : batch.write(input.events), - ) - : immediate + const withProviderBatch: EventSinkV2Shape["withProviderBatch"] = Effect.fn( + "orchestrationV2.EventSink.providerBatch", + )(function* (effect: Effect.Effect) { + if ((yield* CurrentProviderBatch) !== undefined) return yield* effect; + const batch: ProviderBatchPublication = { publish: [], eventCount: 0 }; + let started = false; + const result = yield* commitThenPublishNow( + Effect.sync(() => { + started = true; + }).pipe(Effect.andThen(Effect.provideService(effect, CurrentProviderBatch, batch))), + () => Effect.forEach(batch.publish, (publish) => publish, { discard: true }), ).pipe( + // Normalization mutates run-local state, so only retry before its transaction body starts. + Effect.retry({ + while: (cause) => + !started && SqlError.isSqlError(cause) && cause.reason._tag === "LockTimeoutError", + times: 2, + }), Effect.mapError( - (cause) => - new EventSinkWriteError({ - eventCount: input.events.length, - ...(input.commandId === undefined ? {} : { commandId: input.commandId }), - cause, - }), + (cause) => new EventSinkWriteError({ eventCount: batch.eventCount, cause }), ), ); - }; + yield* Effect.annotateCurrentSpan({ + "orchestration_v2.event_count": batch.eventCount, + "orchestration_v2.batch_writes": batch.publish.length, + }); + return result; + }); return EventSinkV2.of({ - write, + withProviderBatch, + write: (input) => + writeEffect({ ...input, effects: [] }).pipe( + Effect.mapError( + (cause) => + new EventSinkWriteError({ + eventCount: input.events.length, + ...(input.commandId === undefined ? {} : { commandId: input.commandId }), + cause, + }), + ), + ), writeWithEffects: (input) => writeEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 176d873eee834..9414f77bff966 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -30,7 +30,6 @@ import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Arr from "effect/Array"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; -import * as ConfigProvider from "effect/ConfigProvider"; import * as Exit from "effect/Exit"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; @@ -328,34 +327,27 @@ it.effect("paces scheduled compaction while preserving retention state and fixed ); it.effect( - "group commits streaming writes atomically, preserves barriers and bypasses existing transactions", + "commits an ordered provider batch atomically and publishes only after the outer commit", () => Effect.gen(function* () { const sink = yield* EventSink.EventSinkV2; - const store = yield* EventStore.EventStoreV2; - const sql = yield* SqlClient.SqlClient; const projections = yield* ProjectionStore.ProjectionStoreV2; + const store = yield* EventStore.EventStoreV2; const now = yield* DateTime.now; - const threads = [1, 2].map((id) => - makeThread(ThreadId.make(`thread:group-commit:${id}`), now), - ); - yield* sink.write({ - events: threads.map((thread, id) => - threadCreatedEvent({ id: `group-created:${id}`, thread, now }), - ), - }); - const message = (index: number, id: string): OrchestrationV2DomainEvent => ({ + const thread = makeThread(ThreadId.make("thread:single-stream-batch"), now); + yield* sink.write({ events: [threadCreatedEvent({ id: "batch-created", thread, now })] }); + const message = (id: string): OrchestrationV2DomainEvent => ({ id: EventId.make(id), type: "message.updated", - threadId: threads[index]!.id, + threadId: thread.id, occurredAt: now, payload: { - id: MessageId.make(`group-message:${index}`), - threadId: threads[index]!.id, + id: MessageId.make("batch-message"), + threadId: thread.id, runId: null, nodeId: null, createdBy: "agent", - creationSource: "web", + creationSource: "provider", role: "assistant", text: id, streaming: true, @@ -365,72 +357,57 @@ it.effect( }, }); const highWater = yield* sink.latestSequence(); - const first = yield* sink - .write({ batch: true, events: [message(0, "group-duplicate")] }) - .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); - const second = yield* sink - .write({ batch: true, events: [message(1, "group-duplicate")] }) - .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); - yield* TestClock.adjust(0); - assert.equal(yield* sink.latestSequence(), highWater); - yield* TestClock.adjust(25); - assert.isTrue(Exit.isFailure(yield* Fiber.join(first))); - assert.isTrue(Exit.isFailure(yield* Fiber.join(second))); + const failed = yield* sink + .withProviderBatch( + Effect.gen(function* () { + yield* sink.write({ events: [message("batch-duplicate")] }); + yield* sink.write({ events: [message("batch-duplicate")] }); + }), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(failed)); assert.equal(yield* sink.latestSequence(), highWater); - assert.equal( - (yield* projections.getThreadRecords(threads[0]!.id, ["messages"])).messages.length, - 0, - ); - assert.equal( - (yield* projections.getThreadRecords(threads[1]!.id, ["messages"])).messages.length, - 0, - ); + assert.isEmpty((yield* projections.getThreadRecords(thread.id, ["messages"])).messages); - const live = yield* sink - .stream({ afterSequence: highWater }) - .pipe(Stream.take(3), Stream.runCollect, Effect.forkScoped({ startImmediately: true })); - const pending = yield* sink - .write({ batch: true, events: [message(0, "group-first")] }) + const firstPublished = yield* Deferred.make(); + const writesApplied = yield* Deferred.make(); + const release = yield* Deferred.make(); + const live = yield* sink.stream({ afterSequence: highWater }).pipe( + Stream.tap(() => Deferred.succeed(firstPublished, undefined)), + Stream.take(2), + Stream.runCollect, + Effect.forkScoped({ startImmediately: true }), + ); + const batch = yield* sink + .withProviderBatch( + Effect.gen(function* () { + yield* sink.write({ events: [message("batch-first")] }); + yield* sink.write({ events: [message("batch-second")] }); + assert.equal( + (yield* projections.getThreadRecords(thread.id, ["messages"])).messages[0]?.text, + "batch-second", + ); + yield* Deferred.succeed(writesApplied, undefined); + yield* Deferred.await(release); + }), + ) .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(writesApplied); yield* TestClock.adjust(0); - const [control] = yield* sink.write({ - events: [ - { - id: EventId.make("group-control"), - threadId: threads[1]!.id, - occurredAt: now, - payload: threads[1]!, - type: "thread.metadata-updated", - }, - ], - }); - const [committed] = yield* Fiber.join(pending); - assert.isBelow(committed!.sequence, control!.sequence); - // Holding the writer must not wait for a batch worker to acquire that same connection. - yield* sql.withTransaction( - Effect.gen(function* () { - yield* sink.write({ batch: true, events: [message(1, "group-nested")] }); - assert.equal( - (yield* projections.getThreadRecords(threads[1]!.id, ["messages"])).messages[0]?.text, - "group-nested", - ); - }), - ); + assert.isFalse(yield* Deferred.isDone(firstPublished)); + assert.isUndefined(yield* Effect.sync(() => batch.pollUnsafe())); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(batch); const published = yield* Fiber.join(live); - const replayed = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect); assert.deepEqual( published.map((stored) => stored.event.id), - ["group-first", "group-control", "group-nested"], + ["batch-first", "batch-second"], ); - assert.deepEqual(replayed, published); - }).pipe( - Effect.provide(Layer.fresh(layerTest)), - Effect.provideService( - ConfigProvider.ConfigProvider, - ConfigProvider.fromUnknown({ T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: "25" }), - ), - Effect.scoped, - ), + assert.deepEqual( + yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect), + published, + ); + }).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped), ); it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { @@ -3322,87 +3299,101 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { ); }); -it.effect("publishes live events in commit order across concurrent writers", () => - Effect.gen(function* () { - const firstCommitted = yield* Deferred.make(); - const releaseFirst = yield* Deferred.make(); - const layerPausingOutbox = Layer.effect( - EffectOutbox.EffectOutboxV2, - Effect.gen(function* () { - const delegate = yield* EffectOutbox.EffectOutboxV2; - return EffectOutbox.EffectOutboxV2.of({ - ...delegate, - notifyAvailable: (count) => - Deferred.succeed(firstCommitted, undefined).pipe( - Effect.andThen(Deferred.await(releaseFirst)), - Effect.andThen(delegate.notifyAvailable(count)), - ), - }); - }), - ).pipe(Layer.provide(layerEffectOutboxProvided)); - const layerEventSink = EventSink.layerFromStores.pipe( - Layer.provide( - Layer.mergeAll( - layerStoresProvided, - layerPausingOutbox, - layerCommandReceiptStoreProvided, - ProjectStore.layer.pipe(Layer.provide(layerDatabase)), - TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)), +it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)( + "publishes live events in commit order across concurrent writers with %s", + (mode) => + Effect.gen(function* () { + const firstCommitted = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); + const layerPausingOutbox = Layer.effect( + EffectOutbox.EffectOutboxV2, + Effect.gen(function* () { + const delegate = yield* EffectOutbox.EffectOutboxV2; + return EffectOutbox.EffectOutboxV2.of({ + ...delegate, + notifyAvailable: (count) => + Deferred.succeed(firstCommitted, undefined).pipe( + Effect.andThen(Deferred.await(releaseFirst)), + Effect.andThen(delegate.notifyAvailable(count)), + ), + }); + }), + ).pipe(Layer.provide(layerEffectOutboxProvided)); + const layerEventSink = EventSink.layerFromStores.pipe( + Layer.provide( + Layer.mergeAll( + layerStoresProvided, + layerPausingOutbox, + layerCommandReceiptStoreProvided, + ProjectStore.layer.pipe(Layer.provide(layerDatabase)), + TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)), + ), ), - ), - ); + ); - yield* Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const now = yield* DateTime.now; - const first = makeThread(ThreadId.make("thread:foundation-publish-order:first"), now); - const second = makeThread(ThreadId.make("thread:foundation-publish-order:second"), now); - const published = yield* eventSink - .stream({ afterSequence: yield* eventSink.latestSequence() }) - .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped({ startImmediately: true })); - - const firstWrite = yield* eventSink - .writeWithEffects({ - events: [ - threadCreatedEvent({ id: "event:foundation-publish-order:first", thread: first, now }), - ], - effects: [ - { - id: "effect:foundation-publish-order:first", - commandId: CommandId.make("command:foundation-publish-order:first"), - threadId: first.id, - request: { type: "terminal.cleanup" }, - }, - ], - }) - .pipe(Effect.forkScoped); - yield* Deferred.await(firstCommitted); - const secondWrite = yield* eventSink - .write({ - events: [ - threadCreatedEvent({ - id: "event:foundation-publish-order:second", - thread: second, - now, - }), - ], - }) - .pipe( - Effect.provideService(Scheduler.MaxOpsBeforeYield, Number.POSITIVE_INFINITY), - Effect.forkScoped, - ); - yield* Effect.yieldNow; - yield* Deferred.succeed(releaseFirst, undefined); - yield* Fiber.join(firstWrite); - yield* Fiber.join(secondWrite); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const first = makeThread(ThreadId.make("thread:foundation-publish-order:first"), now); + const second = makeThread(ThreadId.make("thread:foundation-publish-order:second"), now); + const published = yield* eventSink + .stream({ afterSequence: yield* eventSink.latestSequence() }) + .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped({ startImmediately: true })); - const sequences = Array.from(yield* Fiber.join(published), (stored) => stored.sequence); - assert.deepEqual( - sequences, - [...sequences].sort((left, right) => left - right), - ); - }).pipe(Effect.provide(layerEventSink)); - }).pipe(Effect.provide(layerDatabase)), + const firstWrite = yield* eventSink + .writeWithEffects({ + events: [ + threadCreatedEvent({ + id: "event:foundation-publish-order:first", + thread: first, + now, + }), + ], + effects: [ + { + id: "effect:foundation-publish-order:first", + commandId: CommandId.make("command:foundation-publish-order:first"), + threadId: first.id, + request: { type: "terminal.cleanup" }, + }, + ], + }) + .pipe( + (write) => (mode === "direct" ? write : eventSink.withProviderBatch(write)), + Effect.forkScoped, + ); + yield* Deferred.await(firstCommitted); + const secondWrite = yield* eventSink + .write({ + events: [ + threadCreatedEvent({ + id: "event:foundation-publish-order:second", + thread: second, + now, + }), + ], + }) + .pipe( + Effect.provideService(Scheduler.MaxOpsBeforeYield, Number.POSITIVE_INFINITY), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const interrupted = + mode === "cancel-provider-batch" + ? yield* Fiber.interrupt(firstWrite).pipe(Effect.forkScoped({ startImmediately: true })) + : undefined; + yield* Deferred.succeed(releaseFirst, undefined); + if (interrupted !== undefined) yield* Fiber.join(interrupted); + else yield* Fiber.join(firstWrite); + yield* Fiber.join(secondWrite); + + const sequences = Array.from(yield* Fiber.join(published), (stored) => stored.sequence); + assert.deepEqual( + sequences, + [...sequences].sort((left, right) => left - right), + ); + }).pipe(Effect.provide(layerEventSink)); + }).pipe(Effect.provide(layerDatabase)), ); it.effect.each(["events", "receipts"] as const)( diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index d6a8e24f1c402..78cc9c5e08391 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -1,34 +1,30 @@ import { assert, it } from "@effect/vitest"; -import { - EventId, - MessageId, - ThreadId, - type OrchestrationV2DomainEvent, - type OrchestrationV2StoredEvent, -} from "@t3tools/contracts"; +import { MessageId, ProviderDriverKind, ProviderTurnId, ThreadId } from "@t3tools/contracts"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; -import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; -function event(id: number, text = "delta"): OrchestrationV2DomainEvent { - const threadId = ThreadId.make(`thread:${id}`); +const driver = ProviderDriverKind.make("codex"); +function event(index: number, text = `delta:${index}`): ProviderAdapter.ProviderAdapterV2Event { + const threadId = ThreadId.make("single-thread"); const now = DateTime.makeUnsafe("2026-11-03T00:00:00Z"); return { - id: EventId.make(`event:${id}`), type: "message.updated", - threadId, - occurredAt: now, - payload: { - id: MessageId.make(`message:${id}`), + driver, + message: { + id: MessageId.make("single-message"), threadId, runId: null, nodeId: null, createdBy: "agent", - creationSource: "web", + creationSource: "provider", role: "assistant", text, attachments: [], @@ -38,170 +34,111 @@ function event(id: number, text = "delta"): OrchestrationV2DomainEvent { }, }; } +const control: ProviderAdapter.ProviderAdapterV2Event = { + type: "background_work.updated", + driver, + providerTurnId: ProviderTurnId.make("turn"), + pending: false, +}; -it.effect.each([false, true])( - "acknowledges shared commits through publication with cancellation=%s", - (cancel) => +it.effect( + "collects successive updates from one stream and flushes before control without waiting for the window", + () => Effect.gen(function* () { - const commits: Array> = []; - const publishing = yield* Deferred.make(); - const published = yield* Deferred.make(); - const batch = yield* ProviderCommitBatch.make( - (events) => - Effect.gen(function* () { - commits.push(events); - yield* Deferred.succeed(publishing, undefined); - yield* Deferred.await(published); - return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); - }), - 25, + const input = yield* Queue.unbounded(); + const output = + yield* Queue.unbounded>(); + const consumer = yield* ProviderCommitBatch.group(Stream.fromQueue(input), 25).pipe( + Stream.runForEach((batch) => Queue.offer(output, batch.events)), + Effect.forkScoped, ); - const first = yield* batch - .write([event(1)]) - .pipe(Effect.forkScoped({ startImmediately: true })); - const second = yield* batch - .write([event(2)]) - .pipe(Effect.forkScoped({ startImmediately: true })); - assert.deepEqual(commits, []); - yield* TestClock.adjust(25); - yield* Deferred.await(publishing); - assert.isUndefined(yield* Effect.sync(() => first.pollUnsafe())); - const interrupted = cancel - ? yield* Fiber.interrupt(second).pipe(Effect.forkScoped({ startImmediately: true })) - : undefined; - yield* Deferred.succeed(published, undefined); - assert.deepEqual( - (yield* Fiber.join(first)).map((stored) => stored.event.id), - ["event:1"], - ); - if (interrupted !== undefined) { - yield* Fiber.join(interrupted); - assert.isTrue(Exit.isFailure(yield* Fiber.await(second))); - } else { - assert.deepEqual( - (yield* Fiber.join(second)).map((stored) => stored.event.id), - ["event:2"], - ); - } - assert.equal(commits.length, 1); + yield* Queue.offer(input, event(1)); + yield* TestClock.adjust(5); + yield* Queue.offer(input, event(2)); + yield* Queue.offer(input, control); + const first = yield* Queue.take(output); + const second = yield* Queue.take(output); + assert.deepEqual(first, [event(1), event(2)]); + assert.deepEqual(second, [control]); + yield* Queue.end(input); + yield* Fiber.join(consumer); }).pipe(Effect.scoped), ); -it.effect("flushes before a control write and does not let an old timer flush a new window", () => +it.effect("flushes a sparse stream on the deadline and restarts the next window", () => Effect.gen(function* () { - const order: string[] = []; - const batch = yield* ProviderCommitBatch.make( - (events) => - Effect.sync(() => { - order.push(...events.map((event) => event.id)); - return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); - }), - 25, + const input = yield* Queue.unbounded(); + const output = yield* Queue.unbounded>(); + const consumer = yield* ProviderCommitBatch.group(Stream.fromQueue(input), 25).pipe( + Stream.runForEach((batch) => Queue.offer(output, batch.events)), + Effect.forkScoped, ); - const first = yield* batch - .write([event(1)]) - .pipe(Effect.forkScoped({ startImmediately: true })); - yield* TestClock.adjust(10); - yield* batch.barrier(Effect.sync(() => order.push("stop"))); - yield* Fiber.join(first); - const next = yield* batch.write([event(2)]).pipe(Effect.forkScoped({ startImmediately: true })); - yield* TestClock.adjust(15); - assert.deepEqual(order, ["event:1", "stop"]); - yield* TestClock.adjust(10); - yield* Fiber.join(next); - assert.deepEqual(order, ["event:1", "stop", "event:2"]); + yield* Queue.offer(input, event(1)); + yield* TestClock.adjust(25); + assert.deepEqual(yield* Queue.take(output), [event(1)]); + yield* Queue.offer(input, event(2)); + yield* TestClock.adjust(24); + assert.isTrue((yield* Queue.poll(output))._tag === "None"); + yield* TestClock.adjust(1); + assert.deepEqual(yield* Queue.take(output), [event(2)]); + yield* Queue.end(input); + yield* Fiber.join(consumer); }).pipe(Effect.scoped), ); -it.effect("fails every affected writer immediately and accepts the next batch", () => +it.effect("bounds transaction groups by count and bytes and flushes the final partial group", () => Effect.gen(function* () { - let fail = true; - const batch = yield* ProviderCommitBatch.make( - (events) => - fail - ? Effect.fail("disk full") - : Effect.succeed( - events.map((event, index) => ({ sequence: index + 1, commandId: null, event })), - ), - 25, + const events = Array.from({ length: 129 }, (_, index) => event(index)); + const batches = yield* ProviderCommitBatch.group(Stream.fromArray(events), 25).pipe( + Stream.runCollect, ); - const first = yield* batch - .write([event(1)]) - .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); - const second = yield* batch - .write([event(2)]) - .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })); - yield* TestClock.adjust(25); - assert.deepEqual(yield* Fiber.join(first), Exit.fail("disk full")); - assert.deepEqual(yield* Fiber.join(second), Exit.fail("disk full")); - fail = false; - const next = yield* batch.write([event(3)]).pipe(Effect.forkScoped({ startImmediately: true })); - yield* batch.barrier(Effect.void); - assert.equal((yield* Fiber.join(next))[0]?.event.id, "event:3"); - }).pipe(Effect.scoped), + assert.deepEqual( + batches.map((batch) => batch.events.length), + [64, 64, 1], + ); + assert.deepEqual( + batches.flatMap((batch) => batch.events), + events, + ); + const large = event(1, "x".repeat(1024 * 1024)); + const sized = yield* ProviderCommitBatch.group(Stream.make(event(0), large, event(2)), 25).pipe( + Stream.runCollect, + ); + assert.deepEqual( + sized.map((batch) => batch.events.length), + [1, 1, 1], + ); + assert.deepEqual( + sized.flatMap((batch) => batch.events), + [event(0), large, event(2)], + ); + }), ); -it.effect("bounds groups by event count and JSON bytes without waiting for the timer", () => +it.effect("delivers the pending prefix before propagating a source failure", () => Effect.gen(function* () { - const commits: Array> = []; - const batch = yield* ProviderCommitBatch.make( - (events) => - Effect.sync(() => { - commits.push(events); - return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); - }), + const observed: Array = []; + const exit = yield* ProviderCommitBatch.group( + Stream.concat(Stream.make(event(1), event(2)), Stream.fail("provider failed")), 25, + ).pipe( + Stream.runForEach((batch) => Effect.sync(() => observed.push(...batch.events))), + Effect.exit, ); - const first = yield* batch - .write(Array.from({ length: 63 }, (_, i) => event(i))) - .pipe(Effect.forkScoped({ startImmediately: true })); - const second = yield* batch - .write([event(64)]) - .pipe(Effect.forkScoped({ startImmediately: true })); - yield* Fiber.join(first); - yield* Fiber.join(second); - assert.equal(commits[0]?.length, 64); - const small = yield* batch - .write([event(65)]) - .pipe(Effect.forkScoped({ startImmediately: true })); - const large = yield* batch.write([event(66, "x".repeat(1024 * 1024))]); - yield* Fiber.join(small); - assert.deepEqual( - commits.map((events) => events.length), - [64, 1, 1], - ); - assert.equal(large.length, 1); - }).pipe(Effect.scoped), + assert.deepEqual(observed, [event(1), event(2)]); + assert.deepEqual(exit, Exit.fail("provider failed")); + }), ); -it.effect( - "removes interrupted queued writes and flushes outstanding writes when its scope closes", - () => - Effect.gen(function* () { - const commits: string[] = []; - const write = ( - events: ReadonlyArray, - ): Effect.Effect> => - Effect.sync(() => { - commits.push(...events.map((event) => event.id)); - return events.map((event, index) => ({ sequence: index + 1, commandId: null, event })); - }); - const batch = yield* ProviderCommitBatch.make(write, 25); - const cancelled = yield* batch - .write([event(1)]) - .pipe(Effect.forkScoped({ startImmediately: true })); - yield* Fiber.interrupt(cancelled); - yield* batch.barrier(Effect.void); - assert.deepEqual(commits, []); - const parentScope = yield* Effect.scope; - yield* Effect.scoped( - Effect.gen(function* () { - const closing = yield* ProviderCommitBatch.make(write, 25); - yield* closing - .write([event(2)]) - .pipe(Effect.forkIn(parentScope, { startImmediately: true })); - }), - ); - assert.deepEqual(commits, ["event:2"]); - }).pipe(Effect.scoped), +it.effect("keeps controls and individual writes immediate when the window is disabled", () => + Effect.gen(function* () { + const batches = yield* ProviderCommitBatch.group( + Stream.make(event(1), control, event(2)), + 0, + ).pipe(Stream.runCollect); + assert.deepEqual( + batches.map((batch) => batch.events), + [[event(1)], [control], [event(2)]], + ); + }), ); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 760fcb589848c..366b4d20e2bfe 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -1,131 +1,114 @@ -import type { OrchestrationV2DomainEvent, OrchestrationV2StoredEvent } from "@t3tools/contracts"; -import * as Deferred from "effect/Deferred"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; -import * as Semaphore from "effect/Semaphore"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; -import * as Clock from "effect/Clock"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; const MAX_EVENTS = 64; const MAX_BYTES = 1024 * 1024; const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); -export const make = Effect.fn("ProviderCommitBatch.make")(function* ( - commit: ( - events: ReadonlyArray, - ) => Effect.Effect, E>, - windowMs: number, -) { - const scope = yield* Effect.scope; - const lane = yield* Semaphore.make(1); - type Pending = { - readonly events: ReadonlyArray; - readonly bytes: number; - readonly result: Deferred.Deferred, E>; - }; - let pending: Array = []; - let eventCount = 0; - let byteCount = 0; - let generation = 0; - let closed = false; - let openedAt = 0; +export const windowMs = Config.String("T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS").pipe( + Config.withDefault("0"), + Effect.map((raw) => { + const value = Number(raw); + return Number.isInteger(value) && value > 0 ? Math.min(value, 25) : 0; + }), + Effect.orDie, +); - // The lane stays held through commit, publication and acknowledgement. - const flush = Effect.fn("orchestrationV2.EventSink.flushProviderBatch")(function* ( - reason: "deadline" | "limit" | "barrier" | "shutdown", - ) { - if (pending.length === 0) return; - const batch = pending; - pending = []; - yield* Effect.annotateCurrentSpan({ - "orchestration_v2.batch_requests": batch.length, - "orchestration_v2.event_count": eventCount, - "orchestration_v2.queued_json_bytes": byteCount, - "orchestration_v2.batch_threads": new Set( - batch.flatMap((entry) => entry.events.map((event) => event.threadId)), - ).size, - "orchestration_v2.commit_window_ms": windowMs, - "orchestration_v2.batch_wait_ms": (yield* Clock.currentTimeMillis) - openedAt, - "orchestration_v2.batch_flush_reason": reason, - }); - eventCount = 0; - byteCount = 0; - generation++; - const exit = yield* Effect.exit(commit(batch.flatMap((entry) => entry.events))); - // Unguarded snapshots preserve one stored event per input, in input order. - let offset = 0; - for (const entry of batch) { - yield* Deferred.done( - entry.result, - Exit.isFailure(exit) - ? exit - : Exit.succeed(exit.value.slice(offset, offset + entry.events.length)), - ); - offset += entry.events.length; - } - }, Effect.uninterruptible); +type Event = ProviderAdapter.ProviderAdapterV2Event; +interface Batch { + readonly events: ReadonlyArray; + readonly bytes: number; + readonly openedAt: number; + readonly reason: "deadline" | "limit" | "control" | "end" | "failure"; +} - yield* Effect.addFinalizer(() => - lane.withPermits(1)( - Effect.sync(() => { - closed = true; - }).pipe(Effect.andThen(flush("shutdown"))), +const isSnapshot = (event: Event) => + event.type === "node.updated" || + event.type === "message.updated" || + event.type === "turn_item.updated" || + event.type === "plan.updated"; + +export function group(source: Stream.Stream, window: number) { + if (window === 0) + return source.pipe( + Stream.map((event): Batch => ({ events: [event], bytes: 0, openedAt: 0, reason: "control" })), + ); + type Signal = + | { readonly _tag: "Event"; readonly event: Event; readonly bytes: number } + | { readonly _tag: "Failure"; readonly cause: Cause.Cause } + | { readonly _tag: "End" }; + const signals = source.pipe( + Stream.mapEffect((event) => + (isSnapshot(event) + ? encodeQueue(event).pipe(Effect.map((encoded) => Buffer.byteLength(encoded))) + : Effect.succeed(0) + ).pipe(Effect.map((bytes): Signal => ({ _tag: "Event", event, bytes }))), ), + // A failed source must not discard the prefix already collected by the sink. + Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure", cause })), + Stream.concat(Stream.succeed({ _tag: "End" })), ); - - // Per-write calls share the batch's tracing boundary. - const write = Effect.fnUntraced(function* (events: ReadonlyArray) { - const result = yield* Deferred.make, E>(); - const entry: Pending = { events, result, bytes: Buffer.byteLength(yield* encodeQueue(events)) }; - yield* lane.withPermits(1)( + const collect = Sink.fold( + () => ({ signals: [] as Array, bytes: 0, openedAt: 0, stop: false }), + (state) => !state.stop, + (state, signal: Signal) => Effect.gen(function* () { - if (closed) return yield* Effect.interrupt; - if (eventCount + events.length > MAX_EVENTS || byteCount + entry.bytes > MAX_BYTES) { - yield* flush("limit"); - } - const opensWindow = pending.length === 0; - if (opensWindow) openedAt = yield* Clock.currentTimeMillis; - pending.push(entry); - eventCount += events.length; - byteCount += entry.bytes; - if (eventCount >= MAX_EVENTS || byteCount >= MAX_BYTES) { - yield* flush("limit"); - } else if (opensWindow) { - const token = ++generation; - yield* Effect.sleep(windowMs).pipe( - Effect.andThen( - lane.withPermits(1)( - Effect.suspend(() => (token === generation ? flush("deadline") : Effect.void)), - ), - ), - Effect.interruptible, - Effect.forkIn(scope, { startImmediately: true }), - ); - } + if (state.signals.length === 0) state.openedAt = yield* Clock.currentTimeMillis; + state.signals.push(signal); + if (signal._tag === "Event") state.bytes += signal.bytes; + state.stop = + signal._tag !== "Event" || + !isSnapshot(signal.event) || + state.signals.length >= MAX_EVENTS || + state.bytes >= MAX_BYTES; + return state; }), - ); - return yield* Deferred.await(result).pipe( - Effect.onInterrupt(() => - lane.withPermits(1)( - Effect.sync(() => { - const index = pending.indexOf(entry); - if (index < 0) return; - pending.splice(index, 1); - eventCount -= events.length; - byteCount -= entry.bytes; - if (pending.length === 0) generation++; - }), + ); + return signals.pipe( + Stream.aggregateWithin(collect, Schedule.spaced(window)), + Stream.flatMap((state) => { + const groups: Array> = []; + let events: Array = []; + let bytes = 0; + const flush = (reason: Batch["reason"]) => { + if (events.length === 0) return; + groups.push({ events, bytes, openedAt: state.openedAt, reason }); + events = []; + bytes = 0; + }; + for (const signal of state.signals) { + if (signal._tag === "Failure") { + flush("failure"); + groups.push(signal); + } else if (signal._tag === "End") flush("end"); + else if (!isSnapshot(signal.event)) { + flush("control"); + groups.push({ + events: [signal.event], + bytes: signal.bytes, + openedAt: state.openedAt, + reason: "control", + }); + } else { + if (bytes + signal.bytes > MAX_BYTES) flush("limit"); + events.push(signal.event); + bytes += signal.bytes; + if (bytes >= MAX_BYTES || events.length >= MAX_EVENTS) flush("limit"); + } + } + flush("deadline"); + return Stream.fromArray(groups).pipe( + Stream.mapEffect((batch) => + "_tag" in batch ? Effect.failCause(batch.cause) : Effect.succeed(batch), ), - ), - ); - }); - - const barrier = (effect: Effect.Effect) => - lane.withPermits(1)( - Effect.suspend(() => - closed ? Effect.interrupt : flush("barrier").pipe(Effect.andThen(effect)), - ), - ); - - return { write, barrier }; -}); + ); + }), + ); +} diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 568129c56cb8f..dc34b4650148e 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -546,8 +546,12 @@ layer("ProviderEventIngestorV2", (it) => { event: { type: "node.updated", driver: CODEX_DRIVER, node: sent }, }); - assert.lengthOf(yield* ingestNode(node), 1); - assert.isEmpty(yield* ingestNode({ ...node })); + yield* eventSink.withProviderBatch( + Effect.gen(function* () { + assert.lengthOf(yield* ingestNode(node), 1); + assert.isEmpty(yield* ingestNode({ ...node })); + }), + ); yield* eventSink.write({ events: [ { diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 97d7c1680ef00..f27580d909aff 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -697,7 +697,6 @@ export const layer: Layer.Layer< if (input.writeIfRunCurrent === undefined) { return yield* eventSink .write({ - batch: true, guardPendingUserInputCancellations: true, ...(input.commandId === undefined ? {} : { commandId: input.commandId }), events, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 6a52fac447926..27bc06aef2812 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -46,6 +46,8 @@ import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; +import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; +import * as Clock from "effect/Clock"; import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; import { @@ -611,6 +613,7 @@ export const layer: Layer.Layer< const eventSink = yield* EventSink.EventSinkV2; const idAllocator = yield* IdAllocator.IdAllocatorV2; const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const commitWindowMs = yield* ProviderCommitBatch.windowMs; const serverSettings = yield* ServerSettings.ServerSettingsService; const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext; const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity; @@ -1322,210 +1325,229 @@ export const layer: Layer.Layer< // Ends once the root turn and the children it drives settle. Background commands such // as a dev server keep this consumer alive afterwards and must not pin preview focus. const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId); - const providerEventFiber = yield* eventSubscription.events.pipe( - Stream.mapEffect((event) => - Effect.gen(function* () { - const previous = yield* Ref.get(eventRouting); - const [accepted, routing] = routeProviderEvent(event, routeIdentity, previous); - yield* Ref.set(eventRouting, routing); - if ( - event.type === "subagent.updated" && - event.subagent.childThreadId !== null && - previous.ownedThreadIds.has(event.subagent.childThreadId) && - !routing.ownedThreadIds.has(event.subagent.childThreadId) - ) { - const moved = event.subagent; - const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => { - const subagentIds = new Set([moved.id]); - for (const subagent of current.subagents.values()) { - if (subagent.threadId === moved.childThreadId) subagentIds.add(subagent.id); - } - const childTurnItems = new Map( - [...current.childTurnItems].filter( - ([, item]) => item.threadId !== moved.childThreadId, - ), - ); - const turnItems = new Map(current.turnItems); - turnItems.delete(moved.id); - const detachedItems = new Set( - [...current.childTurnItems.values(), ...current.turnItems.values()] - .filter( - (item) => item.threadId === moved.childThreadId || item.nodeId === moved.id, - ) - .map((item) => item.id), - ); - return [ - { itemIds: detachedItems, subagentIds }, - { - subagents: new Map( - [...current.subagents].filter(([id]) => !subagentIds.has(id)), - ), - turnItems, - childTurnItems, - nodes: new Map( - [...current.nodes].filter( - ([id, node]) => id !== moved.id && node.threadId !== moved.childThreadId, - ), - ), - linkedChildThreadIds: new Set( - [...current.linkedChildThreadIds].filter( - (id) => id !== moved.childThreadId, - ), - ), - }, - ]; - }); - yield* Ref.update( - activeChildSubagents, - (current) => - new Map( - [...current].filter( - ([id, threadId]) => - !detached.subagentIds.has(id) && threadId !== moved.childThreadId, - ), - ), - ); - yield* Ref.update( - activeChildProviderTurns, - (current) => - new Set([...current].filter((id) => routing.ownedProviderTurns.has(id))), - ); - for (const id of pendingBackgroundProviderTurns) { - if (!routing.ownedProviderTurns.has(id)) { - pendingBackgroundProviderTurns.delete(id); - } - } - yield* Ref.update( - activeBackgroundTurnItems, - (current) => new Set([...current].filter((id) => !detached.itemIds.has(id))), - ); - } - return { event, accepted }; - }), - ), - Stream.tap(({ event, accepted }) => - Effect.gen(function* () { - if (!accepted) return; - if ( - event.type === "provider_turn.updated" && - event.providerTurn.runAttemptId === input.attempt.id && - event.providerTurn.providerThreadId === input.providerThread.id - ) { - yield* Ref.set(latestRootProviderTurn, event.providerTurn); + // Per-event routing and lifecycle bookkeeping share the run's tracing boundary. + const routeEvent = Effect.fnUntraced(function* ( + event: ProviderAdapter.ProviderAdapterV2Event, + ) { + const previous = yield* Ref.get(eventRouting); + const [accepted, routing] = routeProviderEvent(event, routeIdentity, previous); + yield* Ref.set(eventRouting, routing); + if ( + event.type === "subagent.updated" && + event.subagent.childThreadId !== null && + previous.ownedThreadIds.has(event.subagent.childThreadId) && + !routing.ownedThreadIds.has(event.subagent.childThreadId) + ) { + const moved = event.subagent; + const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => { + const subagentIds = new Set([moved.id]); + for (const subagent of current.subagents.values()) { + if (subagent.threadId === moved.childThreadId) subagentIds.add(subagent.id); } - let storedEventCount = 0; - const deliveredEvent = filterAssistantEvent( - event, - DateTime.toEpochMillis(yield* DateTime.now), + const childTurnItems = new Map( + [...current.childTurnItems].filter( + ([, item]) => item.threadId !== moved.childThreadId, + ), ); - if (deliveredEvent) { - const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); - const isRootProviderThreadUpdate = - event.type === "provider_thread.updated" && - event.providerThread.id === input.providerThread.id; - const nativeChildTurn = - event.type === "provider_turn.updated" && - event.providerTurn.providerThreadId !== input.providerThread.id; - if (event.type === "provider_turn.updated" && !nativeChildTurn) { - firstRootProviderTurnId ??= event.providerTurn.id; - } - const storedEvents = yield* providerEventIngestor.ingestNormalized({ - analyticsContext: nativeChildTurn - ? { - ...analyticsContext, - origin: NATIVE_SUBAGENT_TURN_ORIGIN, - segment: "native_child", - } - : { - ...analyticsContext, - segment: - event.type === "provider_turn.updated" && - event.providerTurn.id !== firstRootProviderTurnId - ? "continuation" - : input.attempt.reason, - }, - providerSessionId: input.providerSessionId, - providerInstanceId: input.run.providerInstanceId, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - event: deliveredEvent, - ...(isRootProviderThreadUpdate - ? rootTerminalAlreadySeen - ? { - writeIfProviderThreadOwner: { - providerThreadId: input.providerThread.id, - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedLastRunOrdinal: input.run.ordinal, - }, - } - : { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } - : {}), - }); - storedEventCount = storedEvents.length; - if ( - isRootProviderThreadUpdate && - rootTerminalAlreadySeen && - storedEventCount === 0 - ) { - yield* Ref.set(providerThreadOwnerLost, true); - } - } else if ( - event.type === "turn_item.updated" && - (event.turnItem.type === "assistant_message" || - event.turnItem.type === "reasoning") && - event.turnItem.streaming && - !reservedHeldTurnItemIds.has(event.turnItem.id) - ) { - yield* turnItemPositions.allocate({ - threadId: event.turnItem.threadId, - turnItemId: event.turnItem.id, - runId: event.turnItem.runId, - }); - reservedHeldTurnItemIds.add(event.turnItem.id); - } - if (event.type === "provider_thread.updated") { - if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { - yield* Ref.set(latestProviderThread, event.providerThread); - } - } - if ( - event.type === "turn_item.updated" && - event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId - ) { - yield* Ref.update(latestTurnItemOrdinal, (current) => - Math.max(current, event.turnItem.ordinal), - ); - } - if (event.type === "turn.terminal") { - yield* Ref.set(terminalEvent, event); - yield* Ref.set(rootTerminalSeen, true); - yield* finalizeRootRun(event); - } - if (event.type === "background_work.updated") { - if (event.pending) { - pendingBackgroundProviderTurns.add(event.providerTurnId); - } else { - pendingBackgroundProviderTurns.delete(event.providerTurnId); - } + const turnItems = new Map(current.turnItems); + turnItems.delete(moved.id); + const detachedItems = new Set( + [...current.childTurnItems.values(), ...current.turnItems.values()] + .filter( + (item) => item.threadId === moved.childThreadId || item.nodeId === moved.id, + ) + .map((item) => item.id), + ); + return [ + { itemIds: detachedItems, subagentIds }, + { + subagents: new Map([...current.subagents].filter(([id]) => !subagentIds.has(id))), + turnItems, + childTurnItems, + nodes: new Map( + [...current.nodes].filter( + ([id, node]) => id !== moved.id && node.threadId !== moved.childThreadId, + ), + ), + linkedChildThreadIds: new Set( + [...current.linkedChildThreadIds].filter((id) => id !== moved.childThreadId), + ), + }, + ]; + }); + yield* Ref.update( + activeChildSubagents, + (current) => + new Map( + [...current].filter( + ([id, threadId]) => + !detached.subagentIds.has(id) && threadId !== moved.childThreadId, + ), + ), + ); + yield* Ref.update( + activeChildProviderTurns, + (current) => new Set([...current].filter((id) => routing.ownedProviderTurns.has(id))), + ); + for (const id of pendingBackgroundProviderTurns) { + if (!routing.ownedProviderTurns.has(id)) { + pendingBackgroundProviderTurns.delete(id); } - yield* trackChildLifecycle(event, deliveredEvent !== null); - }), - ), - // Also after events this run does not accept, such as a child handed to another run. - Stream.tap(() => + } + yield* Ref.update( + activeBackgroundTurnItems, + (current) => new Set([...current].filter((id) => !detached.itemIds.has(id))), + ); + } + return { event, accepted }; + }); + const ingestEvent = Effect.fnUntraced(function* ({ + event, + accepted, + }: { + event: ProviderAdapter.ProviderAdapterV2Event; + accepted: boolean; + }) { + if (!accepted) return; + if ( + event.type === "provider_turn.updated" && + event.providerTurn.runAttemptId === input.attempt.id && + event.providerTurn.providerThreadId === input.providerThread.id + ) { + yield* Ref.set(latestRootProviderTurn, event.providerTurn); + } + let storedEventCount = 0; + const deliveredEvent = filterAssistantEvent( + event, + DateTime.toEpochMillis(yield* DateTime.now), + ); + if (deliveredEvent) { + const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); + const isRootProviderThreadUpdate = + event.type === "provider_thread.updated" && + event.providerThread.id === input.providerThread.id; + const nativeChildTurn = + event.type === "provider_turn.updated" && + event.providerTurn.providerThreadId !== input.providerThread.id; + if (event.type === "provider_turn.updated" && !nativeChildTurn) { + firstRootProviderTurnId ??= event.providerTurn.id; + } + const storedEvents = yield* providerEventIngestor.ingestNormalized({ + analyticsContext: nativeChildTurn + ? { + ...analyticsContext, + origin: NATIVE_SUBAGENT_TURN_ORIGIN, + segment: "native_child", + } + : { + ...analyticsContext, + segment: + event.type === "provider_turn.updated" && + event.providerTurn.id !== firstRootProviderTurnId + ? "continuation" + : input.attempt.reason, + }, + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: deliveredEvent, + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } + : {}), + }); + storedEventCount = storedEvents.length; + if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEventCount === 0) { + yield* Ref.set(providerThreadOwnerLost, true); + } + } else if ( + event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") && + event.turnItem.streaming && + !reservedHeldTurnItemIds.has(event.turnItem.id) + ) { + yield* turnItemPositions.allocate({ + threadId: event.turnItem.threadId, + turnItemId: event.turnItem.id, + runId: event.turnItem.runId, + }); + reservedHeldTurnItemIds.add(event.turnItem.id); + } + if (event.type === "provider_thread.updated") { + if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { + yield* Ref.set(latestProviderThread, event.providerThread); + } + } + if ( + event.type === "turn_item.updated" && + event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId + ) { + yield* Ref.update(latestTurnItemOrdinal, (current) => + Math.max(current, event.turnItem.ordinal), + ); + } + if (event.type === "turn.terminal") { + yield* Ref.set(terminalEvent, event); + yield* Ref.set(rootTerminalSeen, true); + yield* finalizeRootRun(event); + } + if (event.type === "background_work.updated") { + if (event.pending) { + pendingBackgroundProviderTurns.add(event.providerTurnId); + } else { + pendingBackgroundProviderTurns.delete(event.providerTurnId); + } + } + yield* trackChildLifecycle(event, deliveredEvent !== null); + }); + const providerEventFiber = yield* ProviderCommitBatch.group( + eventSubscription.events, + commitWindowMs, + ).pipe( + Stream.mapEffect((batch) => Effect.gen(function* () { - if (yield* agentTurnsSettled) yield* endPreviewActivity; + const process = Effect.gen(function* () { + for (const event of batch.events) { + yield* ingestEvent(yield* routeEvent(event)); + if (yield* agentTurnsSettled) yield* endPreviewActivity; + if (yield* shouldStopProviderEventIngestion) return true; + } + return false; + }); + if (batch.events.length < 2 || (yield* Ref.get(rootTerminalSeen))) + return yield* process; + return yield* eventSink.withProviderBatch( + Effect.gen(function* () { + yield* Effect.annotateCurrentSpan({ + "orchestration_v2.provider_event_count": batch.events.length, + "orchestration_v2.raw_json_bytes": batch.bytes, + "orchestration_v2.batch_flush_reason": batch.reason, + "orchestration_v2.commit_window_ms": commitWindowMs, + "orchestration_v2.batch_wait_ms": + (yield* Clock.currentTimeMillis) - batch.openedAt, + }); + return yield* process; + }), + ); }), ), - Stream.takeUntilEffect(() => shouldStopProviderEventIngestion), + Stream.takeUntil((stop) => stop), Stream.runDrain, Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), Effect.flatMap(() => diff --git a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts index c0593351f20d7..334d469dec6a3 100644 --- a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts @@ -1,9 +1,15 @@ -import { describe } from "@effect/vitest"; +import { assert, describe, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Tracer from "effect/Tracer"; import * as Effect from "effect/Effect"; import { materializeReplayTranscriptWorkspace } from "@t3tools/provider-testing/replayTranscript"; import { CodexOrchestratorReplayHarness } from "../Adapters/CodexAdapterV2.testkit.ts"; -import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts"; +import { + findReplayFixture, + itRunsReplayFixtures, + runReplayFixture, +} from "./ProviderReplayFixtures.ts"; describe("Codex replay fixtures", () => { itRunsReplayFixtures("codex", (input) => @@ -15,3 +21,42 @@ describe("Codex replay fixtures", () => { }), ); }); + +it.effect( + "batches consecutive snapshots from one recorded Codex stream in the real run consumer", + () => + Effect.gen(function* () { + const fixture = findReplayFixture("multi_turn")!; + const driver = fixture.providers.find((provider) => provider.driver === "codex")!; + const writes: number[] = []; + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (time, exit) => { + end(time, exit); + if (span.name === "orchestrationV2.EventSink.providerBatch") { + writes.push(Number(span.attributes.get("orchestration_v2.batch_writes"))); + } + }; + return span; + }, + }); + yield* runReplayFixture({ + fixtureName: fixture.name, + buildInput: fixture.buildInput, + driver, + harness: CodexOrchestratorReplayHarness, + materializeWorkspace: (transcript, workspace) => + Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)), + }).pipe( + Effect.withTracer(tracer), + Effect.withTracerEnabled(true), + Effect.provideService( + ConfigProvider.ConfigProvider, + ConfigProvider.fromUnknown({ T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: "10" }), + ), + ); + assert.deepEqual(writes, [3, 3]); + }), +); From 6d4dfc22bb30593c5b82143620ac0909afc5d854 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Wed, 4 Nov 2026 03:27:52 -0600 Subject: [PATCH 3/9] perf(server): share provider commit windows across runs --- apps/server/src/orchestration-v2/EventSink.ts | 32 +- .../FoundationPersistence.test.ts | 119 +++++++ .../ProviderCommitBatch.test.ts | 234 +++++++++++++ .../orchestration-v2/ProviderCommitBatch.ts | 245 +++++++++++++- .../RunExecutionService.test.ts | 32 +- .../orchestration-v2/RunExecutionService.ts | 319 ++++++++---------- .../CodexReplayFixtures.integration.test.ts | 12 +- 7 files changed, 798 insertions(+), 195 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index f110ef513a919..661e8b47c91b1 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -12,6 +12,8 @@ import { type ProjectId, ThreadId, } from "@t3tools/contracts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -77,6 +79,13 @@ const CurrentProviderBatch = Context.Reference( + source: Stream.Stream, + process: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect, + after: Effect.Effect, + canBatch: Effect.Effect, + ) => Effect.Effect; + /** Normalize and persist an ordered group against one transaction; publish after its commit. */ readonly withProviderBatch: ( effect: Effect.Effect, @@ -894,10 +903,17 @@ const layerBase: Layer.Layer< ); }; - const commitThenPublish: typeof commitThenPublishNow = (transaction, publish) => + const commitThenPublish = ( + transaction: Effect.Effect, + publish: (committed: A) => Effect.Effect, + ) => Effect.flatMap(CurrentProviderBatch, (batch) => batch === undefined - ? commitThenPublishNow(transaction, publish) + ? Effect.flatMap(Effect.serviceOption(sql.transactionService), (existing) => + Option.isSome(existing) + ? commitThenPublishNow(transaction, publish) + : coordinator.barrier(commitThenPublishNow(transaction, publish)), + ) : transaction.pipe( Effect.tap((committed) => Effect.sync(() => { @@ -936,7 +952,19 @@ const layerBase: Layer.Layer< return result; }); + const coordinator = yield* ProviderCommitBatch.make( + (work) => + withProviderBatch(work).pipe( + Effect.mapError((cause) => new ProviderCommitBatch.CommitBatchError({ cause })), + ), + yield* ProviderCommitBatch.windowMs, + ); + return EventSinkV2.of({ + consumeProviderEvents: (source, process, after, canBatch) => + coordinator + .consume(source, process, after, canBatch) + .pipe(Effect.mapError((cause) => new EventSinkWriteError({ eventCount: 0, cause }))), withProviderBatch, write: (input) => writeEffect({ ...input, effects: [] }).pipe( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 9414f77bff966..012bc39373830 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -27,6 +27,9 @@ import { TurnItemId, } from "@t3tools/contracts"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Queue from "effect/Queue"; import * as Arr from "effect/Array"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; @@ -410,6 +413,122 @@ it.effect( }).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped), ); +it.effect.each([false, true])( + "shares the writer across independent consumers with rollback=%s", + (rollback) => { + const writes: number[] = []; + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (time, exit) => { + end(time, exit); + if (span.name === "orchestrationV2.EventSink.providerBatch" && Exit.isSuccess(exit)) { + writes.push(Number(span.attributes.get("orchestration_v2.batch_writes"))); + } + }; + return span; + }, + }); + return Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threads = Array.from({ length: 10 }, (_, index) => + makeThread(ThreadId.make(`shared:${index}`), now), + ); + yield* sink.write({ + events: threads.map((thread, index) => + threadCreatedEvent({ id: `created:${index}`, thread, now }), + ), + }); + const highWater = yield* sink.latestSequence(); + const pulled = yield* Deferred.make(); + let arrivals = 0; + const inputs = yield* Effect.forEach(threads, () => + Queue.unbounded(), + ); + const consumers = yield* Effect.forEach(inputs, (input, index) => + sink + .consumeProviderEvents( + Stream.fromQueue(input).pipe( + Stream.tap(() => + ++arrivals === 10 ? Deferred.succeed(pulled, undefined) : Effect.void, + ), + ), + (raw) => + raw.type === "message.updated" + ? sink + .write({ + events: [ + { + id: EventId.make( + rollback ? "duplicate-shared-event" : `shared-event:${index}`, + ), + threadId: raw.message.threadId, + occurredAt: now, + type: "message.updated", + payload: raw.message, + }, + ], + }) + .pipe(Effect.asVoid) + : Effect.void, + Effect.succeed(false), + Effect.succeed(true), + ) + .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })), + ); + for (let index = 0; index < threads.length; index++) { + yield* Queue.offer(inputs[index]!, { + type: "message.updated", + driver: ProviderDriverKind.make("codex"), + message: { + id: MessageId.make(`message:${index}`), + threadId: threads[index]!.id, + runId: null, + nodeId: null, + createdBy: "agent", + creationSource: "provider", + role: "assistant", + text: `stream:${index}`, + streaming: true, + attachments: [], + createdAt: now, + updatedAt: now, + }, + }); + } + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + yield* sink.write({ events: [] }); + if (rollback) { + for (const consumer of consumers) + assert.isTrue(Exit.isFailure(yield* Fiber.join(consumer))); + assert.equal(yield* sink.latestSequence(), highWater); + } else { + assert.deepEqual(writes, [10]); + assert.equal(yield* sink.latestSequence(), highWater + 10); + } + for (const thread of threads) { + assert.lengthOf( + (yield* projections.getThreadRecords(thread.id, ["messages"])).messages, + rollback ? 0 : 1, + ); + } + }).pipe( + Effect.scoped, + Effect.provide(Layer.fresh(layerTest)), + Effect.withTracer(tracer), + Effect.withTracerEnabled(true), + Effect.provideService( + ConfigProvider.ConfigProvider, + ConfigProvider.fromUnknown({ T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: "10" }), + ), + ); + }, +); + it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { it.effect("projects oversized tool bodies before both replay and live RPC retention", () => Effect.scoped( diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index 78cc9c5e08391..5c7c9c08d2b88 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -3,6 +3,7 @@ import { MessageId, ProviderDriverKind, ProviderTurnId, ThreadId } from "@t3tool import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; @@ -142,3 +143,236 @@ it.effect("keeps controls and individual writes immediate when the window is dis ); }), ); + +it.effect.each([ + { streams: 1, updates: 10 }, + { streams: 10, updates: 1 }, + { streams: 10, updates: 10 }, +])( + "shares bounded transactions across $streams streams with $updates updates each", + ({ streams, updates }) => + Effect.gen(function* () { + const commits = yield* Queue.unbounded>(); + let transaction: string[] = []; + const coordinator = yield* ProviderCommitBatch.make( + (work) => + Effect.gen(function* () { + transaction = []; + yield* work; + yield* Queue.offer(commits, transaction); + }), + 10, + ); + const inputs = yield* Effect.forEach(Array.from({ length: streams }), () => + Queue.unbounded(), + ); + const pulled = yield* Deferred.make(); + let arrivals = 0; + const consumers = yield* Effect.forEach(inputs, (input) => + coordinator + .consume( + Stream.fromQueue(input).pipe( + Stream.tap(() => { + arrivals++; + return arrivals === streams * updates + ? Deferred.succeed(pulled, undefined) + : Effect.void; + }), + ), + (raw) => + Effect.sync(() => { + if (raw.type === "message.updated") transaction.push(raw.message.text); + }), + Effect.succeed(true), + ) + .pipe(Effect.forkScoped({ startImmediately: true })), + ); + for (let n = 0; n < updates; n++) { + for (let stream = 0; stream < streams; stream++) + yield* Queue.offer(inputs[stream]!, event(stream * updates + n)); + } + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + if (updates === 1) assert.equal(yield* Queue.size(commits), 0); + yield* coordinator.barrier(Effect.void); + const batches: ReadonlyArray[] = []; + let count = 0; + while (count < streams * updates) { + const batch = yield* Queue.take(commits); + batches.push(batch); + count += batch.length; + } + assert.deepEqual( + batches.map((batch) => batch.length), + streams * updates > 64 ? [64, 36] : [10], + ); + assert.equal(new Set(batches.flat()).size, streams * updates); + for (const input of inputs) yield* Queue.offer(input, control); + yield* Effect.forEach(consumers, Fiber.join); + }).pipe(Effect.scoped), +); + +it.effect("does not hold the shared worker while one consumer checks background work", () => + Effect.gen(function* () { + const checking = yield* Deferred.make(); + const release = yield* Deferred.make(); + const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const consumer = yield* coordinator + .consume( + Stream.make(control), + () => Effect.void, + Deferred.succeed(checking, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(true), + ), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(checking); + assert.equal( + yield* coordinator.barrier(Effect.succeed("another run can commit")), + "another run can commit", + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(consumer); + }).pipe(Effect.scoped), +); + +it.effect( + "fails every participant of a failed transaction and continues serving unrelated runs", + () => + Effect.gen(function* () { + const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const inputs = yield* Effect.forEach([0, 1], () => + Queue.unbounded(), + ); + const failures = yield* Effect.forEach(inputs, (input, index) => + coordinator + .consume( + Stream.fromQueue(input), + () => (index === 1 ? Effect.fail("disk full") : Effect.void), + Effect.succeed(false), + ) + .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })), + ); + for (const input of inputs) yield* Queue.offer(input, event(0)); + yield* TestClock.adjust(0); + yield* coordinator.barrier(Effect.void); + for (const failed of failures) assert.isTrue(Exit.isFailure(yield* Fiber.join(failed))); + assert.equal(yield* coordinator.barrier(Effect.succeed("recovered")), "recovered"); + assert.deepEqual( + yield* coordinator.barrier(Effect.fail("original error")).pipe(Effect.exit), + Exit.fail("original error"), + ); + }).pipe(Effect.scoped), +); + +it.effect("cancels pending work from one consumer without cancelling its neighbours", () => + Effect.gen(function* () { + const committed = yield* Deferred.make(); + let writes = 0; + const coordinator = yield* ProviderCommitBatch.make( + (work) => work.pipe(Effect.tap(() => Deferred.succeed(committed, undefined))), + 10, + ); + const inputs = yield* Effect.forEach([0, 1], () => + Queue.unbounded(), + ); + const consumers = yield* Effect.forEach(inputs, (input) => + coordinator + .consume( + Stream.fromQueue(input), + () => + Effect.sync(() => { + writes++; + }), + Effect.succeed(true), + ) + .pipe(Effect.forkScoped({ startImmediately: true })), + ); + for (const input of inputs) yield* Queue.offer(input, event(0)); + yield* TestClock.adjust(0); + yield* Fiber.interrupt(consumers[0]!); + yield* coordinator.barrier(Effect.void); + yield* Deferred.await(committed); + assert.equal(writes, 1); + yield* Queue.offer(inputs[1]!, control); + yield* Fiber.join(consumers[1]!); + }).pipe(Effect.scoped), +); + +it.effect.each(["failure", "stop"] as const)( + "flushes the prefix and respects a source %s barrier", + (mode) => + Effect.gen(function* () { + const seen: ProviderAdapter.ProviderAdapterV2Event[] = []; + const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const source = + mode === "failure" + ? Stream.concat(Stream.make(event(1), event(2)), Stream.fail("provider failed")) + : Stream.make(event(1), control, event(2)); + const exit = yield* coordinator + .consume( + source, + (raw) => + Effect.sync(() => { + seen.push(raw); + }), + Effect.succeed(mode === "stop"), + ) + .pipe(Effect.exit); + assert.deepEqual(seen, mode === "failure" ? [event(1), event(2)] : [event(1), control]); + assert.equal(Exit.isFailure(exit), mode === "failure"); + assert.equal( + yield* coordinator.barrier(Effect.succeed("still available")), + "still available", + ); + }).pipe(Effect.scoped), +); + +it.effect("does not force a separate commit for every consumer that ends", () => + Effect.gen(function* () { + const pulled = yield* Deferred.make(); + const writes: number[] = []; + let arrivals = 0; + let count = 0; + const coordinator = yield* ProviderCommitBatch.make( + (work) => + Effect.gen(function* () { + count = 0; + yield* work; + writes.push(count); + }), + 10, + ); + for (let index = 0; index < 10; index++) { + yield* coordinator + .consume( + Stream.make(event(index)).pipe( + Stream.tap(() => + ++arrivals === 10 ? Deferred.succeed(pulled, undefined) : Effect.void, + ), + ), + () => + Effect.sync(() => { + count++; + }), + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + } + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + yield* coordinator.barrier(Effect.void); + assert.deepEqual(writes, [10]); + }).pipe(Effect.scoped), +); + +it.effect("settles calls made after the coordinator's scope closes", () => + Effect.gen(function* () { + const coordinator = yield* ProviderCommitBatch.make((work) => work, 10).pipe(Effect.scoped); + const exit = yield* coordinator + .barrier(Effect.die("must not execute after shutdown")) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)); + }), +); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 366b4d20e2bfe..895148d426a32 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -3,6 +3,10 @@ import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Exit from "effect/Exit"; +import * as Queue from "effect/Queue"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Sink from "effect/Sink"; @@ -22,8 +26,8 @@ export const windowMs = Config.String("T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS").pi ); type Event = ProviderAdapter.ProviderAdapterV2Event; -interface Batch { - readonly events: ReadonlyArray; +interface Batch { + readonly events: ReadonlyArray; readonly bytes: number; readonly openedAt: number; readonly reason: "deadline" | "limit" | "control" | "end" | "failure"; @@ -40,16 +44,33 @@ export function group(source: Stream.Stream, window: number) return source.pipe( Stream.map((event): Batch => ({ events: [event], bytes: 0, openedAt: 0, reason: "control" })), ); + return collect(source, window, (event) => + (isSnapshot(event) + ? encodeQueue(event).pipe(Effect.map(Buffer.byteLength)) + : Effect.succeed(0) + ).pipe(Effect.map((bytes) => ({ snapshot: isSnapshot(event), bytes }))), + ); +} + +function collect( + source: Stream.Stream, + window: number, + describe: (item: T) => Effect.Effect<{ snapshot: boolean; bytes: number }, Schema.SchemaError>, +) { type Signal = - | { readonly _tag: "Event"; readonly event: Event; readonly bytes: number } + | { + readonly _tag: "Event"; + readonly event: T; + readonly bytes: number; + readonly snapshot: boolean; + } | { readonly _tag: "Failure"; readonly cause: Cause.Cause } | { readonly _tag: "End" }; const signals = source.pipe( Stream.mapEffect((event) => - (isSnapshot(event) - ? encodeQueue(event).pipe(Effect.map((encoded) => Buffer.byteLength(encoded))) - : Effect.succeed(0) - ).pipe(Effect.map((bytes): Signal => ({ _tag: "Event", event, bytes }))), + describe(event).pipe( + Effect.map((description): Signal => ({ _tag: "Event", event, ...description })), + ), ), // A failed source must not discard the prefix already collected by the sink. Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure", cause })), @@ -65,7 +86,7 @@ export function group(source: Stream.Stream, window: number) if (signal._tag === "Event") state.bytes += signal.bytes; state.stop = signal._tag !== "Event" || - !isSnapshot(signal.event) || + !signal.snapshot || state.signals.length >= MAX_EVENTS || state.bytes >= MAX_BYTES; return state; @@ -74,8 +95,8 @@ export function group(source: Stream.Stream, window: number) return signals.pipe( Stream.aggregateWithin(collect, Schedule.spaced(window)), Stream.flatMap((state) => { - const groups: Array> = []; - let events: Array = []; + const groups: Array | Extract> = []; + let events: Array = []; let bytes = 0; const flush = (reason: Batch["reason"]) => { if (events.length === 0) return; @@ -88,7 +109,7 @@ export function group(source: Stream.Stream, window: number) flush("failure"); groups.push(signal); } else if (signal._tag === "End") flush("end"); - else if (!isSnapshot(signal.event)) { + else if (!signal.snapshot) { flush("control"); groups.push({ events: [signal.event], @@ -112,3 +133,205 @@ export function group(source: Stream.Stream, window: number) }), ); } + +export class CommitBatchError extends Schema.TaggedError()("CommitBatchError", { + cause: Schema.Defect(), +}) {} + +interface Job { + readonly owner: object; + readonly queuedAt: number; + readonly snapshot: boolean; + readonly bytes: number; + readonly active: () => boolean; + readonly execute: Effect.Effect; + readonly settle: (exit: Exit.Exit) => Effect.Effect; + readonly cancel: Effect.Effect; +} +const CurrentWorker = Context.Reference( + "t3/ProviderCommitBatch/CurrentWorker", + { + defaultValue: () => undefined, + }, +); + +export const make = Effect.fn("ProviderCommitBatch.make")(function* ( + commit: (work: Effect.Effect) => Effect.Effect, + window: number, +) { + const queue = yield* Queue.bounded(MAX_EVENTS); + const pending = new Set(); + const shutdown = Effect.gen(function* () { + yield* Queue.shutdown(queue); + for (const job of pending) yield* job.cancel; + }); + yield* Effect.addFinalizer(() => shutdown); + // These helpers run once per raw event; the shared transaction owns the tracing span. + const enqueue = Effect.fnUntraced(function* ( + effect: Effect.Effect, + token: { active: boolean }, + snapshot: boolean, + bytes: number, + ) { + const caller = yield* Effect.context(); + const reply = yield* Deferred.make(); + let result: A; + const job: Job = { + owner: token, + queuedAt: yield* Clock.currentTimeMillis, + snapshot, + bytes, + active: () => token.active, + execute: effect.pipe( + Effect.mapError((cause) => new CommitBatchError({ cause })), + Effect.provideContext(caller), + Effect.tap((value) => + Effect.sync(() => { + result = value; + }), + ), + Effect.asVoid, + ), + settle: Effect.fnUntraced(function* (exit: Exit.Exit) { + pending.delete(job); + if (Exit.isFailure(exit)) { + token.active = false; + yield* Deferred.failCause(reply, exit.cause); + } else yield* Deferred.succeed(reply, result); + }), + cancel: Effect.gen(function* () { + pending.delete(job); + yield* Deferred.interrupt(reply); + }), + }; + yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + pending.add(job); + const accepted = yield* restore(Queue.offer(queue, job)).pipe( + Effect.onError(() => job.cancel), + ); + if (!accepted) yield* job.cancel; + }), + ); + return Deferred.await(reply); + }); + if (window > 0) { + yield* collect(Stream.fromQueue(queue), window, (job) => + Effect.succeed({ snapshot: job.snapshot, bytes: job.bytes }), + ).pipe( + Stream.runForEach((batch) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const jobs = batch.events.filter((job) => job.active()); + for (const job of batch.events) if (!job.active()) yield* job.cancel; + if (jobs.length === 0) return; + yield* Effect.annotateCurrentSpan({ + "orchestration_v2.work_item_count": jobs.length, + "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.owner)).size, + "orchestration_v2.raw_json_bytes": batch.bytes, + "orchestration_v2.batch_flush_reason": batch.reason, + "orchestration_v2.commit_window_ms": window, + "orchestration_v2.batch_wait_ms": + (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)), + }); + const work = Effect.forEach(jobs, (job) => (job.active() ? job.execute : job.cancel), { + discard: true, + }); + const exit = yield* restore( + (batch.reason === "control" && jobs.length === 1 && !jobs[0]!.snapshot + ? work + : commit(work) + ).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)), + ).pipe(Effect.exit); + for (const job of jobs) yield* job.settle(exit); + }), + ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")), + ), + Effect.ensuring(shutdown), + Effect.forkScoped, + ); + } + const barrier = Effect.fnUntraced(function* (effect: Effect.Effect) { + if (window === 0 || (yield* CurrentWorker) === (yield* Effect.fiberId)) return yield* effect; + const token = { active: true }; + return yield* Effect.gen(function* () { + const exit = yield* enqueue(Effect.exit(effect), token, false, 0).pipe( + Effect.flatten, + Effect.orDie, + ); + return yield* exit; + }).pipe( + Effect.ensuring( + Effect.sync(() => { + token.active = false; + }), + ), + ); + }); + const consume = Effect.fnUntraced(function* ( + source: Stream.Stream, + process: (event: Event) => Effect.Effect, + after: Effect.Effect, + canBatch: Effect.Effect = Effect.succeed(true), + ) { + if (window === 0) + return yield* source.pipe( + Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))), + Stream.takeUntil(Boolean), + Stream.runDrain, + ); + const token = { active: true }; + const signals = source.pipe( + Stream.map((event) => ({ _tag: "Event" as const, event })), + Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure" as const, cause })), + Stream.concat(Stream.succeed({ _tag: "End" as const })), + ); + let resume: Effect.Effect = Effect.void; + yield* signals.pipe( + Stream.mapEffect((signal) => + Effect.gen(function* () { + yield* resume; + const processed = yield* Deferred.make(); + const event = signal._tag === "Event" ? signal.event : undefined; + // EOF only acknowledges prior work; it can share a commit with other consumers. + const snapshot = + signal._tag === "End" || + (event !== undefined && isSnapshot(event) && (yield* canBatch)); + const bytes = + snapshot && event !== undefined ? Buffer.byteLength(yield* encodeQueue(event)) : 0; + const work: Effect.Effect = + signal._tag === "Failure" + ? Effect.failCause(signal.cause) + : event === undefined + ? Effect.succeed(true) + : process(event).pipe(Effect.as(false)); + const reply = yield* enqueue(work, token, snapshot, bytes); + if (!snapshot) resume = Deferred.await(processed); + return { reply, processed, check: !snapshot }; + }), + ), + Stream.buffer({ capacity: MAX_EVENTS }), + Stream.mapEffect(({ reply, processed, check }) => + reply.pipe( + Effect.flatMap((end) => + end ? Effect.succeed(true) : check ? after : Effect.succeed(false), + ), + Effect.tap((stop) => + Effect.sync(() => { + if (stop) token.active = false; + }), + ), + Effect.ensuring(Deferred.succeed(processed, undefined)), + ), + ), + Stream.takeUntil(Boolean), + Stream.runDrain, + Effect.ensuring( + Effect.sync(() => { + token.active = false; + }), + ), + ); + }); + return { barrier, consume }; +}); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index f3f1542146c66..b6e69f72de24d 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -55,6 +55,18 @@ import * as RunExecutionService from "./RunExecutionService.ts"; import * as RunFinalizationService from "./RunFinalizationService.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +const mockEventSink = (implementation: Partial) => + Layer.mock(EventSink.EventSinkV2)({ + consumeProviderEvents: (source, process, after) => + source.pipe( + Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))), + Stream.takeUntil(Boolean), + Stream.runDrain, + Effect.mapError((cause) => new EventSink.EventSinkWriteError({ eventCount: 0, cause })), + ), + ...implementation, + }); + const driver = ProviderDriverKind.make("codex"); const rootAppThread = (id: ThreadId) => ({ @@ -73,7 +85,7 @@ const layerRunExecutionTest = RunExecutionService.layer.pipe( PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({}), + mockEventSink({}), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ ingestNormalized: () => Effect.succeed([]), @@ -900,7 +912,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key" captureBaseline: () => Effect.void, }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({}), + mockEventSink({}), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ ingestNormalized: () => Effect.succeed([]), @@ -929,7 +941,7 @@ it.effect("fails the run when its ownership check cannot be read before calling PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ writeIfRunCurrent: (input) => Ref.update(writes, (current) => [...current, ...input.events]).pipe( Effect.as({ committed: true, storedEvents: [] }), @@ -1140,7 +1152,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () => ), }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ writeWithEffects: (input) => Ref.update(writes, (current) => [ ...current, @@ -1254,7 +1266,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co scenario === "start-guard" ? Effect.void : Effect.die("not reached"), }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ writeIfRunCurrent: (input) => Effect.gen(function* () { assert.equal(input.threadId, threadId); @@ -1434,7 +1446,7 @@ it.effect("ingests new member threads of an owned child after the root turn term PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ write: () => Effect.succeed([]), writeWithEffects: (input) => Effect.gen(function* () { @@ -1694,7 +1706,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int PreviewAgentActivity.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ write: () => Effect.succeed([]), writeWithEffects: (input) => Effect.as( @@ -2107,7 +2119,7 @@ it.effect( captureBaseline: () => Effect.void, }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ write: () => Effect.succeed([]), writeWithEffects: (input) => Effect.gen(function* () { @@ -3193,7 +3205,7 @@ function captureRootRunTermination(input: { }), Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ write: (payload) => Effect.gen(function* () { for (const event of payload.events) { @@ -3715,7 +3727,7 @@ function runBackgroundItemScenario( Layer.succeed(PreviewAgentActivity.PreviewAgentActivity, previewActivity), Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, - Layer.mock(EventSink.EventSinkV2)({ + mockEventSink({ write: () => Effect.succeed([]), writeWithEffects: (input) => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 27bc06aef2812..304eeaba43421 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -35,7 +35,6 @@ import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; -import * as Stream from "effect/Stream"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; @@ -46,8 +45,6 @@ import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; -import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; -import * as Clock from "effect/Clock"; import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; import { @@ -613,7 +610,6 @@ export const layer: Layer.Layer< const eventSink = yield* EventSink.EventSinkV2; const idAllocator = yield* IdAllocator.IdAllocatorV2; const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; - const commitWindowMs = yield* ProviderCommitBatch.windowMs; const serverSettings = yield* ServerSettings.ServerSettingsService; const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext; const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity; @@ -1516,178 +1512,159 @@ export const layer: Layer.Layer< } yield* trackChildLifecycle(event, deliveredEvent !== null); }); - const providerEventFiber = yield* ProviderCommitBatch.group( - eventSubscription.events, - commitWindowMs, - ).pipe( - Stream.mapEffect((batch) => + const providerEventFiber = yield* eventSink + .consumeProviderEvents( + eventSubscription.events, + (event) => + Effect.gen(function* () { + yield* ingestEvent(yield* routeEvent(event)); + }), Effect.gen(function* () { - const process = Effect.gen(function* () { - for (const event of batch.events) { - yield* ingestEvent(yield* routeEvent(event)); - if (yield* agentTurnsSettled) yield* endPreviewActivity; - if (yield* shouldStopProviderEventIngestion) return true; - } - return false; - }); - if (batch.events.length < 2 || (yield* Ref.get(rootTerminalSeen))) - return yield* process; - return yield* eventSink.withProviderBatch( - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ - "orchestration_v2.provider_event_count": batch.events.length, - "orchestration_v2.raw_json_bytes": batch.bytes, - "orchestration_v2.batch_flush_reason": batch.reason, - "orchestration_v2.commit_window_ms": commitWindowMs, - "orchestration_v2.batch_wait_ms": - (yield* Clock.currentTimeMillis) - batch.openedAt, - }); - return yield* process; - }), - ); + if (yield* agentTurnsSettled) yield* endPreviewActivity; + return yield* shouldStopProviderEventIngestion; }), - ), - Stream.takeUntil((stop) => stop), - Stream.runDrain, - Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), - Effect.flatMap(() => - Effect.gen(function* () { - const terminal = yield* Ref.get(terminalEvent); - if (terminal === null) { - // A session release interrupts or fails the stream; a clean - // end without a terminal means the run would never settle. - return yield* new RunExecutionIngestError({ - runId: input.run.id, - cause: new ProviderAdapter.ProviderAdapterEventStreamError({ - driver: input.providerThread.driver, - providerSessionId: input.providerSessionId, - cause: "Provider event stream ended without a root terminal.", - }), - }); - } - yield* finalizeRootRun(terminal); - }), - ), - // Server shutdown interrupts the subscription and leaves the run to - // restart recovery; every other failure fails the run. - Effect.catchCauseIf( - (cause) => !Cause.hasInterruptsOnly(cause), - (cause) => + Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)), + ) + .pipe( + Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), + Effect.flatMap(() => Effect.gen(function* () { - const completedAt = yield* DateTime.now; - yield* Effect.logWarning("orchestration V2 provider event ingestion failed", { - runId: input.run.id, - cause, - }); - if (yield* Ref.get(rootRunFinalized)) return; - const providerThread = yield* Ref.get(latestProviderThread); - const routing = yield* Ref.get(eventRouting); - const rootProviderTurn = yield* Ref.get(latestRootProviderTurn); - // Stop background work too; the failed consumer cannot persist terminal events. - const abandonedProviderTurn = - routing.rootProviderTurnId !== null && !routing.rootTurnEnded - ? yield* input.session - .interruptTurn({ - providerThread, - providerTurnId: routing.rootProviderTurnId, - requestRuntimeRestart: true, - }) - .pipe( - Effect.as(true), - Effect.catchCauseIf( - (interruptCause) => !Cause.hasInterruptsOnly(interruptCause), - (interruptCause) => - Effect.logWarning( - "orchestration V2 failed to interrupt turn after ingestion failure", - { runId: input.run.id, cause: interruptCause }, - ).pipe(Effect.as(false)), - ), - // A failed interrupt leaves the stored turn running so the next start - // retries it. The snapshot is cached before it is published, so it may - // not be stored: replay a terminal one as-is. - Effect.map((interrupted) => - !interrupted || rootProviderTurn === null - ? null - : isTerminalProviderTurnStatus(rootProviderTurn.status) - ? rootProviderTurn - : { - ...rootProviderTurn, - status: "interrupted" as const, - completedAt, - }, - ), - ) - : null; - const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal); - const openSubagents = yield* Ref.get(openRunOwnedSubagents); - const squashed = Cause.squash(cause); - let nextRetryWarningAt = completedAt; - yield* writeFinalRunEvents({ - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - writeIfRunCurrent: { - activeAttemptId: input.attempt.id, - expectedStatus: "running", - }, - openRunOwnedSubagents: openSubagents, - ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), - completedAt, - terminal: makeFailedTerminalEvent( - makeProviderFailure({ - cause: squashed, - message: - knownProviderFailureMessage(squashed) ?? - userFacingDatabaseErrorMessage(squashed), - class: "unknown", - }), - latestItemOrdinal + 1, - ), - failureItemPersisted: false, - refreshAfterTurn, - }).pipe( - // Storage failures can be defects; retry both failure channels. - Effect.catchCause((writeCause) => - Effect.gen(function* () { - if (Cause.hasInterruptsOnly(writeCause)) { - return yield* Effect.failCause(writeCause); - } - const now = yield* DateTime.now; - if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) { - yield* Effect.logWarning( - "orchestration V2 run failure finalization retry", - { - runId: input.run.id, - attemptId: input.attempt.id, - cause: writeCause, - }, - ); - // Persistent storage failures must not flood the same disk. - nextRetryWarningAt = DateTime.add(now, { minutes: 10 }); - } - return yield* Effect.fail(writeCause); + const terminal = yield* Ref.get(terminalEvent); + if (terminal === null) { + // A session release interrupts or fails the stream; a clean + // end without a terminal means the run would never settle. + return yield* new RunExecutionIngestError({ + runId: input.run.id, + cause: new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: input.providerThread.driver, + providerSessionId: input.providerSessionId, + cause: "Provider event stream ended without a root terminal.", }), - ), - Effect.retry({ - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), - ), - ), - }), - // Startup may cancel the consumer and fail its own terminal - // write. Only service shutdown or the atomic gate ends recovery. - Effect.forkIn(scope), - Effect.flatMap(Fiber.join), - ); + }); + } + yield* finalizeRootRun(terminal); }), - ), - Effect.ensuring(endPreviewActivity), - Effect.ensuring(eventSubscription.close), - Effect.forkDetach, - ); + ), + // Server shutdown interrupts the subscription and leaves the run to + // restart recovery; every other failure fails the run. + Effect.catchCauseIf( + (cause) => !Cause.hasInterruptsOnly(cause), + (cause) => + Effect.gen(function* () { + const completedAt = yield* DateTime.now; + yield* Effect.logWarning("orchestration V2 provider event ingestion failed", { + runId: input.run.id, + cause, + }); + if (yield* Ref.get(rootRunFinalized)) return; + const providerThread = yield* Ref.get(latestProviderThread); + const routing = yield* Ref.get(eventRouting); + const rootProviderTurn = yield* Ref.get(latestRootProviderTurn); + // Stop background work too; the failed consumer cannot persist terminal events. + const abandonedProviderTurn = + routing.rootProviderTurnId !== null && !routing.rootTurnEnded + ? yield* input.session + .interruptTurn({ + providerThread, + providerTurnId: routing.rootProviderTurnId, + requestRuntimeRestart: true, + }) + .pipe( + Effect.as(true), + Effect.catchCauseIf( + (interruptCause) => !Cause.hasInterruptsOnly(interruptCause), + (interruptCause) => + Effect.logWarning( + "orchestration V2 failed to interrupt turn after ingestion failure", + { runId: input.run.id, cause: interruptCause }, + ).pipe(Effect.as(false)), + ), + // A failed interrupt leaves the stored turn running so the next start + // retries it. The snapshot is cached before it is published, so it may + // not be stored: replay a terminal one as-is. + Effect.map((interrupted) => + !interrupted || rootProviderTurn === null + ? null + : isTerminalProviderTurnStatus(rootProviderTurn.status) + ? rootProviderTurn + : { + ...rootProviderTurn, + status: "interrupted" as const, + completedAt, + }, + ), + ) + : null; + const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal); + const openSubagents = yield* Ref.get(openRunOwnedSubagents); + const squashed = Cause.squash(cause); + let nextRetryWarningAt = completedAt; + yield* writeFinalRunEvents({ + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + writeIfRunCurrent: { + activeAttemptId: input.attempt.id, + expectedStatus: "running", + }, + openRunOwnedSubagents: openSubagents, + ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), + completedAt, + terminal: makeFailedTerminalEvent( + makeProviderFailure({ + cause: squashed, + message: + knownProviderFailureMessage(squashed) ?? + userFacingDatabaseErrorMessage(squashed), + class: "unknown", + }), + latestItemOrdinal + 1, + ), + failureItemPersisted: false, + refreshAfterTurn, + }).pipe( + // Storage failures can be defects; retry both failure channels. + Effect.catchCause((writeCause) => + Effect.gen(function* () { + if (Cause.hasInterruptsOnly(writeCause)) { + return yield* Effect.failCause(writeCause); + } + const now = yield* DateTime.now; + if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) { + yield* Effect.logWarning( + "orchestration V2 run failure finalization retry", + { + runId: input.run.id, + attemptId: input.attempt.id, + cause: writeCause, + }, + ); + // Persistent storage failures must not flood the same disk. + nextRetryWarningAt = DateTime.add(now, { minutes: 10 }); + } + return yield* Effect.fail(writeCause); + }), + ), + Effect.retry({ + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + ), + }), + // Startup may cancel the consumer and fail its own terminal + // write. Only service shutdown or the atomic gate ends recovery. + Effect.forkIn(scope), + Effect.flatMap(Fiber.join), + ); + }), + ), + Effect.ensuring(endPreviewActivity), + Effect.ensuring(eventSubscription.close), + Effect.forkDetach, + ); // A provider turn is a sign that the session is still alive. Keep // its already-issued MCP credential valid even when the agent goes diff --git a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts index 334d469dec6a3..bd41b888504b8 100644 --- a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts @@ -18,7 +18,17 @@ describe("Codex replay fixtures", () => { harness: CodexOrchestratorReplayHarness, materializeWorkspace: (transcript, workspace) => Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)), - }), + }).pipe( + Effect.provideService( + ConfigProvider.ConfigProvider, + ConfigProvider.fromUnknown({ + T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: + input.fixtureName.includes("subagent") || input.fixtureName === "multi_turn" + ? "10" + : "0", + }), + ), + ), ); }); From 98ed7064f49eab1c949067d087cc32e20d835c9a Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Wed, 4 Nov 2026 10:43:54 -0600 Subject: [PATCH 4/9] refactor(server): simplify shared provider batching --- .../ProviderCommitBatch.test.ts | 223 +++++++++--------- .../orchestration-v2/ProviderCommitBatch.ts | 185 +++++---------- 2 files changed, 178 insertions(+), 230 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index 5c7c9c08d2b88..c7309ffca88e5 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -42,106 +42,127 @@ const control: ProviderAdapter.ProviderAdapterV2Event = { pending: false, }; +const recordBatches = Effect.fn("test.recordProviderBatches")(function* (window: number) { + const output = yield* Queue.unbounded>(); + let current: ProviderAdapter.ProviderAdapterV2Event[] = []; + const coordinator = yield* ProviderCommitBatch.make( + (work) => + Effect.gen(function* () { + current = []; + yield* work; + if (current.length > 0) yield* Queue.offer(output, current); + }), + window, + ); + const process = (raw: ProviderAdapter.ProviderAdapterV2Event) => + raw.type === "background_work.updated" + ? Queue.offer(output, [raw]).pipe(Effect.asVoid) + : Effect.sync(() => { + current.push(raw); + }); + return { ...coordinator, process, output }; +}); + it.effect( - "collects successive updates from one stream and flushes before control without waiting for the window", + "flushes one stream's pending snapshots before a control without waiting for the window", () => Effect.gen(function* () { - const input = yield* Queue.unbounded(); - const output = - yield* Queue.unbounded>(); - const consumer = yield* ProviderCommitBatch.group(Stream.fromQueue(input), 25).pipe( - Stream.runForEach((batch) => Queue.offer(output, batch.events)), - Effect.forkScoped, - ); + const recorder = yield* recordBatches(25); + const input = yield* Queue.unbounded(); + const consumer = yield* recorder + .consume(Stream.fromQueue(input), recorder.process, Effect.succeed(true)) + .pipe(Effect.forkScoped); yield* Queue.offer(input, event(1)); yield* TestClock.adjust(5); yield* Queue.offer(input, event(2)); yield* Queue.offer(input, control); - const first = yield* Queue.take(output); - const second = yield* Queue.take(output); - assert.deepEqual(first, [event(1), event(2)]); - assert.deepEqual(second, [control]); - yield* Queue.end(input); + assert.deepEqual(yield* Queue.take(recorder.output), [event(1), event(2)]); + assert.deepEqual(yield* Queue.take(recorder.output), [control]); yield* Fiber.join(consumer); }).pipe(Effect.scoped), ); -it.effect("flushes a sparse stream on the deadline and restarts the next window", () => +it.effect("flushes sparse updates on the deadline and restarts the next window", () => Effect.gen(function* () { - const input = yield* Queue.unbounded(); - const output = yield* Queue.unbounded>(); - const consumer = yield* ProviderCommitBatch.group(Stream.fromQueue(input), 25).pipe( - Stream.runForEach((batch) => Queue.offer(output, batch.events)), - Effect.forkScoped, - ); + const recorder = yield* recordBatches(25); + const input = yield* Queue.unbounded(); + const pulled = yield* Queue.unbounded(); + yield* recorder + .consume( + Stream.fromQueue(input).pipe(Stream.tap((raw) => Queue.offer(pulled, raw))), + recorder.process, + Effect.succeed(false), + ) + .pipe(Effect.forkScoped); yield* Queue.offer(input, event(1)); + yield* Queue.take(pulled); + yield* TestClock.adjust(0); yield* TestClock.adjust(25); - assert.deepEqual(yield* Queue.take(output), [event(1)]); + assert.deepEqual(yield* Queue.take(recorder.output), [event(1)]); yield* Queue.offer(input, event(2)); + yield* Queue.take(pulled); + yield* TestClock.adjust(0); yield* TestClock.adjust(24); - assert.isTrue((yield* Queue.poll(output))._tag === "None"); + assert.isTrue((yield* Queue.poll(recorder.output))._tag === "None"); yield* TestClock.adjust(1); - assert.deepEqual(yield* Queue.take(output), [event(2)]); - yield* Queue.end(input); - yield* Fiber.join(consumer); + assert.deepEqual(yield* Queue.take(recorder.output), [event(2)]); }).pipe(Effect.scoped), ); -it.effect("bounds transaction groups by count and bytes and flushes the final partial group", () => +it.effect("bounds transactions by count and bytes, including the final partial group", () => Effect.gen(function* () { - const events = Array.from({ length: 129 }, (_, index) => event(index)); - const batches = yield* ProviderCommitBatch.group(Stream.fromArray(events), 25).pipe( - Stream.runCollect, - ); - assert.deepEqual( - batches.map((batch) => batch.events.length), - [64, 64, 1], - ); - assert.deepEqual( - batches.flatMap((batch) => batch.events), - events, - ); - const large = event(1, "x".repeat(1024 * 1024)); - const sized = yield* ProviderCommitBatch.group(Stream.make(event(0), large, event(2)), 25).pipe( - Stream.runCollect, - ); - assert.deepEqual( - sized.map((batch) => batch.events.length), - [1, 1, 1], - ); - assert.deepEqual( - sized.flatMap((batch) => batch.events), - [event(0), large, event(2)], - ); - }), -); - -it.effect("delivers the pending prefix before propagating a source failure", () => - Effect.gen(function* () { - const observed: Array = []; - const exit = yield* ProviderCommitBatch.group( - Stream.concat(Stream.make(event(1), event(2)), Stream.fail("provider failed")), - 25, - ).pipe( - Stream.runForEach((batch) => Effect.sync(() => observed.push(...batch.events))), - Effect.exit, - ); - assert.deepEqual(observed, [event(1), event(2)]); - assert.deepEqual(exit, Exit.fail("provider failed")); - }), + const workloads = [ + { events: Array.from({ length: 129 }, (_, index) => event(index)), sizes: [64, 64, 1] }, + { events: [event(0), event(1, "x".repeat(1024 * 1024)), event(2)], sizes: [1, 1, 1] }, + { + events: [event(0, "x".repeat(600_000)), event(1, "y".repeat(600_000)), event(2)], + sizes: [1, 1, 1], + }, + ]; + for (const { events, sizes } of workloads) { + const recorder = yield* recordBatches(25); + const pulled = yield* Deferred.make(); + let arrivals = 0; + const consumer = yield* recorder + .consume( + Stream.fromArray(events).pipe( + Stream.tap(() => + ++arrivals === events.length ? Deferred.succeed(pulled, undefined) : Effect.void, + ), + ), + recorder.process, + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + yield* recorder.barrier(Effect.void); + yield* TestClock.adjust(25); + yield* Fiber.join(consumer); + const batches = yield* Queue.takeAll(recorder.output); + assert.deepEqual( + batches.map((batch) => batch.length), + sizes, + ); + assert.deepEqual(batches.flat(), events); + } + }).pipe(Effect.scoped), ); -it.effect("keeps controls and individual writes immediate when the window is disabled", () => +it.effect("keeps updates immediate and ordered when batching is disabled", () => Effect.gen(function* () { - const batches = yield* ProviderCommitBatch.group( + const seen: ProviderAdapter.ProviderAdapterV2Event[] = []; + const coordinator = yield* ProviderCommitBatch.make(() => Effect.die("must not batch"), 0); + yield* coordinator.consume( Stream.make(event(1), control, event(2)), - 0, - ).pipe(Stream.runCollect); - assert.deepEqual( - batches.map((batch) => batch.events), - [[event(1)], [control], [event(2)]], + (raw) => + Effect.sync(() => { + seen.push(raw); + }), + Effect.succeed(false), ); - }), + assert.deepEqual(seen, [event(1), control, event(2)]); + }).pipe(Effect.scoped), ); it.effect.each([ @@ -152,17 +173,7 @@ it.effect.each([ "shares bounded transactions across $streams streams with $updates updates each", ({ streams, updates }) => Effect.gen(function* () { - const commits = yield* Queue.unbounded>(); - let transaction: string[] = []; - const coordinator = yield* ProviderCommitBatch.make( - (work) => - Effect.gen(function* () { - transaction = []; - yield* work; - yield* Queue.offer(commits, transaction); - }), - 10, - ); + const coordinator = yield* recordBatches(10); const inputs = yield* Effect.forEach(Array.from({ length: streams }), () => Queue.unbounded(), ); @@ -179,10 +190,7 @@ it.effect.each([ : Effect.void; }), ), - (raw) => - Effect.sync(() => { - if (raw.type === "message.updated") transaction.push(raw.message.text); - }), + coordinator.process, Effect.succeed(true), ) .pipe(Effect.forkScoped({ startImmediately: true })), @@ -193,12 +201,12 @@ it.effect.each([ } yield* Deferred.await(pulled); yield* TestClock.adjust(0); - if (updates === 1) assert.equal(yield* Queue.size(commits), 0); + if (updates === 1) assert.equal(yield* Queue.size(coordinator.output), 0); yield* coordinator.barrier(Effect.void); - const batches: ReadonlyArray[] = []; + const batches: Array> = []; let count = 0; while (count < streams * updates) { - const batch = yield* Queue.take(commits); + const batch = yield* Queue.take(coordinator.output); batches.push(batch); count += batch.length; } @@ -206,7 +214,14 @@ it.effect.each([ batches.map((batch) => batch.length), streams * updates > 64 ? [64, 36] : [10], ); - assert.equal(new Set(batches.flat()).size, streams * updates); + assert.equal( + new Set( + batches + .flat() + .flatMap((raw) => (raw.type === "message.updated" ? [raw.message.text] : [])), + ).size, + streams * updates, + ); for (const input of inputs) yield* Queue.offer(input, control); yield* Effect.forEach(consumers, Fiber.join); }).pipe(Effect.scoped), @@ -332,18 +347,8 @@ it.effect.each(["failure", "stop"] as const)( it.effect("does not force a separate commit for every consumer that ends", () => Effect.gen(function* () { const pulled = yield* Deferred.make(); - const writes: number[] = []; let arrivals = 0; - let count = 0; - const coordinator = yield* ProviderCommitBatch.make( - (work) => - Effect.gen(function* () { - count = 0; - yield* work; - writes.push(count); - }), - 10, - ); + const coordinator = yield* recordBatches(10); for (let index = 0; index < 10; index++) { yield* coordinator .consume( @@ -352,10 +357,7 @@ it.effect("does not force a separate commit for every consumer that ends", () => ++arrivals === 10 ? Deferred.succeed(pulled, undefined) : Effect.void, ), ), - () => - Effect.sync(() => { - count++; - }), + coordinator.process, Effect.succeed(false), ) .pipe(Effect.forkScoped({ startImmediately: true })); @@ -363,7 +365,10 @@ it.effect("does not force a separate commit for every consumer that ends", () => yield* Deferred.await(pulled); yield* TestClock.adjust(0); yield* coordinator.barrier(Effect.void); - assert.deepEqual(writes, [10]); + assert.deepEqual( + (yield* Queue.takeAll(coordinator.output)).map((batch) => batch.length), + [10], + ); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 895148d426a32..2e2468574bcf3 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -1,5 +1,4 @@ import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; -import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; @@ -26,128 +25,74 @@ export const windowMs = Config.String("T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS").pi ); type Event = ProviderAdapter.ProviderAdapterV2Event; -interface Batch { - readonly events: ReadonlyArray; - readonly bytes: number; - readonly openedAt: number; - readonly reason: "deadline" | "limit" | "control" | "end" | "failure"; -} - const isSnapshot = (event: Event) => event.type === "node.updated" || event.type === "message.updated" || event.type === "turn_item.updated" || event.type === "plan.updated"; -export function group(source: Stream.Stream, window: number) { - if (window === 0) - return source.pipe( - Stream.map((event): Batch => ({ events: [event], bytes: 0, openedAt: 0, reason: "control" })), - ); - return collect(source, window, (event) => - (isSnapshot(event) - ? encodeQueue(event).pipe(Effect.map(Buffer.byteLength)) - : Effect.succeed(0) - ).pipe(Effect.map((bytes) => ({ snapshot: isSnapshot(event), bytes }))), - ); -} - -function collect( - source: Stream.Stream, - window: number, - describe: (item: T) => Effect.Effect<{ snapshot: boolean; bytes: number }, Schema.SchemaError>, -) { - type Signal = - | { - readonly _tag: "Event"; - readonly event: T; - readonly bytes: number; - readonly snapshot: boolean; - } - | { readonly _tag: "Failure"; readonly cause: Cause.Cause } - | { readonly _tag: "End" }; - const signals = source.pipe( - Stream.mapEffect((event) => - describe(event).pipe( - Effect.map((description): Signal => ({ _tag: "Event", event, ...description })), - ), - ), - // A failed source must not discard the prefix already collected by the sink. - Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure", cause })), - Stream.concat(Stream.succeed({ _tag: "End" })), - ); - const collect = Sink.fold( - () => ({ signals: [] as Array, bytes: 0, openedAt: 0, stop: false }), - (state) => !state.stop, - (state, signal: Signal) => - Effect.gen(function* () { - if (state.signals.length === 0) state.openedAt = yield* Clock.currentTimeMillis; - state.signals.push(signal); - if (signal._tag === "Event") state.bytes += signal.bytes; - state.stop = - signal._tag !== "Event" || - !signal.snapshot || - state.signals.length >= MAX_EVENTS || - state.bytes >= MAX_BYTES; - return state; - }), - ); - return signals.pipe( - Stream.aggregateWithin(collect, Schedule.spaced(window)), - Stream.flatMap((state) => { - const groups: Array | Extract> = []; - let events: Array = []; - let bytes = 0; - const flush = (reason: Batch["reason"]) => { - if (events.length === 0) return; - groups.push({ events, bytes, openedAt: state.openedAt, reason }); - events = []; - bytes = 0; - }; - for (const signal of state.signals) { - if (signal._tag === "Failure") { - flush("failure"); - groups.push(signal); - } else if (signal._tag === "End") flush("end"); - else if (!signal.snapshot) { - flush("control"); - groups.push({ - events: [signal.event], - bytes: signal.bytes, - openedAt: state.openedAt, - reason: "control", - }); - } else { - if (bytes + signal.bytes > MAX_BYTES) flush("limit"); - events.push(signal.event); - bytes += signal.bytes; - if (bytes >= MAX_BYTES || events.length >= MAX_EVENTS) flush("limit"); - } - } - flush("deadline"); - return Stream.fromArray(groups).pipe( - Stream.mapEffect((batch) => - "_tag" in batch ? Effect.failCause(batch.cause) : Effect.succeed(batch), - ), - ); - }), - ); -} - export class CommitBatchError extends Schema.TaggedError()("CommitBatchError", { cause: Schema.Defect(), }) {} interface Job { - readonly owner: object; + readonly owner: { active: boolean }; readonly queuedAt: number; readonly snapshot: boolean; readonly bytes: number; - readonly active: () => boolean; readonly execute: Effect.Effect; readonly settle: (exit: Exit.Exit) => Effect.Effect; readonly cancel: Effect.Effect; } +interface Batch { + readonly events: ReadonlyArray; + readonly bytes: number; + readonly reason: "deadline" | "limit" | "control"; +} + +function collect(source: Stream.Stream, window: number) { + const sink = Sink.fold( + () => ({ events: [] as Array, bytes: 0, stop: false }), + (state) => !state.stop, + (state, job: Job) => + Effect.sync(() => { + state.events.push(job); + state.bytes += job.bytes; + state.stop = !job.snapshot || state.events.length >= MAX_EVENTS || state.bytes >= MAX_BYTES; + return state; + }), + ); + return source.pipe( + Stream.aggregateWithin(sink, Schedule.spaced(window)), + Stream.flatMap((state) => { + const last = state.events.at(-1); + if (last === undefined) return Stream.empty; + if (last.snapshot && (state.events.length === 1 || state.bytes <= MAX_BYTES)) { + return Stream.succeed({ + events: state.events, + bytes: state.bytes, + reason: + state.events.length >= MAX_EVENTS || state.bytes >= MAX_BYTES ? "limit" : "deadline", + }); + } + // The fold can include one overflowing job or a trailing control; isolate it from the prefix. + const batches: Array = []; + if (state.events.length > 1) + batches.push({ + events: state.events.slice(0, -1), + bytes: state.bytes - last.bytes, + reason: last.snapshot ? "limit" : "control", + }); + batches.push({ + events: [last], + bytes: last.bytes, + reason: !last.snapshot ? "control" : last.bytes >= MAX_BYTES ? "limit" : "deadline", + }); + return Stream.fromArray(batches); + }), + ); +} + const CurrentWorker = Context.Reference( "t3/ProviderCommitBatch/CurrentWorker", { @@ -181,7 +126,6 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( queuedAt: yield* Clock.currentTimeMillis, snapshot, bytes, - active: () => token.active, execute: effect.pipe( Effect.mapError((cause) => new CommitBatchError({ cause })), Effect.provideContext(caller), @@ -216,14 +160,12 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( return Deferred.await(reply); }); if (window > 0) { - yield* collect(Stream.fromQueue(queue), window, (job) => - Effect.succeed({ snapshot: job.snapshot, bytes: job.bytes }), - ).pipe( + yield* collect(Stream.fromQueue(queue), window).pipe( Stream.runForEach((batch) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { - const jobs = batch.events.filter((job) => job.active()); - for (const job of batch.events) if (!job.active()) yield* job.cancel; + const jobs = batch.events.filter((job) => job.owner.active); + for (const job of batch.events) if (!job.owner.active) yield* job.cancel; if (jobs.length === 0) return; yield* Effect.annotateCurrentSpan({ "orchestration_v2.work_item_count": jobs.length, @@ -234,9 +176,13 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( "orchestration_v2.batch_wait_ms": (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)), }); - const work = Effect.forEach(jobs, (job) => (job.active() ? job.execute : job.cancel), { - discard: true, - }); + const work = Effect.forEach( + jobs, + (job) => (job.owner.active ? job.execute : job.cancel), + { + discard: true, + }, + ); const exit = yield* restore( (batch.reason === "control" && jobs.length === 1 && !jobs[0]!.snapshot ? work @@ -254,13 +200,10 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( const barrier = Effect.fnUntraced(function* (effect: Effect.Effect) { if (window === 0 || (yield* CurrentWorker) === (yield* Effect.fiberId)) return yield* effect; const token = { active: true }; - return yield* Effect.gen(function* () { - const exit = yield* enqueue(Effect.exit(effect), token, false, 0).pipe( - Effect.flatten, - Effect.orDie, - ); - return yield* exit; - }).pipe( + return yield* enqueue(Effect.exit(effect), token, false, 0).pipe( + Effect.flatten, + Effect.orDie, + Effect.flatMap((exit) => exit), Effect.ensuring( Effect.sync(() => { token.active = false; From 34ecac049bc79b06302cc409bef82a92528ffdac Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 5 Nov 2026 14:38:42 -0600 Subject: [PATCH 5/9] fix(server): isolate batched updates and avoid thread lock deadlocks --- apps/server/src/orchestration-v2/EventSink.ts | 40 +++++++++-- .../FoundationPersistence.test.ts | 57 +++++++++++---- .../ProviderCommitBatch.test.ts | 71 ++++++++++++++++--- .../orchestration-v2/ProviderCommitBatch.ts | 55 +++++++++++--- .../RunExecutionService.test.ts | 1 - 5 files changed, 183 insertions(+), 41 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 661e8b47c91b1..e2d9e337a5962 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -84,7 +84,11 @@ export interface EventSinkV2Shape { process: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect, after: Effect.Effect, canBatch: Effect.Effect, - ) => Effect.Effect; + ) => Effect.Effect< + void, + E | E2 | ProviderCommitBatch.CommitBatchError | Schema.SchemaError, + R | R2 + >; /** Normalize and persist an ordered group against one transaction; publish after its commit. */ readonly withProviderBatch: ( @@ -941,8 +945,10 @@ const layerBase: Layer.Layer< !started && SqlError.isSqlError(cause) && cause.reason._tag === "LockTimeoutError", times: 2, }), - Effect.mapError( - (cause) => new EventSinkWriteError({ eventCount: batch.eventCount, cause }), + Effect.mapError((cause) => + SqlError.isSqlError(cause) + ? new EventSinkWriteError({ eventCount: batch.eventCount, cause }) + : cause, ), ); yield* Effect.annotateCurrentSpan({ @@ -958,13 +964,33 @@ const layerBase: Layer.Layer< Effect.mapError((cause) => new ProviderCommitBatch.CommitBatchError({ cause })), ), yield* ProviderCommitBatch.windowMs, + // Each snapshot owns a savepoint and its deferred publications within the shared commit. + Effect.fnUntraced(function* ( + work: Effect.Effect, + ) { + const batch = yield* CurrentProviderBatch; + const publicationCount = batch?.publish.length ?? 0; + const eventCount = batch?.eventCount ?? 0; + return yield* sql.withTransaction(work).pipe( + Effect.onError(() => + Effect.sync(() => { + if (batch !== undefined) { + batch.publish.length = publicationCount; + batch.eventCount = eventCount; + } + }), + ), + Effect.mapError((cause) => + ProviderCommitBatch.isCommitBatchError(cause) + ? cause + : new ProviderCommitBatch.CommitBatchError({ cause }), + ), + ); + }), ); return EventSinkV2.of({ - consumeProviderEvents: (source, process, after, canBatch) => - coordinator - .consume(source, process, after, canBatch) - .pipe(Effect.mapError((cause) => new EventSinkWriteError({ eventCount: 0, cause }))), + consumeProviderEvents: coordinator.consume, withProviderBatch, write: (input) => writeEffect({ ...input, effects: [] }).pipe( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 012bc39373830..44fb40a5c023e 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -31,6 +31,7 @@ import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAda import * as ConfigProvider from "effect/ConfigProvider"; import * as Queue from "effect/Queue"; import * as Arr from "effect/Array"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Exit from "effect/Exit"; @@ -360,6 +361,11 @@ it.effect( }, }); const highWater = yield* sink.latestSequence(); + const callerError = { _tag: "CallerError" } as const; + assert.strictEqual( + yield* sink.withProviderBatch(Effect.fail(callerError)).pipe(Effect.flip), + callerError, + ); const failed = yield* sink .withProviderBatch( Effect.gen(function* () { @@ -413,9 +419,11 @@ it.effect( }).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped), ); -it.effect.each([false, true])( - "shares the writer across independent consumers with rollback=%s", - (rollback) => { +it.effect.each(["none", "duplicate", "after-write"] as const)( + "shares the writer across independent consumers with failure=%s", + (failure) => { + const rollback = failure !== "none"; + const jobError = { _tag: "InvalidUpdate" } as const; const writes: number[] = []; const tracer = Tracer.make({ span(options) { @@ -443,6 +451,12 @@ it.effect.each([false, true])( ), }); const highWater = yield* sink.latestSequence(); + const store = yield* EventStore.EventStoreV2; + const publications = yield* Queue.unbounded(); + yield* sink.stream({ afterSequence: highWater }).pipe( + Stream.runForEach((stored) => Queue.offer(publications, stored.event.id)), + Effect.forkScoped({ startImmediately: true }), + ); const pulled = yield* Deferred.make(); let arrivals = 0; const inputs = yield* Effect.forEach(threads, () => @@ -463,7 +477,9 @@ it.effect.each([false, true])( events: [ { id: EventId.make( - rollback ? "duplicate-shared-event" : `shared-event:${index}`, + failure === "duplicate" && index === 9 + ? "shared-event:0" + : `shared-event:${index}`, ), threadId: raw.message.threadId, occurredAt: now, @@ -472,7 +488,13 @@ it.effect.each([false, true])( }, ], }) - .pipe(Effect.asVoid) + .pipe( + Effect.andThen( + failure === "after-write" && index === 9 + ? Effect.fail(jobError) + : Effect.void, + ), + ) : Effect.void, Effect.succeed(false), Effect.succeed(true), @@ -503,17 +525,24 @@ it.effect.each([false, true])( yield* TestClock.adjust(0); yield* sink.write({ events: [] }); if (rollback) { - for (const consumer of consumers) - assert.isTrue(Exit.isFailure(yield* Fiber.join(consumer))); - assert.equal(yield* sink.latestSequence(), highWater); - } else { - assert.deepEqual(writes, [10]); - assert.equal(yield* sink.latestSequence(), highWater + 10); + const exit = yield* Fiber.join(consumers[9]!); + assert.isTrue(Exit.isFailure(exit)); + if (failure === "after-write" && Exit.isFailure(exit)) + assert.strictEqual(Cause.squash(exit.cause), jobError); } - for (const thread of threads) { + yield* TestClock.adjust(0); + assert.deepEqual( + yield* Queue.takeAll(publications), + (yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect)).map( + (stored) => stored.event.id, + ), + ); + assert.deepEqual(writes, [rollback ? 9 : 10]); + assert.equal(yield* sink.latestSequence(), highWater + (rollback ? 9 : 10)); + for (let index = 0; index < threads.length; index++) { assert.lengthOf( - (yield* projections.getThreadRecords(thread.id, ["messages"])).messages, - rollback ? 0 : 1, + (yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages, + rollback && index === 9 ? 0 : 1, ); } }).pipe( diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index c7309ffca88e5..0560663623375 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -1,5 +1,6 @@ import { assert, it } from "@effect/vitest"; import { MessageId, ProviderDriverKind, ProviderTurnId, ThreadId } from "@t3tools/contracts"; +import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; @@ -252,27 +253,43 @@ it.effect("does not hold the shared worker while one consumer checks background }).pipe(Effect.scoped), ); -it.effect( - "fails every participant of a failed transaction and continues serving unrelated runs", - () => +it.effect.each(["job", "commit"] as const)( + "isolates a %s failure at its transaction boundary", + (mode) => Effect.gen(function* () { - const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const coordinator = yield* ProviderCommitBatch.make( + (work) => + mode === "commit" + ? work.pipe( + Effect.andThen( + Effect.fail(new ProviderCommitBatch.CommitBatchError({ cause: "COMMIT failed" })), + ), + ) + : work, + 10, + ); const inputs = yield* Effect.forEach([0, 1], () => Queue.unbounded(), ); - const failures = yield* Effect.forEach(inputs, (input, index) => + const consumers = yield* Effect.forEach(inputs, (input, index) => coordinator .consume( Stream.fromQueue(input), - () => (index === 1 ? Effect.fail("disk full") : Effect.void), - Effect.succeed(false), + () => (mode === "job" && index === 1 ? Effect.fail("invalid update") : Effect.void), + Effect.succeed(true), ) .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })), ); for (const input of inputs) yield* Queue.offer(input, event(0)); yield* TestClock.adjust(0); yield* coordinator.barrier(Effect.void); - for (const failed of failures) assert.isTrue(Exit.isFailure(yield* Fiber.join(failed))); + assert.isTrue(Exit.isFailure(yield* Fiber.join(consumers[1]!))); + if (mode === "commit") assert.isTrue(Exit.isFailure(yield* Fiber.join(consumers[0]!))); + else { + assert.isUndefined(consumers[0]!.pollUnsafe()); + yield* Queue.offer(inputs[0]!, control); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(consumers[0]!))); + } assert.equal(yield* coordinator.barrier(Effect.succeed("recovered")), "recovered"); assert.deepEqual( yield* coordinator.barrier(Effect.fail("original error")).pipe(Effect.exit), @@ -281,6 +298,44 @@ it.effect( }).pipe(Effect.scoped), ); +it.effect( + "allows a command holding the thread lock to commit while a provider control waits for it", + () => + Effect.gen(function* () { + const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const locks = yield* KeyedLock.make(); + const threadId = ThreadId.make("locked-thread"); + const held = yield* Deferred.make(); + const waiting = yield* Deferred.make(); + const committed = yield* Deferred.make(); + const command = yield* locks + .withLock( + threadId, + Deferred.succeed(held, undefined).pipe( + Effect.andThen(Deferred.await(waiting)), + Effect.andThen(coordinator.barrier(Deferred.succeed(committed, undefined))), + ), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(held); + const consumer = yield* coordinator + .consume( + Stream.make(control), + () => + Deferred.succeed(waiting, undefined).pipe( + Effect.andThen(locks.withLock(threadId, Effect.void)), + ), + Effect.succeed(true), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(waiting); + yield* TestClock.adjust(0); + assert.isTrue(yield* Deferred.isDone(committed)); + yield* Fiber.join(command); + yield* Fiber.join(consumer); + }).pipe(Effect.scoped), +); + it.effect("cancels pending work from one consumer without cancelling its neighbours", () => Effect.gen(function* () { const committed = yield* Deferred.make(); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 2e2468574bcf3..6ea0fb6672a77 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -1,4 +1,5 @@ import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; @@ -34,6 +35,7 @@ const isSnapshot = (event: Event) => export class CommitBatchError extends Schema.TaggedError()("CommitBatchError", { cause: Schema.Defect(), }) {} +export const isCommitBatchError = Schema.is(CommitBatchError); interface Job { readonly owner: { active: boolean }; @@ -41,7 +43,10 @@ interface Job { readonly snapshot: boolean; readonly bytes: number; readonly execute: Effect.Effect; - readonly settle: (exit: Exit.Exit) => Effect.Effect; + readonly settle: ( + exit: Exit.Exit, + committed: boolean, + ) => Effect.Effect; readonly cancel: Effect.Effect; } interface Batch { @@ -103,6 +108,9 @@ const CurrentWorker = Context.Reference( export const make = Effect.fn("ProviderCommitBatch.make")(function* ( commit: (work: Effect.Effect) => Effect.Effect, window: number, + isolate: ( + work: Effect.Effect, + ) => Effect.Effect = (work) => work, ) { const queue = yield* Queue.bounded(MAX_EVENTS); const pending = new Set(); @@ -119,7 +127,8 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( bytes: number, ) { const caller = yield* Effect.context(); - const reply = yield* Deferred.make(); + const reply = yield* Deferred.make(); + let ownFailure: Cause.Cause | undefined; let result: A; const job: Job = { owner: token, @@ -127,6 +136,11 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( snapshot, bytes, execute: effect.pipe( + Effect.tapCause((cause) => + Effect.sync(() => { + ownFailure = cause; + }), + ), Effect.mapError((cause) => new CommitBatchError({ cause })), Effect.provideContext(caller), Effect.tap((value) => @@ -136,11 +150,17 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ), Effect.asVoid, ), - settle: Effect.fnUntraced(function* (exit: Exit.Exit) { + settle: Effect.fnUntraced(function* ( + exit: Exit.Exit, + committed: boolean, + ) { pending.delete(job); if (Exit.isFailure(exit)) { token.active = false; - yield* Deferred.failCause(reply, exit.cause); + yield* Deferred.failCause( + reply, + committed && ownFailure !== undefined ? ownFailure : exit.cause, + ); } else yield* Deferred.succeed(reply, result); }), cancel: Effect.gen(function* () { @@ -176,12 +196,19 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( "orchestration_v2.batch_wait_ms": (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)), }); + const outcomes = new Map>(); const work = Effect.forEach( jobs, - (job) => (job.owner.active ? job.execute : job.cancel), - { - discard: true, - }, + (job) => + Effect.gen(function* () { + if (!job.owner.active) return yield* job.cancel; + const outcome = yield* (job.bytes > 0 ? isolate(job.execute) : job.execute).pipe( + Effect.exit, + ); + outcomes.set(job, outcome); + if (Exit.isFailure(outcome)) job.owner.active = false; + }), + { discard: true }, ); const exit = yield* restore( (batch.reason === "control" && jobs.length === 1 && !jobs[0]!.snapshot @@ -189,7 +216,11 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( : commit(work) ).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)), ).pipe(Effect.exit); - for (const job of jobs) yield* job.settle(exit); + for (const job of jobs) + yield* job.settle( + Exit.isSuccess(exit) ? (outcomes.get(job) ?? Exit.void) : exit, + Exit.isSuccess(exit), + ); }), ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")), ), @@ -203,7 +234,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( return yield* enqueue(Effect.exit(effect), token, false, 0).pipe( Effect.flatten, Effect.orDie, - Effect.flatMap((exit) => exit), + Effect.flatten, Effect.ensuring( Effect.sync(() => { token.active = false; @@ -248,7 +279,9 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( : event === undefined ? Effect.succeed(true) : process(event).pipe(Effect.as(false)); - const reply = yield* enqueue(work, token, snapshot, bytes); + const reply = snapshot + ? yield* enqueue(work, token, true, bytes) + : (yield* enqueue(Effect.void, token, false, 0)).pipe(Effect.andThen(work)); if (!snapshot) resume = Deferred.await(processed); return { reply, processed, check: !snapshot }; }), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index b6e69f72de24d..5bfb4fe34dc6a 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -62,7 +62,6 @@ const mockEventSink = (implementation: Partial Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))), Stream.takeUntil(Boolean), Stream.runDrain, - Effect.mapError((cause) => new EventSink.EventSinkWriteError({ eventCount: 0, cause })), ), ...implementation, }); From cede1c1c065e4f7b1e77a23efdcc589a32c6d6ba Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 5 Nov 2026 22:02:52 -0600 Subject: [PATCH 6/9] perf(server): batch queued provider writes without waiting --- apps/server/src/orchestration-v2/EventSink.ts | 8 +- .../FoundationPersistence.test.ts | 39 ++-- .../ProviderCommitBatch.test.ts | 200 ++++++++++-------- .../orchestration-v2/ProviderCommitBatch.ts | 122 +++++------ .../CodexReplayFixtures.integration.test.ts | 22 +- 5 files changed, 203 insertions(+), 188 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index e2d9e337a5962..e1ee796e47e1a 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -961,9 +961,13 @@ const layerBase: Layer.Layer< const coordinator = yield* ProviderCommitBatch.make( (work) => withProviderBatch(work).pipe( - Effect.mapError((cause) => new ProviderCommitBatch.CommitBatchError({ cause })), + Effect.mapError((cause) => + ProviderCommitBatch.isCommitBatchError(cause) + ? cause + : new ProviderCommitBatch.CommitBatchError({ cause }), + ), ), - yield* ProviderCommitBatch.windowMs, + yield* ProviderCommitBatch.enabled, // Each snapshot owns a savepoint and its deferred publications within the shared commit. Effect.fnUntraced(function* ( work: Effect.Effect, diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 44fb40a5c023e..21ad0c5e19a1d 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -419,10 +419,15 @@ it.effect( }).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped), ); -it.effect.each(["none", "duplicate", "after-write"] as const)( - "shares the writer across independent consumers with failure=%s", - (failure) => { +it.effect.each( + (["none", "duplicate", "after-write"] as const).flatMap((failure) => + [1, 10].map((streams) => ({ failure, streams })), + ), +)( + "shares the writer and isolates failure=$failure across $streams independent consumers", + ({ failure, streams }) => { const rollback = failure !== "none"; + const failedIndex = streams - 1; const jobError = { _tag: "InvalidUpdate" } as const; const writes: number[] = []; const tracer = Tracer.make({ @@ -442,7 +447,7 @@ it.effect.each(["none", "duplicate", "after-write"] as const)( const sink = yield* EventSink.EventSinkV2; const projections = yield* ProjectionStore.ProjectionStoreV2; const now = yield* DateTime.now; - const threads = Array.from({ length: 10 }, (_, index) => + const threads = Array.from({ length: streams }, (_, index) => makeThread(ThreadId.make(`shared:${index}`), now), ); yield* sink.write({ @@ -467,7 +472,7 @@ it.effect.each(["none", "duplicate", "after-write"] as const)( .consumeProviderEvents( Stream.fromQueue(input).pipe( Stream.tap(() => - ++arrivals === 10 ? Deferred.succeed(pulled, undefined) : Effect.void, + ++arrivals === streams ? Deferred.succeed(pulled, undefined) : Effect.void, ), ), (raw) => @@ -477,8 +482,8 @@ it.effect.each(["none", "duplicate", "after-write"] as const)( events: [ { id: EventId.make( - failure === "duplicate" && index === 9 - ? "shared-event:0" + failure === "duplicate" && index === failedIndex + ? "created:0" : `shared-event:${index}`, ), threadId: raw.message.threadId, @@ -490,7 +495,7 @@ it.effect.each(["none", "duplicate", "after-write"] as const)( }) .pipe( Effect.andThen( - failure === "after-write" && index === 9 + failure === "after-write" && index === failedIndex ? Effect.fail(jobError) : Effect.void, ), @@ -525,24 +530,24 @@ it.effect.each(["none", "duplicate", "after-write"] as const)( yield* TestClock.adjust(0); yield* sink.write({ events: [] }); if (rollback) { - const exit = yield* Fiber.join(consumers[9]!); + const exit = yield* Fiber.join(consumers[failedIndex]!); assert.isTrue(Exit.isFailure(exit)); if (failure === "after-write" && Exit.isFailure(exit)) assert.strictEqual(Cause.squash(exit.cause), jobError); } yield* TestClock.adjust(0); + const replay = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect); assert.deepEqual( - yield* Queue.takeAll(publications), - (yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect)).map( - (stored) => stored.event.id, - ), + replay.length === 0 ? [] : yield* Queue.takeN(publications, replay.length), + replay.map((stored) => stored.event.id), ); - assert.deepEqual(writes, [rollback ? 9 : 10]); - assert.equal(yield* sink.latestSequence(), highWater + (rollback ? 9 : 10)); + assert.equal(yield* Queue.size(publications), 0); + assert.deepEqual(writes, rollback && streams === 1 ? [] : [streams - (rollback ? 1 : 0)]); + assert.equal(yield* sink.latestSequence(), highWater + streams - (rollback ? 1 : 0)); for (let index = 0; index < threads.length; index++) { assert.lengthOf( (yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages, - rollback && index === 9 ? 0 : 1, + rollback && index === failedIndex ? 0 : 1, ); } }).pipe( @@ -552,7 +557,7 @@ it.effect.each(["none", "duplicate", "after-write"] as const)( Effect.withTracerEnabled(true), Effect.provideService( ConfigProvider.ConfigProvider, - ConfigProvider.fromUnknown({ T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: "10" }), + ConfigProvider.fromUnknown({ T3CODE_PROVIDER_COMMIT_BATCHING: true }), ), ); }, diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index 0560663623375..6655d1554533f 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -43,17 +43,20 @@ const control: ProviderAdapter.ProviderAdapterV2Event = { pending: false, }; -const recordBatches = Effect.fn("test.recordProviderBatches")(function* (window: number) { +const recordBatches = Effect.fn("test.recordProviderBatches")(function* ( + beforeCommit: Effect.Effect = Effect.void, +) { const output = yield* Queue.unbounded>(); let current: ProviderAdapter.ProviderAdapterV2Event[] = []; const coordinator = yield* ProviderCommitBatch.make( (work) => Effect.gen(function* () { + yield* beforeCommit; current = []; yield* work; if (current.length > 0) yield* Queue.offer(output, current); }), - window, + true, ); const process = (raw: ProviderAdapter.ProviderAdapterV2Event) => raw.type === "background_work.updated" @@ -64,28 +67,26 @@ const recordBatches = Effect.fn("test.recordProviderBatches")(function* (window: return { ...coordinator, process, output }; }); -it.effect( - "flushes one stream's pending snapshots before a control without waiting for the window", - () => - Effect.gen(function* () { - const recorder = yield* recordBatches(25); - const input = yield* Queue.unbounded(); - const consumer = yield* recorder - .consume(Stream.fromQueue(input), recorder.process, Effect.succeed(true)) - .pipe(Effect.forkScoped); - yield* Queue.offer(input, event(1)); - yield* TestClock.adjust(5); - yield* Queue.offer(input, event(2)); - yield* Queue.offer(input, control); - assert.deepEqual(yield* Queue.take(recorder.output), [event(1), event(2)]); - assert.deepEqual(yield* Queue.take(recorder.output), [control]); - yield* Fiber.join(consumer); - }).pipe(Effect.scoped), +it.effect("publishes preceding snapshots before a control", () => + Effect.gen(function* () { + const recorder = yield* recordBatches(); + const input = yield* Queue.unbounded(); + const consumer = yield* recorder + .consume(Stream.fromQueue(input), recorder.process, Effect.succeed(true)) + .pipe(Effect.forkScoped); + yield* Queue.offer(input, event(1)); + assert.deepEqual(yield* Queue.take(recorder.output), [event(1)]); + yield* Queue.offer(input, event(2)); + yield* Queue.offer(input, control); + assert.deepEqual(yield* Queue.take(recorder.output), [event(2)]); + assert.deepEqual(yield* Queue.take(recorder.output), [control]); + yield* Fiber.join(consumer); + }).pipe(Effect.scoped), ); -it.effect("flushes sparse updates on the deadline and restarts the next window", () => +it.effect("publishes isolated snapshots without advancing the clock", () => Effect.gen(function* () { - const recorder = yield* recordBatches(25); + const recorder = yield* recordBatches(); const input = yield* Queue.unbounded(); const pulled = yield* Queue.unbounded(); yield* recorder @@ -95,33 +96,28 @@ it.effect("flushes sparse updates on the deadline and restarts the next window", Effect.succeed(false), ) .pipe(Effect.forkScoped); - yield* Queue.offer(input, event(1)); - yield* Queue.take(pulled); - yield* TestClock.adjust(0); - yield* TestClock.adjust(25); - assert.deepEqual(yield* Queue.take(recorder.output), [event(1)]); - yield* Queue.offer(input, event(2)); - yield* Queue.take(pulled); - yield* TestClock.adjust(0); - yield* TestClock.adjust(24); - assert.isTrue((yield* Queue.poll(recorder.output))._tag === "None"); - yield* TestClock.adjust(1); - assert.deepEqual(yield* Queue.take(recorder.output), [event(2)]); + for (const raw of [event(1), event(2)]) { + yield* Queue.offer(input, raw); + yield* Queue.take(pulled); + yield* TestClock.adjust(0); + const published = yield* Queue.poll(recorder.output); + assert.equal(published._tag, "Some"); + if (published._tag === "Some") assert.deepEqual(published.value, [raw]); + } }).pipe(Effect.scoped), ); it.effect("bounds transactions by count and bytes, including the final partial group", () => Effect.gen(function* () { const workloads = [ - { events: Array.from({ length: 129 }, (_, index) => event(index)), sizes: [64, 64, 1] }, - { events: [event(0), event(1, "x".repeat(1024 * 1024)), event(2)], sizes: [1, 1, 1] }, + { events: Array.from({ length: 129 }, (_, index) => event(index)) }, + { events: [event(0), event(1, "x".repeat(1024 * 1024)), event(2)] }, { events: [event(0, "x".repeat(600_000)), event(1, "y".repeat(600_000)), event(2)], - sizes: [1, 1, 1], }, ]; - for (const { events, sizes } of workloads) { - const recorder = yield* recordBatches(25); + for (const { events } of workloads) { + const recorder = yield* recordBatches(); const pulled = yield* Deferred.make(); let arrivals = 0; const consumer = yield* recorder @@ -138,12 +134,17 @@ it.effect("bounds transactions by count and bytes, including the final partial g yield* Deferred.await(pulled); yield* TestClock.adjust(0); yield* recorder.barrier(Effect.void); - yield* TestClock.adjust(25); yield* Fiber.join(consumer); const batches = yield* Queue.takeAll(recorder.output); - assert.deepEqual( - batches.map((batch) => batch.length), - sizes, + assert.isTrue(batches.every((batch) => batch.length <= 64)); + if (events.length === 129) assert.isTrue(batches.some((batch) => batch.length > 1)); + assert.isTrue( + batches.every( + (batch) => + batch.length === 1 || + batch.reduce((bytes, raw) => bytes + Buffer.byteLength(JSON.stringify(raw)), 0) <= + 1024 * 1024, + ), ); assert.deepEqual(batches.flat(), events); } @@ -153,7 +154,7 @@ it.effect("bounds transactions by count and bytes, including the final partial g it.effect("keeps updates immediate and ordered when batching is disabled", () => Effect.gen(function* () { const seen: ProviderAdapter.ProviderAdapterV2Event[] = []; - const coordinator = yield* ProviderCommitBatch.make(() => Effect.die("must not batch"), 0); + const coordinator = yield* ProviderCommitBatch.make(() => Effect.die("must not batch"), false); yield* coordinator.consume( Stream.make(event(1), control, event(2)), (raw) => @@ -174,7 +175,7 @@ it.effect.each([ "shares bounded transactions across $streams streams with $updates updates each", ({ streams, updates }) => Effect.gen(function* () { - const coordinator = yield* recordBatches(10); + const coordinator = yield* recordBatches(); const inputs = yield* Effect.forEach(Array.from({ length: streams }), () => Queue.unbounded(), ); @@ -202,7 +203,7 @@ it.effect.each([ } yield* Deferred.await(pulled); yield* TestClock.adjust(0); - if (updates === 1) assert.equal(yield* Queue.size(coordinator.output), 0); + yield* coordinator.barrier(Effect.void); const batches: Array> = []; let count = 0; @@ -232,7 +233,7 @@ it.effect("does not hold the shared worker while one consumer checks background Effect.gen(function* () { const checking = yield* Deferred.make(); const release = yield* Deferred.make(); - const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const coordinator = yield* ProviderCommitBatch.make((work) => work, true); const consumer = yield* coordinator .consume( Stream.make(control), @@ -266,7 +267,7 @@ it.effect.each(["job", "commit"] as const)( ), ) : work, - 10, + true, ); const inputs = yield* Effect.forEach([0, 1], () => Queue.unbounded(), @@ -298,11 +299,30 @@ it.effect.each(["job", "commit"] as const)( }).pipe(Effect.scoped), ); +it.effect("keeps a transaction failure when rolling back one failed snapshot", () => + Effect.gen(function* () { + const transactionError = new ProviderCommitBatch.CommitBatchError({ cause: "ROLLBACK failed" }); + const coordinator = yield* ProviderCommitBatch.make( + (work) => work.pipe(Effect.mapError(() => transactionError)), + true, + ); + const exit = yield* coordinator + .consume( + Stream.concat(Stream.make(event(1)), Stream.never), + () => Effect.fail("invalid snapshot"), + Effect.succeed(false), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(exit)); + if (Exit.isFailure(exit)) assert.strictEqual(Cause.squash(exit.cause), transactionError); + }).pipe(Effect.scoped), +); + it.effect( "allows a command holding the thread lock to commit while a provider control waits for it", () => Effect.gen(function* () { - const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const coordinator = yield* ProviderCommitBatch.make((work) => work, true); const locks = yield* KeyedLock.make(); const threadId = ThreadId.make("locked-thread"); const held = yield* Deferred.make(); @@ -336,38 +356,52 @@ it.effect( }).pipe(Effect.scoped), ); -it.effect("cancels pending work from one consumer without cancelling its neighbours", () => - Effect.gen(function* () { - const committed = yield* Deferred.make(); - let writes = 0; - const coordinator = yield* ProviderCommitBatch.make( - (work) => work.pipe(Effect.tap(() => Deferred.succeed(committed, undefined))), - 10, - ); - const inputs = yield* Effect.forEach([0, 1], () => - Queue.unbounded(), - ); - const consumers = yield* Effect.forEach(inputs, (input) => - coordinator - .consume( - Stream.fromQueue(input), - () => - Effect.sync(() => { - writes++; - }), - Effect.succeed(true), - ) - .pipe(Effect.forkScoped({ startImmediately: true })), - ); - for (const input of inputs) yield* Queue.offer(input, event(0)); - yield* TestClock.adjust(0); - yield* Fiber.interrupt(consumers[0]!); - yield* coordinator.barrier(Effect.void); - yield* Deferred.await(committed); - assert.equal(writes, 1); - yield* Queue.offer(inputs[1]!, control); - yield* Fiber.join(consumers[1]!); - }).pipe(Effect.scoped), +it.effect.each([false, true])( + "batches work accumulated behind a busy writer with cancellation=%s", + (cancel) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + let first = true; + const recorder = yield* recordBatches( + Effect.gen(function* () { + if (!first) return; + first = false; + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(release); + }), + ); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(started); + const pulled = yield* Deferred.make(); + let arrivals = 0; + const consumers = yield* Effect.forEach(Array.from({ length: 10 }), (_, index) => + recorder + .consume( + Stream.make(event(index)).pipe( + Stream.tap(() => + ++arrivals === 10 ? Deferred.succeed(pulled, undefined) : Effect.void, + ), + ), + recorder.process, + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })), + ); + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + if (cancel) yield* Fiber.interrupt(consumers[0]!); + assert.equal(yield* Queue.size(recorder.output), 0); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(leading); + yield* Effect.forEach(consumers.slice(cancel ? 1 : 0), Fiber.join); + assert.deepEqual(yield* Queue.takeAll(recorder.output), [ + [event(-1)], + Array.from({ length: cancel ? 9 : 10 }, (_, index) => event(index + (cancel ? 1 : 0))), + ]); + }).pipe(Effect.scoped), ); it.effect.each(["failure", "stop"] as const)( @@ -375,7 +409,7 @@ it.effect.each(["failure", "stop"] as const)( (mode) => Effect.gen(function* () { const seen: ProviderAdapter.ProviderAdapterV2Event[] = []; - const coordinator = yield* ProviderCommitBatch.make((work) => work, 10); + const coordinator = yield* ProviderCommitBatch.make((work) => work, true); const source = mode === "failure" ? Stream.concat(Stream.make(event(1), event(2)), Stream.fail("provider failed")) @@ -403,7 +437,7 @@ it.effect("does not force a separate commit for every consumer that ends", () => Effect.gen(function* () { const pulled = yield* Deferred.make(); let arrivals = 0; - const coordinator = yield* recordBatches(10); + const coordinator = yield* recordBatches(); for (let index = 0; index < 10; index++) { yield* coordinator .consume( @@ -429,7 +463,7 @@ it.effect("does not force a separate commit for every consumer that ends", () => it.effect("settles calls made after the coordinator's scope closes", () => Effect.gen(function* () { - const coordinator = yield* ProviderCommitBatch.make((work) => work, 10).pipe(Effect.scoped); + const coordinator = yield* ProviderCommitBatch.make((work) => work, true).pipe(Effect.scoped); const exit = yield* coordinator .barrier(Effect.die("must not execute after shutdown")) .pipe(Effect.exit); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 6ea0fb6672a77..60cee2b632f66 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -7,21 +7,15 @@ import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Exit from "effect/Exit"; import * as Queue from "effect/Queue"; -import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; -import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; const MAX_EVENTS = 64; const MAX_BYTES = 1024 * 1024; const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); -export const windowMs = Config.String("T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS").pipe( - Config.withDefault("0"), - Effect.map((raw) => { - const value = Number(raw); - return Number.isInteger(value) && value > 0 ? Math.min(value, 25) : 0; - }), +export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( + Config.withDefault(true), Effect.orDie, ); @@ -45,54 +39,39 @@ interface Job { readonly execute: Effect.Effect; readonly settle: ( exit: Exit.Exit, - committed: boolean, + preserveFailure: boolean, ) => Effect.Effect; readonly cancel: Effect.Effect; } interface Batch { readonly events: ReadonlyArray; readonly bytes: number; - readonly reason: "deadline" | "limit" | "control"; + readonly reason: "ready" | "limit" | "control"; } -function collect(source: Stream.Stream, window: number) { - const sink = Sink.fold( - () => ({ events: [] as Array, bytes: 0, stop: false }), - (state) => !state.stop, - (state, job: Job) => - Effect.sync(() => { - state.events.push(job); - state.bytes += job.bytes; - state.stop = !job.snapshot || state.events.length >= MAX_EVENTS || state.bytes >= MAX_BYTES; - return state; - }), - ); - return source.pipe( - Stream.aggregateWithin(sink, Schedule.spaced(window)), - Stream.flatMap((state) => { - const last = state.events.at(-1); - if (last === undefined) return Stream.empty; - if (last.snapshot && (state.events.length === 1 || state.bytes <= MAX_BYTES)) { - return Stream.succeed({ - events: state.events, - bytes: state.bytes, - reason: - state.events.length >= MAX_EVENTS || state.bytes >= MAX_BYTES ? "limit" : "deadline", - }); - } - // The fold can include one overflowing job or a trailing control; isolate it from the prefix. +function collect(queue: Queue.Queue) { + return Stream.fromEffectRepeat(Queue.takeBetween(queue, 1, MAX_EVENTS)).pipe( + Stream.flatMap((ready) => { const batches: Array = []; - if (state.events.length > 1) - batches.push({ - events: state.events.slice(0, -1), - bytes: state.bytes - last.bytes, - reason: last.snapshot ? "limit" : "control", - }); - batches.push({ - events: [last], - bytes: last.bytes, - reason: !last.snapshot ? "control" : last.bytes >= MAX_BYTES ? "limit" : "deadline", - }); + let events: Array = []; + let bytes = 0; + const flush = (reason: Batch["reason"]) => { + if (events.length > 0) batches.push({ events, bytes, reason }); + events = []; + bytes = 0; + }; + for (const job of ready) { + if (!job.snapshot) { + flush("control"); + batches.push({ events: [job], bytes: 0, reason: "control" }); + } else { + if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit"); + events.push(job); + bytes += job.bytes; + if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit"); + } + } + flush("ready"); return Stream.fromArray(batches); }), ); @@ -107,14 +86,16 @@ const CurrentWorker = Context.Reference( export const make = Effect.fn("ProviderCommitBatch.make")(function* ( commit: (work: Effect.Effect) => Effect.Effect, - window: number, + enabled: boolean, isolate: ( work: Effect.Effect, ) => Effect.Effect = (work) => work, ) { const queue = yield* Queue.bounded(MAX_EVENTS); const pending = new Set(); + let closed = false; const shutdown = Effect.gen(function* () { + closed = true; yield* Queue.shutdown(queue); for (const job of pending) yield* job.cancel; }); @@ -152,14 +133,14 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ), settle: Effect.fnUntraced(function* ( exit: Exit.Exit, - committed: boolean, + preserveFailure: boolean, ) { pending.delete(job); if (Exit.isFailure(exit)) { token.active = false; yield* Deferred.failCause( reply, - committed && ownFailure !== undefined ? ownFailure : exit.cause, + preserveFailure && ownFailure !== undefined ? ownFailure : exit.cause, ); } else yield* Deferred.succeed(reply, result); }), @@ -179,8 +160,8 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ); return Deferred.await(reply); }); - if (window > 0) { - yield* collect(Stream.fromQueue(queue), window).pipe( + if (enabled) { + yield* collect(queue).pipe( Stream.runForEach((batch) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { @@ -192,7 +173,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.owner)).size, "orchestration_v2.raw_json_bytes": batch.bytes, "orchestration_v2.batch_flush_reason": batch.reason, - "orchestration_v2.commit_window_ms": window, + "orchestration_v2.backlog_batching": true, "orchestration_v2.batch_wait_ms": (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)), }); @@ -202,25 +183,32 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( (job) => Effect.gen(function* () { if (!job.owner.active) return yield* job.cancel; - const outcome = yield* (job.bytes > 0 ? isolate(job.execute) : job.execute).pipe( - Effect.exit, - ); + const outcome = yield* ( + jobs.length > 1 && job.bytes > 0 ? isolate(job.execute) : job.execute + ).pipe(Effect.exit); outcomes.set(job, outcome); - if (Exit.isFailure(outcome)) job.owner.active = false; + if (Exit.isFailure(outcome)) { + job.owner.active = false; + if (jobs.length === 1) return yield* Effect.failCause(outcome.cause); + } }), { discard: true }, ); const exit = yield* restore( - (batch.reason === "control" && jobs.length === 1 && !jobs[0]!.snapshot - ? work - : commit(work) - ).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)), + (jobs.every((job) => job.bytes === 0) ? work : commit(work)).pipe( + Effect.provideService(CurrentWorker, yield* Effect.fiberId), + ), ).pipe(Effect.exit); - for (const job of jobs) + for (const job of jobs) { + const outcome = outcomes.get(job) ?? Exit.void; yield* job.settle( - Exit.isSuccess(exit) ? (outcomes.get(job) ?? Exit.void) : exit, - Exit.isSuccess(exit), + Exit.isSuccess(exit) ? outcome : exit, + Exit.isSuccess(exit) || + (Exit.isFailure(exit) && + Exit.isFailure(outcome) && + Cause.squash(exit.cause) === Cause.squash(outcome.cause)), ); + } }), ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")), ), @@ -229,7 +217,9 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ); } const barrier = Effect.fnUntraced(function* (effect: Effect.Effect) { - if (window === 0 || (yield* CurrentWorker) === (yield* Effect.fiberId)) return yield* effect; + if (closed) return yield* Effect.interrupt; + if (!enabled || pending.size === 0 || (yield* CurrentWorker) === (yield* Effect.fiberId)) + return yield* effect; const token = { active: true }; return yield* enqueue(Effect.exit(effect), token, false, 0).pipe( Effect.flatten, @@ -248,7 +238,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( after: Effect.Effect, canBatch: Effect.Effect = Effect.succeed(true), ) { - if (window === 0) + if (!enabled) return yield* source.pipe( Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))), Stream.takeUntil(Boolean), diff --git a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts index bd41b888504b8..1c60af2d7bdc3 100644 --- a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts @@ -1,5 +1,4 @@ import { assert, describe, it } from "@effect/vitest"; -import * as ConfigProvider from "effect/ConfigProvider"; import * as Tracer from "effect/Tracer"; import * as Effect from "effect/Effect"; @@ -18,17 +17,7 @@ describe("Codex replay fixtures", () => { harness: CodexOrchestratorReplayHarness, materializeWorkspace: (transcript, workspace) => Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)), - }).pipe( - Effect.provideService( - ConfigProvider.ConfigProvider, - ConfigProvider.fromUnknown({ - T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: - input.fixtureName.includes("subagent") || input.fixtureName === "multi_turn" - ? "10" - : "0", - }), - ), - ), + }), ); }); @@ -59,14 +48,7 @@ it.effect( harness: CodexOrchestratorReplayHarness, materializeWorkspace: (transcript, workspace) => Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)), - }).pipe( - Effect.withTracer(tracer), - Effect.withTracerEnabled(true), - Effect.provideService( - ConfigProvider.ConfigProvider, - ConfigProvider.fromUnknown({ T3CODE_EXPERIMENTAL_COMMIT_WINDOW_MS: "10" }), - ), - ); + }).pipe(Effect.withTracer(tracer), Effect.withTracerEnabled(true)); assert.deepEqual(writes, [3, 3]); }), ); From 8b0d7bdd8072e2c739770bda0ddca7fe5e7e38b7 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 5 Nov 2026 22:14:08 -0600 Subject: [PATCH 7/9] fix(opencode): publish approval items before actionable requests --- packages/provider-opencode/src/server/adapter.ts | 11 ++++++----- packages/provider-opencode/src/server/v2/adapter.ts | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index c4386e8485411..455e1544e171a 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -1900,15 +1900,16 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function }, }); yield* emitProviderEvent({ - type: "runtime_request.updated", + type: "turn_item.updated", driver: OPENCODE_PROVIDER, - threadId: turn.threadId, - runtimeRequest, + turnItem: runtimeRequestTurnItem(pending, "waiting", null, now), }); + // A response can commit as soon as the actionable request is published. yield* emitProviderEvent({ - type: "turn_item.updated", + type: "runtime_request.updated", driver: OPENCODE_PROVIDER, - turnItem: runtimeRequestTurnItem(pending, "waiting", null, now), + threadId: turn.threadId, + runtimeRequest, }); yield* updateProviderSession("waiting", null); }); diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index ddca16ed8fed3..3cbd4d893b6f1 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -1888,13 +1888,14 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr native, }); yield* emit({ type: "node.updated", driver, node }); + yield* emit({ type: "turn_item.updated", driver, turnItem: item }); + // A response can commit as soon as the actionable request is published. yield* emit({ type: "runtime_request.updated", driver, threadId: turn.input.threadId, runtimeRequest: request, }); - yield* emit({ type: "turn_item.updated", driver, turnItem: item }); yield* setSessionStatus("waiting", null); }); From 5b2be36e138d45a4c340ebcffaf199a8bbc9a048 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 5 Nov 2026 22:23:56 -0600 Subject: [PATCH 8/9] test(opencode): require items before actionable requests --- packages/provider-opencode/src/server/adapter.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts index 6fb5263b6e3bc..dca94c861146a 100644 --- a/packages/provider-opencode/src/server/adapter.test.ts +++ b/packages/provider-opencode/src/server/adapter.test.ts @@ -712,6 +712,10 @@ describe("OpenCodeAdapterV2", () => { }), ); const events = yield* Fiber.join(received); + assert.deepEqual( + events.map((event) => event.type), + ["turn_item.updated", "runtime_request.updated"], + ); const item = events.find((event) => event.type === "turn_item.updated"); const request = events.find((event) => event.type === "runtime_request.updated"); assert.equal(item?.type, "turn_item.updated"); From 45b5b0cb55c3d9543af2f7d55ba242c0462d68f3 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 5 Nov 2026 22:35:35 -0600 Subject: [PATCH 9/9] test(server): allow valid replay batch partitions --- .../testkit/CodexReplayFixtures.integration.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts index 1c60af2d7bdc3..1274c9eaee2e9 100644 --- a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts @@ -22,7 +22,7 @@ describe("Codex replay fixtures", () => { }); it.effect( - "batches consecutive snapshots from one recorded Codex stream in the real run consumer", + "persists recorded Codex snapshots through provider transactions in the real run consumer", () => Effect.gen(function* () { const fixture = findReplayFixture("multi_turn")!; @@ -49,6 +49,10 @@ it.effect( materializeWorkspace: (transcript, workspace) => Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)), }).pipe(Effect.withTracer(tracer), Effect.withTracerEnabled(true)); - assert.deepEqual(writes, [3, 3]); + assert.equal( + writes.reduce((total, count) => total + count, 0), + 6, + ); + assert.isTrue(writes.every((count) => count <= 3)); }), );