From eaf8034671bcb27ad41b0d98d6da41173fb44104 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Fri, 6 Nov 2026 00:50:39 -0600 Subject: [PATCH] perf(server): batch shutdown recovery commits - Defer cancellation signals until each group commits. --- .../FoundationPersistence.test.ts | 153 ++++++++++++++++++ .../ProviderRuntimeRecoveryService.ts | 70 +++++--- 2 files changed, 199 insertions(+), 24 deletions(-) diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index e1b2e7610663..acf3cc27f621 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -563,6 +563,159 @@ it.effect.each( }, ); +it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => { + let commits = 0; + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (time, exit) => { + end(time, exit); + if (span.name === "COMMIT") commits++; + }; + return span; + }, + }); + return Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const events = yield* EventStore.EventStoreV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const threads = Array.from({ length: 65 }, (_, i) => + ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`), + ); + const sharedSession = { + id: ProviderSessionId.make("shutdown-shared"), + driver: providerDriver, + providerInstanceId, + status: "ready" as const, + cwd: process.cwd(), + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }; + yield* sink.write({ + events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [ + threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }), + ...(i < 2 + ? [ + { + id: EventId.make(`shutdown-session:${i}`), + type: "provider-session.attached" as const, + threadId, + occurredAt: now, + payload: sharedSession, + }, + ] + : []), + { + id: EventId.make(`shutdown-provider:${i}`), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: { + id: ProviderThreadId.make(`shutdown-provider:${i}`), + driver: providerDriver, + providerInstanceId, + providerSessionId: i < 2 ? sharedSession.id : null, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: i === 1 || i === 32 ? "idle" : "active", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }, + ]), + }); + yield* outbox.enqueue( + threads.map((threadId, i) => ({ + id: `shutdown-effect:${i}`, + commandId: CommandId.make(`shutdown-command:${i}`), + threadId, + request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) }, + })), + ); + let eventlessNotifications = 0; + let armed = fail; + const recovery = yield* ProviderRuntimeRecovery.make.pipe( + Effect.provide(ServerSettings.layerTest()), + Effect.provideService(EffectOutbox.EffectOutboxV2, { + ...outbox, + signalCancellations: (ids) => + Effect.gen(function* () { + assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService))); + eventlessNotifications += ids.length; + yield* outbox.signalCancellations(ids); + }), + }), + Effect.provideService( + EffectWorker.OrchestrationEffectWorkerV2, + EffectWorker.OrchestrationEffectWorkerV2.of({ + awaitWork: Effect.void, + runRecoveryOnce: Effect.succeed(false), + runOnce: Effect.succeed(false), + nextClaimableAt: Effect.succeedNone, + drain: () => Effect.succeed(0), + }), + ), + Effect.provideService(EventSink.EventSinkV2, { + ...sink, + commitCommand: (input) => + armed && input.threadId === threads[33] + ? Effect.die("injected shutdown failure") + : sink.commitCommand(input), + }), + ); + commits = 0; + const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit); + assert.equal(Exit.isFailure(result), fail); + assert.equal(commits, fail ? 1 : 3); + const receipts = + yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`; + assert.deepEqual( + receipts.map((r) => r.aggregate_id), + threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32), + ); + assert.deepEqual( + yield* projections.getRecoveryThreadIds("runtime"), + fail ? threads.slice(32) : [], + ); + assert.equal(eventlessNotifications, fail ? 1 : 2); + for (const [i] of threads.entries()) { + const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`)); + assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending"); + } + assert.equal( + (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter( + (e) => + e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id, + ).length, + 1, + ); + armed = false; + yield* recovery.reconcile("shutdown"); + assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []); + const highWater = yield* sink.latestSequence(); + yield* recovery.reconcile("shutdown"); + assert.equal(yield* sink.latestSequence(), highWater); + }).pipe( + Effect.provide(Layer.fresh(layerTest)), + Effect.withTracer(tracer), + Effect.withTracerEnabled(true), + Effect.provideService(Statement.SpanPropagationEnabled, true), + ); +}); + it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { it.effect("projects oversized tool bodies once before replay and live RPC retention", () => Effect.scoped( diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index c374db2abfb9..222747c41398 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -51,6 +51,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError candidate.status !== "stopped" && candidate.status !== "error", ).length; let retiredEffects: number; + let cancelledEffectIds: ReadonlyArray = []; if (events.length === 0) { const retiredEffectIds = yield* outbox .cancelUnsettled({ @@ -779,7 +782,7 @@ export const make = Effect.gen(function* () { }), ), ); - yield* outbox.signalCancellations(retiredEffectIds); + cancelledEffectIds = retiredEffectIds; retiredEffects = retiredEffectIds.length; } else { const result = yield* eventSink @@ -812,6 +815,7 @@ export const make = Effect.gen(function* () { stoppedSessions, closedRequests: requests.length, retiredEffects, + cancelledEffectIds, }; }, ); @@ -832,31 +836,49 @@ export const make = Effect.gen(function* () { let closedRequests = 0; let retiredEffects = 0; const stoppedThisPass = new Map(); - for (const threadId of threadIds) { - const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe( - Effect.mapError( - (cause) => - new ProviderRuntimeRecoveryError({ - operation: "read-projections", - threadId, - cause, - }), + const grouped = trigger === "shutdown" && threadIds.length > 1; + const groupSize = grouped ? 32 : 1; + for (let offset = 0; offset < threadIds.length; offset += groupSize) { + const cancelledEffectIds: string[] = []; + // Read and decide on the writer so shared sessions see earlier updates in this group. + const reconcileGroup = Effect.gen(function* () { + for (const threadId of threadIds.slice(offset, offset + groupSize)) { + const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe( + Effect.mapError( + (cause) => + new ProviderRuntimeRecoveryError({ + operation: "read-projections", + threadId, + cause, + }), + ), + ); + const enabled = + continueAfterRestart !== null && + resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings + .continueThreadsAfterServerUpdate; + const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass); + for (const session of projection.providerSessions) { + if (session.status !== "stopped" && session.status !== "error") { + stoppedThisPass.set(session.id, session); + } + } + cancelledEffectIds.push(...result.cancelledEffectIds); + terminalizedRuns += result.terminalizedRuns; + stoppedSessions += result.stoppedSessions; + closedRequests += result.closedRequests; + retiredEffects += result.retiredEffects; + } + }); + yield* (grouped ? eventSink.withProviderBatch(reconcileGroup) : reconcileGroup).pipe( + Effect.mapError((cause) => + isProviderRuntimeRecoveryError(cause) + ? cause + : new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }), ), ); - const enabled = - continueAfterRestart !== null && - resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings - .continueThreadsAfterServerUpdate; - const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass); - for (const session of projection.providerSessions) { - if (session.status !== "stopped" && session.status !== "error") { - stoppedThisPass.set(session.id, session); - } - } - terminalizedRuns += result.terminalizedRuns; - stoppedSessions += result.stoppedSessions; - closedRequests += result.closedRequests; - retiredEffects += result.retiredEffects; + // Eventless outbox cancellations also wait for the group's durable commit. + if (cancelledEffectIds.length > 0) yield* outbox.signalCancellations(cancelledEffectIds); } const outboxReconciliation = yield* outbox.reconcileAfterProcessLoss.pipe( Effect.mapError(