Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 153 additions & 0 deletions apps/server/src/orchestration-v2/FoundationPersistence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,159 @@ it.effect.each(
},
);

it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
let commits = 0;
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (time, exit) => {
end(time, exit);
if (span.name === "COMMIT") commits++;
};
return span;
},
});
return Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const events = yield* EventStore.EventStoreV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const now = yield* DateTime.now;
const threads = Array.from({ length: 65 }, (_, i) =>
ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
);
const sharedSession = {
id: ProviderSessionId.make("shutdown-shared"),
driver: providerDriver,
providerInstanceId,
status: "ready" as const,
cwd: process.cwd(),
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
};
yield* sink.write({
events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
...(i < 2
? [
{
id: EventId.make(`shutdown-session:${i}`),
type: "provider-session.attached" as const,
threadId,
occurredAt: now,
payload: sharedSession,
},
]
: []),
{
id: EventId.make(`shutdown-provider:${i}`),
type: "provider-thread.updated",
threadId,
occurredAt: now,
payload: {
id: ProviderThreadId.make(`shutdown-provider:${i}`),
driver: providerDriver,
providerInstanceId,
providerSessionId: i < 2 ? sharedSession.id : null,
appThreadId: threadId,
ownerNodeId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
status: i === 1 || i === 32 ? "idle" : "active",
firstRunOrdinal: null,
lastRunOrdinal: null,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
},
},
]),
});
yield* outbox.enqueue(
threads.map((threadId, i) => ({
id: `shutdown-effect:${i}`,
commandId: CommandId.make(`shutdown-command:${i}`),
threadId,
request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
})),
);
let eventlessNotifications = 0;
let armed = fail;
const recovery = yield* ProviderRuntimeRecovery.make.pipe(
Effect.provide(ServerSettings.layerTest()),
Effect.provideService(EffectOutbox.EffectOutboxV2, {
...outbox,
signalCancellations: (ids) =>
Effect.gen(function* () {
assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
eventlessNotifications += ids.length;
yield* outbox.signalCancellations(ids);
}),
}),
Effect.provideService(
EffectWorker.OrchestrationEffectWorkerV2,
EffectWorker.OrchestrationEffectWorkerV2.of({
awaitWork: Effect.void,
runRecoveryOnce: Effect.succeed(false),
runOnce: Effect.succeed(false),
nextClaimableAt: Effect.succeedNone,
drain: () => Effect.succeed(0),
}),
),
Effect.provideService(EventSink.EventSinkV2, {
...sink,
commitCommand: (input) =>
armed && input.threadId === threads[33]
? Effect.die("injected shutdown failure")
: sink.commitCommand(input),
}),
);
commits = 0;
const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
assert.equal(Exit.isFailure(result), fail);
assert.equal(commits, fail ? 1 : 3);
const receipts =
yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
assert.deepEqual(
receipts.map((r) => r.aggregate_id),
threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
);
assert.deepEqual(
yield* projections.getRecoveryThreadIds("runtime"),
fail ? threads.slice(32) : [],
);
assert.equal(eventlessNotifications, fail ? 1 : 2);
for (const [i] of threads.entries()) {
const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
}
assert.equal(
(yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
(e) =>
e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
).length,
1,
);
armed = false;
yield* recovery.reconcile("shutdown");
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
const highWater = yield* sink.latestSequence();
yield* recovery.reconcile("shutdown");
assert.equal(yield* sink.latestSequence(), highWater);
}).pipe(
Effect.provide(Layer.fresh(layerTest)),
Effect.withTracer(tracer),
Effect.withTracerEnabled(true),
Effect.provideService(Statement.SpanPropagationEnabled, true),
);
});

it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
Effect.scoped(
Expand Down
70 changes: 46 additions & 24 deletions apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
}
}

const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);

export interface ProviderRuntimeRecoverySummary {
readonly terminalizedRuns: number;
readonly stoppedSessions: number;
Expand Down Expand Up @@ -762,6 +764,7 @@ export const make = Effect.gen(function* () {
(candidate) => candidate.status !== "stopped" && candidate.status !== "error",
).length;
let retiredEffects: number;
let cancelledEffectIds: ReadonlyArray<string> = [];
if (events.length === 0) {
const retiredEffectIds = yield* outbox
.cancelUnsettled({
Expand All @@ -779,7 +782,7 @@ export const make = Effect.gen(function* () {
}),
),
);
yield* outbox.signalCancellations(retiredEffectIds);
cancelledEffectIds = retiredEffectIds;
retiredEffects = retiredEffectIds.length;
} else {
const result = yield* eventSink
Expand Down Expand Up @@ -812,6 +815,7 @@ export const make = Effect.gen(function* () {
stoppedSessions,
closedRequests: requests.length,
retiredEffects,
cancelledEffectIds,
};
},
);
Expand All @@ -832,31 +836,49 @@ export const make = Effect.gen(function* () {
let closedRequests = 0;
let retiredEffects = 0;
const stoppedThisPass = new Map<string, RecoverySession>();
for (const threadId of threadIds) {
const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
Effect.mapError(
(cause) =>
new ProviderRuntimeRecoveryError({
operation: "read-projections",
threadId,
cause,
}),
const grouped = trigger === "shutdown" && threadIds.length > 1;
const groupSize = grouped ? 32 : 1;
for (let offset = 0; offset < threadIds.length; offset += groupSize) {
const cancelledEffectIds: string[] = [];
// Read and decide on the writer so shared sessions see earlier updates in this group.
const reconcileGroup = Effect.gen(function* () {
for (const threadId of threadIds.slice(offset, offset + groupSize)) {
const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
Effect.mapError(
(cause) =>
new ProviderRuntimeRecoveryError({
operation: "read-projections",
threadId,
cause,
}),
),
);
const enabled =
continueAfterRestart !== null &&
resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
.continueThreadsAfterServerUpdate;
const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
for (const session of projection.providerSessions) {
if (session.status !== "stopped" && session.status !== "error") {
stoppedThisPass.set(session.id, session);
}
}
cancelledEffectIds.push(...result.cancelledEffectIds);
terminalizedRuns += result.terminalizedRuns;
stoppedSessions += result.stoppedSessions;
closedRequests += result.closedRequests;
retiredEffects += result.retiredEffects;
}
});
yield* (grouped ? eventSink.withProviderBatch(reconcileGroup) : reconcileGroup).pipe(
Effect.mapError((cause) =>
isProviderRuntimeRecoveryError(cause)
? cause
: new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
),
);
const enabled =
continueAfterRestart !== null &&
resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
.continueThreadsAfterServerUpdate;
const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
for (const session of projection.providerSessions) {
if (session.status !== "stopped" && session.status !== "error") {
stoppedThisPass.set(session.id, session);
}
}
terminalizedRuns += result.terminalizedRuns;
stoppedSessions += result.stoppedSessions;
closedRequests += result.closedRequests;
retiredEffects += result.retiredEffects;
// Eventless outbox cancellations also wait for the group's durable commit.
if (cancelledEffectIds.length > 0) yield* outbox.signalCancellations(cancelledEffectIds);
}
const outboxReconciliation = yield* outbox.reconcileAfterProcessLoss.pipe(
Effect.mapError(
Expand Down
Loading