Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 117 additions & 2 deletions apps/server/src/orchestration-v2/EventSink.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
CommandId,
type OrchestrationV2Command,
type OrchestrationV2Run,
OrchestrationV2DomainEvent,
type OrchestrationV2ProviderThread,
Expand Down Expand Up @@ -34,6 +35,7 @@ import * as EventStore from "./EventStore.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as ProjectStore from "./ProjectStore.ts";
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
import { prepareFreshProviderThread } from "./freshProviderContext.ts";

export class EventSinkWriteError extends Schema.TaggedError<EventSinkWriteError>()(
"EventSinkWriteError",
Expand Down Expand Up @@ -67,6 +69,15 @@ export const EventSinkV2Error = Schema.Union([EventSinkWriteError, EventSinkStre
export type EventSinkV2Error = typeof EventSinkV2Error.Type;

export interface EventSinkV2Shape {
readonly acceptFreshContext: (
event: Extract<OrchestrationV2DomainEvent, { type: "provider-thread.updated" }>,
) => Effect.Effect<
{
readonly committed: boolean;
readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
},
EventSinkV2Error
>;
readonly write: (input: {
/** Return no events if another thread owns this native session at commit time. */
readonly unlessNativeThreadOwned?: Pick<
Expand Down Expand Up @@ -142,6 +153,11 @@ export interface EventSinkV2Shape {
readonly acceptedAt: DateTime.Utc;
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
readonly effects: ReadonlyArray<EffectOutbox.PendingOrchestrationEffectV2>;
/** Recheck renewal safety against ingestion in the same transaction as its events. */
readonly freshProviderRenewal?: Extract<
OrchestrationV2Command,
{ type: "provider-session.renew" }
>;
readonly cancelUnsettledEffects?: {
readonly effectTypes: ReadonlyArray<EffectOutbox.OrchestrationEffectRequestV2["type"]>;
readonly reason: string;
Expand Down Expand Up @@ -616,6 +632,54 @@ const layerBase: Layer.Layer<
);
});

const acceptFreshContextEffect = Effect.fn("orchestrationV2.EventSink.acceptFreshContext")(
function* (event: Parameters<EventSinkV2Shape["acceptFreshContext"]>[0]) {
return yield* commitThenPublish(
Effect.gen(function* () {
const records = yield* projectionStore.getThreadRecords(event.threadId, [
"providerThreads",
]);
const current = records.providerThreads.find((row) => row.id === event.payload.id);
const accepted = event.payload.freshContext;
const nativeId = event.payload.nativeThreadRef?.nativeId;
if (
nativeId == null ||
accepted?.acceptedNativeThreadId !== nativeId ||
current?.nativeThreadRef?.nativeId !== nativeId ||
current.freshContext?.bootstrap !== accepted.bootstrap ||
current.freshContext.throughRunOrdinal !== accepted.throughRunOrdinal ||
current.freshContext.acceptedNativeThreadId === nativeId
) {
return {
committed: false as const,
storedEvents: [] as ReadonlyArray<OrchestrationV2StoredEvent>,
};
}
// Read and merge under the transaction so ingestion cannot invalidate
// acceptance by changing unrelated provider-thread fields.
const storedEvents = yield* eventStore.append({
events: [
{
...event,
payload: {
...current,
freshContext: { ...current.freshContext, acceptedNativeThreadId: nativeId },
updatedAt: event.occurredAt,
},
},
],
});
const outcomes = yield* applyStoredEvents(storedEvents);
return { committed: true as const, storedEvents, outcomes };
}),
(result) =>
result.committed
? publishStoredEvents(result.storedEvents, result.outcomes)
: Effect.void,
);
},
);

const existingCommandResult =
/* This helper runs for individual orchestration events; the enclosing operation owns the span. */
Effect.fnUntraced(function* (commandId: CommandId) {
Expand Down Expand Up @@ -651,11 +715,58 @@ const layerBase: Layer.Layer<
return { ...existing, committed: false as const, cancelledEffectIds: [] };
}

let events = input.events;
if (input.freshProviderRenewal !== undefined) {
const records = yield* projectionStore.getThreadRecords(input.threadId, [
"providerThreads",
"runs",
"providerTurns",
"runtimeRequests",
"subagents",
"turnItems",
]);
const prepared = prepareFreshProviderThread(
records,
input.freshProviderRenewal,
input.acceptedAt,
);
if (typeof prepared === "string") {
const receipt: CommandReceiptStore.CommandReceiptV2 = {
commandId: input.commandId,
threadId: input.threadId,
commandType: input.commandType,
acceptedAt: input.acceptedAt,
resultSequence: yield* eventStore.latestSequence({ threadId: input.threadId }),
status: "rejected",
error: prepared,
};
yield* commandReceipts.upsert(receipt);
return {
receipt,
storedEvents: [] as ReadonlyArray<OrchestrationV2StoredEvent>,
committed: false as const,
cancelledEffectIds: [],
};
}
// Ingestion does not take the command lock. Rebuild from the current
// row so even an unrelated adapter update keeps all of its fields.
events = events.map((event) =>
event.type !== "provider-thread.updated"
? event
: {
...event,
payload:
event.payload.id === prepared.previous.id
? prepared.previous
: prepared.fresh,
},
);
}
const runId = input.updateDelegatedCompletionOnlyRunId;
const normalized = yield* normalizeEvents(
runId === undefined
? input.events
: yield* updateDelegatedCompletionOnly(input.events, (id) => id === runId),
? events
: yield* updateDelegatedCompletionOnly(events, (id) => id === runId),
);
const storedEvents = yield* eventStore.append({
commandId: input.commandId,
Expand Down Expand Up @@ -922,6 +1033,10 @@ const layerBase: Layer.Layer<
}),
),
),
acceptFreshContext: (event) =>
acceptFreshContextEffect(event).pipe(
Effect.mapError((cause) => new EventSinkWriteError({ eventCount: 1, cause })),
),
writeIfProviderThreadUnchanged: (input) =>
writeIfProviderThreadUnchangedEffect(input).pipe(
Effect.mapError(
Expand Down
96 changes: 96 additions & 0 deletions apps/server/src/orchestration-v2/FoundationPersistence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
type OrchestrationV2AppThread,
type OrchestrationV2DomainEvent,
type OrchestrationV2Run,
type OrchestrationV2ProviderThread,
type OrchestrationV2ThreadProjection,
type OrchestrationV2TurnItem,
ProjectId,
Expand Down Expand Up @@ -147,6 +148,101 @@ function threadCreatedEvent(input: {
};
}

it.effect("persists fresh acceptance despite three provider-thread receipt collisions", () =>
Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const store = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:fresh-acceptance-collisions");
const providerThread: OrchestrationV2ProviderThread = {
id: ProviderThreadId.make("provider-thread:fresh-acceptance-collisions"),
driver: providerDriver,
providerInstanceId,
providerSessionId: null,
appThreadId: threadId,
ownerNodeId: null,
nativeThreadRef: { driver: providerDriver, nativeId: "renewed-native", strength: "strong" },
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: 2,
lastRunOrdinal: 2,
handoffIds: [],
forkedFrom: null,
freshContext: { bootstrap: "role and lessons", throughRunOrdinal: 1 },
createdAt: now,
updatedAt: now,
};
const update = (id: string, payload: OrchestrationV2ProviderThread) => ({
id: EventId.make(id),
type: "provider-thread.updated" as const,
threadId,
occurredAt: now,
payload,
});
yield* sink.write({
events: [
threadCreatedEvent({
id: "event:fresh-acceptance:thread",
thread: makeThread(threadId, now),
now,
}),
update("event:fresh-acceptance:initial", providerThread),
],
});
const receipt = update("event:fresh-acceptance:receipt", {
...providerThread,
freshContext: { ...providerThread.freshContext!, acceptedNativeThreadId: "renewed-native" },
});
for (let collision = 0; collision < 3; collision += 1) {
const snapshot = (yield* store.getThreadRecords(threadId, ["providerThreads"]))
.providerThreads[0]!;
yield* sink.write({
events: [
update(`event:fresh-acceptance:roster-${collision}`, {
...snapshot,
pendingBackgroundTasks: [
{ taskId: `roster-${collision}`, description: "ingestion update", kind: "command" },
],
}),
],
});
const staleReceipt = yield* sink.writeIfProviderThreadUnchanged({
expected: snapshot,
events: [receipt],
});
assert.isFalse(staleReceipt.committed);
}
const accepted = yield* sink.acceptFreshContext(receipt);
assert.isTrue(accepted.committed);
const durable = (yield* store.getThreadRecords(threadId, ["providerThreads"]))
.providerThreads[0]!;
assert.equal(durable.freshContext?.acceptedNativeThreadId, "renewed-native");
assert.equal(durable.pendingBackgroundTasks?.[0]?.taskId, "roster-2");
assert.isFalse((yield* sink.acceptFreshContext(receipt)).committed);

for (const replacement of [
{
...providerThread,
nativeThreadRef: { ...providerThread.nativeThreadRef!, nativeId: "replacement-native" },
},
{ ...providerThread, freshContext: { bootstrap: "new generation", throughRunOrdinal: 2 } },
]) {
yield* sink.write({
events: [
update(
`event:fresh-acceptance:replacement-${replacement.freshContext?.throughRunOrdinal}`,
replacement,
),
],
});
assert.isFalse((yield* sink.acceptFreshContext(receipt)).committed);
const current = (yield* store.getThreadRecords(threadId, ["providerThreads"]))
.providerThreads[0]!;
assert.isUndefined(current.freshContext?.acceptedNativeThreadId);
}
}).pipe(Effect.provide(layerTest)),
);

it.effect("rebuilds event history one bounded page at a time", () =>
Effect.gen(function* () {
const eventStore = yield* EventStore.EventStoreV2;
Expand Down
Loading
Loading