From d10e8db7a32659dcae66025185ac6c162c5049da Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 29 Oct 2026 21:28:50 +0000 Subject: [PATCH 1/5] test(server): reproduce missing restart notice on recovered runs (#27048) These tests reproduce #27048: startup recovery cancels a running run without writing a system_notice turn item that says the server restarted. Co-Authored-By: Claude Opus 5.5 --- .../ProviderRuntimeRecoveryService.test.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 2fd9b13a35c33..da24260af792e 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -418,6 +418,74 @@ it.effect("cancels a stale waiting run when no checkpoint capture can finish it" }).pipe(Effect.provide(layer)); }); +it.effect("tells the thread a running run stopped because the server restarted", () => { + const threadId = ThreadId.make("thread_restart_notice"); + const runId = RunId.make("run_restart_notice"); + const rootNodeId = NodeId.make("node_restart_notice"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId }, + runtimeRequests: [], + providerSessions: [], + providerThreads: [], + providerTurns: [], + runs: [{ id: runId, status: "running", providerInstanceId: ProviderInstanceId.make("codex") }], + attempts: [ + { + id: RunAttemptId.make("attempt_restart_notice"), + runId, + rootNodeId, + status: "running", + }, + ], + nodes: [{ id: rootNodeId, runId, status: "running" }], + subagents: [], + messages: [], + turnItems: [], + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ + runRecoveryOnce: Effect.succeed(false), + }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + + return Effect.gen(function* () { + const summary = + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + assert.equal(summary.terminalizedRuns, 1); + const notices = (committedInput?.events ?? []).flatMap((event) => + event.type === "turn-item.updated" && + event.payload.type === "system_notice" && + event.payload.runId === runId + ? [event.payload] + : [], + ); + assert.equal(notices.length, 1, "the restart-cancelled run has no notice saying why"); + assert.equal(notices[0]?.status, "completed"); + assert.include(notices[0]?.message ?? "", "restarted"); + }).pipe(Effect.provide(layer)); +}); + it.effect("recovery closes a secret card's form but leaves a link card open", () => { const threadId = ThreadId.make("thread_secret_recovery"); const runId = RunId.make("run_secret_recovery"); From 71ef26e7d19a21e7fd3b00cf5d349fa0c2b13a13 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 29 Oct 2026 21:41:55 +0000 Subject: [PATCH 2/5] fix(server): runs cancelled by a server restart say so in the thread Startup and shutdown recovery cancelled running work with no item saying why, so the run looked like a provider cancel or a user Stop. Recovery now adds a completed system_notice ("Server restarted" / "Server shut down") to each run it cancels. Its id derives from the run, so replays update it in place, and EventSink slots it after the run's existing items. Queued runs that never started get no notice. Fixes pingdotgg/t3code#27048 Co-Authored-By: Claude Opus 5.5 --- .../ProviderRuntimeRecoveryService.test.ts | 1 + .../ProviderRuntimeRecoveryService.ts | 36 +++++++++++++++++++ .../src/orchestration-v2/runtimeLayer.test.ts | 14 +++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index da24260af792e..fbf1f8e0e9e4a 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -795,6 +795,7 @@ it.effect( ["provider-turn.updated", "cancelled"], ["message.updated", null], ["turn-item.updated", "cancelled"], + ["turn-item.updated", "completed"], ["provider-thread.updated", "idle"], ["provider-session.updated", "stopped"], ], diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index c344357972268..c2ff021588750 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -489,6 +489,42 @@ export const make = Effect.gen(function* () { payload: cancelledItem(item, now), }); } + // A queued run never did provider work, so there is nothing to explain. + if (run.status === "queued") continue; + events.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: projection.thread.id, + runId: run.id, + ...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }), + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { + id: ids.derive.runSignalTurnItem({ runId: run.id, signal: "server-recovery" }), + threadId: projection.thread.id, + runId: run.id, + nodeId: run.rootNodeId, + providerThreadId: run.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + // EventSink places the item after the run's existing items. + ordinal: + Math.max( + 0, + ...projection.turnItems + .filter((item) => item.runId === run.id) + .map((item) => item.ordinal), + ) + 1, + status: "completed", + title: trigger === "startup" ? "Server restarted" : "Server shut down", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "system_notice", + message: detail, + }, + }); } // Blocking waits die with their turn; let child results wake the parent continuation as async tasks. const cutRunIds = new Set(runs.map((run) => run.id)); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 428202d17b651..983372ec57402 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -1324,10 +1324,16 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { assert.equal(native.runtimeRequests[0]?.responseCapability.type, "not_resumable"); yield* recovery.reconcile("shutdown"); yield* recovery.reconcile("startup"); - assert.deepEqual( - (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests, - recovered.runtimeRequests, - ); + const rerecovered = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual(rerecovered.runtimeRequests, recovered.runtimeRequests); + const runItems = rerecovered.turnItems.filter((item) => item.runId === runId); + const notices = runItems.filter((item) => item.type === "system_notice"); + assert.lengthOf(notices, scope === "run" ? 1 : 0); + if (scope === "run") { + assert.equal(notices[0]?.title, "Server restarted"); + assert.equal(notices[0]?.nodeId, recovered.runs[0]?.rootNodeId); + assert.equal(notices[0]?.ordinal, Math.max(...runItems.map((item) => item.ordinal))); + } yield* orchestrator.dispatch({ type: "runtime-request.respond", commandId: CommandId.make(`recovery-answer-${trigger}-${scope}`), From f2a3e04d8b1d228a6f712b5c22d9736719f62418 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 00:32:46 +0000 Subject: [PATCH 3/5] fix(server): a hidden unsent resume run gets no restart notice Recovery hides the prompt of a usage-limit resume that never reached the provider and lets the worker send it again, so a notice on that run would explain a turn the thread no longer shows. AI-assisted; verified with the tests and tools listed above. --- .../src/orchestration-v2/ProviderRuntimeRecoveryService.ts | 4 ++-- apps/server/src/orchestration-v2/runtimeLayer.test.ts | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 824386b483bad..fd64373b370a9 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -634,8 +634,8 @@ export const make = Effect.gen(function* () { payload: cancelledItem(item, now), }); } - // A queued run never did provider work, so there is nothing to explain. - if (run.status === "queued") continue; + // Queued runs and unsent resumes leave no visible turn behind, so there is nothing to explain. + if (run.status === "queued" || unsentResumeRunIds.includes(run.id)) continue; events.push({ id: yield* allocateEventId(), type: "turn-item.updated", diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index b88c7582dc4d1..be6a414a1017d 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -8957,6 +8957,12 @@ it.layer(layerTest)("usage-limit recovery", (it) => { ), [run.id, resumed.runs.at(-1)!.id], ); + assert.deepEqual( + resumed.turnItems.flatMap((item) => + item.type === "system_notice" && item.title === "Server restarted" ? [item.runId] : [], + ), + scenario === "resume" ? [] : [continuation.id], + ); assert.deepEqual( resumed.runs.map((row) => row.status), scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"], From e972b3489142f72877734727addf142be8cb0487 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 00:34:36 +0000 Subject: [PATCH 4/5] test(server): keep restart-notice coverage in the real-store recovery test AI-assisted; verified with the tests and tools listed above. --- .../ProviderRuntimeRecoveryService.test.ts | 68 ------------------- 1 file changed, 68 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 7599bc7531a33..e628fc5bcc4bd 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -418,74 +418,6 @@ it.effect("cancels a stale waiting run when no checkpoint capture can finish it" }).pipe(Effect.provide(layer)); }); -it.effect("tells the thread a running run stopped because the server restarted", () => { - const threadId = ThreadId.make("thread_restart_notice"); - const runId = RunId.make("run_restart_notice"); - const rootNodeId = NodeId.make("node_restart_notice"); - let committedInput: Parameters[0] | null = - null; - const projection = { - thread: { id: threadId }, - runtimeRequests: [], - providerSessions: [], - providerThreads: [], - providerTurns: [], - runs: [{ id: runId, status: "running", providerInstanceId: ProviderInstanceId.make("codex") }], - attempts: [ - { - id: RunAttemptId.make("attempt_restart_notice"), - runId, - rootNodeId, - status: "running", - }, - ], - nodes: [{ id: rootNodeId, runId, status: "running" }], - subagents: [], - messages: [], - turnItems: [], - } as unknown as OrchestrationV2ThreadProjection; - const layer = ProviderRuntimeRecovery.layer.pipe( - Layer.provide(ServerSettings.layerTest()), - Layer.provide( - Layer.mergeAll( - Layer.mock(ProjectionStore.ProjectionStoreV2)({ - getRecoveryThreadIds: () => Effect.succeed([threadId]), - getRuntimeRecoveryProjection: () => Effect.succeed(projection), - }), - Layer.mock(EventSink.EventSinkV2)({ - commitCommand: (input) => { - committedInput = input; - return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); - }, - }), - IdAllocator.layer, - Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ - runRecoveryOnce: Effect.succeed(false), - }), - Layer.mock(EffectOutbox.EffectOutboxV2)({ - reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), - }), - ), - ), - ); - - return Effect.gen(function* () { - const summary = - yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); - assert.equal(summary.terminalizedRuns, 1); - const notices = (committedInput?.events ?? []).flatMap((event) => - event.type === "turn-item.updated" && - event.payload.type === "system_notice" && - event.payload.runId === runId - ? [event.payload] - : [], - ); - assert.equal(notices.length, 1, "the restart-cancelled run has no notice saying why"); - assert.equal(notices[0]?.status, "completed"); - assert.include(notices[0]?.message ?? "", "restarted"); - }).pipe(Effect.provide(layer)); -}); - it.effect("recovery closes a secret card's form but leaves a link card open", () => { const threadId = ThreadId.make("thread_secret_recovery"); const runId = RunId.make("run_secret_recovery"); From cabafde0cc81e20531d10fb1d852d897d556a94b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 03:18:48 +0000 Subject: [PATCH 5/5] refactor(server): let EventSink place the restart notice The event sink allocates every new turn item's position, so the notice no longer computes an ordinal it would discard. AI-assisted; verified with the tests and tools listed above. --- .../orchestration-v2/ProviderRuntimeRecoveryService.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index fd64373b370a9..46b739bec5732 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -653,14 +653,8 @@ export const make = Effect.gen(function* () { providerTurnId: null, nativeItemRef: null, parentItemId: null, - // EventSink places the item after the run's existing items. - ordinal: - Math.max( - 0, - ...projection.turnItems - .filter((item) => item.runId === run.id) - .map((item) => item.ordinal), - ) + 1, + // EventSink allocates the real position, after the run's existing items. + ordinal: 0, status: "completed", title: trigger === "startup" ? "Server restarted" : "Server shut down", startedAt: now,