From 94279ba85ed9206cb0b10b6f8a61afce661b73b9 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Wed, 7 Oct 2026 17:03:21 +0200 Subject: [PATCH 01/12] fix(server): restore the checkpoint before rolling back the provider Rollback changed the provider conversation before a fallible filesystem restore, and restored files left workspace search using the old entry index. Restore the validated, isolated checkpoint first and refresh its workspace entries immediately. Then compute the provider turns to remove and roll back the provider. Provide WorkspaceEntries through the existing runtime layer composition and update the affected test layers. Regression tests cover a failed restore preserving the provider conversation and a successful restore updating the real search index. Model: gpt-6.1-sol (Codex) --- .../CheckpointRestoreSafety.test.ts | 4 +- .../CheckpointRollbackService.test.ts | 150 +++++++++++++++++- .../CheckpointRollbackService.ts | 59 ++++--- .../src/orchestration-v2/runtimeLayer.ts | 3 + .../testkit/ProviderReplayHarness.ts | 2 + 5 files changed, 190 insertions(+), 28 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts index 5f79aefdd0b7b..2164e3862847c 100644 --- a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts @@ -24,6 +24,7 @@ import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; it.effect.each([ "nested", @@ -102,6 +103,7 @@ it.effect.each([ Layer.provide( Layer.mergeAll( NodeServices.layer, + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), IdAllocator.layer, Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeed(Option.some({ workspaceRoot: parent } as never)), @@ -190,7 +192,7 @@ it.effect.each([ assert.equal(yield* fs.readFileString(otherFile), "other thread's uncommitted work"); } else { yield* service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }); - assert.deepEqual(calls, restoreFiles ? ["provider", "files"] : ["provider"]); + assert.deepEqual(calls, restoreFiles ? ["files", "provider"] : ["provider"]); assert.equal(yield* fs.exists(otherFile), !restoreFiles); } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index 433a3be43ac3c..f53b984247eba 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -16,6 +16,10 @@ import * as Option from "effect/Option"; import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; import * as Layer from "effect/Layer"; +import * as ServerConfig from "../config.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; + import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts"; import { isCheckpointRestoreIsolated } from "./CheckpointRestoreSafety.ts"; import * as CheckpointService from "./CheckpointService.ts"; @@ -36,6 +40,7 @@ const layerCheckpointRollbackService = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( NodeServices.layer, + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeed(unrelatedProject) }), ), ), @@ -464,11 +469,154 @@ it.effect.each([ yield* service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }); assert.deepEqual( calls, - restoreFiles ? ["provider", "files", "projection"] : ["provider", "projection"], + restoreFiles ? ["files", "provider", "projection"] : ["provider", "projection"], ); }).pipe(Effect.provide(layerTest)); }); +it.effect.each([false, true])( + "keeps provider conversation and indexed files consistent, restoreFails=%s", + (restoreFails) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-index-" }); + const currentFile = path.join(cwd, "rollback-current.txt"); + const restoredFile = path.join(cwd, "rollback-restored.txt"); + yield* fs.writeFileString(currentFile, "current turn"); + const search = () => workspaceEntries.search({ cwd, query: "rollback", limit: 10 }); + assert.deepEqual((yield* search()).entries, [{ path: "rollback-current.txt", kind: "file" }]); + + const threadId = ThreadId.make("rollback-restore-state"); + const providerThreadId = ProviderThreadId.make("rollback-restore-provider"); + const providerSessionId = ProviderSessionId.make("rollback-restore-session"); + const instanceId = ProviderInstanceId.make("rollback-restore-instance"); + const checkpointId = CheckpointId.make("rollback-restore-checkpoint"); + const scopeId = CheckpointScopeId.make("rollback-restore-scope"); + const providerThread = { + id: providerThreadId, + providerSessionId, + providerInstanceId: instanceId, + }; + const projection = { + thread: { + worktreePath: cwd, + activeProviderThreadId: providerThreadId, + modelSelection: { instanceId, model: "test" }, + }, + providerThreads: [providerThread], + providerSessions: [], + providerTurns: [ + { + id: "turn-1", + providerThreadId, + runAttemptId: "attempt-1", + ordinal: 1, + status: "completed", + }, + ], + nodes: [], + attempts: [{ id: "attempt-1", runId: "run-1" }], + checkpoints: [{ id: checkpointId, scopeId, status: "ready", appRunOrdinal: null }], + checkpointScopes: [{ id: scopeId, cwd }], + runs: [{ id: "run-1", ordinal: 1, status: "completed", rootNodeId: null }], + } as unknown as OrchestrationV2ThreadProjection; + const restoreFailure = new CheckpointService.CheckpointRestoreError({ + scopeId, + checkpointId, + cause: new Error("git restore failed"), + }); + const providerConversation = ["turn-1"]; + const persistedEvents: string[] = []; + const testLayer = CheckpointRollbackService.layer.pipe( + Layer.provide( + Layer.mergeAll( + IdAllocator.layer, + Layer.succeed(WorkspaceEntries.WorkspaceEntries, workspaceEntries), + Layer.mock(ProjectStore.ProjectStoreV2)({ + get: () => Effect.succeed(unrelatedProject), + }), + Layer.mock(CheckpointService.CheckpointServiceV2)({ + restore: () => + restoreFails + ? Effect.fail(restoreFailure) + : Effect.gen(function* () { + yield* fs.remove(currentFile); + yield* fs.writeFileString(restoredFile, "checkpoint turn"); + }).pipe(Effect.orDie), + }), + Layer.mock(EventSink.EventSinkV2)({ + write: ({ events }) => + Effect.sync(() => { + persistedEvents.push(...events.map((event) => event.type)); + return []; + }), + }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThreadRecords: () => Effect.succeed(projection), + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 1, + snapshotSequence: 0, + threads: [], + archivedThreads: [], + }), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + open: () => + Effect.succeed({ + rollbackThread: (input: ProviderAdapterV2RollbackThreadInput) => + Effect.gen(function* () { + const count = yield* resolveCodexRollbackTurnCount(input); + assert.equal(count, 1); + providerConversation.splice(providerConversation.length - count, count); + return { providerThread }; + }), + } as never), + }), + Layer.mock(RuntimePolicy.RuntimePolicyV2)({ + resolve: () => Effect.succeed({} as never), + }), + ), + ), + ); + const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2.pipe( + Effect.provide(testLayer), + ); + const rollback = service.execute({ threadId, providerThreadId, checkpointId, scopeId }); + if (restoreFails) { + const error = yield* Effect.flip(rollback); + assert.equal(error.reason, "unexpected-failure"); + assert.equal(error.cause, restoreFailure); + assert.deepEqual(providerConversation, ["turn-1"]); + assert.deepEqual(persistedEvents, []); + assert.equal(yield* fs.readFileString(currentFile), "current turn"); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-current.txt", kind: "file" }, + ]); + } else { + yield* rollback; + assert.deepEqual(providerConversation, []); + assert.include(persistedEvents, "run.updated"); + assert.equal(yield* fs.exists(currentFile), false); + assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-restored.txt", kind: "file" }, + ]); + } + }).pipe( + Effect.scoped, + Effect.provide( + WorkspaceEntries.layer.pipe( + Layer.provide(WorkspacePaths.layer), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-rollback-config-" })), + Layer.provideMerge(NodeServices.layer), + ), + ), + ), +); + it.effect.skipIf(!symlinksSupported)( "rejects an archived thread sharing a worktree through a symlink", () => diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 1710e7a8cd844..006e92ae0101d 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -14,6 +14,7 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; import { isCheckpointRestoreIsolated, SHARED_WORKSPACE_RESTORE_MESSAGE, @@ -91,6 +92,7 @@ export const layer: Layer.Layer< | FileSystem.FileSystem | Path.Path | ProjectStore.ProjectStoreV2 + | WorkspaceEntries.WorkspaceEntries > = Layer.effect( CheckpointRollbackServiceV2, Effect.gen(function* () { @@ -103,6 +105,7 @@ export const layer: Layer.Layer< const fileSystem = yield* FileSystem.FileSystem; const projects = yield* ProjectStore.ProjectStoreV2; const path = yield* Path.Path; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const execute = Effect.fn("orchestrationV2.checkpointRollback.execute")(function* (input: { readonly threadId: ThreadId; @@ -197,31 +200,6 @@ export const layer: Layer.Layer< }); const targetOrdinal = checkpoint.appRunOrdinal ?? 0; - // Stopped and failed runs after the target leave the provider - // conversation too, so they must not stay visible. - const runsToRollback = projection.runs.filter( - (run) => - run.ordinal > targetOrdinal && - (run.status === "completed" || - run.status === "interrupted" || - run.status === "failed" || - run.status === "cancelled"), - ); - // Rolled-back turns stay in the audit history, but no longer exist in - // the provider conversation and must not be counted by a later rewind. - const rolledBackRunIds = new Set( - projection.runs.filter((run) => run.status === "rolled_back").map((run) => run.id), - ); - const rolledBackAttemptIds = new Set( - projection.attempts - .filter((attempt) => rolledBackRunIds.has(attempt.runId)) - .map((attempt) => attempt.id), - ); - const providerThreadTurns = projection.providerTurns.filter( - (turn) => - turn.providerThreadId === providerThread.id && - (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)), - ); const rollbackTarget: ProviderAdapterV2RollbackTarget = targetOrdinal === 0 ? { @@ -254,6 +232,36 @@ export const layer: Layer.Layer< }; }); + if (input.restoreFiles !== false) { + yield* checkpoints.restore({ scope, checkpoint }); + yield* workspaceEntries.refresh(scope.cwd); + } + + // Stopped and failed runs after the target leave the provider + // conversation too, so they must not stay visible. + const runsToRollback = projection.runs.filter( + (run) => + run.ordinal > targetOrdinal && + (run.status === "completed" || + run.status === "interrupted" || + run.status === "failed" || + run.status === "cancelled"), + ); + // Rolled-back turns stay in the audit history, but no longer exist in + // the provider conversation and must not be counted by a later rewind. + const rolledBackRunIds = new Set( + projection.runs.filter((run) => run.status === "rolled_back").map((run) => run.id), + ); + const rolledBackAttemptIds = new Set( + projection.attempts + .filter((attempt) => rolledBackRunIds.has(attempt.runId)) + .map((attempt) => attempt.id), + ); + const providerThreadTurns = projection.providerTurns.filter( + (turn) => + turn.providerThreadId === providerThread.id && + (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)), + ); const snapshot = runsToRollback.length === 0 ? { providerThread } @@ -262,7 +270,6 @@ export const layer: Layer.Layer< target: rollbackTarget, providerThreadTurns, }); - if (input.restoreFiles !== false) yield* checkpoints.restore({ scope, checkpoint }); const staleCheckpoints = projection.checkpoints.filter( (candidate) => candidate.scopeId === scope.id && diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 917ad3a30b8cb..d4944ffdb88bb 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -11,6 +11,8 @@ import * as AgentSessionScanner from "../project/AgentSessionScanner.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as CheckpointCaptureService from "./CheckpointCaptureService.ts"; import * as CheckpointService from "./CheckpointService.ts"; import * as CheckpointRollbackService from "./CheckpointRollbackService.ts"; @@ -173,6 +175,7 @@ const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe( const layerCheckpointRollbackServiceProvided = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( + WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)), ProjectStore.layer, layerCheckpointServiceProvided, layerEventSinkProvided, diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 687e881c46563..81902d54e34fa 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -22,6 +22,7 @@ import * as VcsProcess from "../../vcs/VcsProcess.ts"; import * as CheckpointCaptureService from "../CheckpointCaptureService.ts"; import * as CheckpointService from "../CheckpointService.ts"; import * as CheckpointRollbackService from "../CheckpointRollbackService.ts"; +import * as WorkspaceEntries from "../../workspace/WorkspaceEntries.ts"; import * as CommandPolicy from "../CommandPolicy.ts"; import * as CommandReceiptStore from "../CommandReceiptStore.ts"; import * as ContextHandoffService from "../ContextHandoffService.ts"; @@ -388,6 +389,7 @@ export function layerWithRegistry( const layerCheckpointRollbackServiceProvided = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, From 6e615b211d84c3e1564d8587df7498c46ba6a4f6 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Wed, 7 Oct 2026 21:31:52 +0200 Subject: [PATCH 02/12] fix(server): report restored files when provider rollback fails Model: gpt-6.1-sol (Codex harness) --- .../CheckpointRollbackService.test.ts | 59 +++++++-- .../CheckpointRollbackService.ts | 29 ++++- .../src/orchestration-v2/EffectWorker.ts | 49 ++++---- .../src/orchestration-v2/runtimeLayer.test.ts | 117 +++++++++++++++++- 4 files changed, 210 insertions(+), 44 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index f53b984247eba..69d09e8078adc 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -474,9 +474,9 @@ it.effect.each([ }).pipe(Effect.provide(layerTest)); }); -it.effect.each([false, true])( - "keeps provider conversation and indexed files consistent, restoreFails=%s", - (restoreFails) => +it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fails"] as const)( + "reports checkpoint rollback state with %s", + (outcome) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -494,6 +494,7 @@ it.effect.each([false, true])( const instanceId = ProviderInstanceId.make("rollback-restore-instance"); const checkpointId = CheckpointId.make("rollback-restore-checkpoint"); const scopeId = CheckpointScopeId.make("rollback-restore-scope"); + const laterCheckpointId = CheckpointId.make("rollback-later-checkpoint"); const providerThread = { id: providerThreadId, providerSessionId, @@ -518,7 +519,10 @@ it.effect.each([false, true])( ], nodes: [], attempts: [{ id: "attempt-1", runId: "run-1" }], - checkpoints: [{ id: checkpointId, scopeId, status: "ready", appRunOrdinal: null }], + checkpoints: [ + { id: checkpointId, scopeId, status: "ready", appRunOrdinal: null }, + { id: laterCheckpointId, scopeId, status: "ready", appRunOrdinal: 1 }, + ], checkpointScopes: [{ id: scopeId, cwd }], runs: [{ id: "run-1", ordinal: 1, status: "completed", rootNodeId: null }], } as unknown as OrchestrationV2ThreadProjection; @@ -528,6 +532,8 @@ it.effect.each([false, true])( cause: new Error("git restore failed"), }); const providerConversation = ["turn-1"]; + const providerFailure = new Error("provider rollback failed"); + const deletedRefs: CheckpointId[] = []; const persistedEvents: string[] = []; const testLayer = CheckpointRollbackService.layer.pipe( Layer.provide( @@ -539,12 +545,16 @@ it.effect.each([false, true])( }), Layer.mock(CheckpointService.CheckpointServiceV2)({ restore: () => - restoreFails + outcome === "restore-fails" ? Effect.fail(restoreFailure) : Effect.gen(function* () { yield* fs.remove(currentFile); yield* fs.writeFileString(restoredFile, "checkpoint turn"); }).pipe(Effect.orDie), + deleteStaleRefs: ({ checkpoints }) => + Effect.sync(() => { + deletedRefs.push(...checkpoints.map((checkpoint) => checkpoint.id)); + }), }), Layer.mock(EventSink.EventSinkV2)({ write: ({ events }) => @@ -570,6 +580,9 @@ it.effect.each([false, true])( Effect.gen(function* () { const count = yield* resolveCodexRollbackTurnCount(input); assert.equal(count, 1); + if (outcome === "provider-fails" || outcome === "provider-only-fails") { + return yield* Effect.fail(providerFailure); + } providerConversation.splice(providerConversation.length - count, count); return { providerThread }; }), @@ -584,21 +597,45 @@ it.effect.each([false, true])( const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2.pipe( Effect.provide(testLayer), ); - const rollback = service.execute({ threadId, providerThreadId, checkpointId, scopeId }); - if (restoreFails) { + const rollback = service.execute({ + threadId, + providerThreadId, + checkpointId, + scopeId, + ...(outcome === "provider-only-fails" ? { restoreFiles: false } : {}), + }); + if (outcome !== "success") { const error = yield* Effect.flip(rollback); - assert.equal(error.reason, "unexpected-failure"); - assert.equal(error.cause, restoreFailure); + assert.equal( + error.reason, + outcome === "provider-fails" ? "files-restored-provider-failed" : "unexpected-failure", + ); + assert.equal(error.cause, outcome === "restore-fails" ? restoreFailure : providerFailure); assert.deepEqual(providerConversation, ["turn-1"]); assert.deepEqual(persistedEvents, []); - assert.equal(yield* fs.readFileString(currentFile), "current turn"); + assert.deepEqual(deletedRefs, []); + assert.equal(projection.runs[0]?.status, "completed"); + assert.equal(projection.checkpoints[1]?.status, "ready"); + if (outcome === "provider-fails") { + assert.include(error.message, "Files were restored to the selected checkpoint"); + assert.equal(yield* fs.exists(currentFile), false); + assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); + } else { + assert.equal(error.message, CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE); + assert.equal(yield* fs.readFileString(currentFile), "current turn"); + assert.equal(yield* fs.exists(restoredFile), false); + } assert.deepEqual((yield* search()).entries, [ - { path: "rollback-current.txt", kind: "file" }, + { + path: outcome === "provider-fails" ? "rollback-restored.txt" : "rollback-current.txt", + kind: "file", + }, ]); } else { yield* rollback; assert.deepEqual(providerConversation, []); assert.include(persistedEvents, "run.updated"); + assert.deepEqual(deletedRefs, [laterCheckpointId]); assert.equal(yield* fs.exists(currentFile), false); assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); assert.deepEqual((yield* search()).entries, [ diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 006e92ae0101d..061dd21eed011 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -39,6 +39,7 @@ export class CheckpointRollbackExecutionError extends Schema.TaggedError + new CheckpointRollbackExecutionError({ + reason: + input.restoreFiles === false + ? "unexpected-failure" + : "files-restored-provider-failed", + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), + ), + ); const staleCheckpoints = projection.checkpoints.filter( (candidate) => candidate.scopeId === scope.id && diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index b3ee780334d33..6f895e6851c00 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -380,28 +380,33 @@ export const layerExecutor: Layer.Layer< }) .pipe( // The last failed attempt tells waiting clients it failed, - // instead of leaving them to time out. Clients get a fixed - // message; the worker logs the full cause for each attempt. - Effect.tapCause((cause) => - willRetry || Cause.hasInterruptsOnly(cause) - ? Effect.void - : threads - .dispatch({ - type: "checkpoint.rollback.fail", - commandId: CommandId.make(`${effect.commandId}:rollback-failed`), - threadId: effect.threadId, - requestId: effect.commandId, - message: CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE, - }) - .pipe( - Effect.catchCause((recordCause) => - Effect.logWarning("Failed to record rollback failure", { - effectId: effect.id, - cause: recordCause, - }), - ), - ), - ), + // instead of leaving them to time out. Preserve the fixed + // partial-restore warning; log the full cause for each attempt. + Effect.tapCause((cause) => { + if (willRetry || Cause.hasInterruptsOnly(cause)) return Effect.void; + const failure = Cause.findErrorOption(cause); + const message = + Option.isSome(failure) && + failure.value.reason === "files-restored-provider-failed" + ? failure.value.message + : CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE; + return threads + .dispatch({ + type: "checkpoint.rollback.fail", + commandId: CommandId.make(`${effect.commandId}:rollback-failed`), + threadId: effect.threadId, + requestId: effect.commandId, + message, + }) + .pipe( + Effect.catchCause((recordCause) => + Effect.logWarning("Failed to record rollback failure", { + effectId: effect.id, + cause: recordCause, + }), + ), + ); + }), Effect.mapError( (cause) => new OrchestrationEffectExecutionError({ diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 131f05034051b..2c9d16e3e4615 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -31,6 +31,7 @@ import * as Effect from "effect/Effect"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import * as Path from "effect/Path"; @@ -67,7 +68,11 @@ import * as PullRequestWatchReactor from "./PullRequestWatchReactor.ts"; import { PullRequestProviderError } from "../pullRequest/PullRequestProvider.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as ProjectStore from "./ProjectStore.ts"; -import type { ProviderAdapterV2SessionRuntime, ProviderAdapterV2Shape } from "./ProviderAdapter.ts"; +import { + ProviderAdapterRollbackThreadError, + type ProviderAdapterV2SessionRuntime, + type ProviderAdapterV2Shape, +} from "./ProviderAdapter.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as RuntimeLayer from "./runtimeLayer.ts"; import { shellStreamItemFromThreadShell } from "./ShellStream.ts"; @@ -273,7 +278,7 @@ const layerTest = Layer.mergeAll( ).pipe( Layer.provide(McpSessionRegistryTestkit.layer), Layer.provide(SqlitePersistence.layerMemory), - Layer.provide(layerCheckpointStoreTest), + Layer.provideMerge(layerCheckpointStoreTest), Layer.provide(layerServerConfig), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerTestProviderInstanceRegistry), @@ -525,7 +530,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { * A thread with one ready checkpoint and no queued turn start. Every provider * rollback on it fails, so each rollback effect retries until it gives up. */ - const seedFailingRollbackThread = (name: string) => + const seedFailingRollbackThread = (name: string, cwd?: string) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const eventSink = yield* EventSink.EventSinkV2; @@ -544,7 +549,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { interactionMode: "default", branch: null, // Its own path, so other rollback tests keep an isolated worktree. - worktreePath: `/tmp/t3-${name}`, + worktreePath: cwd ?? `/tmp/t3-${name}`, }); yield* orchestrator.dispatch({ type: "message.dispatch", @@ -593,14 +598,15 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }); return { threadId, - rollback: (commandId: CommandId) => + checkpointId, + rollback: (commandId: CommandId, restoreFiles = false) => orchestrator.dispatch({ type: "checkpoint.rollback", commandId, threadId, checkpointId, scopeId: scope.id, - restoreFiles: false, + restoreFiles, }), }; }); @@ -634,6 +640,105 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); + it.effect("persists restored-file warning when provider rollback exhausts its attempts", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-partial-" }); + const file = path.join(cwd, "workspace.txt"); + yield* fs.writeFileString(file, "current uncheckpointed contents"); + const { threadId, checkpointId, rollback } = yield* seedFailingRollbackThread( + "runtime-rollback-partial", + cwd, + ); + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const providerThread = before.providerThreads[0]!; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make("runtime-rollback-partial-completed"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", completedAt: now }, + }, + ], + }); + let restored = 0; + let rejected = 0; + const restoreSpy = vi + .spyOn(checkpointStore, "restoreCheckpoint") + .mockImplementation((input) => + Effect.gen(function* () { + assert.equal(path.resolve(input.cwd), path.resolve(cwd)); + restored += 1; + yield* fs.writeFileString(file, "checkpoint contents"); + return true; + }).pipe(Effect.orDie), + ); + const deleteSpy = vi.spyOn(checkpointStore, "deleteCheckpointRefs"); + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + rollbackThread: () => + Effect.gen(function* () { + rejected += 1; + assert.equal( + yield* fs.readFileString(file).pipe(Effect.orDie), + "checkpoint contents", + ); + return yield* new ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: providerThread.id, + cause: new Error("legacy history cannot be reverted"), + }); + }), + } as unknown as ProviderAdapterV2SessionRuntime), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + restoreSpy.mockRestore(); + deleteSpy.mockRestore(); + sessionSpy.mockRestore(); + }), + ); + const commandId = CommandId.make("runtime-rollback-partial-request"); + yield* rollback(commandId, true); + for (let attempt = 0; attempt < 5; attempt++) { + yield* worker.drain(); + yield* TestClock.adjust("30 seconds"); + } + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "failed"); + assert.equal(restored, 5); + assert.equal(rejected, 5); + assert.equal(yield* fs.readFileString(file), "checkpoint contents"); + const failed = yield* orchestrator.getThreadProjection(threadId); + assert.equal(failed.thread.rollbackFailure?.requestId, commandId); + assert.include( + failed.thread.rollbackFailure?.message, + "Files were restored to the selected checkpoint", + ); + assert.include(failed.thread.rollbackFailure?.message, "conversation rollback failed"); + assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); + assert.equal( + failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, + "ready", + ); + assert.equal(deleteSpy.mock.calls.length, 0); + yield* rollback(CommandId.make("runtime-rollback-partial-next"), true); + assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); + }).pipe(Effect.scoped), + ); + it.effect("ignores a late failure from a rollback that a newer one superseded", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; From b6076556d56ddd46b0bedde964461b5c49d3c30a Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Wed, 7 Oct 2026 22:58:01 +0200 Subject: [PATCH 03/12] test(server): isolate checkpoint rollback runtime fixture Model: gpt-6.1-sol (Codex harness) --- apps/server/src/orchestration-v2/runtimeLayer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 2c9d16e3e4615..e1dc45a68dd50 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -736,7 +736,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { assert.equal(deleteSpy.mock.calls.length, 0); yield* rollback(CommandId.make("runtime-rollback-partial-next"), true); assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); - }).pipe(Effect.scoped), + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), ); it.effect("ignores a late failure from a rollback that a newer one superseded", () => From 2fbce9616959e43a2a2d70c3e39bad3797cddc4c Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Wed, 7 Oct 2026 23:18:30 +0200 Subject: [PATCH 04/12] fix(server): tag partial checkpoint rollback failures Model: gpt-6.1-sol (Codex harness) --- .../CheckpointRestoreSafety.test.ts | 1 + .../CheckpointRollbackService.test.ts | 35 +++++++++--- .../CheckpointRollbackService.ts | 55 +++++++++++++------ .../src/orchestration-v2/EffectWorker.ts | 2 +- .../src/orchestration-v2/runtimeLayer.test.ts | 5 +- 5 files changed, 68 insertions(+), 30 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts index 2164e3862847c..d31f7ac2d1f32 100644 --- a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts @@ -187,6 +187,7 @@ it.effect.each([ const error = yield* service .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }) .pipe(Effect.flip); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "shared-workspace"); assert.deepEqual(calls, []); assert.equal(yield* fs.readFileString(otherFile), "other thread's uncommitted work"); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index 69d09e8078adc..5ca86dd08161b 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -4,6 +4,7 @@ import { CheckpointId, CheckpointScopeId, type OrchestrationV2ThreadProjection, + ProviderDriverKind, ProviderInstanceId, ProviderSessionId, ProviderThreadId, @@ -28,7 +29,10 @@ import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "./IdAllocator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; -import type { ProviderAdapterV2RollbackThreadInput } from "./ProviderAdapter.ts"; +import { + ProviderAdapterRollbackThreadError, + type ProviderAdapterV2RollbackThreadInput, +} from "./ProviderAdapter.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as RuntimePolicy from "./RuntimePolicy.ts"; @@ -99,6 +103,7 @@ it.effect("rejects a non-ready checkpoint before opening a session or restoring }) .pipe(Effect.flip); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "rollback-target-invalid"); assert.equal( error.message, @@ -176,6 +181,7 @@ it.effect("rejects a rollback when another provider thread became active", () => }) .pipe(Effect.flip); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "active-provider-changed"); assert.equal( error.message, @@ -256,6 +262,7 @@ it.effect("rejects a rollback when provider selection changed before execution", }) .pipe(Effect.flip); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "active-provider-changed"); assert.equal( error.message, @@ -331,6 +338,7 @@ it.effect("reports a missing provider turn as a structured rollback failure", () }) .pipe(Effect.flip); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "provider-turn-unavailable"); assert.equal( error.message, @@ -462,6 +470,7 @@ it.effect.each([ const error = yield* Effect.flip( service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }), ); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "shared-workspace"); assert.deepEqual(calls, []); return; @@ -532,7 +541,12 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai cause: new Error("git restore failed"), }); const providerConversation = ["turn-1"]; - const providerFailure = new Error("provider rollback failed"); + const providerFailure = new ProviderAdapterRollbackThreadError({ + driver: ProviderDriverKind.make("codex"), + providerThreadId, + checkpointId, + cause: new Error("provider rollback failed"), + }); const deletedRefs: CheckpointId[] = []; const persistedEvents: string[] = []; const testLayer = CheckpointRollbackService.layer.pipe( @@ -581,7 +595,7 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai const count = yield* resolveCodexRollbackTurnCount(input); assert.equal(count, 1); if (outcome === "provider-fails" || outcome === "provider-only-fails") { - return yield* Effect.fail(providerFailure); + return yield* providerFailure; } providerConversation.splice(providerConversation.length - count, count); return { providerThread }; @@ -606,10 +620,9 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai }); if (outcome !== "success") { const error = yield* Effect.flip(rollback); - assert.equal( - error.reason, - outcome === "provider-fails" ? "files-restored-provider-failed" : "unexpected-failure", - ); + assert.equal(error.threadId, threadId); + assert.equal(error.providerThreadId, providerThreadId); + assert.equal(error.checkpointId, checkpointId); assert.equal(error.cause, outcome === "restore-fails" ? restoreFailure : providerFailure); assert.deepEqual(providerConversation, ["turn-1"]); assert.deepEqual(persistedEvents, []); @@ -617,10 +630,16 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai assert.equal(projection.runs[0]?.status, "completed"); assert.equal(projection.checkpoints[1]?.status, "ready"); if (outcome === "provider-fails") { - assert.include(error.message, "Files were restored to the selected checkpoint"); + assert(error._tag === "CheckpointRollbackPartialRestoreError"); + assert.equal( + error.message, + "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying.", + ); assert.equal(yield* fs.exists(currentFile), false); assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); } else { + assert(error._tag === "CheckpointRollbackExecutionError"); + assert.equal(error.reason, "unexpected-failure"); assert.equal(error.message, CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE); assert.equal(yield* fs.readFileString(currentFile), "current turn"); assert.equal(yield* fs.exists(restoredFile), false); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 061dd21eed011..43f890bc02743 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -39,7 +39,6 @@ export class CheckpointRollbackExecutionError extends Schema.TaggedError()( + "CheckpointRollbackPartialRestoreError", + { + threadId: ThreadId, + providerThreadId: ProviderThreadId, + checkpointId: CheckpointId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying."; + } +} + +const isCheckpointRollbackError = Schema.is( + Schema.Union([CheckpointRollbackExecutionError, CheckpointRollbackPartialRestoreError]), +); export interface CheckpointRollbackServiceV2Shape { readonly execute: (input: { @@ -75,7 +88,10 @@ export interface CheckpointRollbackServiceV2Shape { readonly checkpointId: CheckpointId; readonly scopeId: CheckpointScopeId; readonly restoreFiles?: boolean; - }) => Effect.Effect; + }) => Effect.Effect< + void, + CheckpointRollbackExecutionError | CheckpointRollbackPartialRestoreError + >; } export class CheckpointRollbackServiceV2 extends Context.Service< @@ -275,18 +291,21 @@ export const layer: Layer.Layer< providerThreadTurns, }) .pipe( - Effect.mapError( - (cause) => - new CheckpointRollbackExecutionError({ - reason: - input.restoreFiles === false - ? "unexpected-failure" - : "files-restored-provider-failed", - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - cause, - }), + Effect.mapError((cause) => + input.restoreFiles === false + ? new CheckpointRollbackExecutionError({ + reason: "unexpected-failure", + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }) + : new CheckpointRollbackPartialRestoreError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), ), ); const staleCheckpoints = projection.checkpoints.filter( @@ -372,7 +391,7 @@ export const layer: Layer.Layer< execute: (input) => execute(input).pipe( Effect.mapError((cause) => - isCheckpointRollbackExecutionError(cause) + isCheckpointRollbackError(cause) ? cause : new CheckpointRollbackExecutionError({ reason: "unexpected-failure", diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 6f895e6851c00..2599381f62bef 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -387,7 +387,7 @@ export const layerExecutor: Layer.Layer< const failure = Cause.findErrorOption(cause); const message = Option.isSome(failure) && - failure.value.reason === "files-restored-provider-failed" + failure.value._tag === "CheckpointRollbackPartialRestoreError" ? failure.value.message : CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE; return threads diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index e1dc45a68dd50..a9c22bbdebe69 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -723,11 +723,10 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { assert.equal(yield* fs.readFileString(file), "checkpoint contents"); const failed = yield* orchestrator.getThreadProjection(threadId); assert.equal(failed.thread.rollbackFailure?.requestId, commandId); - assert.include( + assert.equal( failed.thread.rollbackFailure?.message, - "Files were restored to the selected checkpoint", + "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying.", ); - assert.include(failed.thread.rollbackFailure?.message, "conversation rollback failed"); assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); assert.equal( failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, From 75081c2e1cafc2c768c2acf4e1139f028d26f193 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Thu, 8 Oct 2026 07:17:21 +0200 Subject: [PATCH 05/12] fix(server): stop retrying partial checkpoint restores Model: gpt-6.1-sol (Codex harness) --- .../CheckpointRollbackService.ts | 4 + .../src/orchestration-v2/EffectWorker.ts | 19 +- .../src/orchestration-v2/runtimeLayer.test.ts | 229 +++++++++++------- 3 files changed, 154 insertions(+), 98 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 43f890bc02743..22e8fffc85ef8 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -77,6 +77,10 @@ export class CheckpointRollbackPartialRestoreError extends Schema.TaggedError { - if (willRetry || Cause.hasInterruptsOnly(cause)) return Effect.void; + if (Cause.hasInterruptsOnly(cause)) return Effect.void; const failure = Cause.findErrorOption(cause); + const partialRestore = + Option.isSome(failure) && + failure.value._tag === "CheckpointRollbackPartialRestoreError"; + if (willRetry && !partialRestore) return Effect.void; const message = Option.isSome(failure) && failure.value._tag === "CheckpointRollbackPartialRestoreError" @@ -709,6 +713,11 @@ export const layerWithOptions = ( const error = Cause.pretty(exit.cause); const nonRetryable = isNonRetryableProviderTurnControlFailure(effect.request.type, error); + const failure = Cause.findErrorOption(exit.cause); + const partialRestore = + effect.request.type === "provider-thread.rollback" && + Option.isSome(failure) && + CheckpointRollbackService.isCheckpointRollbackPartialRestoreError(failure.value.cause); yield* Effect.logWarning("Orchestration effect execution failed", { effectId: effect.id, effectType: effect.request.type, @@ -722,7 +731,7 @@ export const layerWithOptions = ( ? yield* outbox .succeed({ effectId: effect.id, workerId }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) - : effect.attemptCount >= maxAttempts + : partialRestore || effect.attemptCount >= maxAttempts ? yield* outbox .fail({ effectId: effect.id, workerId, error }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index a9c22bbdebe69..88d563d2178f7 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -628,6 +628,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { const [rollbackEffect] = yield* outbox.listByCommandId(rollbackCommandId); assert.equal(rollbackEffect?.status, "failed"); + assert.equal(rollbackEffect?.attemptCount, 5); const failed = yield* orchestrator.getThreadProjection(threadId); assert.deepEqual(failed.thread.rollbackFailure, { requestId: rollbackCommandId, @@ -640,102 +641,144 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); - it.effect("persists restored-file warning when provider rollback exhausts its attempts", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const outbox = yield* EffectOutbox.EffectOutboxV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const checkpointStore = yield* CheckpointStore.CheckpointStore; - const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-partial-" }); - const file = path.join(cwd, "workspace.txt"); - yield* fs.writeFileString(file, "current uncheckpointed contents"); - const { threadId, checkpointId, rollback } = yield* seedFailingRollbackThread( - "runtime-rollback-partial", - cwd, - ); - const before = yield* orchestrator.getThreadProjection(threadId); - const run = before.runs[0]!; - const providerThread = before.providerThreads[0]!; - const now = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: EventId.make("runtime-rollback-partial-completed"), - type: "run.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...run, status: "completed", completedAt: now }, - }, - ], - }); - let restored = 0; - let rejected = 0; - const restoreSpy = vi - .spyOn(checkpointStore, "restoreCheckpoint") - .mockImplementation((input) => - Effect.gen(function* () { - assert.equal(path.resolve(input.cwd), path.resolve(cwd)); - restored += 1; - yield* fs.writeFileString(file, "checkpoint contents"); - return true; - }).pipe(Effect.orDie), + it.effect.each(["partial-restore", "provider-only", "restore-fails"] as const)( + "settles checkpoint rollback safely for %s", + (outcome) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-partial-" }); + const file = path.join(cwd, "workspace.txt"); + yield* fs.writeFileString(file, "current uncheckpointed contents"); + const { threadId, checkpointId, rollback } = yield* seedFailingRollbackThread( + `runtime-rollback-${outcome}`, + cwd, ); - const deleteSpy = vi.spyOn(checkpointStore, "deleteCheckpointRefs"); - const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( - Effect.succeed({ - rollbackThread: () => + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const providerThread = before.providerThreads[0]!; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make("runtime-rollback-partial-completed"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", completedAt: now }, + }, + ], + }); + let restored = 0; + let rejected = 0; + const restoreSpy = vi + .spyOn(checkpointStore, "restoreCheckpoint") + .mockImplementation((input) => Effect.gen(function* () { - rejected += 1; - assert.equal( - yield* fs.readFileString(file).pipe(Effect.orDie), - "checkpoint contents", - ); - return yield* new ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: providerThread.id, - cause: new Error("legacy history cannot be reverted"), - }); - }), - } as unknown as ProviderAdapterV2SessionRuntime), - ); - yield* Effect.addFinalizer(() => - Effect.sync(() => { - restoreSpy.mockRestore(); - deleteSpy.mockRestore(); - sessionSpy.mockRestore(); - }), - ); - const commandId = CommandId.make("runtime-rollback-partial-request"); - yield* rollback(commandId, true); - for (let attempt = 0; attempt < 5; attempt++) { + assert.equal(path.resolve(input.cwd), path.resolve(cwd)); + restored += 1; + if (outcome === "restore-fails") return false; + yield* fs.writeFileString(file, "checkpoint contents"); + return true; + }).pipe(Effect.orDie), + ); + const deleteSpy = vi.spyOn(checkpointStore, "deleteCheckpointRefs"); + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + rollbackThread: () => + Effect.gen(function* () { + rejected += 1; + assert.equal( + yield* fs.readFileString(file).pipe(Effect.orDie), + outcome === "provider-only" + ? "current uncheckpointed contents" + : "checkpoint contents", + ); + return yield* new ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: providerThread.id, + cause: new Error("legacy history cannot be reverted"), + }); + }), + } as unknown as ProviderAdapterV2SessionRuntime), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + restoreSpy.mockRestore(); + deleteSpy.mockRestore(); + sessionSpy.mockRestore(); + }), + ); + const commandId = CommandId.make("runtime-rollback-partial-request"); + yield* rollback(commandId, outcome !== "provider-only"); yield* worker.drain(); - yield* TestClock.adjust("30 seconds"); - } - const [effect] = yield* outbox.listByCommandId(commandId); - assert.equal(effect?.status, "failed"); - assert.equal(restored, 5); - assert.equal(rejected, 5); - assert.equal(yield* fs.readFileString(file), "checkpoint contents"); - const failed = yield* orchestrator.getThreadProjection(threadId); - assert.equal(failed.thread.rollbackFailure?.requestId, commandId); - assert.equal( - failed.thread.rollbackFailure?.message, - "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying.", - ); - assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); - assert.equal( - failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, - "ready", - ); - assert.equal(deleteSpy.mock.calls.length, 0); - yield* rollback(CommandId.make("runtime-rollback-partial-next"), true); - assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); - }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), + const [firstAttempt] = yield* outbox.listByCommandId(commandId); + assert.equal(firstAttempt?.attemptCount, 1); + assert.equal(firstAttempt?.status, outcome === "partial-restore" ? "failed" : "pending"); + if (outcome !== "partial-restore") { + assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); + assert.equal(yield* fs.readFileString(file), "current uncheckpointed contents"); + for (let attempt = 1; attempt < 5; attempt++) { + yield* TestClock.adjust("30 seconds"); + yield* worker.drain(); + } + } + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "failed"); + const expectedAttempts = outcome === "partial-restore" ? 1 : 5; + assert.equal(effect?.attemptCount, expectedAttempts); + assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); + assert.equal(rejected, outcome === "restore-fails" ? 0 : expectedAttempts); + assert.equal( + yield* fs.readFileString(file), + outcome === "partial-restore" ? "checkpoint contents" : "current uncheckpointed contents", + ); + const failed = yield* orchestrator.getThreadProjection(threadId); + assert.equal(failed.thread.rollbackFailure?.requestId, commandId); + assert.equal( + failed.thread.rollbackFailure?.message, + outcome === "partial-restore" + ? "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying." + : ROLLBACK_FAILED_MESSAGE, + ); + assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); + assert.equal( + failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, + "ready", + ); + assert.equal(deleteSpy.mock.calls.length, 0); + yield* fs.writeFileString(file, "user edits after rollback failure"); + for (let attempt = 0; attempt < 5; attempt++) { + yield* TestClock.adjust("30 seconds"); + assert.equal(yield* worker.drain(), 0); + } + assert.equal(yield* fs.readFileString(file), "user edits after rollback failure"); + assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); + assert.equal(rejected, outcome === "restore-fails" ? 0 : expectedAttempts); + if (outcome === "partial-restore") { + const nextCommandId = CommandId.make("runtime-rollback-partial-next"); + yield* rollback(nextCommandId, true); + assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); + yield* worker.drain(); + const [nextEffect] = yield* outbox.listByCommandId(nextCommandId); + assert.equal(nextEffect?.status, "failed"); + assert.equal(nextEffect?.attemptCount, 1); + assert.equal(restored, 2); + assert.equal(rejected, 2); + assert.equal(yield* fs.readFileString(file), "checkpoint contents"); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.requestId, + nextCommandId, + ); + } + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), ); it.effect("ignores a late failure from a rollback that a newer one superseded", () => From 3d5e88b74b555a363b2a168dbb409389caf1457b Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Thu, 8 Oct 2026 14:23:05 +0200 Subject: [PATCH 06/12] fix(server): stop checkpoint replay after index refresh defects --- .../CheckpointRollbackService.test.ts | 76 +++++++++++++++---- .../CheckpointRollbackService.ts | 43 +++++++++-- .../src/orchestration-v2/EffectWorker.ts | 16 ++-- .../src/orchestration-v2/runtimeLayer.test.ts | 33 +++++--- 4 files changed, 132 insertions(+), 36 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index 8ab1ee4348ca0..cfc5f2673804d 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -11,6 +11,8 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Cause from "effect/Cause"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Option from "effect/Option"; @@ -502,7 +504,16 @@ it.effect.each([ }).pipe(Effect.provide(layerTest)); }); -it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fails"] as const)( +const checkpointRollbackStateOutcomes = [ + "success", + "restore-fails", + "refresh-defect", + "refresh-interrupted", + "provider-fails", + "provider-only-fails", +] as const; + +it.effect.each(checkpointRollbackStateOutcomes)( "reports checkpoint rollback state with %s", (outcome) => Effect.gen(function* () { @@ -560,6 +571,8 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai cause: new Error("git restore failed"), }); const providerConversation = ["turn-1"]; + const refreshFailure = new Error("index refresh defect"); + let providerCalls = 0; const providerFailure = new ProviderAdapterRollbackThreadError({ driver: ProviderDriverKind.make("codex"), providerThreadId, @@ -573,7 +586,15 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai Layer.mergeAll( IdAllocator.layer, ThreadCommandExecutor.layer, - Layer.succeed(WorkspaceEntries.WorkspaceEntries, workspaceEntries), + Layer.succeed(WorkspaceEntries.WorkspaceEntries, { + ...workspaceEntries, + refresh: + outcome === "refresh-defect" + ? () => Effect.die(refreshFailure) + : outcome === "refresh-interrupted" + ? () => Effect.interrupt + : workspaceEntries.refresh, + }), Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeed(unrelatedProject), }), @@ -612,6 +633,7 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai Effect.succeed({ rollbackThread: (input: ProviderAdapterV2RollbackThreadInput) => Effect.gen(function* () { + providerCalls += 1; const count = yield* resolveCodexRollbackTurnCount(input); assert.equal(count, 1); if (outcome === "provider-fails" || outcome === "provider-only-fails") { @@ -638,22 +660,49 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai scopeId, ...(outcome === "provider-only-fails" ? { restoreFiles: false } : {}), }); - if (outcome !== "success") { + if (outcome === "refresh-interrupted") { + const exit = yield* Effect.exit(rollback); + assert(Exit.isFailure(exit)); + assert(Cause.hasInterruptsOnly(exit.cause)); + assert.equal(providerCalls, 0); + assert.deepEqual(providerConversation, ["turn-1"]); + assert.deepEqual(persistedEvents, []); + assert.deepEqual(deletedRefs, []); + assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); + } else if (outcome !== "success") { const error = yield* Effect.flip(rollback); assert.equal(error.threadId, threadId); assert.equal(error.providerThreadId, providerThreadId); assert.equal(error.checkpointId, checkpointId); - assert.equal(error.cause, outcome === "restore-fails" ? restoreFailure : providerFailure); + assert.equal( + error.cause, + outcome === "restore-fails" + ? restoreFailure + : outcome === "refresh-defect" + ? refreshFailure + : providerFailure, + ); + assert.equal( + providerCalls, + outcome === "restore-fails" || outcome === "refresh-defect" ? 0 : 1, + ); assert.deepEqual(providerConversation, ["turn-1"]); assert.deepEqual(persistedEvents, []); assert.deepEqual(deletedRefs, []); assert.equal(projection.runs[0]?.status, "completed"); assert.equal(projection.checkpoints[1]?.status, "ready"); - if (outcome === "provider-fails") { - assert(error._tag === "CheckpointRollbackPartialRestoreError"); + if (outcome === "provider-fails" || outcome === "refresh-defect") { + assert.equal( + error._tag, + outcome === "refresh-defect" + ? "CheckpointRollbackIndexRefreshError" + : "CheckpointRollbackPartialRestoreError", + ); assert.equal( error.message, - "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying.", + outcome === "refresh-defect" + ? "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying." + : "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying.", ); assert.equal(yield* fs.exists(currentFile), false); assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); @@ -664,12 +713,13 @@ it.effect.each(["success", "restore-fails", "provider-fails", "provider-only-fai assert.equal(yield* fs.readFileString(currentFile), "current turn"); assert.equal(yield* fs.exists(restoredFile), false); } - assert.deepEqual((yield* search()).entries, [ - { - path: outcome === "provider-fails" ? "rollback-restored.txt" : "rollback-current.txt", - kind: "file", - }, - ]); + if (outcome !== "refresh-defect") + assert.deepEqual((yield* search()).entries, [ + { + path: outcome === "provider-fails" ? "rollback-restored.txt" : "rollback-current.txt", + kind: "file", + }, + ]); } else { yield* rollback; assert.deepEqual(providerConversation, []); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 1448344b1e86a..bffb9ff21ab41 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -78,12 +78,30 @@ export class CheckpointRollbackPartialRestoreError extends Schema.TaggedError()( + "CheckpointRollbackIndexRefreshError", + { + threadId: ThreadId, + providerThreadId: ProviderThreadId, + checkpointId: CheckpointId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying."; + } +} + +export const isCheckpointRollbackPostRestoreError = Schema.is( + Schema.Union([CheckpointRollbackPartialRestoreError, CheckpointRollbackIndexRefreshError]), ); const isCheckpointRollbackError = Schema.is( - Schema.Union([CheckpointRollbackExecutionError, CheckpointRollbackPartialRestoreError]), + Schema.Union([ + CheckpointRollbackExecutionError, + CheckpointRollbackPartialRestoreError, + CheckpointRollbackIndexRefreshError, + ]), ); export interface CheckpointRollbackServiceV2Shape { @@ -95,7 +113,9 @@ export interface CheckpointRollbackServiceV2Shape { readonly restoreFiles?: boolean; }) => Effect.Effect< void, - CheckpointRollbackExecutionError | CheckpointRollbackPartialRestoreError + | CheckpointRollbackExecutionError + | CheckpointRollbackPartialRestoreError + | CheckpointRollbackIndexRefreshError >; } @@ -260,7 +280,20 @@ export const layer: Layer.Layer< if (input.restoreFiles !== false) { yield* checkpoints.restore({ scope, checkpoint }); - yield* workspaceEntries.refresh(scope.cwd); + // Refresh already recovers typed index errors. An escaping defect must + // not cause the worker to replay this completed file restoration. + yield* workspaceEntries.refresh(scope.cwd).pipe( + Effect.catchDefect((cause) => + Effect.fail( + new CheckpointRollbackIndexRefreshError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), + ), + ), + ); } // Stopped and failed runs after the target leave the provider diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 38f6e112f51bb..c8401f7fb2d8d 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -379,19 +379,19 @@ export const layerExecutor: Layer.Layer< : { restoreFiles: effect.request.restoreFiles }), }) .pipe( - // A provider failure after restore is terminal: replaying the + // Known failures after restore are terminal: replaying the // restore could overwrite edits made after this attempt. // Other failures notify waiting clients only after retries end. Effect.tapCause((cause) => { if (Cause.hasInterruptsOnly(cause)) return Effect.void; const failure = Cause.findErrorOption(cause); - const partialRestore = + const postRestoreFailure = Option.isSome(failure) && - failure.value._tag === "CheckpointRollbackPartialRestoreError"; - if (willRetry && !partialRestore) return Effect.void; + CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure.value); + if (willRetry && !postRestoreFailure) return Effect.void; const message = Option.isSome(failure) && - failure.value._tag === "CheckpointRollbackPartialRestoreError" + CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure.value) ? failure.value.message : CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE; return threads @@ -714,10 +714,10 @@ export const layerWithOptions = ( const error = Cause.pretty(exit.cause); const nonRetryable = isNonRetryableProviderTurnControlFailure(effect.request.type, error); const failure = Cause.findErrorOption(exit.cause); - const partialRestore = + const postRestoreFailure = effect.request.type === "provider-thread.rollback" && Option.isSome(failure) && - CheckpointRollbackService.isCheckpointRollbackPartialRestoreError(failure.value.cause); + CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure.value.cause); yield* Effect.logWarning("Orchestration effect execution failed", { effectId: effect.id, effectType: effect.request.type, @@ -731,7 +731,7 @@ export const layerWithOptions = ( ? yield* outbox .succeed({ effectId: effect.id, workerId }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) - : partialRestore || effect.attemptCount >= maxAttempts + : postRestoreFailure || effect.attemptCount >= maxAttempts ? yield* outbox .fail({ effectId: effect.id, workerId, error }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 82ef497406420..dbcf4926e6f5e 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -55,6 +55,7 @@ import type { ProviderInstance } from "../provider/ProviderDriver.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts"; import * as Orchestrator from "./Orchestrator.ts"; import { ROLLBACK_FAILED_MESSAGE } from "./CheckpointRollbackService.ts"; @@ -275,6 +276,7 @@ const layerTest = Layer.mergeAll( ProjectionStore.layer, EffectOutbox.layer, ThreadCommandExecutor.layer, + WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)), ).pipe( Layer.provide(McpSessionRegistryTestkit.layer), Layer.provide(SqlitePersistence.layerMemory), @@ -641,7 +643,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); - it.effect.each(["partial-restore", "provider-only", "restore-fails"] as const)( + it.effect.each(["partial-restore", "refresh-defect", "provider-only", "restore-fails"] as const)( "settles checkpoint rollback safely for %s", (outcome) => Effect.gen(function* () { @@ -653,6 +655,8 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; const checkpointStore = yield* CheckpointStore.CheckpointStore; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const postRestoreFailure = outcome === "partial-restore" || outcome === "refresh-defect"; const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-partial-" }); const file = path.join(cwd, "workspace.txt"); yield* fs.writeFileString(file, "current uncheckpointed contents"); @@ -690,6 +694,10 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }).pipe(Effect.orDie), ); const deleteSpy = vi.spyOn(checkpointStore, "deleteCheckpointRefs"); + const refreshSpy = vi.spyOn(workspaceEntries, "refresh"); + if (outcome === "refresh-defect") { + refreshSpy.mockImplementation(() => Effect.die(new Error("index refresh defect"))); + } const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( Effect.succeed({ rollbackThread: () => @@ -713,6 +721,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { Effect.sync(() => { restoreSpy.mockRestore(); deleteSpy.mockRestore(); + refreshSpy.mockRestore(); sessionSpy.mockRestore(); }), ); @@ -721,8 +730,8 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { yield* worker.drain(); const [firstAttempt] = yield* outbox.listByCommandId(commandId); assert.equal(firstAttempt?.attemptCount, 1); - assert.equal(firstAttempt?.status, outcome === "partial-restore" ? "failed" : "pending"); - if (outcome !== "partial-restore") { + assert.equal(firstAttempt?.status, postRestoreFailure ? "failed" : "pending"); + if (!postRestoreFailure) { assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); assert.equal(yield* fs.readFileString(file), "current uncheckpointed contents"); for (let attempt = 1; attempt < 5; attempt++) { @@ -732,13 +741,15 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { } const [effect] = yield* outbox.listByCommandId(commandId); assert.equal(effect?.status, "failed"); - const expectedAttempts = outcome === "partial-restore" ? 1 : 5; + const expectedAttempts = postRestoreFailure ? 1 : 5; + const expectedProviderCalls = + outcome === "restore-fails" || outcome === "refresh-defect" ? 0 : expectedAttempts; assert.equal(effect?.attemptCount, expectedAttempts); assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); - assert.equal(rejected, outcome === "restore-fails" ? 0 : expectedAttempts); + assert.equal(rejected, expectedProviderCalls); assert.equal( yield* fs.readFileString(file), - outcome === "partial-restore" ? "checkpoint contents" : "current uncheckpointed contents", + postRestoreFailure ? "checkpoint contents" : "current uncheckpointed contents", ); const failed = yield* orchestrator.getThreadProjection(threadId); assert.equal(failed.thread.rollbackFailure?.requestId, commandId); @@ -746,7 +757,9 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { failed.thread.rollbackFailure?.message, outcome === "partial-restore" ? "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying." - : ROLLBACK_FAILED_MESSAGE, + : outcome === "refresh-defect" + ? "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying." + : ROLLBACK_FAILED_MESSAGE, ); assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); assert.equal( @@ -761,8 +774,8 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { } assert.equal(yield* fs.readFileString(file), "user edits after rollback failure"); assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); - assert.equal(rejected, outcome === "restore-fails" ? 0 : expectedAttempts); - if (outcome === "partial-restore") { + assert.equal(rejected, expectedProviderCalls); + if (postRestoreFailure) { const nextCommandId = CommandId.make("runtime-rollback-partial-next"); yield* rollback(nextCommandId, true); assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); @@ -771,7 +784,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { assert.equal(nextEffect?.status, "failed"); assert.equal(nextEffect?.attemptCount, 1); assert.equal(restored, 2); - assert.equal(rejected, 2); + assert.equal(rejected, outcome === "refresh-defect" ? 0 : 2); assert.equal(yield* fs.readFileString(file), "checkpoint contents"); assert.equal( (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.requestId, From 23ca6abdc278d2751ea07f55b69f33bff252c842 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 03:03:36 +0000 Subject: [PATCH 07/12] test(server): consolidate checkpoint rollback regression coverage --- .../CheckpointRollbackService.test.ts | 115 ++++-------------- .../src/orchestration-v2/runtimeLayer.test.ts | 26 +++- 2 files changed, 43 insertions(+), 98 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index cfc5f2673804d..00bb5bcf33cd1 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -4,15 +4,12 @@ import { CheckpointId, CheckpointScopeId, type OrchestrationV2ThreadProjection, - ProviderDriverKind, ProviderInstanceId, ProviderSessionId, ProviderThreadId, ThreadId, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; -import * as Cause from "effect/Cause"; -import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Option from "effect/Option"; @@ -33,10 +30,7 @@ import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "./IdAllocator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; -import { - ProviderAdapterRollbackThreadError, - type ProviderAdapterV2RollbackThreadInput, -} from "./ProviderAdapter.ts"; +import type { ProviderAdapterV2RollbackThreadInput } from "./ProviderAdapter.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as RuntimePolicy from "./RuntimePolicy.ts"; @@ -504,17 +498,8 @@ it.effect.each([ }).pipe(Effect.provide(layerTest)); }); -const checkpointRollbackStateOutcomes = [ - "success", - "restore-fails", - "refresh-defect", - "refresh-interrupted", - "provider-fails", - "provider-only-fails", -] as const; - -it.effect.each(checkpointRollbackStateOutcomes)( - "reports checkpoint rollback state with %s", +it.effect.each(["success", "restore-fails"] as const)( + "restores files and refreshes the workspace index before the provider: %s", (outcome) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -571,14 +556,7 @@ it.effect.each(checkpointRollbackStateOutcomes)( cause: new Error("git restore failed"), }); const providerConversation = ["turn-1"]; - const refreshFailure = new Error("index refresh defect"); let providerCalls = 0; - const providerFailure = new ProviderAdapterRollbackThreadError({ - driver: ProviderDriverKind.make("codex"), - providerThreadId, - checkpointId, - cause: new Error("provider rollback failed"), - }); const deletedRefs: CheckpointId[] = []; const persistedEvents: string[] = []; const testLayer = CheckpointRollbackService.layer.pipe( @@ -586,15 +564,7 @@ it.effect.each(checkpointRollbackStateOutcomes)( Layer.mergeAll( IdAllocator.layer, ThreadCommandExecutor.layer, - Layer.succeed(WorkspaceEntries.WorkspaceEntries, { - ...workspaceEntries, - refresh: - outcome === "refresh-defect" - ? () => Effect.die(refreshFailure) - : outcome === "refresh-interrupted" - ? () => Effect.interrupt - : workspaceEntries.refresh, - }), + Layer.succeed(WorkspaceEntries.WorkspaceEntries, workspaceEntries), Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeed(unrelatedProject), }), @@ -636,9 +606,14 @@ it.effect.each(checkpointRollbackStateOutcomes)( providerCalls += 1; const count = yield* resolveCodexRollbackTurnCount(input); assert.equal(count, 1); - if (outcome === "provider-fails" || outcome === "provider-only-fails") { - return yield* providerFailure; - } + assert.equal(yield* fs.exists(currentFile).pipe(Effect.orDie), false); + assert.equal( + yield* fs.readFileString(restoredFile).pipe(Effect.orDie), + "checkpoint turn", + ); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-restored.txt", kind: "file" }, + ]); providerConversation.splice(providerConversation.length - count, count); return { providerThread }; }), @@ -658,70 +633,22 @@ it.effect.each(checkpointRollbackStateOutcomes)( providerThreadId, checkpointId, scopeId, - ...(outcome === "provider-only-fails" ? { restoreFiles: false } : {}), }); - if (outcome === "refresh-interrupted") { - const exit = yield* Effect.exit(rollback); - assert(Exit.isFailure(exit)); - assert(Cause.hasInterruptsOnly(exit.cause)); - assert.equal(providerCalls, 0); - assert.deepEqual(providerConversation, ["turn-1"]); - assert.deepEqual(persistedEvents, []); - assert.deepEqual(deletedRefs, []); - assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); - } else if (outcome !== "success") { + if (outcome === "restore-fails") { const error = yield* Effect.flip(rollback); - assert.equal(error.threadId, threadId); - assert.equal(error.providerThreadId, providerThreadId); - assert.equal(error.checkpointId, checkpointId); - assert.equal( - error.cause, - outcome === "restore-fails" - ? restoreFailure - : outcome === "refresh-defect" - ? refreshFailure - : providerFailure, - ); - assert.equal( - providerCalls, - outcome === "restore-fails" || outcome === "refresh-defect" ? 0 : 1, - ); + assert.equal(error.cause, restoreFailure); + assert.equal(providerCalls, 0); assert.deepEqual(providerConversation, ["turn-1"]); assert.deepEqual(persistedEvents, []); assert.deepEqual(deletedRefs, []); - assert.equal(projection.runs[0]?.status, "completed"); - assert.equal(projection.checkpoints[1]?.status, "ready"); - if (outcome === "provider-fails" || outcome === "refresh-defect") { - assert.equal( - error._tag, - outcome === "refresh-defect" - ? "CheckpointRollbackIndexRefreshError" - : "CheckpointRollbackPartialRestoreError", - ); - assert.equal( - error.message, - outcome === "refresh-defect" - ? "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying." - : "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying.", - ); - assert.equal(yield* fs.exists(currentFile), false); - assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); - } else { - assert(error._tag === "CheckpointRollbackExecutionError"); - assert.equal(error.reason, "unexpected-failure"); - assert.equal(error.message, CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE); - assert.equal(yield* fs.readFileString(currentFile), "current turn"); - assert.equal(yield* fs.exists(restoredFile), false); - } - if (outcome !== "refresh-defect") - assert.deepEqual((yield* search()).entries, [ - { - path: outcome === "provider-fails" ? "rollback-restored.txt" : "rollback-current.txt", - kind: "file", - }, - ]); + assert.equal(yield* fs.readFileString(currentFile), "current turn"); + assert.equal(yield* fs.exists(restoredFile), false); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-current.txt", kind: "file" }, + ]); } else { yield* rollback; + assert.equal(providerCalls, 1); assert.deepEqual(providerConversation, []); assert.include(persistedEvents, "run.updated"); assert.deepEqual(deletedRefs, [laterCheckpointId]); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index dbcf4926e6f5e..c4a830f33881b 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -668,8 +668,22 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { const run = before.runs[0]!; const providerThread = before.providerThreads[0]!; const now = yield* DateTime.now; + const laterCheckpoint = { + ...before.checkpoints[0]!, + id: CheckpointId.make("runtime-rollback-later-checkpoint"), + ref: CheckpointRef.make("refs/t3/runtime-rollback-later"), + appRunOrdinal: run.ordinal, + ordinalWithinScope: 1, + }; yield* eventSink.write({ events: [ + { + id: EventId.make("runtime-rollback-later-captured"), + type: "checkpoint.captured", + threadId, + occurredAt: now, + payload: laterCheckpoint, + }, { id: EventId.make("runtime-rollback-partial-completed"), type: "run.updated", @@ -766,12 +780,16 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, "ready", ); + assert.deepEqual( + failed.checkpoints.find((candidate) => candidate.id === laterCheckpoint.id), + laterCheckpoint, + ); + assert.deepEqual(failed.nodes, before.nodes); + assert.deepEqual(failed.providerThreads, before.providerThreads); assert.equal(deleteSpy.mock.calls.length, 0); yield* fs.writeFileString(file, "user edits after rollback failure"); - for (let attempt = 0; attempt < 5; attempt++) { - yield* TestClock.adjust("30 seconds"); - assert.equal(yield* worker.drain(), 0); - } + yield* TestClock.adjust("30 seconds"); + assert.equal(yield* worker.drain(), 0); assert.equal(yield* fs.readFileString(file), "user edits after rollback failure"); assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); assert.equal(rejected, expectedProviderCalls); From f8ffa08459ace2a93fcbb31322b35dd416d604ef Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Tue, 27 Oct 2026 14:27:51 +0100 Subject: [PATCH 08/12] fix(server): guard checkpoint restore against active turns --- .../Adapters/AcpAdapterV2.test.ts | 13 + .../Adapters/ClaudeAdapterV2.test.ts | 171 +++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 38 ++- .../Adapters/CodexAdapterV2.test.ts | 17 ++ .../Adapters/CodexAdapterV2.ts | 70 +++-- .../CheckpointRestoreSafety.test.ts | 13 + .../CheckpointRollbackService.test.ts | 78 ++++-- .../CheckpointRollbackService.ts | 231 +++++++++++------ .../src/orchestration-v2/EffectWorker.ts | 12 +- .../src/orchestration-v2/Orchestrator.ts | 8 + .../src/orchestration-v2/ProjectionStore.ts | 18 ++ .../ProviderTurnControlService.test.ts | 1 + .../src/orchestration-v2/runtimeLayer.test.ts | 241 ++++++++++++++++-- docs/user/composer.md | 4 +- docs/user/providers-acp.md | 3 +- packages/provider-acp/src/server/adapter.ts | 201 ++++++++------- .../src/server/ProviderAdapter.ts | 9 + 17 files changed, 888 insertions(+), 240 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index a919726c9a56e..2fa05bc72d7f2 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -3087,6 +3087,19 @@ describe("AcpAdapterV2", () => { if (activeRollback._tag === "Failure") { assert.include(String(activeRollback.failure.cause), "while turn"); } + const activeAdmission = yield* runtime.withRollbackAdmission!( + { + providerThread, + providerThreadTurns: [], + target: { + type: "thread_start", + checkpointId: CheckpointId.make("checkpoint-acp-active-admission"), + appRunOrdinal: 0, + }, + }, + () => Effect.die("active ACP turn must not admit restore"), + ).pipe(Effect.result); + assert.equal(activeAdmission._tag, "Failure"); const firstProviderTurnId = idAllocator.derive.providerTurn({ driver: ACP_TEST_DRIVER, nativeTurnId: acpScopedNativeId(instanceId, `${firstSessionId}:turn:1`), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 8de0f78af88c0..562e379a4b47f 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2618,6 +2618,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarnessWithOptions = (options?: { + readonly afterTerminalEvent?: Effect.Effect; readonly close?: (sdkMessages: Queue.Queue) => Effect.Effect; readonly interrupt?: Effect.Effect; readonly setPermissionMode?: ClaudeAdapterV2.ClaudeAgentSdkQuerySession["setPermissionMode"]; @@ -2688,6 +2689,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { >(); let openedOptions: ClaudeAdapterV2.ClaudeAgentSdkQueryOptions | undefined; const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + ...(options?.afterTerminalEvent === undefined + ? {} + : { testHooks: { afterTerminalEvent: options.afterTerminalEvent } }), instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, settings: options?.settings ?? DEFAULT_CLAUDE_SETTINGS, environment: options?.environment ?? {}, @@ -4924,6 +4928,173 @@ describe("ClaudeAdapterV2 background wake turns", () => { close: (sdkMessages) => Queue.shutdown(sdkMessages), }); + it.effect("rejects rollback while a Claude provider turn is active", () => + Effect.gen(function* () { + const harness = yield* makeRollbackHarness; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("active-rollback-proof"), + text: "Keep the turn active.", + attachments: [], + }), + ); + const error = yield* harness.runtime + .rollbackThread({ + providerThread: harness.providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("active-rollback-checkpoint"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }) + .pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterRollbackThreadError"); + if (error._tag !== "ProviderAdapterRollbackThreadError") return; + assert.instanceOf(error.cause, ProviderAdapter.ProviderAdapterProtocolError); + assert.include(String(error.cause), "while provider turn"); + const admissionError = yield* harness.runtime.withRollbackAdmission!( + { + providerThread: harness.providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("active-admission-checkpoint"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }, + () => Effect.die("active turns must not enter rollback admission"), + ).pipe(Effect.flip); + assert.equal(admissionError._tag, "ProviderAdapterProtocolError"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + + it.effect("keeps admission closed after terminal until Claude clears the active turn", () => + Effect.gen(function* () { + const terminalEmitted = yield* Deferred.make(); + const releaseTerminal = yield* Deferred.make(); + const harness = yield* makeWakeHarnessWithOptions({ + close: (messages) => Queue.shutdown(messages), + afterTerminalEvent: Deferred.succeed(terminalEmitted, undefined).pipe( + Effect.andThen(Deferred.await(releaseTerminal)), + ), + }); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("terminal-admission-turn"), + text: "Finish this turn.", + attachments: [], + }), + ); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ uuid: "terminal-admission-result", result: "Done." }), + ); + yield* Deferred.await(terminalEmitted); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "completed"); + const rejected = yield* harness.runtime.withRollbackAdmission!( + { + providerThread: harness.providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("terminal-admission"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }, + () => Effect.die("terminal is not native cleanup"), + ).pipe(Effect.flip); + assert.equal(rejected._tag, "ProviderAdapterProtocolError"); + yield* Deferred.succeed(releaseTerminal, undefined); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + + it.effect.each(["complete", "interrupt"] as const)( + "waits for rollback admission to %s before starting Claude", + (outcome) => + Effect.gen(function* () { + const harness = yield* makeRollbackHarness; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const admitted = yield* harness.runtime.withRollbackAdmission!( + { + providerThread: harness.providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("exclusive-admission"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }, + () => Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ).pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(entered); + const starting = yield* harness.runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("exclusive-admission-turn"), + text: "Start after the rollback.", + attachments: [], + }), + ) + .pipe(Effect.forkChild({ startImmediately: true })); + assert.isUndefined(harness.getOpenedOptions()); + if (outcome === "interrupt") { + yield* Fiber.interrupt(admitted); + } else { + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(admitted); + } + yield* Fiber.join(starting); + assert.isDefined(harness.getOpenedOptions()); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + + it.effect("preserves failures from inside rollback admission", () => + Effect.gen(function* () { + const harness = yield* makeRollbackHarness; + const expected = { _tag: "host-restore-failure" as const }; + const error = yield* harness.runtime.withRollbackAdmission!( + { + providerThread: harness.providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("admission-host-failure"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }, + () => Effect.fail(expected), + ).pipe(Effect.flip); + assert.strictEqual(error, expected); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect.each([ { kind: "fork with inherited head", fork: true, headMessageId: "remapped-inherited-head" }, { kind: "fork without recorded head", fork: true, headMessageId: undefined }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index b66e85e14d7c0..79ca2ef829c8e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4217,6 +4217,7 @@ function claudeProposedPlan(input: unknown): string | null { } export interface ClaudeAdapterV2Options { + readonly testHooks?: { readonly afterTerminalEvent?: Effect.Effect }; readonly instanceId: ProviderInstanceId; readonly settings: ClaudeSettings; readonly environment: NodeJS.ProcessEnv; @@ -4419,6 +4420,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); const events = yield* Queue.unbounded(); const activeTurn = yield* Ref.make(null); + const rollbackAdmission = yield* Semaphore.make(1); const taskOutputTails = new Map< string, { readonly fiber: Fiber.Fiber; readonly stop: Deferred.Deferred } @@ -7313,6 +7315,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ], { concurrency: 1 }, ); + yield* adapterOptions.testHooks?.afterTerminalEvent ?? Effect.void; yield* Ref.update(activeTurn, (current) => current?.providerTurnId === input.context.providerTurnId ? null : current, ); @@ -10636,12 +10639,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), ), ), - startTurn, + startTurn: (turnInput) => rollbackAdmission.withPermit(startTurn(turnInput)), compactThread: (turnInput) => - startTurn({ - ...turnInput, - message: { ...turnInput.message, text: "/compact" }, - }), + rollbackAdmission.withPermit( + startTurn({ + ...turnInput, + message: { ...turnInput.message, text: "/compact" }, + }), + ), steerTurn, stopTask: Effect.fn("ClaudeAdapterV2.stopTask")(function* ({ providerThread, taskId }) { const liveQuery = yield* Ref.get(queryContext); @@ -10828,6 +10833,29 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( cause: "Claude V2 adapter does not implement snapshots.", }), ), + withRollbackAdmission: (rollbackInput, use) => + rollbackAdmission.withPermit( + Effect.gen(function* () { + const currentTurn = yield* Ref.get(activeTurn); + const nativeThreadId = rollbackInput.providerThread.nativeThreadRef?.nativeId; + const goal = + nativeThreadId == null + ? rollbackInput.providerThread.goal + : (goalsByNativeThread.get(nativeThreadId) ?? + rollbackInput.providerThread.goal); + if ( + currentTurn !== null || + goal?.status === "active" || + (yield* runtime.hasPendingBackgroundWork!) + ) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CLAUDE_PROVIDER, + detail: `Cannot roll back Claude provider thread ${rollbackInput.providerThread.id} while provider work is active.`, + }); + } + return yield* use(() => runtime.rollbackThread(rollbackInput)); + }), + ), rollbackThread: Effect.fn("ClaudeAdapterV2.rollbackThread")( function* (rollbackInput) { const currentTurn = yield* Ref.get(activeTurn); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 602e1a4d56f29..87a40578fecea 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -5700,6 +5700,23 @@ describe("CodexAdapterV2 post-settle continuation", () => { now, }); + const activeGoalAdmission = yield* harness.runtime.withRollbackAdmission!( + { + providerThread: { + ...harness.providerThread, + goal: { objective: "Continue autonomously", status: "active" }, + }, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("checkpoint-codex-active-goal"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }, + () => Effect.die("an active goal must not admit restoration"), + ).pipe(Effect.result); + assert.equal(activeGoalAdmission._tag, "Failure"); + const error = yield* Effect.flip( harness.runtime.rollbackThread({ providerThread: harness.providerThread, diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 6dba639b63b01..381c5b5699e86 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1723,6 +1723,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( new Map>(), ); const activeTurns = yield* Ref.make(new Map()); + const rollbackAdmission = yield* Semaphore.make(1); const turnTokenUsageByThread = new Map(); const usageStateForThread = (nativeThreadId: string) => { let state = turnTokenUsageByThread.get(nativeThreadId); @@ -6697,6 +6698,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }, (effect, turnInput) => effect.pipe( + (effect) => rollbackAdmission.withPermit(effect), Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterTurnStartError({ @@ -6740,24 +6742,27 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( turnInput.restartContinuationOfRunId === undefined ? parseGoalCommand(turnInput.message) : null; - if (goalCommand !== null) return runGoalCommand(turnInput, goalCommand); - return Effect.gen(function* () { - yield* startNativeTurn( - turnInput, - turnInput.restartContinuationOfRunId === undefined - ? yield* toCodexInput(turnInput) - : [], - ); - }).pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterTurnStartError({ - driver: CODEX_PROVIDER, - threadId: turnInput.threadId, - providerThreadId: turnInput.providerThread.id, - runId: turnInput.runId, - cause, - }), + if (goalCommand !== null) + return rollbackAdmission.withPermit(runGoalCommand(turnInput, goalCommand)); + return rollbackAdmission.withPermit( + Effect.gen(function* () { + yield* startNativeTurn( + turnInput, + turnInput.restartContinuationOfRunId === undefined + ? yield* toCodexInput(turnInput) + : [], + ); + }).pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: CODEX_PROVIDER, + threadId: turnInput.threadId, + providerThreadId: turnInput.providerThread.id, + runId: turnInput.runId, + cause, + }), + ), ), ); }, @@ -7316,6 +7321,35 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), ), ), + withRollbackAdmission: (rollbackInput, use) => + rollbackAdmission.withPermit( + Effect.gen(function* () { + const nativeThreadId = rollbackInput.providerThread.nativeThreadRef?.nativeId; + const active = [...(yield* Ref.get(activeTurns)).values()].some( + (turn) => turn.providerThread.id === rollbackInput.providerThread.id, + ); + const pending = + nativeThreadId != null && (yield* Ref.get(pendingRootTurns)).has(nativeThreadId); + const goalRunning = + (nativeThreadId == null && + rollbackInput.providerThread.goal?.status === "active") || + (nativeThreadId != null && + (goalActivations.has(nativeThreadId) || + goalHolds.has(nativeThreadId) || + (goalsByNativeThread.get(nativeThreadId) ?? rollbackInput.providerThread.goal) + ?.status === "active")); + const background = yield* runtime.hasPendingBackgroundWorkForThread!( + rollbackInput.providerThread, + ); + if (active || pending || background || goalRunning) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CODEX_PROVIDER, + detail: "Cannot roll back while the provider thread has active work.", + }); + } + return yield* use(() => runtime.rollbackThread(rollbackInput)); + }), + ), rollbackThread: Effect.fn("CodexAdapterV2.rollbackThread")( function* (threadInput) { const threadId = yield* getNativeThreadId(threadInput.providerThread); diff --git a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts index 77e4e90f7dc43..a5301974dbe9a 100644 --- a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts @@ -107,6 +107,7 @@ it.effect.each([ Layer.mock(ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), getCheckpointRollbackContext: () => Effect.succeed(projection), + hasLiveRun: () => Effect.succeed(false), getShellSnapshot: () => Effect.succeed({ schemaVersion: 1, @@ -152,6 +153,18 @@ it.effect.each([ Layer.mock(ProviderSessionManagerV2)({ open: () => Effect.succeed({ + withRollbackAdmission: ( + _input: unknown, + use: ( + rollback: () => Effect.Effect<{ providerThread: typeof providerThread }>, + ) => Effect.Effect, + ) => + use(() => + Effect.sync(() => { + calls.push("provider"); + return { providerThread }; + }), + ), rollbackThread: () => Effect.sync(() => { calls.push("provider"); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index ab20c4cd1977e..71ff3b23886f3 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -52,6 +52,15 @@ import * as RuntimePolicy from "./RuntimePolicy.ts"; const unrelatedProject = Option.some({ workspaceRoot: "/nonexistent/t3-rollback-project", } as never); +const admittedSession = ( + session: Pick, +) => ({ + ...session, + withRollbackAdmission: ((input, use) => + use(() => session.rollbackThread(input))) satisfies NonNullable< + ProviderAdapter.ProviderAdapterV2SessionRuntime["withRollbackAdmission"] + >, +}); const layerCheckpointRollbackService = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( @@ -178,6 +187,7 @@ it.effect.each(["initial", "waiting"])( it.effect.each([ { order: "cleanup-first", restoreFiles: true }, { order: "rollback-first", restoreFiles: true }, + { order: "interrupt", restoreFiles: true }, { order: "cleanup-first", restoreFiles: false, unmanaged: true }, ])("locks rollback with %s", ({ order, restoreFiles, unmanaged = false }) => Effect.gen(function* () { @@ -274,6 +284,7 @@ it.effect.each([ Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), getCheckpointRollbackContext: () => Effect.succeed(projection), + hasLiveRun: () => Effect.succeed(false), getShellSnapshot: () => Effect.succeed({ schemaVersion: 1, @@ -291,8 +302,9 @@ it.effect.each([ } calls.push("session"); yield* Deferred.succeed(openEntered, undefined); - if (order === "rollback-first") yield* Deferred.await(release); - return {} as never; + if (order === "rollback-first" || order === "interrupt") + yield* Deferred.await(release); + return admittedSession({ rollbackThread: () => Effect.die("unused") }) as never; }), }), Layer.mock(RuntimePolicy.RuntimePolicyV2)({ @@ -322,6 +334,13 @@ it.effect.each([ const rollback = yield* service .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }) .pipe(Effect.forkChild({ startImmediately: true })); + if (order === "interrupt") { + yield* Deferred.await(openEntered); + yield* Fiber.interrupt(rollback); + yield* cleanupEffect; + assert.deepEqual(calls, ["session", "cleanup"]); + return; + } if (order === "cleanup-first") assert.deepEqual(calls, []); if (order === "rollback-first") yield* Deferred.await(openEntered); const laterCleanup = @@ -600,13 +619,15 @@ const seedRollbackHistory = Effect.gen(function* () { }), Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open: () => - Effect.succeed({ - rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => - Effect.gen(function* () { - rollbackCounts.push(yield* resolveCodexRollbackTurnCount(input)); - return { providerThread: input.providerThread }; - }), - } as never), + Effect.succeed( + admittedSession({ + rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => + Effect.gen(function* () { + rollbackCounts.push(yield* resolveCodexRollbackTurnCount(input)); + return { providerThread: input.providerThread }; + }), + } as never) as never, + ), }), Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: () => Effect.succeed({} as never) }), ), @@ -835,6 +856,7 @@ it.effect.each(["success", "restore-fails"] as const)( }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), + hasLiveRun: () => Effect.succeed(false), getCheckpointRollbackContext: () => Effect.succeed({ thread: projection.thread, @@ -859,24 +881,26 @@ it.effect.each(["success", "restore-fails"] as const)( }), Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open: () => - Effect.succeed({ - rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => - Effect.gen(function* () { - providerCalls += 1; - const count = yield* resolveCodexRollbackTurnCount(input); - assert.equal(count, 1); - assert.equal(yield* fs.exists(currentFile).pipe(Effect.orDie), false); - assert.equal( - yield* fs.readFileString(restoredFile).pipe(Effect.orDie), - "checkpoint turn", - ); - assert.deepEqual((yield* search()).entries, [ - { path: "rollback-restored.txt", kind: "file" }, - ]); - providerConversation.splice(providerConversation.length - count, count); - return { providerThread }; - }), - } as never), + Effect.succeed( + admittedSession({ + rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => + Effect.gen(function* () { + providerCalls += 1; + const count = yield* resolveCodexRollbackTurnCount(input); + assert.equal(count, 1); + assert.equal(yield* fs.exists(currentFile).pipe(Effect.orDie), false); + assert.equal( + yield* fs.readFileString(restoredFile).pipe(Effect.orDie), + "checkpoint turn", + ); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-restored.txt", kind: "file" }, + ]); + providerConversation.splice(providerConversation.length - count, count); + return { providerThread }; + }), + } as never) as never, + ), }), Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: () => Effect.succeed({} as never), diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 854df7bb1d543..62c958d3a0f1e 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -32,7 +32,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import { GIT_INDEX_LOCKED_DETAIL } from "../vcs/VcsProcess.ts"; import * as ProjectStore from "./ProjectStore.ts"; -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; @@ -47,6 +47,8 @@ export class CheckpointRollbackExecutionError extends Schema.TaggedError + isCheckpointRollbackExecutionError(error) && + (error.reason === "active-turn" || error.reason === "rollback-admission-unavailable"); + const isCheckpointRollbackError = Schema.is( Schema.Union([ CheckpointRollbackExecutionError, @@ -126,6 +139,9 @@ export function rollbackFailureClientMessage( | CheckpointRollbackIndexRefreshError, ): string { if (isCheckpointRollbackPostRestoreError(error)) return error.message; + if (error.reason === "rollback-admission-unavailable" || error.reason === "active-turn") { + return error.message; + } const restoreCause = isCheckpointRestoreError(error.cause) ? error.cause.cause : undefined; return isVcsProcessExitError(restoreCause) && restoreCause.detail === GIT_INDEX_LOCKED_DETAIL ? GIT_INDEX_LOCKED_DETAIL @@ -321,27 +337,51 @@ export const layer: Layer.Layer< (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)), ); - if (input.restoreFiles !== false) { - yield* checkpoints.restore({ scope, checkpoint }); - yield* workspaceEntries.refresh(scope.cwd).pipe( - Effect.catchDefect((cause) => - Effect.fail( - new CheckpointRollbackIndexRefreshError({ - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - cause, - }), + const rollbackInput = { providerThread, target: rollbackTarget, providerThreadTurns }; + let enteredAdmission = false; + const admitted = Effect.fnUntraced(function* ( + rollback: () => ReturnType, + ) { + enteredAdmission = true; + if ( + input.restoreFiles !== false && + session.canRestoreRollbackFiles !== undefined && + !(yield* session.canRestoreRollbackFiles) + ) { + return yield* new CheckpointRollbackExecutionError({ + reason: "rollback-admission-unavailable", + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + }); + } + if (yield* projections.hasLiveRun(input.threadId)) { + return yield* new CheckpointRollbackExecutionError({ + reason: "active-turn", + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + }); + } + if (input.restoreFiles !== false) { + yield* checkpoints.restore({ scope, checkpoint }); + yield* workspaceEntries.refresh(scope.cwd).pipe( + Effect.catchDefect((cause) => + Effect.fail( + new CheckpointRollbackIndexRefreshError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), + ), ), - ), - ); - } - const snapshot = - runsToRollback.length === 0 - ? { providerThread } - : yield* session - .rollbackThread({ providerThread, target: rollbackTarget, providerThreadTurns }) - .pipe( + ); + } + const snapshot = + runsToRollback.length === 0 + ? { providerThread } + : yield* rollback().pipe( Effect.mapError((cause) => input.restoreFiles === false ? cause @@ -353,82 +393,113 @@ export const layer: Layer.Layer< }), ), ); - if (staleCheckpoints.length > 0) { - yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }); - } + if (staleCheckpoints.length > 0) { + yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }); + } - // Lock against full-row thread writes that could restore the old native session after rollback. - yield* threadCommands.withLock( - input.threadId, - Effect.gen(function* () { - const now = yield* DateTime.now; - const makeEvent = (event: Omit) => - Effect.map( - ids.allocate.event({ threadId: event.threadId }), - (id) => - ({ - ...event, - id, - }) as Event, - ); - const events: Array = []; - events.push( - yield* makeEvent({ - type: "provider-thread.updated", - threadId: input.threadId, - driver: providerThread.driver, - providerInstanceId: providerThread.providerInstanceId, - occurredAt: now, - payload: { - ...snapshot.providerThread, - lastRunOrdinal: targetOrdinal === 0 ? null : targetOrdinal, - updatedAt: now, - }, - }), - ); - for (const staleCheckpoint of staleCheckpoints) { + // Lock against full-row thread writes that could restore the old native session after rollback. + yield* threadCommands.withLock( + input.threadId, + Effect.gen(function* () { + const now = yield* DateTime.now; + const makeEvent = ( + event: Omit, + ) => + Effect.map( + ids.allocate.event({ threadId: event.threadId }), + (id) => + ({ + ...event, + id, + }) as Event, + ); + const events: Array = []; events.push( yield* makeEvent({ - type: "checkpoint.captured", + type: "provider-thread.updated", threadId: input.threadId, - ...(staleCheckpoint.runId === null ? {} : { runId: staleCheckpoint.runId }), - nodeId: staleCheckpoint.nodeId, + driver: providerThread.driver, providerInstanceId: providerThread.providerInstanceId, occurredAt: now, - payload: { ...staleCheckpoint, status: "stale" }, + payload: { + ...snapshot.providerThread, + lastRunOrdinal: targetOrdinal === 0 ? null : targetOrdinal, + updatedAt: now, + }, }), ); - } - for (const run of runsToRollback) { - const rootNode = context.rootNodes.find((candidate) => candidate.id === run.rootNodeId); - events.push( - yield* makeEvent({ - type: "run.updated", - threadId: input.threadId, - runId: run.id, - ...(rootNode === undefined ? {} : { nodeId: rootNode.id }), - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: { ...run, status: "rolled_back", completedAt: now }, - }), - ); - if (rootNode !== undefined) { + for (const staleCheckpoint of staleCheckpoints) { events.push( yield* makeEvent({ - type: "node.updated", + type: "checkpoint.captured", + threadId: input.threadId, + ...(staleCheckpoint.runId === null ? {} : { runId: staleCheckpoint.runId }), + nodeId: staleCheckpoint.nodeId, + providerInstanceId: providerThread.providerInstanceId, + occurredAt: now, + payload: { ...staleCheckpoint, status: "stale" }, + }), + ); + } + for (const run of runsToRollback) { + const rootNode = context.rootNodes.find( + (candidate) => candidate.id === run.rootNodeId, + ); + events.push( + yield* makeEvent({ + type: "run.updated", threadId: input.threadId, runId: run.id, - nodeId: rootNode.id, + ...(rootNode === undefined ? {} : { nodeId: rootNode.id }), providerInstanceId: run.providerInstanceId, occurredAt: now, - payload: { ...rootNode, status: "rolled_back", completedAt: now }, + payload: { ...run, status: "rolled_back", completedAt: now }, }), ); + if (rootNode !== undefined) { + events.push( + yield* makeEvent({ + type: "node.updated", + threadId: input.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...rootNode, status: "rolled_back", completedAt: now }, + }), + ); + } } - } - yield* eventSink.write({ events }); - }), - ); + yield* eventSink.write({ events }); + }), + ); + }); + if (session.withRollbackAdmission !== undefined) { + yield* session.withRollbackAdmission(rollbackInput, admitted).pipe( + Effect.catchIf( + (cause) => !enteredAdmission && isProviderAdapterProtocolError(cause), + (cause) => + Effect.fail( + new CheckpointRollbackExecutionError({ + reason: "active-turn", + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), + ), + ), + ); + } else if (input.restoreFiles === false) { + yield* admitted(() => session.rollbackThread(rollbackInput)); + } else { + return yield* new CheckpointRollbackExecutionError({ + reason: "rollback-admission-unavailable", + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + }); + } }); return CheckpointRollbackServiceV2.of({ diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 11234545e99bc..23e11a4eb7619 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -437,7 +437,11 @@ export const layerExecutor: Layer.Layer< CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure) ? failure : undefined; - if (willRetry && postRestoreFailure === undefined) return Effect.void; + // A later retry must not restore after the active turn has made new edits. + const admissionRejected = + CheckpointRollbackService.isCheckpointRollbackAdmissionRejection(failure); + if (willRetry && postRestoreFailure === undefined && !admissionRejected) + return Effect.void; return threads .dispatch({ type: "checkpoint.rollback.fail", @@ -783,6 +787,10 @@ export const layerWithOptions = ( effect.request.type === "provider-thread.rollback" && Option.isSome(failure) && CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure.value.cause); + const admissionRejected = + effect.request.type === "provider-thread.rollback" && + Option.isSome(failure) && + CheckpointRollbackService.isCheckpointRollbackAdmissionRejection(failure.value.cause); yield* Effect.logWarning("Orchestration effect execution failed", { effectId: effect.id, effectType: effect.request.type, @@ -794,7 +802,7 @@ export const layerWithOptions = ( ? yield* outbox .succeed({ effectId: effect.id, workerId }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) - : postRestoreFailure || effect.attemptCount >= maxAttempts + : postRestoreFailure || admissionRejected || effect.attemptCount >= maxAttempts ? yield* outbox .fail({ effectId: effect.id, workerId, error }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index e27f32ef84420..6148b1054f93d 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -10816,6 +10816,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } } + if (hasLiveRun(projection)) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Cannot restore a checkpoint while a turn is active.", + }); + } + const now = yield* DateTime.now; yield* emit( events, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index ed064325bfe16..23ca307a9e8cb 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -613,6 +613,7 @@ export interface ProjectionStoreV2Shape { readonly canStartQueuedRun: ( threadId: ThreadId, ) => Effect.Effect; + readonly hasLiveRun: (threadId: ThreadId) => Effect.Effect; readonly getRecoveryThreadIds: ( kind: ProjectionRecoveryKind, ) => Effect.Effect, ProjectionStoreV2Error>; @@ -5284,6 +5285,13 @@ export const layer: Layer.Layer = ) .pipe(Effect.mapError(controlReadError(threadId))); + const hasLiveRun: ProjectionStoreV2Shape["hasLiveRun"] = (threadId) => + sql<{ readonly found: number }>`SELECT 1 AS found FROM orchestration_v2_projection_runs + WHERE thread_id = ${threadId} AND status IN ('preparing', 'starting', 'running') LIMIT 1`.pipe( + Effect.map((rows) => rows.length > 0), + Effect.mapError(controlReadError(threadId)), + ); + const getMessageCount: ProjectionStoreV2Shape["getMessageCount"] = (threadId) => sql<{ count: number; @@ -6793,6 +6801,7 @@ export const layer: Layer.Layer = getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext), getRunMessage: readOnReader(getRunMessage), canStartQueuedRun: readOnReader(canStartQueuedRun), + hasLiveRun: readOnReader(hasLiveRun), getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs), hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest), isNodeStored: readOnReader(isNodeStored), @@ -7500,6 +7509,15 @@ export const layerMemory: Layer.Layer = Layer.effect( const run = projection.runs.find((candidate) => candidate.id === runId); return projection.messages.find((message) => message.id === run?.userMessageId); }), + hasLiveRun: Effect.fn("ProjectionStore.hasLiveRun")(function* (threadId) { + const projection = (yield* Ref.get(replayState)).projections.get(threadId); + if (projection === undefined) + return yield* new ProjectionStoreThreadNotFoundError({ threadId }); + return projection.runs.some( + (run) => + run.status === "preparing" || run.status === "starting" || run.status === "running", + ); + }), canStartQueuedRun: Effect.fn("ProjectionStore.canStartQueuedRun")(function* (threadId) { const projection = (yield* Ref.get(replayState)).projections.get(threadId); if (projection === undefined) diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 4837c4a2ac1c6..57e1b61e70108 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -260,6 +260,7 @@ it.effect( getRunMessage: () => Effect.die("not used"), isNodeStored: () => Effect.die("not used"), canStartQueuedRun: () => Effect.die("not used"), + hasLiveRun: () => Effect.die("not used"), getRecoveryThreadIds: () => Effect.die("unused getRecoveryThreadIds"), getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"), getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"), diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index d6e105a61db4f..be421d65ce5ea 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -670,12 +670,22 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { attachments: [], dispatchMode: { type: "start_immediately" }, }); - const scope = (yield* orchestrator.getThreadProjection(threadId)).checkpointScopes[0]!; + const seededProjection = yield* orchestrator.getThreadProjection(threadId); + const scope = seededProjection.checkpointScopes[0]!; + const initialRun = seededProjection.runs[0]!; const now = yield* DateTime.now; const checkpointId = CheckpointId.make(`${name}-checkpoint`); yield* eventSink.write({ commandId: CommandId.make(`${name}-seed`), events: [ + { + id: EventId.make(`${name}-run-completed`), + type: "run.updated", + threadId, + runId: initialRun.id, + occurredAt: now, + payload: { ...initialRun, status: "completed", completedAt: now }, + }, { id: EventId.make(`${name}-checkpoint-event`), type: "checkpoint.captured", @@ -747,6 +757,187 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); + it.effect.each(["starting", "running"] as const)( + "does not restore files while a turn is %s", + (status) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-active-rollback-" }); + const file = path.join(cwd, "workspace.txt"); + yield* fs.writeFileString(file, "in-flight changes"); + const { threadId, rollback } = yield* seedFailingRollbackThread("active-rollback", cwd); + const before = yield* orchestrator.getThreadProjection(threadId); + assert.equal(before.runs[0]?.status, "completed"); + const restoreSpy = vi + .spyOn(checkpointStore, "restoreCheckpoint") + .mockImplementation(() => + fs.writeFileString(file, "checkpoint contents").pipe(Effect.as(true), Effect.orDie), + ); + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + withRollbackAdmission: ( + _input: unknown, + use: (rollback: () => Effect.Effect) => Effect.Effect, + ) => use(() => Effect.die("rollback is not needed for an empty history")), + rollbackThread: () => + Effect.fail( + new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: before.providerThreads[0]!.id, + cause: "active turn", + }), + ), + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + restoreSpy.mockRestore(); + sessionSpy.mockRestore(); + }), + ); + const commandId = CommandId.make("active-rollback-request"); + yield* rollback(commandId, true); + const run = before.runs[0]!; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make("active-rollback-running"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { + ...run, + status, + completedAt: null, + startedAt: status === "running" ? now : null, + }, + }, + ], + }); + const rejectedCommand = CommandId.make("active-rollback-rejected-request"); + const rejected = yield* rollback(rejectedCommand, true).pipe(Effect.flip); + assert.include(String(rejected.cause), "while a turn is active"); + assert.lengthOf(yield* outbox.listByCommandId(rejectedCommand), 0); + yield* worker.drain(); + assert.equal(yield* fs.readFileString(file), "in-flight changes"); + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "failed"); + assert.equal(effect?.attemptCount, 1); + yield* TestClock.adjust("30 seconds"); + assert.equal(yield* worker.drain(), 0); + assert.equal(yield* fs.readFileString(file), "in-flight changes"); + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), + ); + + it.effect.each([ + { restoreFiles: true, admission: false }, + { restoreFiles: false, admission: false }, + { restoreFiles: true, admission: true }, + { restoreFiles: false, admission: true }, + { restoreFiles: true, admission: true, nativeActive: true }, + ])( + "requires provider admission for file restore: %s", + ({ restoreFiles, admission, nativeActive = false }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-no-rollback-admission-" }); + const file = path.join(cwd, "workspace.txt"); + yield* fs.writeFileString(file, "current"); + const { threadId, rollback } = yield* seedFailingRollbackThread( + "no-rollback-admission", + cwd, + ); + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make("no-rollback-admission-completed"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", completedAt: now }, + }, + ], + }); + let providerCalls = 0; + const providerThread = before.providerThreads[0]!; + const rollbackThread = () => + Effect.sync(() => { + providerCalls++; + return { providerThread }; + }); + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + ...(admission + ? { + canRestoreRollbackFiles: Effect.succeed(false), + withRollbackAdmission: ( + _input: unknown, + use: (rollback: typeof rollbackThread) => Effect.Effect, + ) => + nativeActive + ? Effect.fail( + new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: "native turn is still active", + }), + ) + : use(rollbackThread), + } + : {}), + rollbackThread, + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), + ); + const restoreSpy = vi + .spyOn(checkpointStore, "restoreCheckpoint") + .mockImplementation(() => + fs.writeFileString(file, "checkpoint").pipe(Effect.as(true), Effect.orDie), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + sessionSpy.mockRestore(); + restoreSpy.mockRestore(); + }), + ); + const commandId = CommandId.make("no-rollback-admission-request"); + yield* rollback(commandId, restoreFiles); + yield* worker.drain(); + assert.equal(yield* fs.readFileString(file), "current"); + assert.equal(restoreSpy.mock.calls.length, 0); + assert.equal(providerCalls, restoreFiles ? 0 : 1); + if (restoreFiles) { + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "failed"); + assert.equal(effect?.attemptCount, 1); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.message, + nativeActive + ? "Cannot restore a checkpoint while a turn is active." + : "This provider cannot safely restore files during conversation rollback. Rewind the conversation without restoring files instead.", + ); + } + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), + ); + it.effect.each(["partial-restore", "refresh-defect", "provider-only", "restore-fails"] as const)( "settles checkpoint rollback safely for %s", (outcome) => @@ -816,23 +1007,28 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { if (outcome === "refresh-defect") { refreshSpy.mockImplementation(() => Effect.die(new Error("index refresh defect"))); } + const rejectRollback = () => + Effect.gen(function* () { + rejected += 1; + assert.equal( + yield* fs.readFileString(file).pipe(Effect.orDie), + outcome === "provider-only" + ? "current uncheckpointed contents" + : "checkpoint contents", + ); + return yield* new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: providerThread.id, + cause: new Error("legacy history cannot be reverted"), + }); + }); const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( Effect.succeed({ - rollbackThread: () => - Effect.gen(function* () { - rejected += 1; - assert.equal( - yield* fs.readFileString(file).pipe(Effect.orDie), - outcome === "provider-only" - ? "current uncheckpointed contents" - : "checkpoint contents", - ); - return yield* new ProviderAdapter.ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: providerThread.id, - cause: new Error("legacy history cannot be reverted"), - }); - }), + withRollbackAdmission: ( + _input: unknown, + use: (rollback: typeof rejectRollback) => Effect.Effect, + ) => use(rejectRollback), + rollbackThread: rejectRollback, } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), ); yield* Effect.addFinalizer(() => @@ -950,6 +1146,19 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { const projection = yield* orchestrator.getThreadProjection(threadId); const scope = projection.checkpointScopes[0]!; const now = yield* DateTime.now; + const readinessRun = projection.runs[0]!; + yield* eventSink.write({ + events: [ + { + id: EventId.make("runtime-rollback-readiness-completed"), + type: "run.updated", + threadId, + runId: readinessRun.id, + occurredAt: now, + payload: { ...readinessRun, status: "completed", completedAt: now }, + }, + ], + }); for (const status of ["missing", "error", "stale", "ready"] as const) { const checkpointId = CheckpointId.make("runtime-rollback-checkpoint"); diff --git a/docs/user/composer.md b/docs/user/composer.md index 82cc005d061f1..00cfcf79b7aab 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -155,7 +155,9 @@ File restore is only offered for threads running in a worktree, and it is refused when another thread or agent session also uses that directory, a folder inside it, or a folder that contains it, since restoring would erase their changes. A thread that works in the project directory -rewinds the conversation only. The selected prompt and its attachments return to the composer for editing and +rewinds the conversation only. File restore is unavailable for Pi, OpenCode, and some ACP agents +that can resume work independently; choose conversation-only rewind for those providers. +Stop an active turn before rewinding. The selected prompt and its attachments return to the composer for editing and resending. Any unsent draft stays above the restored prompt. This removes the selected message and later conversation from the active thread diff --git a/docs/user/providers-acp.md b/docs/user/providers-acp.md index cf7872d8c96c2..94a98754a3981 100644 --- a/docs/user/providers-acp.md +++ b/docs/user/providers-acp.md @@ -126,7 +126,8 @@ remain agent-specific. ## Checkpoints -Checkpoint rollback restores your files as usual. ACP agents cannot rewind their own conversation, +Checkpoint rollback can restore files when the agent cannot resume work independently. For agents +that can resume on their own, use conversation-only rollback. ACP agents cannot rewind their own conversation, so the next turn after a rollback starts a fresh agent session that no longer remembers the conversation from before the checkpoint. diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 6fa99a058cae6..77951d0469e3e 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -7728,6 +7728,97 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }), ); + const rollbackUnlocked = Effect.fn("AcpAdapterV2.rollbackThread.unlocked")( + function* (rollbackInput: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) { + const currentTurn = yield* Ref.get(activeTurn); + if (currentTurn !== null) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: `Cannot roll back ACP provider thread ${rollbackInput.providerThread.id} while turn ${currentTurn.providerTurnId} is active`, + }); + } + yield* awaitRuntimeTeardown(); + itemIdentityVersion = 2; + prepareTerminalEnvironment( + yield* readMcpContext(rollbackInput.providerThread.appThreadId), + ); + const replacement = yield* startReplacementAcpRuntime( + rollbackInput.providerThread.appThreadId, + (candidate) => + Effect.gen(function* () { + rememberTerminalEnvironment( + candidate.sessionId, + yield* readMcpContext(rollbackInput.providerThread.appThreadId), + ); + yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(activeSessionId, candidate.sessionId); + yield* setActiveSessionSetup(candidate); + yield* Ref.set(activeSelection, null); + yield* Ref.set(activeInteractionMode, null); + yield* Ref.set(promptInstructionStates, new Map()); + yield* Ref.set(compactionMarkers, new Map()); + yield* Ref.set(providerTurns, new Map()); + yield* Ref.set(snapshot, { + order: [], + messages: new Map(), + loadingRole: null, + loadingMessageId: null, + loadingIndex: 0, + }); + yield* continuationPermit.withPermit( + Effect.gen(function* () { + yield* Ref.update(continuationGeneration, (value) => value + 1); + yield* Ref.set(stoppedRunQuarantine, false); + yield* Ref.set(wakeBuffer, []); + yield* Ref.set(wakeReports, noWakeReports); + yield* Ref.set(continuationRequested, false); + yield* Ref.set(runningBackgroundTaskIds, new Set()); + yield* Ref.set(endedBackgroundTaskIds, new Set()); + yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); + yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); + yield* Ref.set(carryoverSubagents, null); + yield* Ref.set(suppressPostSettleMonitorPrompt, false); + yield* Ref.set(lastTurnRoute, null); + }), + ); + }), + ).pipe(Effect.retry({ times: 1 })); + const now = yield* DateTime.now; + return { + providerThread: { + ...rollbackInput.providerThread, + nativeThreadRef: { + driver, + nativeId: replacement.sessionId, + strength: "strong" as const, + }, + nativeConversationHeadRef: null, + nativeMetadata: { + ...rollbackInput.providerThread.nativeMetadata, + itemIdentityVersion: 2 as const, + }, + status: "idle" as const, + updatedAt: now, + }, + providerTurns: [], + messages: [], + runtimeRequests: [], + }; + }, + (effect, rollbackInput) => + effect.pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: rollbackInput.providerThread.id, + checkpointId: rollbackInput.target.checkpointId, + cause, + }), + ), + ), + ); + const sessionRuntime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { instanceId: options.instanceId, driver, @@ -8329,97 +8420,27 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ), ), ), + canRestoreRollbackFiles: Effect.sync( + () => !postSettleContinuationEnabled && !followsAgentWorkState, + ), + withRollbackAdmission: (rollbackInput, use) => + runtimeTransitionPermit.withPermit( + Effect.gen(function* () { + const currentTurn = yield* Ref.get(activeTurn); + if ( + currentTurn !== null || + (yield* sessionRuntime.hasPendingBackgroundWork ?? Effect.succeed(false)) + ) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: `Cannot roll back ACP provider thread ${rollbackInput.providerThread.id} while provider work is active`, + }); + } + return yield* use(() => rollbackUnlocked(rollbackInput)); + }), + ), rollbackThread: (rollbackInput) => - runtimeTransitionPermit - .withPermit( - Effect.gen(function* () { - const currentTurn = yield* Ref.get(activeTurn); - if (currentTurn !== null) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: `Cannot roll back ACP provider thread ${rollbackInput.providerThread.id} while turn ${currentTurn.providerTurnId} is active`, - }); - } - yield* awaitRuntimeTeardown(); - itemIdentityVersion = 2; - prepareTerminalEnvironment( - yield* readMcpContext(rollbackInput.providerThread.appThreadId), - ); - const replacement = yield* startReplacementAcpRuntime( - rollbackInput.providerThread.appThreadId, - (candidate) => - Effect.gen(function* () { - rememberTerminalEnvironment( - candidate.sessionId, - yield* readMcpContext(rollbackInput.providerThread.appThreadId), - ); - yield* Ref.set(runtimeRestartRequired, false); - yield* Ref.set(activeSessionId, candidate.sessionId); - yield* setActiveSessionSetup(candidate); - yield* Ref.set(activeSelection, null); - yield* Ref.set(activeInteractionMode, null); - yield* Ref.set(promptInstructionStates, new Map()); - yield* Ref.set(compactionMarkers, new Map()); - yield* Ref.set(providerTurns, new Map()); - yield* Ref.set(snapshot, { - order: [], - messages: new Map(), - loadingRole: null, - loadingMessageId: null, - loadingIndex: 0, - }); - yield* continuationPermit.withPermit( - Effect.gen(function* () { - yield* Ref.update(continuationGeneration, (value) => value + 1); - yield* Ref.set(stoppedRunQuarantine, false); - yield* Ref.set(wakeBuffer, []); - yield* Ref.set(wakeReports, noWakeReports); - yield* Ref.set(continuationRequested, false); - yield* Ref.set(runningBackgroundTaskIds, new Set()); - yield* Ref.set(endedBackgroundTaskIds, new Set()); - yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); - yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); - yield* Ref.set(carryoverSubagents, null); - yield* Ref.set(suppressPostSettleMonitorPrompt, false); - yield* Ref.set(lastTurnRoute, null); - }), - ); - }), - ).pipe(Effect.retry({ times: 1 })); - const now = yield* DateTime.now; - return { - providerThread: { - ...rollbackInput.providerThread, - nativeThreadRef: { - driver, - nativeId: replacement.sessionId, - strength: "strong" as const, - }, - nativeConversationHeadRef: null, - nativeMetadata: { - ...rollbackInput.providerThread.nativeMetadata, - itemIdentityVersion: 2 as const, - }, - status: "idle" as const, - updatedAt: now, - }, - providerTurns: [], - messages: [], - runtimeRequests: [], - }; - }), - ) - .pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: rollbackInput.providerThread.id, - checkpointId: rollbackInput.target.checkpointId, - cause, - }), - ), - ), + runtimeTransitionPermit.withPermit(rollbackUnlocked(rollbackInput)), forkThread: Effect.fn("AcpAdapterV2.forkThread")( function* (forkInput) { return yield* runtimeTransitionPermit.withPermit( diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 7c77b7bb0ac6f..97a23dcd32ac0 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -624,6 +624,15 @@ export interface ProviderAdapterV2SessionRuntime { readonly rollbackThread: ( input: ProviderAdapterV2RollbackThreadInput, ) => Effect.Effect; + /** A provider may refuse file restore while still allowing conversation-only rollback. */ + readonly canRestoreRollbackFiles?: Effect.Effect; + /** Reject active native work, then hold admission through `use` and its unlocked rollback. Absent means file restore is unsafe. */ + readonly withRollbackAdmission?: ( + input: ProviderAdapterV2RollbackThreadInput, + use: ( + rollback: () => Effect.Effect, + ) => Effect.Effect, + ) => Effect.Effect; readonly forkThread: ( input: ProviderAdapterV2ForkThreadInput, ) => Effect.Effect; From 725739427daad92efbf04d7d8bbb71d4562bc173 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Tue, 27 Oct 2026 14:35:41 +0100 Subject: [PATCH 09/12] fix(server): stop retrying incomplete checkpoint finalization --- .../Adapters/AcpAdapterV2.test.ts | 21 +++ .../CheckpointRollbackService.ts | 61 +++++++- .../src/orchestration-v2/runtimeLayer.test.ts | 147 ++++++++++++++++++ packages/provider-acp/src/server/adapter.ts | 11 ++ 4 files changed, 236 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 2fa05bc72d7f2..7192948b4a646 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -7112,6 +7112,27 @@ describe("AcpAdapterV2", () => { if (Exit.isSuccess(interruptExit)) assert.fail("missing hard teardown must fail interrupt"); assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + let admitted = false; + const admissionExit = yield* runtime.withRollbackAdmission!( + { + providerThread, + target: { + type: "thread_start", + checkpointId: CheckpointId.make("poisoned-rollback"), + appRunOrdinal: 0, + }, + providerThreadTurns: [], + }, + () => + Effect.sync(() => { + admitted = true; + }), + ).pipe(Effect.exit); + if (Exit.isSuccess(admissionExit)) + assert.fail("poisoned session must reject rollback admission"); + assert.include(Cause.pretty(admissionExit.cause), "session is poisoned"); + assert.isFalse(admitted); + const startExit = yield* runtime .startTurn( makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 62c958d3a0f1e..0baaba3dff757 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -106,8 +106,31 @@ export class CheckpointRollbackIndexRefreshError extends Schema.TaggedError()( + "CheckpointRollbackFinalizationError", + { + threadId: ThreadId, + providerThreadId: ProviderThreadId, + checkpointId: CheckpointId, + stage: Schema.Literals(["stale-refs", "event-commit"]), + providerRolledBack: Schema.Boolean, + cause: Schema.Defect(), + }, +) { + override get message(): string { + const prefix = this.providerRolledBack + ? "Files were restored and the provider conversation was rolled back" + : "Files were restored"; + return `${prefix}, but T3 could not finish finalizing checkpoint history. Checkpoint references and conversation history may not match. Check the server logs before retrying.`; + } +} + export const isCheckpointRollbackPostRestoreError = Schema.is( - Schema.Union([CheckpointRollbackPartialRestoreError, CheckpointRollbackIndexRefreshError]), + Schema.Union([ + CheckpointRollbackPartialRestoreError, + CheckpointRollbackIndexRefreshError, + CheckpointRollbackFinalizationError, + ]), ); const isCheckpointRollbackExecutionError = Schema.is(CheckpointRollbackExecutionError); @@ -122,6 +145,7 @@ const isCheckpointRollbackError = Schema.is( CheckpointRollbackExecutionError, CheckpointRollbackPartialRestoreError, CheckpointRollbackIndexRefreshError, + CheckpointRollbackFinalizationError, ]), ); @@ -136,7 +160,8 @@ export function rollbackFailureClientMessage( error: | CheckpointRollbackExecutionError | CheckpointRollbackPartialRestoreError - | CheckpointRollbackIndexRefreshError, + | CheckpointRollbackIndexRefreshError + | CheckpointRollbackFinalizationError, ): string { if (isCheckpointRollbackPostRestoreError(error)) return error.message; if (error.reason === "rollback-admission-unavailable" || error.reason === "active-turn") { @@ -160,6 +185,7 @@ export interface CheckpointRollbackServiceV2Shape { | CheckpointRollbackExecutionError | CheckpointRollbackPartialRestoreError | CheckpointRollbackIndexRefreshError + | CheckpointRollbackFinalizationError >; } @@ -394,11 +420,24 @@ export const layer: Layer.Layer< ), ); if (staleCheckpoints.length > 0) { - yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }); + yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }).pipe( + Effect.mapError((cause) => + input.restoreFiles === false + ? cause + : new CheckpointRollbackFinalizationError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + stage: "stale-refs", + providerRolledBack: runsToRollback.length > 0, + cause, + }), + ), + ); } // Lock against full-row thread writes that could restore the old native session after rollback. - yield* threadCommands.withLock( + const commitEvents = threadCommands.withLock( input.threadId, Effect.gen(function* () { const now = yield* DateTime.now; @@ -473,6 +512,20 @@ export const layer: Layer.Layer< yield* eventSink.write({ events }); }), ); + yield* commitEvents.pipe( + Effect.mapError((cause) => + input.restoreFiles === false + ? cause + : new CheckpointRollbackFinalizationError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + stage: "event-commit", + providerRolledBack: runsToRollback.length > 0, + cause, + }), + ), + ); }); if (session.withRollbackAdmission !== undefined) { yield* session.withRollbackAdmission(rollbackInput, admitted).pipe( diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index be421d65ce5ea..2ca579b5a657d 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -27,6 +27,7 @@ import { ProviderSessionId, ProviderThreadId, ProviderTurnId, + VcsProcessSpawnError, RunId, ScheduledTaskId, ThreadId, @@ -1112,6 +1113,152 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), ); + it.effect.each([ + { stage: "stale-refs", restoreFiles: true }, + { stage: "event-commit", restoreFiles: true }, + { stage: "stale-refs", restoreFiles: false }, + { stage: "event-commit", restoreFiles: false }, + ] as const)("handles rollback finalization failure %s", ({ stage, restoreFiles }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const checkpoints = yield* CheckpointStore.CheckpointStore; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-finalization-" }); + const file = path.join(cwd, "workspace.txt"); + yield* fs.writeFileString(file, "before restore"); + const { threadId, rollback } = yield* seedFailingRollbackThread( + `rollback-finalization-${stage}`, + cwd, + ); + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const providerThread = before.providerThreads[0]!; + const now = yield* DateTime.now; + const laterCheckpoint = { + ...before.checkpoints[0]!, + id: CheckpointId.make(`finalization-later-${stage}`), + ref: CheckpointRef.make(`refs/t3/finalization-later-${stage}`), + appRunOrdinal: run.ordinal, + ordinalWithinScope: 1, + }; + yield* sink.write({ + events: [ + { + id: EventId.make(`finalization-later-event-${stage}`), + type: "checkpoint.captured", + threadId, + occurredAt: now, + payload: laterCheckpoint, + }, + ], + }); + let restores = 0; + let providerCalls = 0; + let refsRemain = true; + let commitCalls = 0; + const providerRollback = () => + Effect.sync(() => { + providerCalls++; + return { providerThread }; + }); + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + rollbackThread: providerRollback, + withRollbackAdmission: ( + _input: unknown, + use: (rollback: typeof providerRollback) => Effect.Effect, + ) => use(providerRollback), + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), + ); + const restoreSpy = vi.spyOn(checkpoints, "restoreCheckpoint").mockImplementation(() => + Effect.gen(function* () { + restores++; + yield* fs.writeFileString(file, "restored"); + return true; + }).pipe(Effect.orDie), + ); + const deleteSpy = vi.spyOn(checkpoints, "deleteCheckpointRefs").mockImplementation(() => + stage === "stale-refs" + ? Effect.fail( + new VcsProcessSpawnError({ + operation: "delete refs", + command: "git", + cwd, + cause: "failure", + }), + ) + : Effect.sync(() => { + refsRemain = false; + }), + ); + const originalWrite = sink.write; + const writeSpy = vi.spyOn(sink, "write").mockImplementation((input) => { + if (input.events.some((event) => event.type === "provider-thread.updated")) { + commitCalls++; + if (stage === "event-commit") + return Effect.fail( + new EventSink.EventSinkWriteError({ + eventCount: input.events.length, + cause: "failure", + }), + ); + } + return originalWrite(input); + }); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + sessionSpy.mockRestore(); + restoreSpy.mockRestore(); + deleteSpy.mockRestore(); + writeSpy.mockRestore(); + }), + ); + const commandId = CommandId.make(`rollback-finalization-request-${stage}`); + yield* rollback(commandId, restoreFiles); + yield* worker.drain(); + const [first] = yield* outbox.listByCommandId(commandId); + assert.equal(first?.status, restoreFiles ? "failed" : "pending"); + if (!restoreFiles) { + for (let attempt = 1; attempt < 5; attempt++) { + yield* TestClock.adjust("30 seconds"); + yield* worker.drain(); + } + } + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "failed"); + assert.equal(effect?.attemptCount, restoreFiles ? 1 : 5); + assert.equal(restores, restoreFiles ? 1 : 0); + assert.equal(providerCalls, restoreFiles ? 1 : 5); + assert.equal(refsRemain, stage === "stale-refs"); + assert.equal(commitCalls, stage === "event-commit" ? (restoreFiles ? 1 : 5) : 0); + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal(after.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); + assert.equal( + after.checkpoints.find((candidate) => candidate.id === laterCheckpoint.id)?.status, + "ready", + ); + if (restoreFiles) { + assert.equal( + after.thread.rollbackFailure?.message, + "Files were restored and the provider conversation was rolled back, but T3 could not finish finalizing checkpoint history. Checkpoint references and conversation history may not match. Check the server logs before retrying.", + ); + } else { + assert.equal(after.thread.rollbackFailure?.message, ROLLBACK_FAILED_MESSAGE); + } + yield* fs.writeFileString(file, "new edits"); + yield* TestClock.adjust("30 seconds"); + assert.equal(yield* worker.drain(), 0); + assert.equal(yield* fs.readFileString(file), "new edits"); + assert.equal(restores, restoreFiles ? 1 : 0); + assert.equal(providerCalls, restoreFiles ? 1 : 5); + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), + ); + it.effect("rejects non-ready rollback targets before persisting events or effects", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 77951d0469e3e..deacc67573c58 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -8426,6 +8426,17 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( withRollbackAdmission: (rollbackInput, use) => runtimeTransitionPermit.withPermit( Effect.gen(function* () { + yield* awaitRuntimeTeardown().pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: rollbackInput.providerThread.id, + checkpointId: rollbackInput.target.checkpointId, + cause, + }), + ), + ); const currentTurn = yield* Ref.get(activeTurn); if ( currentTurn !== null || From b756b4c483be99d54a1839faa887af0013085548 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Tue, 27 Oct 2026 19:05:51 +0100 Subject: [PATCH 10/12] fix(server): preserve provider checkpoint restore support --- .../Adapters/AcpAdapterV2.test.ts | 25 +++++++++++----- .../CheckpointRollbackService.ts | 29 ++----------------- .../src/orchestration-v2/runtimeLayer.test.ts | 21 ++++++++------ docs/user/composer.md | 4 +-- docs/user/providers-acp.md | 3 +- packages/provider-acp/src/server/adapter.ts | 3 -- .../src/server/ProviderAdapter.ts | 4 +-- 7 files changed, 35 insertions(+), 54 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 7192948b4a646..bcfd133ac652e 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -3286,15 +3286,24 @@ describe("AcpAdapterV2", () => { runtimePolicy, }); - yield* runtime.rollbackThread({ - providerThread, - providerThreadTurns: [], - target: { - type: "thread_start", - checkpointId: CheckpointId.make("checkpoint-acp-rollback-retry-generation"), - appRunOrdinal: 0, + assert.isFalse(yield* hasPendingBackgroundWork); + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-acp-rollback-admission-" }); + const file = path.join(directory, "workspace.txt"); + yield* fs.writeFileString(file, "current"); + yield* runtime.withRollbackAdmission!( + { + providerThread, + providerThreadTurns: [], + target: { + type: "thread_start", + checkpointId: CheckpointId.make("checkpoint-acp-rollback-retry-generation"), + appRunOrdinal: 0, + }, }, - }); + (rollback) => fs.writeFileString(file, "restored").pipe(Effect.andThen(rollback())), + ); + assert.equal(yield* fs.readFileString(file), "restored"); assert.equal(runtimeOrdinalSeen, 3); assert.deepEqual( diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 0baaba3dff757..a2e2445ef2046 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -47,7 +47,6 @@ export class CheckpointRollbackExecutionError extends Schema.TaggedError - isCheckpointRollbackExecutionError(error) && - (error.reason === "active-turn" || error.reason === "rollback-admission-unavailable"); + isCheckpointRollbackExecutionError(error) && error.reason === "active-turn"; const isCheckpointRollbackError = Schema.is( Schema.Union([ @@ -164,7 +160,7 @@ export function rollbackFailureClientMessage( | CheckpointRollbackFinalizationError, ): string { if (isCheckpointRollbackPostRestoreError(error)) return error.message; - if (error.reason === "rollback-admission-unavailable" || error.reason === "active-turn") { + if (error.reason === "active-turn") { return error.message; } const restoreCause = isCheckpointRestoreError(error.cause) ? error.cause.cause : undefined; @@ -369,18 +365,6 @@ export const layer: Layer.Layer< rollback: () => ReturnType, ) { enteredAdmission = true; - if ( - input.restoreFiles !== false && - session.canRestoreRollbackFiles !== undefined && - !(yield* session.canRestoreRollbackFiles) - ) { - return yield* new CheckpointRollbackExecutionError({ - reason: "rollback-admission-unavailable", - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - }); - } if (yield* projections.hasLiveRun(input.threadId)) { return yield* new CheckpointRollbackExecutionError({ reason: "active-turn", @@ -543,15 +527,8 @@ export const layer: Layer.Layer< ), ), ); - } else if (input.restoreFiles === false) { - yield* admitted(() => session.rollbackThread(rollbackInput)); } else { - return yield* new CheckpointRollbackExecutionError({ - reason: "rollback-admission-unavailable", - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - }); + yield* admitted(() => session.rollbackThread(rollbackInput)); } }); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 2ca579b5a657d..849c2ed0811e9 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -846,7 +846,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { { restoreFiles: false, admission: true }, { restoreFiles: true, admission: true, nativeActive: true }, ])( - "requires provider admission for file restore: %s", + "restores with or without provider admission: %s", ({ restoreFiles, admission, nativeActive = false }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); @@ -890,7 +890,6 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { Effect.succeed({ ...(admission ? { - canRestoreRollbackFiles: Effect.succeed(false), withRollbackAdmission: ( _input: unknown, use: (rollback: typeof rollbackThread) => Effect.Effect, @@ -922,19 +921,23 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { const commandId = CommandId.make("no-rollback-admission-request"); yield* rollback(commandId, restoreFiles); yield* worker.drain(); - assert.equal(yield* fs.readFileString(file), "current"); - assert.equal(restoreSpy.mock.calls.length, 0); - assert.equal(providerCalls, restoreFiles ? 0 : 1); - if (restoreFiles) { + assert.equal( + yield* fs.readFileString(file), + restoreFiles && !nativeActive ? "checkpoint" : "current", + ); + assert.equal(restoreSpy.mock.calls.length, restoreFiles && !nativeActive ? 1 : 0); + assert.equal(providerCalls, nativeActive ? 0 : 1); + if (nativeActive) { const [effect] = yield* outbox.listByCommandId(commandId); assert.equal(effect?.status, "failed"); assert.equal(effect?.attemptCount, 1); assert.equal( (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.message, - nativeActive - ? "Cannot restore a checkpoint while a turn is active." - : "This provider cannot safely restore files during conversation rollback. Rewind the conversation without restoring files instead.", + "Cannot restore a checkpoint while a turn is active.", ); + } else { + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "succeeded"); } }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), ); diff --git a/docs/user/composer.md b/docs/user/composer.md index 00cfcf79b7aab..82cc005d061f1 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -155,9 +155,7 @@ File restore is only offered for threads running in a worktree, and it is refused when another thread or agent session also uses that directory, a folder inside it, or a folder that contains it, since restoring would erase their changes. A thread that works in the project directory -rewinds the conversation only. File restore is unavailable for Pi, OpenCode, and some ACP agents -that can resume work independently; choose conversation-only rewind for those providers. -Stop an active turn before rewinding. The selected prompt and its attachments return to the composer for editing and +rewinds the conversation only. The selected prompt and its attachments return to the composer for editing and resending. Any unsent draft stays above the restored prompt. This removes the selected message and later conversation from the active thread diff --git a/docs/user/providers-acp.md b/docs/user/providers-acp.md index 94a98754a3981..cf7872d8c96c2 100644 --- a/docs/user/providers-acp.md +++ b/docs/user/providers-acp.md @@ -126,8 +126,7 @@ remain agent-specific. ## Checkpoints -Checkpoint rollback can restore files when the agent cannot resume work independently. For agents -that can resume on their own, use conversation-only rollback. ACP agents cannot rewind their own conversation, +Checkpoint rollback restores your files as usual. ACP agents cannot rewind their own conversation, so the next turn after a rollback starts a fresh agent session that no longer remembers the conversation from before the checkpoint. diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index deacc67573c58..eb174fd067389 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -8420,9 +8420,6 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ), ), ), - canRestoreRollbackFiles: Effect.sync( - () => !postSettleContinuationEnabled && !followsAgentWorkState, - ), withRollbackAdmission: (rollbackInput, use) => runtimeTransitionPermit.withPermit( Effect.gen(function* () { diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 97a23dcd32ac0..9cb781937a5e0 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -624,9 +624,7 @@ export interface ProviderAdapterV2SessionRuntime { readonly rollbackThread: ( input: ProviderAdapterV2RollbackThreadInput, ) => Effect.Effect; - /** A provider may refuse file restore while still allowing conversation-only rollback. */ - readonly canRestoreRollbackFiles?: Effect.Effect; - /** Reject active native work, then hold admission through `use` and its unlocked rollback. Absent means file restore is unsafe. */ + /** Reject active native work, then hold admission through `use` and its unlocked rollback. */ readonly withRollbackAdmission?: ( input: ProviderAdapterV2RollbackThreadInput, use: ( From a319205eaa7bac51c80ac410e6bc3c8b65d06e48 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Tue, 27 Oct 2026 19:28:14 +0100 Subject: [PATCH 11/12] chore(server): move rollback safety follow-ups into a stack --- .../Adapters/AcpAdapterV2.test.ts | 59 +-- .../Adapters/ClaudeAdapterV2.test.ts | 171 -------- .../Adapters/ClaudeAdapterV2.ts | 38 +- .../Adapters/CodexAdapterV2.test.ts | 17 - .../Adapters/CodexAdapterV2.ts | 70 +--- .../CheckpointRestoreSafety.test.ts | 13 - .../CheckpointRollbackService.test.ts | 78 ++-- .../CheckpointRollbackService.ts | 265 ++++-------- .../src/orchestration-v2/EffectWorker.ts | 12 +- .../src/orchestration-v2/Orchestrator.ts | 8 - .../src/orchestration-v2/ProjectionStore.ts | 18 - .../ProviderTurnControlService.test.ts | 1 - .../src/orchestration-v2/runtimeLayer.test.ts | 391 +----------------- packages/provider-acp/src/server/adapter.ts | 209 ++++------ .../src/server/ProviderAdapter.ts | 7 - 15 files changed, 248 insertions(+), 1109 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index bcfd133ac652e..a919726c9a56e 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -3087,19 +3087,6 @@ describe("AcpAdapterV2", () => { if (activeRollback._tag === "Failure") { assert.include(String(activeRollback.failure.cause), "while turn"); } - const activeAdmission = yield* runtime.withRollbackAdmission!( - { - providerThread, - providerThreadTurns: [], - target: { - type: "thread_start", - checkpointId: CheckpointId.make("checkpoint-acp-active-admission"), - appRunOrdinal: 0, - }, - }, - () => Effect.die("active ACP turn must not admit restore"), - ).pipe(Effect.result); - assert.equal(activeAdmission._tag, "Failure"); const firstProviderTurnId = idAllocator.derive.providerTurn({ driver: ACP_TEST_DRIVER, nativeTurnId: acpScopedNativeId(instanceId, `${firstSessionId}:turn:1`), @@ -3286,24 +3273,15 @@ describe("AcpAdapterV2", () => { runtimePolicy, }); - assert.isFalse(yield* hasPendingBackgroundWork); - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-acp-rollback-admission-" }); - const file = path.join(directory, "workspace.txt"); - yield* fs.writeFileString(file, "current"); - yield* runtime.withRollbackAdmission!( - { - providerThread, - providerThreadTurns: [], - target: { - type: "thread_start", - checkpointId: CheckpointId.make("checkpoint-acp-rollback-retry-generation"), - appRunOrdinal: 0, - }, + yield* runtime.rollbackThread({ + providerThread, + providerThreadTurns: [], + target: { + type: "thread_start", + checkpointId: CheckpointId.make("checkpoint-acp-rollback-retry-generation"), + appRunOrdinal: 0, }, - (rollback) => fs.writeFileString(file, "restored").pipe(Effect.andThen(rollback())), - ); - assert.equal(yield* fs.readFileString(file), "restored"); + }); assert.equal(runtimeOrdinalSeen, 3); assert.deepEqual( @@ -7121,27 +7099,6 @@ describe("AcpAdapterV2", () => { if (Exit.isSuccess(interruptExit)) assert.fail("missing hard teardown must fail interrupt"); assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); - let admitted = false; - const admissionExit = yield* runtime.withRollbackAdmission!( - { - providerThread, - target: { - type: "thread_start", - checkpointId: CheckpointId.make("poisoned-rollback"), - appRunOrdinal: 0, - }, - providerThreadTurns: [], - }, - () => - Effect.sync(() => { - admitted = true; - }), - ).pipe(Effect.exit); - if (Exit.isSuccess(admissionExit)) - assert.fail("poisoned session must reject rollback admission"); - assert.include(Cause.pretty(admissionExit.cause), "session is poisoned"); - assert.isFalse(admitted); - const startExit = yield* runtime .startTurn( makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 562e379a4b47f..8de0f78af88c0 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2618,7 +2618,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarnessWithOptions = (options?: { - readonly afterTerminalEvent?: Effect.Effect; readonly close?: (sdkMessages: Queue.Queue) => Effect.Effect; readonly interrupt?: Effect.Effect; readonly setPermissionMode?: ClaudeAdapterV2.ClaudeAgentSdkQuerySession["setPermissionMode"]; @@ -2689,9 +2688,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { >(); let openedOptions: ClaudeAdapterV2.ClaudeAgentSdkQueryOptions | undefined; const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ - ...(options?.afterTerminalEvent === undefined - ? {} - : { testHooks: { afterTerminalEvent: options.afterTerminalEvent } }), instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, settings: options?.settings ?? DEFAULT_CLAUDE_SETTINGS, environment: options?.environment ?? {}, @@ -4928,173 +4924,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { close: (sdkMessages) => Queue.shutdown(sdkMessages), }); - it.effect("rejects rollback while a Claude provider turn is active", () => - Effect.gen(function* () { - const harness = yield* makeRollbackHarness; - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("active-rollback-proof"), - text: "Keep the turn active.", - attachments: [], - }), - ); - const error = yield* harness.runtime - .rollbackThread({ - providerThread: harness.providerThread, - target: { - type: "thread_start", - checkpointId: CheckpointId.make("active-rollback-checkpoint"), - appRunOrdinal: 0, - }, - providerThreadTurns: [], - }) - .pipe(Effect.flip); - assert.equal(error._tag, "ProviderAdapterRollbackThreadError"); - if (error._tag !== "ProviderAdapterRollbackThreadError") return; - assert.instanceOf(error.cause, ProviderAdapter.ProviderAdapterProtocolError); - assert.include(String(error.cause), "while provider turn"); - const admissionError = yield* harness.runtime.withRollbackAdmission!( - { - providerThread: harness.providerThread, - target: { - type: "thread_start", - checkpointId: CheckpointId.make("active-admission-checkpoint"), - appRunOrdinal: 0, - }, - providerThreadTurns: [], - }, - () => Effect.die("active turns must not enter rollback admission"), - ).pipe(Effect.flip); - assert.equal(admissionError._tag, "ProviderAdapterProtocolError"); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), - ); - - it.effect("keeps admission closed after terminal until Claude clears the active turn", () => - Effect.gen(function* () { - const terminalEmitted = yield* Deferred.make(); - const releaseTerminal = yield* Deferred.make(); - const harness = yield* makeWakeHarnessWithOptions({ - close: (messages) => Queue.shutdown(messages), - afterTerminalEvent: Deferred.succeed(terminalEmitted, undefined).pipe( - Effect.andThen(Deferred.await(releaseTerminal)), - ), - }); - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("terminal-admission-turn"), - text: "Finish this turn.", - attachments: [], - }), - ); - yield* Queue.offer( - harness.sdkMessages, - makeResultFrame({ uuid: "terminal-admission-result", result: "Done." }), - ); - yield* Deferred.await(terminalEmitted); - assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "completed"); - const rejected = yield* harness.runtime.withRollbackAdmission!( - { - providerThread: harness.providerThread, - target: { - type: "thread_start", - checkpointId: CheckpointId.make("terminal-admission"), - appRunOrdinal: 0, - }, - providerThreadTurns: [], - }, - () => Effect.die("terminal is not native cleanup"), - ).pipe(Effect.flip); - assert.equal(rejected._tag, "ProviderAdapterProtocolError"); - yield* Deferred.succeed(releaseTerminal, undefined); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), - ); - - it.effect.each(["complete", "interrupt"] as const)( - "waits for rollback admission to %s before starting Claude", - (outcome) => - Effect.gen(function* () { - const harness = yield* makeRollbackHarness; - const entered = yield* Deferred.make(); - const release = yield* Deferred.make(); - const admitted = yield* harness.runtime.withRollbackAdmission!( - { - providerThread: harness.providerThread, - target: { - type: "thread_start", - checkpointId: CheckpointId.make("exclusive-admission"), - appRunOrdinal: 0, - }, - providerThreadTurns: [], - }, - () => Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), - ).pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(entered); - const starting = yield* harness.runtime - .startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("exclusive-admission-turn"), - text: "Start after the rollback.", - attachments: [], - }), - ) - .pipe(Effect.forkChild({ startImmediately: true })); - assert.isUndefined(harness.getOpenedOptions()); - if (outcome === "interrupt") { - yield* Fiber.interrupt(admitted); - } else { - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(admitted); - } - yield* Fiber.join(starting); - assert.isDefined(harness.getOpenedOptions()); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), - ); - - it.effect("preserves failures from inside rollback admission", () => - Effect.gen(function* () { - const harness = yield* makeRollbackHarness; - const expected = { _tag: "host-restore-failure" as const }; - const error = yield* harness.runtime.withRollbackAdmission!( - { - providerThread: harness.providerThread, - target: { - type: "thread_start", - checkpointId: CheckpointId.make("admission-host-failure"), - appRunOrdinal: 0, - }, - providerThreadTurns: [], - }, - () => Effect.fail(expected), - ).pipe(Effect.flip); - assert.strictEqual(error, expected); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), - ); - it.effect.each([ { kind: "fork with inherited head", fork: true, headMessageId: "remapped-inherited-head" }, { kind: "fork without recorded head", fork: true, headMessageId: undefined }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 79ca2ef829c8e..b66e85e14d7c0 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4217,7 +4217,6 @@ function claudeProposedPlan(input: unknown): string | null { } export interface ClaudeAdapterV2Options { - readonly testHooks?: { readonly afterTerminalEvent?: Effect.Effect }; readonly instanceId: ProviderInstanceId; readonly settings: ClaudeSettings; readonly environment: NodeJS.ProcessEnv; @@ -4420,7 +4419,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); const events = yield* Queue.unbounded(); const activeTurn = yield* Ref.make(null); - const rollbackAdmission = yield* Semaphore.make(1); const taskOutputTails = new Map< string, { readonly fiber: Fiber.Fiber; readonly stop: Deferred.Deferred } @@ -7315,7 +7313,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ], { concurrency: 1 }, ); - yield* adapterOptions.testHooks?.afterTerminalEvent ?? Effect.void; yield* Ref.update(activeTurn, (current) => current?.providerTurnId === input.context.providerTurnId ? null : current, ); @@ -10639,14 +10636,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), ), ), - startTurn: (turnInput) => rollbackAdmission.withPermit(startTurn(turnInput)), + startTurn, compactThread: (turnInput) => - rollbackAdmission.withPermit( - startTurn({ - ...turnInput, - message: { ...turnInput.message, text: "/compact" }, - }), - ), + startTurn({ + ...turnInput, + message: { ...turnInput.message, text: "/compact" }, + }), steerTurn, stopTask: Effect.fn("ClaudeAdapterV2.stopTask")(function* ({ providerThread, taskId }) { const liveQuery = yield* Ref.get(queryContext); @@ -10833,29 +10828,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( cause: "Claude V2 adapter does not implement snapshots.", }), ), - withRollbackAdmission: (rollbackInput, use) => - rollbackAdmission.withPermit( - Effect.gen(function* () { - const currentTurn = yield* Ref.get(activeTurn); - const nativeThreadId = rollbackInput.providerThread.nativeThreadRef?.nativeId; - const goal = - nativeThreadId == null - ? rollbackInput.providerThread.goal - : (goalsByNativeThread.get(nativeThreadId) ?? - rollbackInput.providerThread.goal); - if ( - currentTurn !== null || - goal?.status === "active" || - (yield* runtime.hasPendingBackgroundWork!) - ) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver: CLAUDE_PROVIDER, - detail: `Cannot roll back Claude provider thread ${rollbackInput.providerThread.id} while provider work is active.`, - }); - } - return yield* use(() => runtime.rollbackThread(rollbackInput)); - }), - ), rollbackThread: Effect.fn("ClaudeAdapterV2.rollbackThread")( function* (rollbackInput) { const currentTurn = yield* Ref.get(activeTurn); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 87a40578fecea..602e1a4d56f29 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -5700,23 +5700,6 @@ describe("CodexAdapterV2 post-settle continuation", () => { now, }); - const activeGoalAdmission = yield* harness.runtime.withRollbackAdmission!( - { - providerThread: { - ...harness.providerThread, - goal: { objective: "Continue autonomously", status: "active" }, - }, - target: { - type: "thread_start", - checkpointId: CheckpointId.make("checkpoint-codex-active-goal"), - appRunOrdinal: 0, - }, - providerThreadTurns: [], - }, - () => Effect.die("an active goal must not admit restoration"), - ).pipe(Effect.result); - assert.equal(activeGoalAdmission._tag, "Failure"); - const error = yield* Effect.flip( harness.runtime.rollbackThread({ providerThread: harness.providerThread, diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 381c5b5699e86..6dba639b63b01 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1723,7 +1723,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( new Map>(), ); const activeTurns = yield* Ref.make(new Map()); - const rollbackAdmission = yield* Semaphore.make(1); const turnTokenUsageByThread = new Map(); const usageStateForThread = (nativeThreadId: string) => { let state = turnTokenUsageByThread.get(nativeThreadId); @@ -6698,7 +6697,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }, (effect, turnInput) => effect.pipe( - (effect) => rollbackAdmission.withPermit(effect), Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterTurnStartError({ @@ -6742,27 +6740,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( turnInput.restartContinuationOfRunId === undefined ? parseGoalCommand(turnInput.message) : null; - if (goalCommand !== null) - return rollbackAdmission.withPermit(runGoalCommand(turnInput, goalCommand)); - return rollbackAdmission.withPermit( - Effect.gen(function* () { - yield* startNativeTurn( - turnInput, - turnInput.restartContinuationOfRunId === undefined - ? yield* toCodexInput(turnInput) - : [], - ); - }).pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterTurnStartError({ - driver: CODEX_PROVIDER, - threadId: turnInput.threadId, - providerThreadId: turnInput.providerThread.id, - runId: turnInput.runId, - cause, - }), - ), + if (goalCommand !== null) return runGoalCommand(turnInput, goalCommand); + return Effect.gen(function* () { + yield* startNativeTurn( + turnInput, + turnInput.restartContinuationOfRunId === undefined + ? yield* toCodexInput(turnInput) + : [], + ); + }).pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: CODEX_PROVIDER, + threadId: turnInput.threadId, + providerThreadId: turnInput.providerThread.id, + runId: turnInput.runId, + cause, + }), ), ); }, @@ -7321,35 +7316,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), ), ), - withRollbackAdmission: (rollbackInput, use) => - rollbackAdmission.withPermit( - Effect.gen(function* () { - const nativeThreadId = rollbackInput.providerThread.nativeThreadRef?.nativeId; - const active = [...(yield* Ref.get(activeTurns)).values()].some( - (turn) => turn.providerThread.id === rollbackInput.providerThread.id, - ); - const pending = - nativeThreadId != null && (yield* Ref.get(pendingRootTurns)).has(nativeThreadId); - const goalRunning = - (nativeThreadId == null && - rollbackInput.providerThread.goal?.status === "active") || - (nativeThreadId != null && - (goalActivations.has(nativeThreadId) || - goalHolds.has(nativeThreadId) || - (goalsByNativeThread.get(nativeThreadId) ?? rollbackInput.providerThread.goal) - ?.status === "active")); - const background = yield* runtime.hasPendingBackgroundWorkForThread!( - rollbackInput.providerThread, - ); - if (active || pending || background || goalRunning) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver: CODEX_PROVIDER, - detail: "Cannot roll back while the provider thread has active work.", - }); - } - return yield* use(() => runtime.rollbackThread(rollbackInput)); - }), - ), rollbackThread: Effect.fn("CodexAdapterV2.rollbackThread")( function* (threadInput) { const threadId = yield* getNativeThreadId(threadInput.providerThread); diff --git a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts index a5301974dbe9a..77e4e90f7dc43 100644 --- a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts @@ -107,7 +107,6 @@ it.effect.each([ Layer.mock(ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), getCheckpointRollbackContext: () => Effect.succeed(projection), - hasLiveRun: () => Effect.succeed(false), getShellSnapshot: () => Effect.succeed({ schemaVersion: 1, @@ -153,18 +152,6 @@ it.effect.each([ Layer.mock(ProviderSessionManagerV2)({ open: () => Effect.succeed({ - withRollbackAdmission: ( - _input: unknown, - use: ( - rollback: () => Effect.Effect<{ providerThread: typeof providerThread }>, - ) => Effect.Effect, - ) => - use(() => - Effect.sync(() => { - calls.push("provider"); - return { providerThread }; - }), - ), rollbackThread: () => Effect.sync(() => { calls.push("provider"); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index 71ff3b23886f3..ab20c4cd1977e 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -52,15 +52,6 @@ import * as RuntimePolicy from "./RuntimePolicy.ts"; const unrelatedProject = Option.some({ workspaceRoot: "/nonexistent/t3-rollback-project", } as never); -const admittedSession = ( - session: Pick, -) => ({ - ...session, - withRollbackAdmission: ((input, use) => - use(() => session.rollbackThread(input))) satisfies NonNullable< - ProviderAdapter.ProviderAdapterV2SessionRuntime["withRollbackAdmission"] - >, -}); const layerCheckpointRollbackService = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( @@ -187,7 +178,6 @@ it.effect.each(["initial", "waiting"])( it.effect.each([ { order: "cleanup-first", restoreFiles: true }, { order: "rollback-first", restoreFiles: true }, - { order: "interrupt", restoreFiles: true }, { order: "cleanup-first", restoreFiles: false, unmanaged: true }, ])("locks rollback with %s", ({ order, restoreFiles, unmanaged = false }) => Effect.gen(function* () { @@ -284,7 +274,6 @@ it.effect.each([ Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), getCheckpointRollbackContext: () => Effect.succeed(projection), - hasLiveRun: () => Effect.succeed(false), getShellSnapshot: () => Effect.succeed({ schemaVersion: 1, @@ -302,9 +291,8 @@ it.effect.each([ } calls.push("session"); yield* Deferred.succeed(openEntered, undefined); - if (order === "rollback-first" || order === "interrupt") - yield* Deferred.await(release); - return admittedSession({ rollbackThread: () => Effect.die("unused") }) as never; + if (order === "rollback-first") yield* Deferred.await(release); + return {} as never; }), }), Layer.mock(RuntimePolicy.RuntimePolicyV2)({ @@ -334,13 +322,6 @@ it.effect.each([ const rollback = yield* service .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }) .pipe(Effect.forkChild({ startImmediately: true })); - if (order === "interrupt") { - yield* Deferred.await(openEntered); - yield* Fiber.interrupt(rollback); - yield* cleanupEffect; - assert.deepEqual(calls, ["session", "cleanup"]); - return; - } if (order === "cleanup-first") assert.deepEqual(calls, []); if (order === "rollback-first") yield* Deferred.await(openEntered); const laterCleanup = @@ -619,15 +600,13 @@ const seedRollbackHistory = Effect.gen(function* () { }), Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open: () => - Effect.succeed( - admittedSession({ - rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => - Effect.gen(function* () { - rollbackCounts.push(yield* resolveCodexRollbackTurnCount(input)); - return { providerThread: input.providerThread }; - }), - } as never) as never, - ), + Effect.succeed({ + rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => + Effect.gen(function* () { + rollbackCounts.push(yield* resolveCodexRollbackTurnCount(input)); + return { providerThread: input.providerThread }; + }), + } as never), }), Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: () => Effect.succeed({} as never) }), ), @@ -856,7 +835,6 @@ it.effect.each(["success", "restore-fails"] as const)( }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), - hasLiveRun: () => Effect.succeed(false), getCheckpointRollbackContext: () => Effect.succeed({ thread: projection.thread, @@ -881,26 +859,24 @@ it.effect.each(["success", "restore-fails"] as const)( }), Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open: () => - Effect.succeed( - admittedSession({ - rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => - Effect.gen(function* () { - providerCalls += 1; - const count = yield* resolveCodexRollbackTurnCount(input); - assert.equal(count, 1); - assert.equal(yield* fs.exists(currentFile).pipe(Effect.orDie), false); - assert.equal( - yield* fs.readFileString(restoredFile).pipe(Effect.orDie), - "checkpoint turn", - ); - assert.deepEqual((yield* search()).entries, [ - { path: "rollback-restored.txt", kind: "file" }, - ]); - providerConversation.splice(providerConversation.length - count, count); - return { providerThread }; - }), - } as never) as never, - ), + Effect.succeed({ + rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => + Effect.gen(function* () { + providerCalls += 1; + const count = yield* resolveCodexRollbackTurnCount(input); + assert.equal(count, 1); + assert.equal(yield* fs.exists(currentFile).pipe(Effect.orDie), false); + assert.equal( + yield* fs.readFileString(restoredFile).pipe(Effect.orDie), + "checkpoint turn", + ); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-restored.txt", kind: "file" }, + ]); + providerConversation.splice(providerConversation.length - count, count); + return { providerThread }; + }), + } as never), }), Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: () => Effect.succeed({} as never), diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index a2e2445ef2046..854df7bb1d543 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -32,7 +32,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import { GIT_INDEX_LOCKED_DETAIL } from "../vcs/VcsProcess.ts"; import * as ProjectStore from "./ProjectStore.ts"; -import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; @@ -47,7 +47,6 @@ export class CheckpointRollbackExecutionError extends Schema.TaggedError()( - "CheckpointRollbackFinalizationError", - { - threadId: ThreadId, - providerThreadId: ProviderThreadId, - checkpointId: CheckpointId, - stage: Schema.Literals(["stale-refs", "event-commit"]), - providerRolledBack: Schema.Boolean, - cause: Schema.Defect(), - }, -) { - override get message(): string { - const prefix = this.providerRolledBack - ? "Files were restored and the provider conversation was rolled back" - : "Files were restored"; - return `${prefix}, but T3 could not finish finalizing checkpoint history. Checkpoint references and conversation history may not match. Check the server logs before retrying.`; - } -} - export const isCheckpointRollbackPostRestoreError = Schema.is( - Schema.Union([ - CheckpointRollbackPartialRestoreError, - CheckpointRollbackIndexRefreshError, - CheckpointRollbackFinalizationError, - ]), + Schema.Union([CheckpointRollbackPartialRestoreError, CheckpointRollbackIndexRefreshError]), ); -const isCheckpointRollbackExecutionError = Schema.is(CheckpointRollbackExecutionError); -const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError); - -export const isCheckpointRollbackAdmissionRejection = (error: unknown): boolean => - isCheckpointRollbackExecutionError(error) && error.reason === "active-turn"; - const isCheckpointRollbackError = Schema.is( Schema.Union([ CheckpointRollbackExecutionError, CheckpointRollbackPartialRestoreError, CheckpointRollbackIndexRefreshError, - CheckpointRollbackFinalizationError, ]), ); @@ -156,13 +123,9 @@ export function rollbackFailureClientMessage( error: | CheckpointRollbackExecutionError | CheckpointRollbackPartialRestoreError - | CheckpointRollbackIndexRefreshError - | CheckpointRollbackFinalizationError, + | CheckpointRollbackIndexRefreshError, ): string { if (isCheckpointRollbackPostRestoreError(error)) return error.message; - if (error.reason === "active-turn") { - return error.message; - } const restoreCause = isCheckpointRestoreError(error.cause) ? error.cause.cause : undefined; return isVcsProcessExitError(restoreCause) && restoreCause.detail === GIT_INDEX_LOCKED_DETAIL ? GIT_INDEX_LOCKED_DETAIL @@ -181,7 +144,6 @@ export interface CheckpointRollbackServiceV2Shape { | CheckpointRollbackExecutionError | CheckpointRollbackPartialRestoreError | CheckpointRollbackIndexRefreshError - | CheckpointRollbackFinalizationError >; } @@ -359,39 +321,27 @@ export const layer: Layer.Layer< (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)), ); - const rollbackInput = { providerThread, target: rollbackTarget, providerThreadTurns }; - let enteredAdmission = false; - const admitted = Effect.fnUntraced(function* ( - rollback: () => ReturnType, - ) { - enteredAdmission = true; - if (yield* projections.hasLiveRun(input.threadId)) { - return yield* new CheckpointRollbackExecutionError({ - reason: "active-turn", - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - }); - } - if (input.restoreFiles !== false) { - yield* checkpoints.restore({ scope, checkpoint }); - yield* workspaceEntries.refresh(scope.cwd).pipe( - Effect.catchDefect((cause) => - Effect.fail( - new CheckpointRollbackIndexRefreshError({ - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - cause, - }), - ), + if (input.restoreFiles !== false) { + yield* checkpoints.restore({ scope, checkpoint }); + yield* workspaceEntries.refresh(scope.cwd).pipe( + Effect.catchDefect((cause) => + Effect.fail( + new CheckpointRollbackIndexRefreshError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), ), - ); - } - const snapshot = - runsToRollback.length === 0 - ? { providerThread } - : yield* rollback().pipe( + ), + ); + } + const snapshot = + runsToRollback.length === 0 + ? { providerThread } + : yield* session + .rollbackThread({ providerThread, target: rollbackTarget, providerThreadTurns }) + .pipe( Effect.mapError((cause) => input.restoreFiles === false ? cause @@ -403,133 +353,82 @@ export const layer: Layer.Layer< }), ), ); - if (staleCheckpoints.length > 0) { - yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }).pipe( - Effect.mapError((cause) => - input.restoreFiles === false - ? cause - : new CheckpointRollbackFinalizationError({ - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - stage: "stale-refs", - providerRolledBack: runsToRollback.length > 0, - cause, - }), - ), - ); - } + if (staleCheckpoints.length > 0) { + yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }); + } - // Lock against full-row thread writes that could restore the old native session after rollback. - const commitEvents = threadCommands.withLock( - input.threadId, - Effect.gen(function* () { - const now = yield* DateTime.now; - const makeEvent = ( - event: Omit, - ) => - Effect.map( - ids.allocate.event({ threadId: event.threadId }), - (id) => - ({ - ...event, - id, - }) as Event, - ); - const events: Array = []; + // Lock against full-row thread writes that could restore the old native session after rollback. + yield* threadCommands.withLock( + input.threadId, + Effect.gen(function* () { + const now = yield* DateTime.now; + const makeEvent = (event: Omit) => + Effect.map( + ids.allocate.event({ threadId: event.threadId }), + (id) => + ({ + ...event, + id, + }) as Event, + ); + const events: Array = []; + events.push( + yield* makeEvent({ + type: "provider-thread.updated", + threadId: input.threadId, + driver: providerThread.driver, + providerInstanceId: providerThread.providerInstanceId, + occurredAt: now, + payload: { + ...snapshot.providerThread, + lastRunOrdinal: targetOrdinal === 0 ? null : targetOrdinal, + updatedAt: now, + }, + }), + ); + for (const staleCheckpoint of staleCheckpoints) { events.push( yield* makeEvent({ - type: "provider-thread.updated", + type: "checkpoint.captured", threadId: input.threadId, - driver: providerThread.driver, + ...(staleCheckpoint.runId === null ? {} : { runId: staleCheckpoint.runId }), + nodeId: staleCheckpoint.nodeId, providerInstanceId: providerThread.providerInstanceId, occurredAt: now, - payload: { - ...snapshot.providerThread, - lastRunOrdinal: targetOrdinal === 0 ? null : targetOrdinal, - updatedAt: now, - }, + payload: { ...staleCheckpoint, status: "stale" }, }), ); - for (const staleCheckpoint of staleCheckpoints) { - events.push( - yield* makeEvent({ - type: "checkpoint.captured", - threadId: input.threadId, - ...(staleCheckpoint.runId === null ? {} : { runId: staleCheckpoint.runId }), - nodeId: staleCheckpoint.nodeId, - providerInstanceId: providerThread.providerInstanceId, - occurredAt: now, - payload: { ...staleCheckpoint, status: "stale" }, - }), - ); - } - for (const run of runsToRollback) { - const rootNode = context.rootNodes.find( - (candidate) => candidate.id === run.rootNodeId, - ); + } + for (const run of runsToRollback) { + const rootNode = context.rootNodes.find((candidate) => candidate.id === run.rootNodeId); + events.push( + yield* makeEvent({ + type: "run.updated", + threadId: input.threadId, + runId: run.id, + ...(rootNode === undefined ? {} : { nodeId: rootNode.id }), + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...run, status: "rolled_back", completedAt: now }, + }), + ); + if (rootNode !== undefined) { events.push( yield* makeEvent({ - type: "run.updated", + type: "node.updated", threadId: input.threadId, runId: run.id, - ...(rootNode === undefined ? {} : { nodeId: rootNode.id }), + nodeId: rootNode.id, providerInstanceId: run.providerInstanceId, occurredAt: now, - payload: { ...run, status: "rolled_back", completedAt: now }, + payload: { ...rootNode, status: "rolled_back", completedAt: now }, }), ); - if (rootNode !== undefined) { - events.push( - yield* makeEvent({ - type: "node.updated", - threadId: input.threadId, - runId: run.id, - nodeId: rootNode.id, - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: { ...rootNode, status: "rolled_back", completedAt: now }, - }), - ); - } } - yield* eventSink.write({ events }); - }), - ); - yield* commitEvents.pipe( - Effect.mapError((cause) => - input.restoreFiles === false - ? cause - : new CheckpointRollbackFinalizationError({ - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - stage: "event-commit", - providerRolledBack: runsToRollback.length > 0, - cause, - }), - ), - ); - }); - if (session.withRollbackAdmission !== undefined) { - yield* session.withRollbackAdmission(rollbackInput, admitted).pipe( - Effect.catchIf( - (cause) => !enteredAdmission && isProviderAdapterProtocolError(cause), - (cause) => - Effect.fail( - new CheckpointRollbackExecutionError({ - reason: "active-turn", - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - cause, - }), - ), - ), - ); - } else { - yield* admitted(() => session.rollbackThread(rollbackInput)); - } + } + yield* eventSink.write({ events }); + }), + ); }); return CheckpointRollbackServiceV2.of({ diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 23e11a4eb7619..11234545e99bc 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -437,11 +437,7 @@ export const layerExecutor: Layer.Layer< CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure) ? failure : undefined; - // A later retry must not restore after the active turn has made new edits. - const admissionRejected = - CheckpointRollbackService.isCheckpointRollbackAdmissionRejection(failure); - if (willRetry && postRestoreFailure === undefined && !admissionRejected) - return Effect.void; + if (willRetry && postRestoreFailure === undefined) return Effect.void; return threads .dispatch({ type: "checkpoint.rollback.fail", @@ -787,10 +783,6 @@ export const layerWithOptions = ( effect.request.type === "provider-thread.rollback" && Option.isSome(failure) && CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure.value.cause); - const admissionRejected = - effect.request.type === "provider-thread.rollback" && - Option.isSome(failure) && - CheckpointRollbackService.isCheckpointRollbackAdmissionRejection(failure.value.cause); yield* Effect.logWarning("Orchestration effect execution failed", { effectId: effect.id, effectType: effect.request.type, @@ -802,7 +794,7 @@ export const layerWithOptions = ( ? yield* outbox .succeed({ effectId: effect.id, workerId }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) - : postRestoreFailure || admissionRejected || effect.attemptCount >= maxAttempts + : postRestoreFailure || effect.attemptCount >= maxAttempts ? yield* outbox .fail({ effectId: effect.id, workerId, error }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 6148b1054f93d..e27f32ef84420 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -10816,14 +10816,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } } - if (hasLiveRun(projection)) { - return yield* new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause: "Cannot restore a checkpoint while a turn is active.", - }); - } - const now = yield* DateTime.now; yield* emit( events, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 23ca307a9e8cb..ed064325bfe16 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -613,7 +613,6 @@ export interface ProjectionStoreV2Shape { readonly canStartQueuedRun: ( threadId: ThreadId, ) => Effect.Effect; - readonly hasLiveRun: (threadId: ThreadId) => Effect.Effect; readonly getRecoveryThreadIds: ( kind: ProjectionRecoveryKind, ) => Effect.Effect, ProjectionStoreV2Error>; @@ -5285,13 +5284,6 @@ export const layer: Layer.Layer = ) .pipe(Effect.mapError(controlReadError(threadId))); - const hasLiveRun: ProjectionStoreV2Shape["hasLiveRun"] = (threadId) => - sql<{ readonly found: number }>`SELECT 1 AS found FROM orchestration_v2_projection_runs - WHERE thread_id = ${threadId} AND status IN ('preparing', 'starting', 'running') LIMIT 1`.pipe( - Effect.map((rows) => rows.length > 0), - Effect.mapError(controlReadError(threadId)), - ); - const getMessageCount: ProjectionStoreV2Shape["getMessageCount"] = (threadId) => sql<{ count: number; @@ -6801,7 +6793,6 @@ export const layer: Layer.Layer = getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext), getRunMessage: readOnReader(getRunMessage), canStartQueuedRun: readOnReader(canStartQueuedRun), - hasLiveRun: readOnReader(hasLiveRun), getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs), hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest), isNodeStored: readOnReader(isNodeStored), @@ -7509,15 +7500,6 @@ export const layerMemory: Layer.Layer = Layer.effect( const run = projection.runs.find((candidate) => candidate.id === runId); return projection.messages.find((message) => message.id === run?.userMessageId); }), - hasLiveRun: Effect.fn("ProjectionStore.hasLiveRun")(function* (threadId) { - const projection = (yield* Ref.get(replayState)).projections.get(threadId); - if (projection === undefined) - return yield* new ProjectionStoreThreadNotFoundError({ threadId }); - return projection.runs.some( - (run) => - run.status === "preparing" || run.status === "starting" || run.status === "running", - ); - }), canStartQueuedRun: Effect.fn("ProjectionStore.canStartQueuedRun")(function* (threadId) { const projection = (yield* Ref.get(replayState)).projections.get(threadId); if (projection === undefined) diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 57e1b61e70108..4837c4a2ac1c6 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -260,7 +260,6 @@ it.effect( getRunMessage: () => Effect.die("not used"), isNodeStored: () => Effect.die("not used"), canStartQueuedRun: () => Effect.die("not used"), - hasLiveRun: () => Effect.die("not used"), getRecoveryThreadIds: () => Effect.die("unused getRecoveryThreadIds"), getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"), getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"), diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 849c2ed0811e9..d6e105a61db4f 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -27,7 +27,6 @@ import { ProviderSessionId, ProviderThreadId, ProviderTurnId, - VcsProcessSpawnError, RunId, ScheduledTaskId, ThreadId, @@ -671,22 +670,12 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { attachments: [], dispatchMode: { type: "start_immediately" }, }); - const seededProjection = yield* orchestrator.getThreadProjection(threadId); - const scope = seededProjection.checkpointScopes[0]!; - const initialRun = seededProjection.runs[0]!; + const scope = (yield* orchestrator.getThreadProjection(threadId)).checkpointScopes[0]!; const now = yield* DateTime.now; const checkpointId = CheckpointId.make(`${name}-checkpoint`); yield* eventSink.write({ commandId: CommandId.make(`${name}-seed`), events: [ - { - id: EventId.make(`${name}-run-completed`), - type: "run.updated", - threadId, - runId: initialRun.id, - occurredAt: now, - payload: { ...initialRun, status: "completed", completedAt: now }, - }, { id: EventId.make(`${name}-checkpoint-event`), type: "checkpoint.captured", @@ -758,190 +747,6 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); - it.effect.each(["starting", "running"] as const)( - "does not restore files while a turn is %s", - (status) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const outbox = yield* EffectOutbox.EffectOutboxV2; - const checkpointStore = yield* CheckpointStore.CheckpointStore; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-active-rollback-" }); - const file = path.join(cwd, "workspace.txt"); - yield* fs.writeFileString(file, "in-flight changes"); - const { threadId, rollback } = yield* seedFailingRollbackThread("active-rollback", cwd); - const before = yield* orchestrator.getThreadProjection(threadId); - assert.equal(before.runs[0]?.status, "completed"); - const restoreSpy = vi - .spyOn(checkpointStore, "restoreCheckpoint") - .mockImplementation(() => - fs.writeFileString(file, "checkpoint contents").pipe(Effect.as(true), Effect.orDie), - ); - const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( - Effect.succeed({ - withRollbackAdmission: ( - _input: unknown, - use: (rollback: () => Effect.Effect) => Effect.Effect, - ) => use(() => Effect.die("rollback is not needed for an empty history")), - rollbackThread: () => - Effect.fail( - new ProviderAdapter.ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: before.providerThreads[0]!.id, - cause: "active turn", - }), - ), - } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), - ); - yield* Effect.addFinalizer(() => - Effect.sync(() => { - restoreSpy.mockRestore(); - sessionSpy.mockRestore(); - }), - ); - const commandId = CommandId.make("active-rollback-request"); - yield* rollback(commandId, true); - const run = before.runs[0]!; - const now = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: EventId.make("active-rollback-running"), - type: "run.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { - ...run, - status, - completedAt: null, - startedAt: status === "running" ? now : null, - }, - }, - ], - }); - const rejectedCommand = CommandId.make("active-rollback-rejected-request"); - const rejected = yield* rollback(rejectedCommand, true).pipe(Effect.flip); - assert.include(String(rejected.cause), "while a turn is active"); - assert.lengthOf(yield* outbox.listByCommandId(rejectedCommand), 0); - yield* worker.drain(); - assert.equal(yield* fs.readFileString(file), "in-flight changes"); - const [effect] = yield* outbox.listByCommandId(commandId); - assert.equal(effect?.status, "failed"); - assert.equal(effect?.attemptCount, 1); - yield* TestClock.adjust("30 seconds"); - assert.equal(yield* worker.drain(), 0); - assert.equal(yield* fs.readFileString(file), "in-flight changes"); - }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), - ); - - it.effect.each([ - { restoreFiles: true, admission: false }, - { restoreFiles: false, admission: false }, - { restoreFiles: true, admission: true }, - { restoreFiles: false, admission: true }, - { restoreFiles: true, admission: true, nativeActive: true }, - ])( - "restores with or without provider admission: %s", - ({ restoreFiles, admission, nativeActive = false }) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const outbox = yield* EffectOutbox.EffectOutboxV2; - const checkpointStore = yield* CheckpointStore.CheckpointStore; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-no-rollback-admission-" }); - const file = path.join(cwd, "workspace.txt"); - yield* fs.writeFileString(file, "current"); - const { threadId, rollback } = yield* seedFailingRollbackThread( - "no-rollback-admission", - cwd, - ); - const before = yield* orchestrator.getThreadProjection(threadId); - const run = before.runs[0]!; - const now = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: EventId.make("no-rollback-admission-completed"), - type: "run.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...run, status: "completed", completedAt: now }, - }, - ], - }); - let providerCalls = 0; - const providerThread = before.providerThreads[0]!; - const rollbackThread = () => - Effect.sync(() => { - providerCalls++; - return { providerThread }; - }); - const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( - Effect.succeed({ - ...(admission - ? { - withRollbackAdmission: ( - _input: unknown, - use: (rollback: typeof rollbackThread) => Effect.Effect, - ) => - nativeActive - ? Effect.fail( - new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: "native turn is still active", - }), - ) - : use(rollbackThread), - } - : {}), - rollbackThread, - } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), - ); - const restoreSpy = vi - .spyOn(checkpointStore, "restoreCheckpoint") - .mockImplementation(() => - fs.writeFileString(file, "checkpoint").pipe(Effect.as(true), Effect.orDie), - ); - yield* Effect.addFinalizer(() => - Effect.sync(() => { - sessionSpy.mockRestore(); - restoreSpy.mockRestore(); - }), - ); - const commandId = CommandId.make("no-rollback-admission-request"); - yield* rollback(commandId, restoreFiles); - yield* worker.drain(); - assert.equal( - yield* fs.readFileString(file), - restoreFiles && !nativeActive ? "checkpoint" : "current", - ); - assert.equal(restoreSpy.mock.calls.length, restoreFiles && !nativeActive ? 1 : 0); - assert.equal(providerCalls, nativeActive ? 0 : 1); - if (nativeActive) { - const [effect] = yield* outbox.listByCommandId(commandId); - assert.equal(effect?.status, "failed"); - assert.equal(effect?.attemptCount, 1); - assert.equal( - (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.message, - "Cannot restore a checkpoint while a turn is active.", - ); - } else { - const [effect] = yield* outbox.listByCommandId(commandId); - assert.equal(effect?.status, "succeeded"); - } - }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), - ); - it.effect.each(["partial-restore", "refresh-defect", "provider-only", "restore-fails"] as const)( "settles checkpoint rollback safely for %s", (outcome) => @@ -1011,28 +816,23 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { if (outcome === "refresh-defect") { refreshSpy.mockImplementation(() => Effect.die(new Error("index refresh defect"))); } - const rejectRollback = () => - Effect.gen(function* () { - rejected += 1; - assert.equal( - yield* fs.readFileString(file).pipe(Effect.orDie), - outcome === "provider-only" - ? "current uncheckpointed contents" - : "checkpoint contents", - ); - return yield* new ProviderAdapter.ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: providerThread.id, - cause: new Error("legacy history cannot be reverted"), - }); - }); const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( Effect.succeed({ - withRollbackAdmission: ( - _input: unknown, - use: (rollback: typeof rejectRollback) => Effect.Effect, - ) => use(rejectRollback), - rollbackThread: rejectRollback, + rollbackThread: () => + Effect.gen(function* () { + rejected += 1; + assert.equal( + yield* fs.readFileString(file).pipe(Effect.orDie), + outcome === "provider-only" + ? "current uncheckpointed contents" + : "checkpoint contents", + ); + return yield* new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: providerThread.id, + cause: new Error("legacy history cannot be reverted"), + }); + }), } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), ); yield* Effect.addFinalizer(() => @@ -1116,152 +916,6 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), ); - it.effect.each([ - { stage: "stale-refs", restoreFiles: true }, - { stage: "event-commit", restoreFiles: true }, - { stage: "stale-refs", restoreFiles: false }, - { stage: "event-commit", restoreFiles: false }, - ] as const)("handles rollback finalization failure %s", ({ stage, restoreFiles }) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); - const orchestrator = yield* Orchestrator.OrchestratorV2; - const sink = yield* EventSink.EventSinkV2; - const outbox = yield* EffectOutbox.EffectOutboxV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const checkpoints = yield* CheckpointStore.CheckpointStore; - const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-finalization-" }); - const file = path.join(cwd, "workspace.txt"); - yield* fs.writeFileString(file, "before restore"); - const { threadId, rollback } = yield* seedFailingRollbackThread( - `rollback-finalization-${stage}`, - cwd, - ); - const before = yield* orchestrator.getThreadProjection(threadId); - const run = before.runs[0]!; - const providerThread = before.providerThreads[0]!; - const now = yield* DateTime.now; - const laterCheckpoint = { - ...before.checkpoints[0]!, - id: CheckpointId.make(`finalization-later-${stage}`), - ref: CheckpointRef.make(`refs/t3/finalization-later-${stage}`), - appRunOrdinal: run.ordinal, - ordinalWithinScope: 1, - }; - yield* sink.write({ - events: [ - { - id: EventId.make(`finalization-later-event-${stage}`), - type: "checkpoint.captured", - threadId, - occurredAt: now, - payload: laterCheckpoint, - }, - ], - }); - let restores = 0; - let providerCalls = 0; - let refsRemain = true; - let commitCalls = 0; - const providerRollback = () => - Effect.sync(() => { - providerCalls++; - return { providerThread }; - }); - const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( - Effect.succeed({ - rollbackThread: providerRollback, - withRollbackAdmission: ( - _input: unknown, - use: (rollback: typeof providerRollback) => Effect.Effect, - ) => use(providerRollback), - } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), - ); - const restoreSpy = vi.spyOn(checkpoints, "restoreCheckpoint").mockImplementation(() => - Effect.gen(function* () { - restores++; - yield* fs.writeFileString(file, "restored"); - return true; - }).pipe(Effect.orDie), - ); - const deleteSpy = vi.spyOn(checkpoints, "deleteCheckpointRefs").mockImplementation(() => - stage === "stale-refs" - ? Effect.fail( - new VcsProcessSpawnError({ - operation: "delete refs", - command: "git", - cwd, - cause: "failure", - }), - ) - : Effect.sync(() => { - refsRemain = false; - }), - ); - const originalWrite = sink.write; - const writeSpy = vi.spyOn(sink, "write").mockImplementation((input) => { - if (input.events.some((event) => event.type === "provider-thread.updated")) { - commitCalls++; - if (stage === "event-commit") - return Effect.fail( - new EventSink.EventSinkWriteError({ - eventCount: input.events.length, - cause: "failure", - }), - ); - } - return originalWrite(input); - }); - yield* Effect.addFinalizer(() => - Effect.sync(() => { - sessionSpy.mockRestore(); - restoreSpy.mockRestore(); - deleteSpy.mockRestore(); - writeSpy.mockRestore(); - }), - ); - const commandId = CommandId.make(`rollback-finalization-request-${stage}`); - yield* rollback(commandId, restoreFiles); - yield* worker.drain(); - const [first] = yield* outbox.listByCommandId(commandId); - assert.equal(first?.status, restoreFiles ? "failed" : "pending"); - if (!restoreFiles) { - for (let attempt = 1; attempt < 5; attempt++) { - yield* TestClock.adjust("30 seconds"); - yield* worker.drain(); - } - } - const [effect] = yield* outbox.listByCommandId(commandId); - assert.equal(effect?.status, "failed"); - assert.equal(effect?.attemptCount, restoreFiles ? 1 : 5); - assert.equal(restores, restoreFiles ? 1 : 0); - assert.equal(providerCalls, restoreFiles ? 1 : 5); - assert.equal(refsRemain, stage === "stale-refs"); - assert.equal(commitCalls, stage === "event-commit" ? (restoreFiles ? 1 : 5) : 0); - const after = yield* orchestrator.getThreadProjection(threadId); - assert.equal(after.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); - assert.equal( - after.checkpoints.find((candidate) => candidate.id === laterCheckpoint.id)?.status, - "ready", - ); - if (restoreFiles) { - assert.equal( - after.thread.rollbackFailure?.message, - "Files were restored and the provider conversation was rolled back, but T3 could not finish finalizing checkpoint history. Checkpoint references and conversation history may not match. Check the server logs before retrying.", - ); - } else { - assert.equal(after.thread.rollbackFailure?.message, ROLLBACK_FAILED_MESSAGE); - } - yield* fs.writeFileString(file, "new edits"); - yield* TestClock.adjust("30 seconds"); - assert.equal(yield* worker.drain(), 0); - assert.equal(yield* fs.readFileString(file), "new edits"); - assert.equal(restores, restoreFiles ? 1 : 0); - assert.equal(providerCalls, restoreFiles ? 1 : 5); - }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), - ); - it.effect("rejects non-ready rollback targets before persisting events or effects", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -1296,19 +950,6 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { const projection = yield* orchestrator.getThreadProjection(threadId); const scope = projection.checkpointScopes[0]!; const now = yield* DateTime.now; - const readinessRun = projection.runs[0]!; - yield* eventSink.write({ - events: [ - { - id: EventId.make("runtime-rollback-readiness-completed"), - type: "run.updated", - threadId, - runId: readinessRun.id, - occurredAt: now, - payload: { ...readinessRun, status: "completed", completedAt: now }, - }, - ], - }); for (const status of ["missing", "error", "stale", "ready"] as const) { const checkpointId = CheckpointId.make("runtime-rollback-checkpoint"); diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index eb174fd067389..6fa99a058cae6 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -7728,97 +7728,6 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }), ); - const rollbackUnlocked = Effect.fn("AcpAdapterV2.rollbackThread.unlocked")( - function* (rollbackInput: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) { - const currentTurn = yield* Ref.get(activeTurn); - if (currentTurn !== null) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: `Cannot roll back ACP provider thread ${rollbackInput.providerThread.id} while turn ${currentTurn.providerTurnId} is active`, - }); - } - yield* awaitRuntimeTeardown(); - itemIdentityVersion = 2; - prepareTerminalEnvironment( - yield* readMcpContext(rollbackInput.providerThread.appThreadId), - ); - const replacement = yield* startReplacementAcpRuntime( - rollbackInput.providerThread.appThreadId, - (candidate) => - Effect.gen(function* () { - rememberTerminalEnvironment( - candidate.sessionId, - yield* readMcpContext(rollbackInput.providerThread.appThreadId), - ); - yield* Ref.set(runtimeRestartRequired, false); - yield* Ref.set(activeSessionId, candidate.sessionId); - yield* setActiveSessionSetup(candidate); - yield* Ref.set(activeSelection, null); - yield* Ref.set(activeInteractionMode, null); - yield* Ref.set(promptInstructionStates, new Map()); - yield* Ref.set(compactionMarkers, new Map()); - yield* Ref.set(providerTurns, new Map()); - yield* Ref.set(snapshot, { - order: [], - messages: new Map(), - loadingRole: null, - loadingMessageId: null, - loadingIndex: 0, - }); - yield* continuationPermit.withPermit( - Effect.gen(function* () { - yield* Ref.update(continuationGeneration, (value) => value + 1); - yield* Ref.set(stoppedRunQuarantine, false); - yield* Ref.set(wakeBuffer, []); - yield* Ref.set(wakeReports, noWakeReports); - yield* Ref.set(continuationRequested, false); - yield* Ref.set(runningBackgroundTaskIds, new Set()); - yield* Ref.set(endedBackgroundTaskIds, new Set()); - yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); - yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); - yield* Ref.set(carryoverSubagents, null); - yield* Ref.set(suppressPostSettleMonitorPrompt, false); - yield* Ref.set(lastTurnRoute, null); - }), - ); - }), - ).pipe(Effect.retry({ times: 1 })); - const now = yield* DateTime.now; - return { - providerThread: { - ...rollbackInput.providerThread, - nativeThreadRef: { - driver, - nativeId: replacement.sessionId, - strength: "strong" as const, - }, - nativeConversationHeadRef: null, - nativeMetadata: { - ...rollbackInput.providerThread.nativeMetadata, - itemIdentityVersion: 2 as const, - }, - status: "idle" as const, - updatedAt: now, - }, - providerTurns: [], - messages: [], - runtimeRequests: [], - }; - }, - (effect, rollbackInput) => - effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: rollbackInput.providerThread.id, - checkpointId: rollbackInput.target.checkpointId, - cause, - }), - ), - ), - ); - const sessionRuntime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { instanceId: options.instanceId, driver, @@ -8420,35 +8329,97 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ), ), ), - withRollbackAdmission: (rollbackInput, use) => - runtimeTransitionPermit.withPermit( - Effect.gen(function* () { - yield* awaitRuntimeTeardown().pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: rollbackInput.providerThread.id, - checkpointId: rollbackInput.target.checkpointId, - cause, - }), - ), - ); - const currentTurn = yield* Ref.get(activeTurn); - if ( - currentTurn !== null || - (yield* sessionRuntime.hasPendingBackgroundWork ?? Effect.succeed(false)) - ) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: `Cannot roll back ACP provider thread ${rollbackInput.providerThread.id} while provider work is active`, - }); - } - return yield* use(() => rollbackUnlocked(rollbackInput)); - }), - ), rollbackThread: (rollbackInput) => - runtimeTransitionPermit.withPermit(rollbackUnlocked(rollbackInput)), + runtimeTransitionPermit + .withPermit( + Effect.gen(function* () { + const currentTurn = yield* Ref.get(activeTurn); + if (currentTurn !== null) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: `Cannot roll back ACP provider thread ${rollbackInput.providerThread.id} while turn ${currentTurn.providerTurnId} is active`, + }); + } + yield* awaitRuntimeTeardown(); + itemIdentityVersion = 2; + prepareTerminalEnvironment( + yield* readMcpContext(rollbackInput.providerThread.appThreadId), + ); + const replacement = yield* startReplacementAcpRuntime( + rollbackInput.providerThread.appThreadId, + (candidate) => + Effect.gen(function* () { + rememberTerminalEnvironment( + candidate.sessionId, + yield* readMcpContext(rollbackInput.providerThread.appThreadId), + ); + yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(activeSessionId, candidate.sessionId); + yield* setActiveSessionSetup(candidate); + yield* Ref.set(activeSelection, null); + yield* Ref.set(activeInteractionMode, null); + yield* Ref.set(promptInstructionStates, new Map()); + yield* Ref.set(compactionMarkers, new Map()); + yield* Ref.set(providerTurns, new Map()); + yield* Ref.set(snapshot, { + order: [], + messages: new Map(), + loadingRole: null, + loadingMessageId: null, + loadingIndex: 0, + }); + yield* continuationPermit.withPermit( + Effect.gen(function* () { + yield* Ref.update(continuationGeneration, (value) => value + 1); + yield* Ref.set(stoppedRunQuarantine, false); + yield* Ref.set(wakeBuffer, []); + yield* Ref.set(wakeReports, noWakeReports); + yield* Ref.set(continuationRequested, false); + yield* Ref.set(runningBackgroundTaskIds, new Set()); + yield* Ref.set(endedBackgroundTaskIds, new Set()); + yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); + yield* Ref.set(handledBackgroundTaskIdsInActiveTurn, new Set()); + yield* Ref.set(carryoverSubagents, null); + yield* Ref.set(suppressPostSettleMonitorPrompt, false); + yield* Ref.set(lastTurnRoute, null); + }), + ); + }), + ).pipe(Effect.retry({ times: 1 })); + const now = yield* DateTime.now; + return { + providerThread: { + ...rollbackInput.providerThread, + nativeThreadRef: { + driver, + nativeId: replacement.sessionId, + strength: "strong" as const, + }, + nativeConversationHeadRef: null, + nativeMetadata: { + ...rollbackInput.providerThread.nativeMetadata, + itemIdentityVersion: 2 as const, + }, + status: "idle" as const, + updatedAt: now, + }, + providerTurns: [], + messages: [], + runtimeRequests: [], + }; + }), + ) + .pipe( + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: rollbackInput.providerThread.id, + checkpointId: rollbackInput.target.checkpointId, + cause, + }), + ), + ), forkThread: Effect.fn("AcpAdapterV2.forkThread")( function* (forkInput) { return yield* runtimeTransitionPermit.withPermit( diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 9cb781937a5e0..7c77b7bb0ac6f 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -624,13 +624,6 @@ export interface ProviderAdapterV2SessionRuntime { readonly rollbackThread: ( input: ProviderAdapterV2RollbackThreadInput, ) => Effect.Effect; - /** Reject active native work, then hold admission through `use` and its unlocked rollback. */ - readonly withRollbackAdmission?: ( - input: ProviderAdapterV2RollbackThreadInput, - use: ( - rollback: () => Effect.Effect, - ) => Effect.Effect, - ) => Effect.Effect; readonly forkThread: ( input: ProviderAdapterV2ForkThreadInput, ) => Effect.Effect; From 2daa507b5b0c64192c7007f29d23ae89739ecbd1 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Thu, 12 Nov 2026 00:04:38 +0100 Subject: [PATCH 12/12] fix(server): stop retrying checkpoint restores after partial mutation --- .../src/checkpointing/CheckpointStore.ts | 4 +- .../CheckpointRollbackService.ts | 43 ++- .../src/orchestration-v2/runtimeLayer.test.ts | 342 ++++++++++-------- apps/server/src/vcs/GitVcsDriver.test.ts | 66 +++- apps/server/src/vcs/GitVcsDriver.ts | 203 ++++++----- apps/server/src/vcs/VcsDriver.ts | 12 +- 6 files changed, 421 insertions(+), 249 deletions(-) diff --git a/apps/server/src/checkpointing/CheckpointStore.ts b/apps/server/src/checkpointing/CheckpointStore.ts index e5d3006568a67..22380e784dc69 100644 --- a/apps/server/src/checkpointing/CheckpointStore.ts +++ b/apps/server/src/checkpointing/CheckpointStore.ts @@ -4,7 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import type { CheckpointStoreError } from "./Errors.ts"; -import type { VcsCheckpointOps } from "../vcs/VcsDriver.ts"; +import type { VcsCheckpointOps, VcsCheckpointPartialRestoreError } from "../vcs/VcsDriver.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; export interface CaptureCheckpointInput { @@ -57,7 +57,7 @@ export class CheckpointStore extends Context.Service< readonly restoreCheckpoint: ( input: RestoreCheckpointInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly diffCheckpoints: ( input: DiffCheckpointsInput, diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index fe874b8275b3c..13ac980d64034 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -16,6 +16,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import { VcsCheckpointPartialRestoreError } from "../vcs/VcsDriver.ts"; import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import type * as ServerConfig from "../config.ts"; @@ -86,6 +87,20 @@ export class CheckpointRollbackPartialRestoreError extends Schema.TaggedError()( + "CheckpointRollbackIncompleteRestoreError", + { + threadId: ThreadId, + providerThreadId: ProviderThreadId, + checkpointId: CheckpointId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "File restoration failed after workspace files may have changed. The conversation rollback was not attempted. Check your files and the server logs before retrying."; + } +} + export class CheckpointRollbackIndexRefreshError extends Schema.TaggedError()( "CheckpointRollbackIndexRefreshError", { @@ -101,7 +116,11 @@ export class CheckpointRollbackIndexRefreshError extends Schema.TaggedError; } @@ -340,7 +363,21 @@ export const layer: Layer.Layer< ); if (input.restoreFiles !== false) { - yield* checkpoints.restore({ scope, checkpoint }); + yield* checkpoints.restore({ scope, checkpoint }).pipe( + Effect.catchTags({ + CheckpointRestoreError: (cause) => + Effect.fail( + isVcsCheckpointPartialRestoreError(cause.cause) + ? new CheckpointRollbackIncompleteRestoreError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }) + : cause, + ), + }), + ); yield* workspaceEntries.refresh(scope.cwd).pipe( Effect.catchDefect((cause) => Effect.fail( diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 7ec2dbc8067ce..a0957b1ba30a1 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -94,6 +94,7 @@ import * as TerminalManager from "../terminal/Manager.ts"; import * as ResourceCleanupService from "./ResourceCleanupService.ts"; import * as ThreadLaunchService from "./ThreadLaunchService.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; +import { VcsCheckpointPartialRestoreError } from "../vcs/VcsDriver.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; const layerPlatformTest = Layer.merge( @@ -1016,173 +1017,196 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); - it.effect.each(["partial-restore", "refresh-defect", "provider-only", "restore-fails"] as const)( - "settles checkpoint rollback safely for %s", - (outcome) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); - const orchestrator = yield* Orchestrator.OrchestratorV2; - const eventSink = yield* EventSink.EventSinkV2; - const outbox = yield* EffectOutbox.EffectOutboxV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const checkpointStore = yield* CheckpointStore.CheckpointStore; - const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; - const postRestoreFailure = outcome === "partial-restore" || outcome === "refresh-defect"; - const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-partial-" }); - const file = path.join(cwd, "workspace.txt"); - yield* fs.writeFileString(file, "current uncheckpointed contents"); - const { threadId, checkpointId, rollback } = yield* seedFailingRollbackThread( - `runtime-rollback-${outcome}`, - cwd, + it.effect.each([ + "partial-restore", + "refresh-defect", + "provider-only", + "restore-fails", + "incomplete-restore", + ] as const)("settles checkpoint rollback safely for %s", (outcome) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const postRestoreFailure = + outcome === "partial-restore" || + outcome === "refresh-defect" || + outcome === "incomplete-restore"; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-partial-" }); + const file = path.join(cwd, "workspace.txt"); + yield* fs.writeFileString(file, "current uncheckpointed contents"); + const { threadId, checkpointId, rollback } = yield* seedFailingRollbackThread( + `runtime-rollback-${outcome}`, + cwd, + ); + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const providerThread = before.providerThreads[0]!; + const now = yield* DateTime.now; + const laterCheckpoint = { + ...before.checkpoints[0]!, + id: CheckpointId.make("runtime-rollback-later-checkpoint"), + ref: CheckpointRef.make("refs/t3/runtime-rollback-later"), + appRunOrdinal: run.ordinal, + ordinalWithinScope: 1, + }; + yield* eventSink.write({ + events: [ + { + id: EventId.make("runtime-rollback-later-captured"), + type: "checkpoint.captured", + threadId, + occurredAt: now, + payload: laterCheckpoint, + }, + { + id: EventId.make("runtime-rollback-partial-completed"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", completedAt: now }, + }, + ], + }); + let restored = 0; + let rejected = 0; + const restoreSpy = vi + .spyOn(checkpointStore, "restoreCheckpoint") + .mockImplementation((input) => + Effect.gen(function* () { + assert.equal(path.resolve(input.cwd), path.resolve(cwd)); + restored += 1; + if (outcome === "restore-fails") return false; + yield* fs.writeFileString(file, "checkpoint contents").pipe(Effect.orDie); + if (outcome === "incomplete-restore") { + return yield* new VcsCheckpointPartialRestoreError({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + cause: new Error("cleanup failed after restoring files"), + }); + } + return true; + }), ); - const before = yield* orchestrator.getThreadProjection(threadId); - const run = before.runs[0]!; - const providerThread = before.providerThreads[0]!; - const now = yield* DateTime.now; - const laterCheckpoint = { - ...before.checkpoints[0]!, - id: CheckpointId.make("runtime-rollback-later-checkpoint"), - ref: CheckpointRef.make("refs/t3/runtime-rollback-later"), - appRunOrdinal: run.ordinal, - ordinalWithinScope: 1, - }; - yield* eventSink.write({ - events: [ - { - id: EventId.make("runtime-rollback-later-captured"), - type: "checkpoint.captured", - threadId, - occurredAt: now, - payload: laterCheckpoint, - }, - { - id: EventId.make("runtime-rollback-partial-completed"), - type: "run.updated", - threadId, - runId: run.id, - occurredAt: now, - payload: { ...run, status: "completed", completedAt: now }, - }, - ], - }); - let restored = 0; - let rejected = 0; - const restoreSpy = vi - .spyOn(checkpointStore, "restoreCheckpoint") - .mockImplementation((input) => + const deleteSpy = vi.spyOn(checkpointStore, "deleteCheckpointRefs"); + const refreshSpy = vi.spyOn(workspaceEntries, "refresh"); + if (outcome === "refresh-defect") { + refreshSpy.mockImplementation(() => Effect.die(new Error("index refresh defect"))); + } + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + rollbackThread: () => Effect.gen(function* () { - assert.equal(path.resolve(input.cwd), path.resolve(cwd)); - restored += 1; - if (outcome === "restore-fails") return false; - yield* fs.writeFileString(file, "checkpoint contents"); - return true; - }).pipe(Effect.orDie), - ); - const deleteSpy = vi.spyOn(checkpointStore, "deleteCheckpointRefs"); - const refreshSpy = vi.spyOn(workspaceEntries, "refresh"); - if (outcome === "refresh-defect") { - refreshSpy.mockImplementation(() => Effect.die(new Error("index refresh defect"))); + rejected += 1; + assert.equal( + yield* fs.readFileString(file).pipe(Effect.orDie), + outcome === "provider-only" + ? "current uncheckpointed contents" + : "checkpoint contents", + ); + return yield* new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: providerThread.id, + cause: new Error("legacy history cannot be reverted"), + }); + }), + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + restoreSpy.mockRestore(); + deleteSpy.mockRestore(); + refreshSpy.mockRestore(); + sessionSpy.mockRestore(); + }), + ); + const commandId = CommandId.make("runtime-rollback-partial-request"); + yield* rollback(commandId, outcome !== "provider-only"); + yield* worker.drain(); + const [firstAttempt] = yield* outbox.listByCommandId(commandId); + assert.equal(firstAttempt?.attemptCount, 1); + assert.equal(firstAttempt?.status, postRestoreFailure ? "failed" : "pending"); + if (!postRestoreFailure) { + assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); + assert.equal(yield* fs.readFileString(file), "current uncheckpointed contents"); + for (let attempt = 1; attempt < 5; attempt++) { + yield* TestClock.adjust("30 seconds"); + yield* worker.drain(); } - const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( - Effect.succeed({ - rollbackThread: () => - Effect.gen(function* () { - rejected += 1; - assert.equal( - yield* fs.readFileString(file).pipe(Effect.orDie), - outcome === "provider-only" - ? "current uncheckpointed contents" - : "checkpoint contents", - ); - return yield* new ProviderAdapter.ProviderAdapterRollbackThreadError({ - driver, - providerThreadId: providerThread.id, - cause: new Error("legacy history cannot be reverted"), - }); - }), - } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), - ); - yield* Effect.addFinalizer(() => - Effect.sync(() => { - restoreSpy.mockRestore(); - deleteSpy.mockRestore(); - refreshSpy.mockRestore(); - sessionSpy.mockRestore(); - }), - ); - const commandId = CommandId.make("runtime-rollback-partial-request"); - yield* rollback(commandId, outcome !== "provider-only"); + } + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "failed"); + const expectedAttempts = postRestoreFailure ? 1 : 5; + const expectedProviderCalls = + outcome === "restore-fails" || + outcome === "refresh-defect" || + outcome === "incomplete-restore" + ? 0 + : expectedAttempts; + assert.equal(effect?.attemptCount, expectedAttempts); + assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); + assert.equal(rejected, expectedProviderCalls); + assert.equal( + yield* fs.readFileString(file), + postRestoreFailure ? "checkpoint contents" : "current uncheckpointed contents", + ); + const failed = yield* orchestrator.getThreadProjection(threadId); + assert.equal(failed.thread.rollbackFailure?.requestId, commandId); + assert.equal( + failed.thread.rollbackFailure?.message, + outcome === "partial-restore" + ? "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying." + : outcome === "refresh-defect" + ? "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying." + : outcome === "incomplete-restore" + ? "File restoration failed after workspace files may have changed. The conversation rollback was not attempted. Check your files and the server logs before retrying." + : ROLLBACK_FAILED_MESSAGE, + ); + assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); + assert.equal( + failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, + "ready", + ); + assert.deepEqual( + failed.checkpoints.find((candidate) => candidate.id === laterCheckpoint.id), + laterCheckpoint, + ); + assert.deepEqual(failed.nodes, before.nodes); + assert.deepEqual(failed.providerThreads, before.providerThreads); + assert.equal(deleteSpy.mock.calls.length, 0); + yield* fs.writeFileString(file, "user edits after rollback failure"); + yield* TestClock.adjust("30 seconds"); + assert.equal(yield* worker.drain(), 0); + assert.equal(yield* fs.readFileString(file), "user edits after rollback failure"); + assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); + assert.equal(rejected, expectedProviderCalls); + if (postRestoreFailure) { + const nextCommandId = CommandId.make("runtime-rollback-partial-next"); + yield* rollback(nextCommandId, true); + assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); yield* worker.drain(); - const [firstAttempt] = yield* outbox.listByCommandId(commandId); - assert.equal(firstAttempt?.attemptCount, 1); - assert.equal(firstAttempt?.status, postRestoreFailure ? "failed" : "pending"); - if (!postRestoreFailure) { - assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); - assert.equal(yield* fs.readFileString(file), "current uncheckpointed contents"); - for (let attempt = 1; attempt < 5; attempt++) { - yield* TestClock.adjust("30 seconds"); - yield* worker.drain(); - } - } - const [effect] = yield* outbox.listByCommandId(commandId); - assert.equal(effect?.status, "failed"); - const expectedAttempts = postRestoreFailure ? 1 : 5; - const expectedProviderCalls = - outcome === "restore-fails" || outcome === "refresh-defect" ? 0 : expectedAttempts; - assert.equal(effect?.attemptCount, expectedAttempts); - assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); - assert.equal(rejected, expectedProviderCalls); - assert.equal( - yield* fs.readFileString(file), - postRestoreFailure ? "checkpoint contents" : "current uncheckpointed contents", - ); - const failed = yield* orchestrator.getThreadProjection(threadId); - assert.equal(failed.thread.rollbackFailure?.requestId, commandId); + const [nextEffect] = yield* outbox.listByCommandId(nextCommandId); + assert.equal(nextEffect?.status, "failed"); + assert.equal(nextEffect?.attemptCount, 1); + assert.equal(restored, 2); assert.equal( - failed.thread.rollbackFailure?.message, - outcome === "partial-restore" - ? "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying." - : outcome === "refresh-defect" - ? "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying." - : ROLLBACK_FAILED_MESSAGE, + rejected, + outcome === "refresh-defect" || outcome === "incomplete-restore" ? 0 : 2, ); - assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); + assert.equal(yield* fs.readFileString(file), "checkpoint contents"); assert.equal( - failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, - "ready", - ); - assert.deepEqual( - failed.checkpoints.find((candidate) => candidate.id === laterCheckpoint.id), - laterCheckpoint, + (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.requestId, + nextCommandId, ); - assert.deepEqual(failed.nodes, before.nodes); - assert.deepEqual(failed.providerThreads, before.providerThreads); - assert.equal(deleteSpy.mock.calls.length, 0); - yield* fs.writeFileString(file, "user edits after rollback failure"); - yield* TestClock.adjust("30 seconds"); - assert.equal(yield* worker.drain(), 0); - assert.equal(yield* fs.readFileString(file), "user edits after rollback failure"); - assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); - assert.equal(rejected, expectedProviderCalls); - if (postRestoreFailure) { - const nextCommandId = CommandId.make("runtime-rollback-partial-next"); - yield* rollback(nextCommandId, true); - assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); - yield* worker.drain(); - const [nextEffect] = yield* outbox.listByCommandId(nextCommandId); - assert.equal(nextEffect?.status, "failed"); - assert.equal(nextEffect?.attemptCount, 1); - assert.equal(restored, 2); - assert.equal(rejected, outcome === "refresh-defect" ? 0 : 2); - assert.equal(yield* fs.readFileString(file), "checkpoint contents"); - assert.equal( - (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.requestId, - nextCommandId, - ); - } - }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), + } + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), ); it.effect("captures and repeatedly rewinds non-Git conversations without losing run links", () => diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 128c8afcd3b4c..20770148406e8 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -1186,7 +1186,10 @@ it.effect("restores empty checkpoints without changing paths outside the workspa const incomplete = yield* driver.checkpoints .restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }) .pipe(Effect.flip); - assert.strictEqual(incomplete._tag, "VcsProcessExitError"); + assert.strictEqual(incomplete._tag, "VcsCheckpointPartialRestoreError"); + if (incomplete._tag === "VcsCheckpointPartialRestoreError") { + assert.instanceOf(incomplete.cause, VcsProcessExitError); + } assert.strictEqual( yield* fileSystem.readFileString(path.join(cwd, "checkpoint.txt")), "checkpoint\n", @@ -1213,6 +1216,67 @@ it.effect("restores empty checkpoints without changing paths outside the workspa }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("reports cleanup failure after restoring checkpoint files as a partial restore", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-partial-restore-" }); + yield* runGit(cwd, ["init"]); + yield* runGit(cwd, ["config", "user.email", "test@test.com"]); + yield* runGit(cwd, ["config", "user.name", "Test"]); + yield* runGit(cwd, ["commit", "--allow-empty", "-m", "initial"]); + const file = path.join(cwd, "file.txt"); + yield* fileSystem.writeFileString(file, "checkpoint"); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/partial"); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(file, "later edits"); + let deepest = cwd; + for (let depth = 0; depth < 5; depth++) { + yield* fileSystem.writeFileString(path.join(deepest, ".gitignore"), "d/.gitignore\n"); + deepest = path.join(deepest, "d"); + yield* fileSystem.makeDirectory(deepest); + } + yield* fileSystem.writeFileString(path.join(deepest, ".gitignore"), "hidden.txt\n"); + yield* fileSystem.writeFileString(path.join(deepest, "hidden.txt"), "untracked"); + const error = yield* driver.checkpoints + .restoreCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.flip); + assert.equal(error._tag, "VcsCheckpointPartialRestoreError"); + if (error._tag === "VcsCheckpointPartialRestoreError") + assert.instanceOf(error.cause, VcsProcessExitError); + assert.equal(yield* fileSystem.readFileString(file), "checkpoint"); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("keeps a restore blocked by the Git index lock retryable", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-restore-locked-" }); + yield* runGit(cwd, ["init"]); + yield* runGit(cwd, ["config", "user.email", "test@test.com"]); + yield* runGit(cwd, ["config", "user.name", "Test"]); + const file = path.join(cwd, "file.txt"); + yield* fileSystem.writeFileString(file, "checkpoint"); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/locked"); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(file, "later edits"); + yield* fileSystem.writeFileString(path.join(cwd, ".git", "index.lock"), ""); + const error = yield* driver.checkpoints + .restoreCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.flip); + assert.equal(error._tag, "VcsProcessExitError"); + if (error._tag === "VcsProcessExitError") + assert.equal(error.detail, VcsProcess.GIT_INDEX_LOCKED_DETAIL); + assert.equal(yield* fileSystem.readFileString(file), "later edits"); + yield* fileSystem.remove(path.join(cwd, ".git", "index.lock")); + assert.isTrue(yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef })); + assert.equal(yield* fileSystem.readFileString(file), "checkpoint"); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("restores a checkpoint before the first commit without staging its files", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index a5dd821d4a8b4..ff3a714be4e75 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -12,6 +12,7 @@ import { ChildProcessSpawner } from "effect/process"; import { GitCommandError, + type VcsError, VcsProcessExitError, VcsUnsupportedOperationError, type VcsSwitchRefInput, @@ -1393,105 +1394,141 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( cwd: input.cwd, args: ["ls-files", "--cached", `--with-tree=${commitOid}`, "-z", "--", "."], }); - // An empty index and checkpoint have nothing for git restore's pathspec to match. - if (tracked.stdout.length > 0) { - yield* execute({ - operation, - cwd: input.cwd, - args: [ - ...longPaths, - "restore", - "--source", - commitOid, - "--worktree", - "--staged", - "--", - ".", - ], + let filesMayHaveChanged = false; + // A mutating Git command can fail after writing some paths. Only known + // pre-execution failures are safe to retry when no earlier mutation ran. + const mutate = (effect: Effect.Effect) => + Effect.suspend(() => { + const changedBefore = filesMayHaveChanged; + filesMayHaveChanged = true; + return effect.pipe( + Effect.tapError((cause) => + Effect.sync(() => { + if ( + !changedBefore && + (cause._tag === "VcsProcessSpawnError" || + (cause._tag === "VcsProcessExitError" && + cause.detail === VcsProcess.GIT_INDEX_LOCKED_DETAIL)) + ) { + filesMayHaveChanged = false; + } + }), + ), + ); }); - } - // Restoring away the last tracked file can remove a nested workspace directory. - const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( - Effect.mapError( - (cause) => - new VcsProcessExitError({ + return yield* Effect.gen(function* () { + // An empty index and checkpoint have nothing for git restore's pathspec to match. + if (tracked.stdout.length > 0) { + yield* mutate( + execute({ operation, - command: "git restore", cwd: input.cwd, - exitCode: 0, - detail: `Could not recreate the checkpoint workspace: ${cause.message}`, + args: [ + ...longPaths, + "restore", + "--source", + commitOid, + "--worktree", + "--staged", + "--", + ".", + ], }), - ), - ); - const resetIndex = Effect.gen(function* () { - // On Windows clean can remove './' and exit 0. - yield* recreateWorkspace; - yield* execute({ - operation, - cwd: input.cwd, - args: [...longPaths, "reset", "--quiet", "--", "."], + ); + } + // Restoring away the last tracked file can remove a nested workspace directory. + const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( + Effect.mapError( + (cause) => + new VcsProcessExitError({ + operation, + command: "git restore", + cwd: input.cwd, + exitCode: 0, + detail: `Could not recreate the checkpoint workspace: ${cause.message}`, + }), + ), + ); + const resetIndex = Effect.gen(function* () { + // On Windows clean can remove './' and exit 0. + yield* recreateWorkspace; + yield* execute({ + operation, + cwd: input.cwd, + args: [...longPaths, "reset", "--quiet", "--", "."], + }); }); - }); - yield* recreateWorkspace; - const clean = execute({ - operation, - cwd: input.cwd, - args: [...longPaths, "clean", "-fd", "--", "."], - env: { LC_ALL: "C" }, - allowNonZeroExit: true, - }); - let cleaned = yield* clean; - // Removing a new .gitignore exposes the files it hid; clean again until nothing is left. - for ( - let pass = 1; - pass < 4 && cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout); - pass++ - ) { yield* recreateWorkspace; - cleaned = yield* clean; - } - if (cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout)) { - yield* recreateWorkspace; - const remaining = yield* execute({ + const clean = execute({ operation, cwd: input.cwd, - args: [...longPaths, "clean", "-dn", "--", "."], + args: [...longPaths, "clean", "-fd", "--", "."], env: { LC_ALL: "C" }, + allowNonZeroExit: true, }); - if (/^Would remove /m.test(remaining.stdout)) { - yield* resetIndex; - return yield* new VcsProcessExitError({ - operation, - command: "git clean", - cwd: input.cwd, - exitCode: 0, - detail: "Could not clean the checkpoint workspace: untracked files remain.", - }); + let cleaned = yield* mutate(clean); + // Removing a new .gitignore exposes the files it hid; clean again until nothing is left. + for ( + let pass = 1; + pass < 4 && cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout); + pass++ + ) { + yield* recreateWorkspace; + cleaned = yield* mutate(clean); } - } - if (cleaned.exitCode !== 0) { - // Git can remove every child, then fail trying to remove './' itself. - const emptiedWorkspace = - cleaned.exitCode === 1 && - /^warning: failed to remove \.\/: [^\n]+$/.test(cleaned.stderr.trim()) && - (yield* fileSystem.readDirectory(input.cwd).pipe( - Effect.map((entries) => entries.length === 0), - Effect.orElseSucceed(() => false), - )); - if (!emptiedWorkspace) { - yield* resetIndex; - return yield* new VcsProcessExitError({ + if (cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout)) { + yield* recreateWorkspace; + const remaining = yield* execute({ operation, - command: "git clean", cwd: input.cwd, - exitCode: cleaned.exitCode, - detail: cleaned.stderr.trim() || "Could not clean the checkpoint workspace.", + args: [...longPaths, "clean", "-dn", "--", "."], + env: { LC_ALL: "C" }, }); + if (/^Would remove /m.test(remaining.stdout)) { + yield* resetIndex; + return yield* new VcsProcessExitError({ + operation, + command: "git clean", + cwd: input.cwd, + exitCode: 0, + detail: "Could not clean the checkpoint workspace: untracked files remain.", + }); + } } - } - yield* resetIndex; + if (cleaned.exitCode !== 0) { + // Git can remove every child, then fail trying to remove './' itself. + const emptiedWorkspace = + cleaned.exitCode === 1 && + /^warning: failed to remove \.\/: [^\n]+$/.test(cleaned.stderr.trim()) && + (yield* fileSystem.readDirectory(input.cwd).pipe( + Effect.map((entries) => entries.length === 0), + Effect.orElseSucceed(() => false), + )); + if (!emptiedWorkspace) { + yield* resetIndex; + return yield* new VcsProcessExitError({ + operation, + command: "git clean", + cwd: input.cwd, + exitCode: cleaned.exitCode, + detail: cleaned.stderr.trim() || "Could not clean the checkpoint workspace.", + }); + } + } + yield* resetIndex; - return true; + return true; + }).pipe( + Effect.mapError((cause) => + filesMayHaveChanged + ? new VcsDriver.VcsCheckpointPartialRestoreError({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + cause, + }) + : cause, + ), + ); }), diffCheckpoints: Effect.fn("GitVcsDriver.checkpoints.diffCheckpoints")(function* (input) { diff --git a/apps/server/src/vcs/VcsDriver.ts b/apps/server/src/vcs/VcsDriver.ts index b382922c5ab27..3361c3b571b64 100644 --- a/apps/server/src/vcs/VcsDriver.ts +++ b/apps/server/src/vcs/VcsDriver.ts @@ -1,3 +1,4 @@ +import * as Schema from "effect/Schema"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; @@ -11,6 +12,15 @@ import type { import { CheckpointRef } from "@t3tools/contracts"; import * as VcsProcess from "./VcsProcess.ts"; +export class VcsCheckpointPartialRestoreError extends Schema.TaggedError()( + "VcsCheckpointPartialRestoreError", + { cwd: Schema.String, checkpointRef: CheckpointRef, cause: Schema.Defect() }, +) { + override get message(): string { + return "Checkpoint restoration failed after workspace files may have changed."; + } +} + export interface VcsCaptureCheckpointInput { readonly cwd: string; readonly checkpointRef: CheckpointRef; @@ -55,7 +65,7 @@ export interface VcsCheckpointOps { ) => Effect.Effect; readonly restoreCheckpoint: ( input: VcsRestoreCheckpointInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly diffCheckpoints: (input: VcsDiffCheckpointsInput) => Effect.Effect; readonly listAuthoredPaths: ( input: VcsListAuthoredPathsInput,