diff --git a/apps/server/src/checkpointing/CheckpointStore.ts b/apps/server/src/checkpointing/CheckpointStore.ts index e5d3006568a67..22380e784dc69 100644 --- a/apps/server/src/checkpointing/CheckpointStore.ts +++ b/apps/server/src/checkpointing/CheckpointStore.ts @@ -4,7 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import type { CheckpointStoreError } from "./Errors.ts"; -import type { VcsCheckpointOps } from "../vcs/VcsDriver.ts"; +import type { VcsCheckpointOps, VcsCheckpointPartialRestoreError } from "../vcs/VcsDriver.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; export interface CaptureCheckpointInput { @@ -57,7 +57,7 @@ export class CheckpointStore extends Context.Service< readonly restoreCheckpoint: ( input: RestoreCheckpointInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly diffCheckpoints: ( input: DiffCheckpointsInput, diff --git a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts index 145de122f387a..6f3fa130728af 100644 --- a/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts @@ -26,6 +26,7 @@ import { ProjectionStoreV2, type ProjectionCheckpointRollbackContext } from "./P import * as ProjectStore from "./ProjectStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; it.effect.each([ "nested", @@ -95,6 +96,7 @@ it.effect.each([ Layer.provide( Layer.mergeAll( NodeServices.layer, + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(ServerConfig.ServerConfig)({ worktreesDir: "/synthetic-worktrees" } as never), ServerSettings.layerTest(), ThreadCommandExecutor.layer, @@ -171,12 +173,13 @@ it.effect.each([ const error = yield* service .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }) .pipe(Effect.flip); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "shared-workspace"); assert.deepEqual(calls, []); assert.equal(yield* fs.readFileString(otherFile), "other thread's uncommitted work"); } else { yield* service.execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles }); - assert.deepEqual(calls, restoreFiles ? ["provider", "files"] : ["provider"]); + assert.deepEqual(calls, restoreFiles ? ["files", "provider"] : ["provider"]); assert.equal(yield* fs.exists(otherFile), !restoreFiles); } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index c61f9bc175edc..ab20c4cd1977e 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -7,6 +7,7 @@ import { MessageId, NodeId, type OrchestrationV2DomainEvent, + type OrchestrationV2ThreadProjection, ProviderDriverKind, ProviderTurnId, RunAttemptId, @@ -29,6 +30,8 @@ import * as Option from "effect/Option"; import * as Layer from "effect/Layer"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as ServerConfig from "../config.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as GitWorkflow from "../git/GitWorkflowService.ts"; @@ -53,6 +56,7 @@ const layerCheckpointRollbackService = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( NodeServices.layer, + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(ServerConfig.ServerConfig)({ worktreesDir: "/synthetic-worktrees" } as never), ServerSettings.layerTest(), Layer.mock(GitWorkflow.GitWorkflowService)({}), @@ -157,6 +161,7 @@ it.effect.each(["initial", "waiting"])( }, ], ]); + assert(error._tag === "CheckpointRollbackExecutionError"); assert.equal(error.reason, "rollback-target-invalid"); assert.equal( error.message, @@ -224,6 +229,7 @@ it.effect.each([ Layer.mergeAll( ThreadCommandExecutor.layer, NodeServices.layer, + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(ServerConfig.ServerConfig)({ worktreesDir } as never), ServerSettings.layerTest(), Layer.mock(ProjectStore.ProjectStoreV2)({ @@ -731,3 +737,194 @@ it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMe ); }, ); + +it.effect.each(["success", "restore-fails"] as const)( + "restores files and refreshes the workspace index before the provider: %s", + (outcome) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-index-" }); + const currentFile = path.join(cwd, "rollback-current.txt"); + const restoredFile = path.join(cwd, "rollback-restored.txt"); + yield* fs.writeFileString(currentFile, "current turn"); + const search = () => workspaceEntries.search({ cwd, query: "rollback", limit: 10 }); + assert.deepEqual((yield* search()).entries, [{ path: "rollback-current.txt", kind: "file" }]); + + const threadId = ThreadId.make("rollback-restore-state"); + const providerThreadId = ProviderThreadId.make("rollback-restore-provider"); + const providerSessionId = ProviderSessionId.make("rollback-restore-session"); + const instanceId = ProviderInstanceId.make("rollback-restore-instance"); + const checkpointId = CheckpointId.make("rollback-restore-checkpoint"); + const scopeId = CheckpointScopeId.make("rollback-restore-scope"); + const laterCheckpointId = CheckpointId.make("rollback-later-checkpoint"); + const providerThread = { + id: providerThreadId, + providerSessionId, + providerInstanceId: instanceId, + }; + const projection = { + thread: { + worktreePath: cwd, + activeProviderThreadId: providerThreadId, + modelSelection: { instanceId, model: "test" }, + }, + providerThreads: [providerThread], + providerSessions: [], + providerTurns: [ + { + id: "turn-1", + providerThreadId, + runAttemptId: "attempt-1", + ordinal: 1, + status: "completed", + }, + ], + nodes: [], + attempts: [{ id: "attempt-1", runId: "run-1" }], + checkpoints: [ + { id: checkpointId, scopeId, status: "ready", appRunOrdinal: null }, + { id: laterCheckpointId, scopeId, status: "ready", appRunOrdinal: 1 }, + ], + checkpointScopes: [{ id: scopeId, cwd }], + runs: [{ id: "run-1", ordinal: 1, status: "completed", rootNodeId: null }], + } as unknown as OrchestrationV2ThreadProjection; + const restoreFailure = new CheckpointService.CheckpointRestoreError({ + scopeId, + checkpointId, + cause: new Error("git restore failed"), + }); + const providerConversation = ["turn-1"]; + let providerCalls = 0; + const deletedRefs: CheckpointId[] = []; + const persistedEvents: string[] = []; + const testLayer = CheckpointRollbackService.layer.pipe( + Layer.provide( + Layer.mergeAll( + IdAllocator.layer, + Layer.mock(GitWorkflow.GitWorkflowService)({}), + Layer.mock(ServerConfig.ServerConfig)({ + worktreesDir: "/synthetic-worktrees", + } as never), + ServerSettings.layerTest(), + ThreadCommandExecutor.layer, + Layer.succeed(WorkspaceEntries.WorkspaceEntries, workspaceEntries), + Layer.mock(ProjectStore.ProjectStoreV2)({ + get: () => Effect.succeed(unrelatedProject), + }), + Layer.mock(CheckpointService.CheckpointServiceV2)({ + restore: () => + outcome === "restore-fails" + ? Effect.fail(restoreFailure) + : Effect.gen(function* () { + yield* fs.remove(currentFile); + yield* fs.writeFileString(restoredFile, "checkpoint turn"); + }).pipe(Effect.orDie), + deleteStaleRefs: ({ checkpoints }) => + Effect.sync(() => { + deletedRefs.push(...checkpoints.map((checkpoint) => checkpoint.id)); + }), + }), + Layer.mock(EventSink.EventSinkV2)({ + write: ({ events }) => + Effect.sync(() => { + persistedEvents.push(...events.map((event) => event.type)); + return []; + }), + }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThread: () => Effect.succeed(projection.thread), + getCheckpointRollbackContext: () => + Effect.succeed({ + thread: projection.thread, + providerThread, + providerSession: undefined, + checkpoint: projection.checkpoints[0], + scope: projection.checkpointScopes[0], + targetAttempt: undefined, + runsToRollback: projection.runs, + staleCheckpoints: [projection.checkpoints[1]], + rootNodes: [], + rolledBackAttemptIds: [], + providerTurns: projection.providerTurns, + } as never), + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 1, + snapshotSequence: 0, + threads: [], + archivedThreads: [], + }), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + open: () => + Effect.succeed({ + rollbackThread: (input: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => + Effect.gen(function* () { + providerCalls += 1; + const count = yield* resolveCodexRollbackTurnCount(input); + assert.equal(count, 1); + assert.equal(yield* fs.exists(currentFile).pipe(Effect.orDie), false); + assert.equal( + yield* fs.readFileString(restoredFile).pipe(Effect.orDie), + "checkpoint turn", + ); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-restored.txt", kind: "file" }, + ]); + providerConversation.splice(providerConversation.length - count, count); + return { providerThread }; + }), + } as never), + }), + Layer.mock(RuntimePolicy.RuntimePolicyV2)({ + resolve: () => Effect.succeed({} as never), + }), + ), + ), + ); + const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2.pipe( + Effect.provide(testLayer), + ); + const rollback = service.execute({ + threadId, + providerThreadId, + checkpointId, + scopeId, + }); + if (outcome === "restore-fails") { + const error = yield* Effect.flip(rollback); + assert.equal(error.cause, restoreFailure); + assert.equal(providerCalls, 0); + assert.deepEqual(providerConversation, ["turn-1"]); + assert.deepEqual(persistedEvents, []); + assert.deepEqual(deletedRefs, []); + assert.equal(yield* fs.readFileString(currentFile), "current turn"); + assert.equal(yield* fs.exists(restoredFile), false); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-current.txt", kind: "file" }, + ]); + } else { + yield* rollback; + assert.equal(providerCalls, 1); + assert.deepEqual(providerConversation, []); + assert.include(persistedEvents, "run.updated"); + assert.deepEqual(deletedRefs, [laterCheckpointId]); + assert.equal(yield* fs.exists(currentFile), false); + assert.equal(yield* fs.readFileString(restoredFile), "checkpoint turn"); + assert.deepEqual((yield* search()).entries, [ + { path: "rollback-restored.txt", kind: "file" }, + ]); + } + }).pipe( + Effect.scoped, + Effect.provide( + WorkspaceEntries.layer.pipe( + Layer.provide(WorkspacePaths.layer), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-rollback-config-" })), + Layer.provideMerge(NodeServices.layer), + ), + ), + ), +); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 4d40641fa54db..13ac980d64034 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -16,6 +16,8 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import { VcsCheckpointPartialRestoreError } from "../vcs/VcsDriver.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import type * as ServerConfig from "../config.ts"; import type * as ServerSettings from "../serverSettings.ts"; @@ -71,17 +73,82 @@ export class CheckpointRollbackExecutionError extends Schema.TaggedError()( + "CheckpointRollbackPartialRestoreError", + { + threadId: ThreadId, + providerThreadId: ProviderThreadId, + checkpointId: CheckpointId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying."; + } +} + +export class CheckpointRollbackIncompleteRestoreError extends Schema.TaggedError()( + "CheckpointRollbackIncompleteRestoreError", + { + threadId: ThreadId, + providerThreadId: ProviderThreadId, + checkpointId: CheckpointId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "File restoration failed after workspace files may have changed. The conversation rollback was not attempted. Check your files and the server logs before retrying."; + } +} + +export class CheckpointRollbackIndexRefreshError extends Schema.TaggedError()( + "CheckpointRollbackIndexRefreshError", + { + threadId: ThreadId, + providerThreadId: ProviderThreadId, + checkpointId: CheckpointId, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying."; + } +} + +export const isCheckpointRollbackPostRestoreError = Schema.is( + Schema.Union([ + CheckpointRollbackPartialRestoreError, + CheckpointRollbackIndexRefreshError, + CheckpointRollbackIncompleteRestoreError, + ]), +); + +const isCheckpointRollbackError = Schema.is( + Schema.Union([ + CheckpointRollbackExecutionError, + CheckpointRollbackPartialRestoreError, + CheckpointRollbackIndexRefreshError, + CheckpointRollbackIncompleteRestoreError, + ]), +); + +const isVcsCheckpointPartialRestoreError = Schema.is(VcsCheckpointPartialRestoreError); const isVcsProcessExitError = Schema.is(VcsProcessExitError); const isCheckpointRestoreError = Schema.is(CheckpointRestoreError); const isRuntimePolicyAboveCeilingError = Schema.is(RuntimePolicyAboveCeilingError); /** - * The message clients see when a rollback fails. Only a held Git index lock - * gets its own explanation; other reasons carry internal ids or raw Git - * output, so they fall back to the generic message. + * Post-restore failures and a held Git index lock have safe, specific messages. + * Other failures may carry internal IDs or raw Git output. */ -export function rollbackFailureClientMessage(error: CheckpointRollbackExecutionError): string { +export function rollbackFailureClientMessage( + error: + | CheckpointRollbackExecutionError + | CheckpointRollbackPartialRestoreError + | CheckpointRollbackIndexRefreshError + | CheckpointRollbackIncompleteRestoreError, +): string { + if (isCheckpointRollbackPostRestoreError(error)) return error.message; if (isRuntimePolicyAboveCeilingError(error.cause)) return error.cause.message; const restoreCause = isCheckpointRestoreError(error.cause) ? error.cause.cause : undefined; return isVcsProcessExitError(restoreCause) && restoreCause.detail === GIT_INDEX_LOCKED_DETAIL @@ -96,7 +163,13 @@ export interface CheckpointRollbackServiceV2Shape { readonly checkpointId: CheckpointId; readonly scopeId: CheckpointScopeId; readonly restoreFiles?: boolean; - }) => Effect.Effect; + }) => Effect.Effect< + void, + | CheckpointRollbackExecutionError + | CheckpointRollbackPartialRestoreError + | CheckpointRollbackIndexRefreshError + | CheckpointRollbackIncompleteRestoreError + >; } export class CheckpointRollbackServiceV2 extends Context.Service< @@ -108,6 +181,7 @@ export const layer: Layer.Layer< CheckpointRollbackServiceV2, never, | CheckpointServiceV2 + | WorkspaceEntries.WorkspaceEntries | GitWorkflowService.GitWorkflowService | EventSinkV2 | IdAllocator.IdAllocatorV2 @@ -124,6 +198,7 @@ export const layer: Layer.Layer< CheckpointRollbackServiceV2, Effect.gen(function* () { const checkpoints = yield* CheckpointServiceV2; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const gitWorkflow = yield* GitWorkflowService.GitWorkflowService; const eventSink = yield* EventSinkV2; const ids = yield* IdAllocator.IdAllocatorV2; @@ -217,6 +292,26 @@ export const layer: Layer.Layer< }; }); + const validateRestoreIsolation = Effect.gen(function* () { + if ( + input.restoreFiles !== false && + !(yield* isCheckpointRestoreIsolated(context.thread, scope, { + fileSystem, + projections, + projects, + path, + })) + ) { + return yield* new CheckpointRollbackExecutionError({ + reason: "shared-workspace", + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + }); + } + }); + yield* validateRestoreIsolation; + const { worktreePath, branch } = context.thread; // Recreated only where the server would have made it, as turn start does. if ( @@ -233,26 +328,10 @@ export const layer: Layer.Layer< refName: branch, path: worktreePath, }); + yield* validateRestoreIsolation; } } - if ( - input.restoreFiles !== false && - !(yield* isCheckpointRestoreIsolated(context.thread, scope, { - fileSystem, - projections, - projects, - path, - })) - ) { - return yield* new CheckpointRollbackExecutionError({ - reason: "shared-workspace", - threadId: input.threadId, - providerThreadId: input.providerThreadId, - checkpointId: input.checkpointId, - }); - } - const modelSelection = context.thread.modelSelection; const resolvedRuntimePolicy = yield* runtimePolicy.resolve({ thread: context.thread, @@ -283,15 +362,52 @@ export const layer: Layer.Layer< (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)), ); + if (input.restoreFiles !== false) { + yield* checkpoints.restore({ scope, checkpoint }).pipe( + Effect.catchTags({ + CheckpointRestoreError: (cause) => + Effect.fail( + isVcsCheckpointPartialRestoreError(cause.cause) + ? new CheckpointRollbackIncompleteRestoreError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }) + : cause, + ), + }), + ); + yield* workspaceEntries.refresh(scope.cwd).pipe( + Effect.catchDefect((cause) => + Effect.fail( + new CheckpointRollbackIndexRefreshError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), + ), + ), + ); + } const snapshot = runsToRollback.length === 0 ? { providerThread } - : yield* session.rollbackThread({ - providerThread, - target: rollbackTarget, - providerThreadTurns, - }); - if (input.restoreFiles !== false) yield* checkpoints.restore({ scope, checkpoint }); + : yield* session + .rollbackThread({ providerThread, target: rollbackTarget, providerThreadTurns }) + .pipe( + Effect.mapError((cause) => + input.restoreFiles === false + ? cause + : new CheckpointRollbackPartialRestoreError({ + threadId: input.threadId, + providerThreadId: input.providerThreadId, + checkpointId: input.checkpointId, + cause, + }), + ), + ); if (staleCheckpoints.length > 0) { yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }); } @@ -383,7 +499,7 @@ export const layer: Layer.Layer< (effect, input) => effect.pipe( Effect.mapError((cause) => - isCheckpointRollbackExecutionError(cause) + isCheckpointRollbackError(cause) ? cause : new CheckpointRollbackExecutionError({ reason: "unexpected-failure", diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 3ec8193d3ef7b..3c436b58152d8 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -466,29 +466,36 @@ export const layerExecutor: Layer.Layer< : { restoreFiles: effect.request.restoreFiles }), }) .pipe( - Effect.tapCause((cause) => - willRetry || Cause.hasInterruptsOnly(cause) - ? Effect.void - : threads - .dispatch({ - type: "checkpoint.rollback.fail", - commandId: CommandId.make(`${effect.commandId}:rollback-failed`), - threadId: effect.threadId, - requestId: effect.commandId, - message: Option.match(Cause.findErrorOption(cause), { - onNone: () => CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE, - onSome: CheckpointRollbackService.rollbackFailureClientMessage, - }), - }) - .pipe( - Effect.catchCause((recordCause) => - Effect.logWarning("Failed to record rollback failure", { - effectId: effect.id, - cause: recordCause, - }), - ), - ), - ), + // Retrying a completed restore could overwrite subsequent edits. + Effect.tapCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.void; + const failure = Option.getOrUndefined(Cause.findErrorOption(cause)); + const postRestoreFailure = + CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure) + ? failure + : undefined; + if (willRetry && postRestoreFailure === undefined) return Effect.void; + return threads + .dispatch({ + type: "checkpoint.rollback.fail", + commandId: CommandId.make(`${effect.commandId}:rollback-failed`), + threadId: effect.threadId, + requestId: effect.commandId, + message: + postRestoreFailure?.message ?? + (failure === undefined + ? CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE + : CheckpointRollbackService.rollbackFailureClientMessage(failure)), + }) + .pipe( + Effect.catchCause((recordCause) => + Effect.logWarning("Failed to record rollback failure", { + effectId: effect.id, + cause: recordCause, + }), + ), + ); + }), Effect.mapError( (cause) => new OrchestrationEffectExecutionError({ @@ -830,6 +837,11 @@ export const layerWithOptions = ( const error = Cause.pretty(exit.cause); const nonRetryable = isNonRetryableProviderTurnControlFailure(effect.request.type, error); + const failure = Cause.findErrorOption(exit.cause); + const postRestoreFailure = + effect.request.type === "provider-thread.rollback" && + Option.isSome(failure) && + CheckpointRollbackService.isCheckpointRollbackPostRestoreError(failure.value.cause); yield* Effect.logWarning("Orchestration effect execution failed", { effectId: effect.id, effectType: effect.request.type, @@ -841,7 +853,7 @@ export const layerWithOptions = ( ? yield* outbox .succeed({ effectId: effect.id, workerId }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) - : effect.attemptCount >= maxAttempts + : postRestoreFailure || effect.attemptCount >= maxAttempts ? yield* outbox .fail({ effectId: effect.id, workerId, error }) .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause))) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index a56616340050c..a0957b1ba30a1 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -63,6 +63,7 @@ import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts"; import * as Orchestrator from "./Orchestrator.ts"; import { ROLLBACK_FAILED_MESSAGE } from "./CheckpointRollbackService.ts"; @@ -93,7 +94,8 @@ import * as TerminalManager from "../terminal/Manager.ts"; import * as ResourceCleanupService from "./ResourceCleanupService.ts"; import * as ThreadLaunchService from "./ThreadLaunchService.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { VcsCheckpointPartialRestoreError } from "../vcs/VcsDriver.ts"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; const layerPlatformTest = Layer.merge( Layer.mergeAll(NodeServices.layer, NodeCrypto.layer), @@ -318,11 +320,12 @@ const layerTest = Layer.mergeAll( ProjectionStore.layer, EffectOutbox.layer, ThreadCommandExecutor.layer, + WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)), ).pipe( Layer.provide(McpSessionRegistryTestkit.layer), Layer.provide(McpProviderSessions.layer), Layer.provide(SqlitePersistence.layerMemory), - Layer.provide(layerCheckpointStoreTest), + Layer.provideMerge(layerCheckpointStoreTest), Layer.provide(layerServerConfig), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerTestProviderInstanceRegistry), @@ -906,7 +909,7 @@ const layerSharedApplicationDataPlaneTest = Layer.mergeAll( ); it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { - const seedFailingRollbackThread = (name: string) => + const seedFailingRollbackThread = (name: string, cwd?: string) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const eventSink = yield* EventSink.EventSinkV2; @@ -924,7 +927,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { runtimeMode: "full-access", interactionMode: "default", branch: null, - worktreePath: `/tmp/t3-${name}`, + worktreePath: cwd ?? `/tmp/t3-${name}`, }); yield* orchestrator.dispatch({ type: "message.dispatch", @@ -972,14 +975,15 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }); return { threadId, - rollback: (commandId: CommandId) => + checkpointId, + rollback: (commandId: CommandId, restoreFiles = false) => orchestrator.dispatch({ type: "checkpoint.rollback", commandId, threadId, checkpointId, scopeId: scope.id, - restoreFiles: false, + restoreFiles, }), }; }); @@ -1000,6 +1004,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { const [rollbackEffect] = yield* outbox.listByCommandId(rollbackCommandId); assert.equal(rollbackEffect?.status, "failed"); + assert.equal(rollbackEffect?.attemptCount, 5); const failed = yield* orchestrator.getThreadProjection(threadId); assert.deepEqual(failed.thread.rollbackFailure, { requestId: rollbackCommandId, @@ -1012,6 +1017,198 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); + it.effect.each([ + "partial-restore", + "refresh-defect", + "provider-only", + "restore-fails", + "incomplete-restore", + ] as const)("settles checkpoint rollback safely for %s", (outcome) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const postRestoreFailure = + outcome === "partial-restore" || + outcome === "refresh-defect" || + outcome === "incomplete-restore"; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-partial-" }); + const file = path.join(cwd, "workspace.txt"); + yield* fs.writeFileString(file, "current uncheckpointed contents"); + const { threadId, checkpointId, rollback } = yield* seedFailingRollbackThread( + `runtime-rollback-${outcome}`, + cwd, + ); + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const providerThread = before.providerThreads[0]!; + const now = yield* DateTime.now; + const laterCheckpoint = { + ...before.checkpoints[0]!, + id: CheckpointId.make("runtime-rollback-later-checkpoint"), + ref: CheckpointRef.make("refs/t3/runtime-rollback-later"), + appRunOrdinal: run.ordinal, + ordinalWithinScope: 1, + }; + yield* eventSink.write({ + events: [ + { + id: EventId.make("runtime-rollback-later-captured"), + type: "checkpoint.captured", + threadId, + occurredAt: now, + payload: laterCheckpoint, + }, + { + id: EventId.make("runtime-rollback-partial-completed"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", completedAt: now }, + }, + ], + }); + let restored = 0; + let rejected = 0; + const restoreSpy = vi + .spyOn(checkpointStore, "restoreCheckpoint") + .mockImplementation((input) => + Effect.gen(function* () { + assert.equal(path.resolve(input.cwd), path.resolve(cwd)); + restored += 1; + if (outcome === "restore-fails") return false; + yield* fs.writeFileString(file, "checkpoint contents").pipe(Effect.orDie); + if (outcome === "incomplete-restore") { + return yield* new VcsCheckpointPartialRestoreError({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + cause: new Error("cleanup failed after restoring files"), + }); + } + return true; + }), + ); + const deleteSpy = vi.spyOn(checkpointStore, "deleteCheckpointRefs"); + const refreshSpy = vi.spyOn(workspaceEntries, "refresh"); + if (outcome === "refresh-defect") { + refreshSpy.mockImplementation(() => Effect.die(new Error("index refresh defect"))); + } + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + rollbackThread: () => + Effect.gen(function* () { + rejected += 1; + assert.equal( + yield* fs.readFileString(file).pipe(Effect.orDie), + outcome === "provider-only" + ? "current uncheckpointed contents" + : "checkpoint contents", + ); + return yield* new ProviderAdapter.ProviderAdapterRollbackThreadError({ + driver, + providerThreadId: providerThread.id, + cause: new Error("legacy history cannot be reverted"), + }); + }), + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + restoreSpy.mockRestore(); + deleteSpy.mockRestore(); + refreshSpy.mockRestore(); + sessionSpy.mockRestore(); + }), + ); + const commandId = CommandId.make("runtime-rollback-partial-request"); + yield* rollback(commandId, outcome !== "provider-only"); + yield* worker.drain(); + const [firstAttempt] = yield* outbox.listByCommandId(commandId); + assert.equal(firstAttempt?.attemptCount, 1); + assert.equal(firstAttempt?.status, postRestoreFailure ? "failed" : "pending"); + if (!postRestoreFailure) { + assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); + assert.equal(yield* fs.readFileString(file), "current uncheckpointed contents"); + for (let attempt = 1; attempt < 5; attempt++) { + yield* TestClock.adjust("30 seconds"); + yield* worker.drain(); + } + } + const [effect] = yield* outbox.listByCommandId(commandId); + assert.equal(effect?.status, "failed"); + const expectedAttempts = postRestoreFailure ? 1 : 5; + const expectedProviderCalls = + outcome === "restore-fails" || + outcome === "refresh-defect" || + outcome === "incomplete-restore" + ? 0 + : expectedAttempts; + assert.equal(effect?.attemptCount, expectedAttempts); + assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); + assert.equal(rejected, expectedProviderCalls); + assert.equal( + yield* fs.readFileString(file), + postRestoreFailure ? "checkpoint contents" : "current uncheckpointed contents", + ); + const failed = yield* orchestrator.getThreadProjection(threadId); + assert.equal(failed.thread.rollbackFailure?.requestId, commandId); + assert.equal( + failed.thread.rollbackFailure?.message, + outcome === "partial-restore" + ? "Files were restored to the selected checkpoint, but the conversation rollback failed. Conversation history may not match your files. Check the provider and server logs before retrying." + : outcome === "refresh-defect" + ? "Files were restored to the selected checkpoint, but the workspace index could not be refreshed. The conversation rollback was not attempted. Conversation history may not match your files. Check the server logs before retrying." + : outcome === "incomplete-restore" + ? "File restoration failed after workspace files may have changed. The conversation rollback was not attempted. Check your files and the server logs before retrying." + : ROLLBACK_FAILED_MESSAGE, + ); + assert.equal(failed.runs.find((candidate) => candidate.id === run.id)?.status, "completed"); + assert.equal( + failed.checkpoints.find((candidate) => candidate.id === checkpointId)?.status, + "ready", + ); + assert.deepEqual( + failed.checkpoints.find((candidate) => candidate.id === laterCheckpoint.id), + laterCheckpoint, + ); + assert.deepEqual(failed.nodes, before.nodes); + assert.deepEqual(failed.providerThreads, before.providerThreads); + assert.equal(deleteSpy.mock.calls.length, 0); + yield* fs.writeFileString(file, "user edits after rollback failure"); + yield* TestClock.adjust("30 seconds"); + assert.equal(yield* worker.drain(), 0); + assert.equal(yield* fs.readFileString(file), "user edits after rollback failure"); + assert.equal(restored, outcome === "provider-only" ? 0 : expectedAttempts); + assert.equal(rejected, expectedProviderCalls); + if (postRestoreFailure) { + const nextCommandId = CommandId.make("runtime-rollback-partial-next"); + yield* rollback(nextCommandId, true); + assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure); + yield* worker.drain(); + const [nextEffect] = yield* outbox.listByCommandId(nextCommandId); + assert.equal(nextEffect?.status, "failed"); + assert.equal(nextEffect?.attemptCount, 1); + assert.equal(restored, 2); + assert.equal( + rejected, + outcome === "refresh-defect" || outcome === "incomplete-restore" ? 0 : 2, + ); + assert.equal(yield* fs.readFileString(file), "checkpoint contents"); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).thread.rollbackFailure?.requestId, + nextCommandId, + ); + } + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), + ); + it.effect("captures and repeatedly rewinds non-Git conversations without losing run links", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 8434de806bf0d..74b7536d49e40 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -17,6 +17,8 @@ import * as AgentSessionScanner from "../project/AgentSessionScanner.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as ScratchThreadPromotion from "../project/ScratchThreadPromotion.ts"; import * as CheckpointCaptureService from "./CheckpointCaptureService.ts"; import * as CheckpointService from "./CheckpointService.ts"; @@ -187,6 +189,7 @@ const layerRuntimeRequestServiceProvided = RuntimeRequestService.layer.pipe( const layerCheckpointRollbackServiceProvided = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( + WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)), ProjectStore.layer, layerCheckpointServiceProvided, layerEventSink, diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 363164b688043..5f930c5d44943 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -31,6 +31,7 @@ import * as VcsProcess from "../../vcs/VcsProcess.ts"; import * as CheckpointCaptureService from "../CheckpointCaptureService.ts"; import * as CheckpointService from "../CheckpointService.ts"; import * as CheckpointRollbackService from "../CheckpointRollbackService.ts"; +import * as WorkspaceEntries from "../../workspace/WorkspaceEntries.ts"; import * as CommandPolicy from "../CommandPolicy.ts"; import * as CommandReceiptStore from "../CommandReceiptStore.ts"; import * as ContextHandoffService from "../ContextHandoffService.ts"; @@ -441,6 +442,7 @@ export function layerWithRegistry( const layerCheckpointRollbackServiceProvided = CheckpointRollbackService.layer.pipe( Layer.provide( Layer.mergeAll( + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), layerCheckpointServiceProvided, layerEventSinkProvided, IdAllocator.layer, diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 128c8afcd3b4c..20770148406e8 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -1186,7 +1186,10 @@ it.effect("restores empty checkpoints without changing paths outside the workspa const incomplete = yield* driver.checkpoints .restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }) .pipe(Effect.flip); - assert.strictEqual(incomplete._tag, "VcsProcessExitError"); + assert.strictEqual(incomplete._tag, "VcsCheckpointPartialRestoreError"); + if (incomplete._tag === "VcsCheckpointPartialRestoreError") { + assert.instanceOf(incomplete.cause, VcsProcessExitError); + } assert.strictEqual( yield* fileSystem.readFileString(path.join(cwd, "checkpoint.txt")), "checkpoint\n", @@ -1213,6 +1216,67 @@ it.effect("restores empty checkpoints without changing paths outside the workspa }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("reports cleanup failure after restoring checkpoint files as a partial restore", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-partial-restore-" }); + yield* runGit(cwd, ["init"]); + yield* runGit(cwd, ["config", "user.email", "test@test.com"]); + yield* runGit(cwd, ["config", "user.name", "Test"]); + yield* runGit(cwd, ["commit", "--allow-empty", "-m", "initial"]); + const file = path.join(cwd, "file.txt"); + yield* fileSystem.writeFileString(file, "checkpoint"); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/partial"); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(file, "later edits"); + let deepest = cwd; + for (let depth = 0; depth < 5; depth++) { + yield* fileSystem.writeFileString(path.join(deepest, ".gitignore"), "d/.gitignore\n"); + deepest = path.join(deepest, "d"); + yield* fileSystem.makeDirectory(deepest); + } + yield* fileSystem.writeFileString(path.join(deepest, ".gitignore"), "hidden.txt\n"); + yield* fileSystem.writeFileString(path.join(deepest, "hidden.txt"), "untracked"); + const error = yield* driver.checkpoints + .restoreCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.flip); + assert.equal(error._tag, "VcsCheckpointPartialRestoreError"); + if (error._tag === "VcsCheckpointPartialRestoreError") + assert.instanceOf(error.cause, VcsProcessExitError); + assert.equal(yield* fileSystem.readFileString(file), "checkpoint"); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("keeps a restore blocked by the Git index lock retryable", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-restore-locked-" }); + yield* runGit(cwd, ["init"]); + yield* runGit(cwd, ["config", "user.email", "test@test.com"]); + yield* runGit(cwd, ["config", "user.name", "Test"]); + const file = path.join(cwd, "file.txt"); + yield* fileSystem.writeFileString(file, "checkpoint"); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/locked"); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(file, "later edits"); + yield* fileSystem.writeFileString(path.join(cwd, ".git", "index.lock"), ""); + const error = yield* driver.checkpoints + .restoreCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.flip); + assert.equal(error._tag, "VcsProcessExitError"); + if (error._tag === "VcsProcessExitError") + assert.equal(error.detail, VcsProcess.GIT_INDEX_LOCKED_DETAIL); + assert.equal(yield* fileSystem.readFileString(file), "later edits"); + yield* fileSystem.remove(path.join(cwd, ".git", "index.lock")); + assert.isTrue(yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef })); + assert.equal(yield* fileSystem.readFileString(file), "checkpoint"); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("restores a checkpoint before the first commit without staging its files", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index a5dd821d4a8b4..ff3a714be4e75 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -12,6 +12,7 @@ import { ChildProcessSpawner } from "effect/process"; import { GitCommandError, + type VcsError, VcsProcessExitError, VcsUnsupportedOperationError, type VcsSwitchRefInput, @@ -1393,105 +1394,141 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( cwd: input.cwd, args: ["ls-files", "--cached", `--with-tree=${commitOid}`, "-z", "--", "."], }); - // An empty index and checkpoint have nothing for git restore's pathspec to match. - if (tracked.stdout.length > 0) { - yield* execute({ - operation, - cwd: input.cwd, - args: [ - ...longPaths, - "restore", - "--source", - commitOid, - "--worktree", - "--staged", - "--", - ".", - ], + let filesMayHaveChanged = false; + // A mutating Git command can fail after writing some paths. Only known + // pre-execution failures are safe to retry when no earlier mutation ran. + const mutate = (effect: Effect.Effect) => + Effect.suspend(() => { + const changedBefore = filesMayHaveChanged; + filesMayHaveChanged = true; + return effect.pipe( + Effect.tapError((cause) => + Effect.sync(() => { + if ( + !changedBefore && + (cause._tag === "VcsProcessSpawnError" || + (cause._tag === "VcsProcessExitError" && + cause.detail === VcsProcess.GIT_INDEX_LOCKED_DETAIL)) + ) { + filesMayHaveChanged = false; + } + }), + ), + ); }); - } - // Restoring away the last tracked file can remove a nested workspace directory. - const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( - Effect.mapError( - (cause) => - new VcsProcessExitError({ + return yield* Effect.gen(function* () { + // An empty index and checkpoint have nothing for git restore's pathspec to match. + if (tracked.stdout.length > 0) { + yield* mutate( + execute({ operation, - command: "git restore", cwd: input.cwd, - exitCode: 0, - detail: `Could not recreate the checkpoint workspace: ${cause.message}`, + args: [ + ...longPaths, + "restore", + "--source", + commitOid, + "--worktree", + "--staged", + "--", + ".", + ], }), - ), - ); - const resetIndex = Effect.gen(function* () { - // On Windows clean can remove './' and exit 0. - yield* recreateWorkspace; - yield* execute({ - operation, - cwd: input.cwd, - args: [...longPaths, "reset", "--quiet", "--", "."], + ); + } + // Restoring away the last tracked file can remove a nested workspace directory. + const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( + Effect.mapError( + (cause) => + new VcsProcessExitError({ + operation, + command: "git restore", + cwd: input.cwd, + exitCode: 0, + detail: `Could not recreate the checkpoint workspace: ${cause.message}`, + }), + ), + ); + const resetIndex = Effect.gen(function* () { + // On Windows clean can remove './' and exit 0. + yield* recreateWorkspace; + yield* execute({ + operation, + cwd: input.cwd, + args: [...longPaths, "reset", "--quiet", "--", "."], + }); }); - }); - yield* recreateWorkspace; - const clean = execute({ - operation, - cwd: input.cwd, - args: [...longPaths, "clean", "-fd", "--", "."], - env: { LC_ALL: "C" }, - allowNonZeroExit: true, - }); - let cleaned = yield* clean; - // Removing a new .gitignore exposes the files it hid; clean again until nothing is left. - for ( - let pass = 1; - pass < 4 && cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout); - pass++ - ) { yield* recreateWorkspace; - cleaned = yield* clean; - } - if (cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout)) { - yield* recreateWorkspace; - const remaining = yield* execute({ + const clean = execute({ operation, cwd: input.cwd, - args: [...longPaths, "clean", "-dn", "--", "."], + args: [...longPaths, "clean", "-fd", "--", "."], env: { LC_ALL: "C" }, + allowNonZeroExit: true, }); - if (/^Would remove /m.test(remaining.stdout)) { - yield* resetIndex; - return yield* new VcsProcessExitError({ - operation, - command: "git clean", - cwd: input.cwd, - exitCode: 0, - detail: "Could not clean the checkpoint workspace: untracked files remain.", - }); + let cleaned = yield* mutate(clean); + // Removing a new .gitignore exposes the files it hid; clean again until nothing is left. + for ( + let pass = 1; + pass < 4 && cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout); + pass++ + ) { + yield* recreateWorkspace; + cleaned = yield* mutate(clean); } - } - if (cleaned.exitCode !== 0) { - // Git can remove every child, then fail trying to remove './' itself. - const emptiedWorkspace = - cleaned.exitCode === 1 && - /^warning: failed to remove \.\/: [^\n]+$/.test(cleaned.stderr.trim()) && - (yield* fileSystem.readDirectory(input.cwd).pipe( - Effect.map((entries) => entries.length === 0), - Effect.orElseSucceed(() => false), - )); - if (!emptiedWorkspace) { - yield* resetIndex; - return yield* new VcsProcessExitError({ + if (cleaned.exitCode === 0 && /^Removing /m.test(cleaned.stdout)) { + yield* recreateWorkspace; + const remaining = yield* execute({ operation, - command: "git clean", cwd: input.cwd, - exitCode: cleaned.exitCode, - detail: cleaned.stderr.trim() || "Could not clean the checkpoint workspace.", + args: [...longPaths, "clean", "-dn", "--", "."], + env: { LC_ALL: "C" }, }); + if (/^Would remove /m.test(remaining.stdout)) { + yield* resetIndex; + return yield* new VcsProcessExitError({ + operation, + command: "git clean", + cwd: input.cwd, + exitCode: 0, + detail: "Could not clean the checkpoint workspace: untracked files remain.", + }); + } } - } - yield* resetIndex; + if (cleaned.exitCode !== 0) { + // Git can remove every child, then fail trying to remove './' itself. + const emptiedWorkspace = + cleaned.exitCode === 1 && + /^warning: failed to remove \.\/: [^\n]+$/.test(cleaned.stderr.trim()) && + (yield* fileSystem.readDirectory(input.cwd).pipe( + Effect.map((entries) => entries.length === 0), + Effect.orElseSucceed(() => false), + )); + if (!emptiedWorkspace) { + yield* resetIndex; + return yield* new VcsProcessExitError({ + operation, + command: "git clean", + cwd: input.cwd, + exitCode: cleaned.exitCode, + detail: cleaned.stderr.trim() || "Could not clean the checkpoint workspace.", + }); + } + } + yield* resetIndex; - return true; + return true; + }).pipe( + Effect.mapError((cause) => + filesMayHaveChanged + ? new VcsDriver.VcsCheckpointPartialRestoreError({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + cause, + }) + : cause, + ), + ); }), diffCheckpoints: Effect.fn("GitVcsDriver.checkpoints.diffCheckpoints")(function* (input) { diff --git a/apps/server/src/vcs/VcsDriver.ts b/apps/server/src/vcs/VcsDriver.ts index b382922c5ab27..3361c3b571b64 100644 --- a/apps/server/src/vcs/VcsDriver.ts +++ b/apps/server/src/vcs/VcsDriver.ts @@ -1,3 +1,4 @@ +import * as Schema from "effect/Schema"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; @@ -11,6 +12,15 @@ import type { import { CheckpointRef } from "@t3tools/contracts"; import * as VcsProcess from "./VcsProcess.ts"; +export class VcsCheckpointPartialRestoreError extends Schema.TaggedError()( + "VcsCheckpointPartialRestoreError", + { cwd: Schema.String, checkpointRef: CheckpointRef, cause: Schema.Defect() }, +) { + override get message(): string { + return "Checkpoint restoration failed after workspace files may have changed."; + } +} + export interface VcsCaptureCheckpointInput { readonly cwd: string; readonly checkpointRef: CheckpointRef; @@ -55,7 +65,7 @@ export interface VcsCheckpointOps { ) => Effect.Effect; readonly restoreCheckpoint: ( input: VcsRestoreCheckpointInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly diffCheckpoints: (input: VcsDiffCheckpointsInput) => Effect.Effect; readonly listAuthoredPaths: ( input: VcsListAuthoredPathsInput,