From 67ff554fb33cfa42072dfa17b4fc4b152c6f9907 Mon Sep 17 00:00:00 2001 From: Tobi <155588579+spiky02plateau@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:40:14 +0200 Subject: [PATCH 1/5] fix(server): a large untracked file no longer fills the disk with checkpoint packs Capture staged every untracked file into a private index. An untracked file above core.bigFileThreshold is streamed into a pack, and a 1.5 GB file outlasts the 30 s Git timeout, leaving a half-written tmp_pack on every turn while no checkpoint ever succeeds. Checkpoints now never capture untracked files over 100 MiB, and restore never cleans them away. One helper lists them, using lstat so a symlink is never oversized. Capture excludes them from the first git add, and nested repository recovery keeps those exclusions on its retry; a truncated listing stages everything as before. Restore lists them before changing anything and passes them to git clean as anchored exclude patterns, so a skipped file survives even inside an untracked directory; a truncated listing fails the restore before any destructive command. Refs #3646 Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/vcs/GitVcsDriver.test.ts | 148 +++++++++++++++++++++-- apps/server/src/vcs/GitVcsDriver.ts | 75 +++++++++++- 2 files changed, 208 insertions(+), 15 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 3a7a16f95d4cc..f86d5aedcff8d 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -163,6 +163,133 @@ it.effect("checkpoint capture skips untracked nested repositories without a comm }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("checkpoint capture skips untracked files over the size cap", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-cap-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.makeDirectory(path.join(cwd, "models")); + yield* fileSystem.writeFileString(path.join(cwd, "models", "over [cap].bin"), "x".repeat(1025)); + yield* fileSystem.writeFileString(path.join(cwd, "at-cap.bin"), "x".repeat(1024)); + yield* fileSystem.symlink("models/over [cap].bin", path.join(cwd, "link.bin")); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + + assert.strictEqual( + (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, + "at-cap.bin\nfile.txt\nlink.bin\n", + ); + assert.match((yield* git(["ls-tree", checkpointRef, "--", "link.bin"])).stdout, /^120000 /); + }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), +); + +// Reports every untracked listing as truncated, as Git does past the output limit. +const truncateUntrackedListings = (liveProcess: VcsProcess.VcsProcess["Service"]) => + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + liveProcess + .run(input) + .pipe( + Effect.map((result) => + input.args.includes("--others") ? { ...result, stdoutTruncated: true } : result, + ), + ), + }); + +it.effect("checkpoint capture stages everything when the untracked listing is truncated", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }).pipe(truncateUntrackedListings(liveProcess)); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-trunc-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.writeFileString(path.join(cwd, "over.bin"), "x".repeat(1025)); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + + assert.strictEqual( + (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, + "file.txt\nover.bin\n", + ); + }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), +); + +it.effect("checkpoint restore refuses a truncated untracked listing before changing files", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-trunc-restore-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "edited\n"); + yield* fileSystem.writeFileString(path.join(cwd, "over.bin"), "x".repeat(1025)); + yield* fileSystem.writeFileString(path.join(cwd, "under.txt"), "later\n"); + const truncatedDriver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }).pipe(truncateUntrackedListings(liveProcess)); + + const result = yield* Effect.result( + truncatedDriver.checkpoints.restoreCheckpoint({ + cwd, + checkpointRef, + fallbackToHead: false, + }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") assert.strictEqual(result.failure._tag, "VcsProcessExitError"); + assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, "file.txt")), "edited\n"); + assert.isTrue(yield* fileSystem.exists(path.join(cwd, "over.bin"))); + assert.isTrue(yield* fileSystem.exists(path.join(cwd, "under.txt"))); + }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), +); + +it.effect.each([{ workspace: "." }, { workspace: "nested" }])( + "checkpoint restore keeps untracked files over the size cap in workspace $workspace", + ({ workspace }) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const root = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-cap-restore-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, root); + const cwd = path.join(root, workspace); + const overCap = path.join(cwd, "models", "over [cap].bin"); + const laterOverCap = path.join(cwd, "models", "later over cap.bin"); + const laterUnderCap = path.join(cwd, "models", "later.txt"); + yield* fileSystem.makeDirectory(path.dirname(overCap), { recursive: true }); + yield* fileSystem.writeFileString(overCap, "x".repeat(1025)); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(laterOverCap, "y".repeat(1025)); + yield* fileSystem.writeFileString(laterUnderCap, "later\n"); + + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(yield* fileSystem.readFileString(overCap), "x".repeat(1025)); + assert.strictEqual(yield* fileSystem.readFileString(laterOverCap), "y".repeat(1025)); + assert.isFalse(yield* fileSystem.exists(laterUnderCap)); + }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), +); + it.effect("checkpoint recovery discovers nested HEAD independently of inherited GIT_DIR", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -311,17 +438,17 @@ it.effect.each([ const clock = yield* Clock.Clock; const privateIndexes = new Set(); let racedAttempts = 0; + let stagingCalls = 0; let discoveries = 0; let stageError: VcsProcessExitError | undefined; const captureProcess = yield* VcsProcess.make.pipe( Effect.provideService(ProcessRunner.ProcessRunner, { run: (input) => { - if (input.args.includes("--others")) discoveries += 1; + if (input.args.includes("add")) stagingCalls += 1; + // Every capture lists untracked files before staging; recovery lists them again after. + if (input.args.includes("--others") && stagingCalls > 0) discoveries += 1; if (input.env?.GIT_INDEX_FILE) privateIndexes.add(input.env.GIT_INDEX_FILE); - const initialStage = - phase === "add" && - nestedRecovery && - !input.args.some((arg) => arg.startsWith(":(exclude,literal)")); + const initialStage = phase === "add" && nestedRecovery && stagingCalls === 1; if (!input.args.includes(phase) || initialStage || ++racedAttempts !== 1) { return liveRunner.run(input); } @@ -418,12 +545,11 @@ it.effect.each(["discovery", "probe", "retry"] as const)( privateIndex = input.env?.GIT_INDEX_FILE; stagingAttempts += 1; } + const discovery = input.args.includes("--others") && stagingAttempts > 0; const block = - (blockedPhase === "discovery" && input.args.includes("--others")) || + (blockedPhase === "discovery" && discovery) || (blockedPhase === "probe" && input.cwd !== cwd && input.args.includes("rev-parse")) || - (blockedPhase === "retry" && - staging && - input.args.some((arg) => arg.startsWith(":(exclude,literal)"))); + (blockedPhase === "retry" && staging && stagingAttempts === 2); if (block) return ( blockedPhase === "retry" @@ -445,7 +571,7 @@ it.effect.each(["discovery", "probe", "retry"] as const)( ); return liveProcess.run(input).pipe( Effect.tap(() => - blockedPhase === "probe" && input.args.includes("--others") + blockedPhase === "probe" && discovery ? Deferred.succeed(discovered, undefined).pipe( Effect.andThen(Effect.sleep("3 seconds")), ) @@ -496,7 +622,7 @@ it.effect("checkpoint recovery preserves interruption and removes the private in run: (input) => { if (input.args.includes("add") && input.args.includes("-A")) privateIndex = input.env?.GIT_INDEX_FILE; - return input.args.includes("--others") + return input.args.includes("--others") && privateIndex !== undefined ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)) : liveProcess.run(input); }, diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 3803c78fc6d2e..f62f3f45f64e6 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -1,4 +1,6 @@ import * as NodeBuffer from "node:buffer"; +// @effect-diagnostics-next-line nodeBuiltinImport:off - FileSystem has no lstat +import * as NodeFSP from "node:fs/promises"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; @@ -421,6 +423,9 @@ export class GitVcsDriver extends Context.Service< >()("t3/vcs/GitVcsDriver") {} const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; +// Checkpoints never capture untracked files over this size, and restore never cleans them away. +// Hashing one into the object store can outlast the Git timeout and orphan a tmp_pack every turn. +export const CHECKPOINT_MAX_UNTRACKED_FILE_BYTES = 100 * 1024 * 1024; const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64; const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds"; const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; @@ -537,7 +542,11 @@ const gitCommand = ( : {}), }); -export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () { +export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (options?: { + readonly checkpointMaxUntrackedFileBytes?: number; +}) { + const checkpointMaxUntrackedFileBytes = + options?.checkpointMaxUntrackedFileBytes ?? CHECKPOINT_MAX_UNTRACKED_FILE_BYTES; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const vcsProcess = yield* VcsProcess.VcsProcess; @@ -807,6 +816,31 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "core.fsyncMethod=fsync", ] as const; + // Lists untracked files over the checkpoint cap, relative to cwd, or undefined when Git's + // listing was truncated. Pass the private index through env to classify against it. + const listOversizedUntrackedFiles = Effect.fn( + "GitVcsDriver.checkpoints.listOversizedUntrackedFiles", + )(function* (operation: string, cwd: string, env?: NodeJS.ProcessEnv) { + const untracked = yield* execute({ + operation, + cwd, + args: ["ls-files", "--others", "--exclude-standard", "-z", "--", "."], + ...(env !== undefined ? { env } : {}), + maxOutputBytes: WORKSPACE_FILES_MAX_OUTPUT_BYTES, + }); + if (untracked.stdoutTruncated) return undefined; + return yield* Effect.filter( + splitNullSeparatedGitStdoutPaths(untracked).filter((entry) => !entry.endsWith("/")), + // lstat: Git stores a symlink, never its target, so a link is never oversized. + (entry) => + Effect.tryPromise(() => NodeFSP.lstat(path.join(cwd, entry))).pipe( + Effect.map((stats) => stats.isFile() && stats.size > checkpointMaxUntrackedFileBytes), + Effect.orElseSucceed(() => false), + ), + { concurrency: 16 }, + ); + }); + const checkpoints: VcsDriver.VcsCheckpointOps = { captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) { const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION; @@ -961,6 +995,10 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } } + // A truncated listing stages everything, as capture did before the size cap. + const oversizedExclusions = ( + (yield* listOversizedUntrackedFiles(operation, input.cwd, commitEnv)) ?? [] + ).map((entry) => `:(exclude,literal)${entry}`); const stageFiles = (exclusions: ReadonlyArray) => execute({ operation, @@ -978,7 +1016,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( ], env: commitEnv, }); - yield* stageFiles([]).pipe( + yield* stageFiles(oversizedExclusions).pipe( Effect.catchTags({ VcsProcessExitError: (error) => Effect.gen(function* () { @@ -1020,7 +1058,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } } if (exclusions.length === 0) return yield* error; - return yield* stageFiles(exclusions); + return yield* stageFiles([...oversizedExclusions, ...exclusions]); }).pipe( // One budget covers discovery, queued Git admission, probes, and the staging retry. Effect.timeoutOrElse({ @@ -1100,6 +1138,35 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( return false; } + // Clean would delete files capture skipped. Exclude patterns, unlike pathspecs, also keep + // them inside an untracked directory; they are anchored at the top level. Listing before + // the restore covers everything clean can see afterwards. + const oversized = yield* listOversizedUntrackedFiles(operation, input.cwd); + if (oversized === undefined) { + return yield* new VcsProcessExitError({ + operation, + command: "git ls-files", + cwd: input.cwd, + exitCode: 0, + detail: "Too many untracked files to restore the checkpoint without risking large files.", + }); + } + const keepOversized: Array = []; + if (oversized.length > 0) { + const prefix = yield* execute({ + operation, + cwd: input.cwd, + args: ["rev-parse", "--show-prefix"], + }); + for (const entry of oversized) { + const literal = `${prefix.stdout.replace(/\n$/, "")}${entry}`.replace( + /[\\*?[ ]/g, + "\\$&", + ); + keepOversized.push("-e", `/${literal}`); + } + } + const tracked = yield* execute({ operation, cwd: input.cwd, @@ -1130,7 +1197,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const cleaned = yield* execute({ operation, cwd: input.cwd, - args: ["clean", "-fd", "--", "."], + args: ["clean", "-fd", ...keepOversized, "--", "."], allowNonZeroExit: true, }); if (cleaned.exitCode !== 0) { From 90feb112431720314ee4fb3b8936f35f627195d0 Mon Sep 17 00:00:00 2001 From: Tobi <155588579+spiky02plateau@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:22:40 +0200 Subject: [PATCH 2/5] fix(server): checkpoint size cap follows the real index and fails safe on undecodable paths The untracked listing now runs against the real index, so "untracked" means what git status shows and a large file that is staged but not committed is still captured. An incomplete listing (Git failed, output truncated, or a path that is not valid UTF-8) now counts as no listing: capture stages everything as before, and restore fails before any destructive command. CHECKPOINT_MAX_UNTRACKED_FILE_BYTES is no longer exported; knip flagged it as unused. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/vcs/GitVcsDriver.test.ts | 95 ++++++++++++++++-------- apps/server/src/vcs/GitVcsDriver.ts | 26 ++++--- 2 files changed, 79 insertions(+), 42 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index f86d5aedcff8d..7e27cba2b64b4 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -187,76 +187,105 @@ it.effect("checkpoint capture skips untracked files over the size cap", () => }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), ); -// Reports every untracked listing as truncated, as Git does past the output limit. -const truncateUntrackedListings = (liveProcess: VcsProcess.VcsProcess["Service"]) => +// Marks every untracked listing incomplete, as Git output past the limit or with a non-UTF-8 path is. +const markUntrackedListings = ( + liveProcess: VcsProcess.VcsProcess["Service"], + incomplete: Partial, +) => Effect.provideService(VcsProcess.VcsProcess, { run: (input) => liveProcess .run(input) .pipe( Effect.map((result) => - input.args.includes("--others") ? { ...result, stdoutTruncated: true } : result, + input.args.includes("--others") ? { ...result, ...incomplete } : result, ), ), }); -it.effect("checkpoint capture stages everything when the untracked listing is truncated", () => +it.effect("checkpoint capture keeps an over-cap file staged in the real index", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const liveProcess = yield* VcsProcess.VcsProcess; const driver = yield* GitVcsDriver.makeVcsDriverShape({ checkpointMaxUntrackedFileBytes: 1024, - }).pipe(truncateUntrackedListings(liveProcess)); - const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-trunc-" }); + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-staged-" }); const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); - yield* fileSystem.writeFileString(path.join(cwd, "over.bin"), "x".repeat(1025)); + yield* fileSystem.writeFileString(path.join(cwd, "staged.bin"), "x".repeat(1025)); + yield* git(["add", "staged.bin"]); yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); assert.strictEqual( (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, - "file.txt\nover.bin\n", + "file.txt\nstaged.bin\n", ); }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), ); -it.effect("checkpoint restore refuses a truncated untracked listing before changing files", () => +it.effect("checkpoint capture stages everything when the untracked listing is truncated", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const liveProcess = yield* VcsProcess.VcsProcess; const driver = yield* GitVcsDriver.makeVcsDriverShape({ checkpointMaxUntrackedFileBytes: 1024, - }); - const cwd = yield* fileSystem.makeTempDirectoryScoped({ - prefix: "t3-checkpoint-size-trunc-restore-", - }); - const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); - yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); - yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "edited\n"); + }).pipe(markUntrackedListings(liveProcess, { stdoutTruncated: true })); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-trunc-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); yield* fileSystem.writeFileString(path.join(cwd, "over.bin"), "x".repeat(1025)); - yield* fileSystem.writeFileString(path.join(cwd, "under.txt"), "later\n"); - const truncatedDriver = yield* GitVcsDriver.makeVcsDriverShape({ - checkpointMaxUntrackedFileBytes: 1024, - }).pipe(truncateUntrackedListings(liveProcess)); - const result = yield* Effect.result( - truncatedDriver.checkpoints.restoreCheckpoint({ - cwd, - checkpointRef, - fallbackToHead: false, - }), - ); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); - assert.strictEqual(result._tag, "Failure"); - if (result._tag === "Failure") assert.strictEqual(result.failure._tag, "VcsProcessExitError"); - assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, "file.txt")), "edited\n"); - assert.isTrue(yield* fileSystem.exists(path.join(cwd, "over.bin"))); - assert.isTrue(yield* fileSystem.exists(path.join(cwd, "under.txt"))); + assert.strictEqual( + (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, + "file.txt\nover.bin\n", + ); }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), ); +it.effect.each([ + { incomplete: { stdoutTruncated: true }, reason: "truncated" }, + { incomplete: { stdoutInvalidUtf8: true }, reason: "non-UTF-8" }, +])( + "checkpoint restore refuses a $reason untracked listing before changing files", + ({ incomplete }) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-trunc-restore-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "edited\n"); + yield* fileSystem.writeFileString(path.join(cwd, "over.bin"), "x".repeat(1025)); + yield* fileSystem.writeFileString(path.join(cwd, "under.txt"), "later\n"); + const incompleteDriver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }).pipe(markUntrackedListings(liveProcess, incomplete)); + + const result = yield* Effect.result( + incompleteDriver.checkpoints.restoreCheckpoint({ + cwd, + checkpointRef, + fallbackToHead: false, + }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") assert.strictEqual(result.failure._tag, "VcsProcessExitError"); + assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, "file.txt")), "edited\n"); + assert.isTrue(yield* fileSystem.exists(path.join(cwd, "over.bin"))); + assert.isTrue(yield* fileSystem.exists(path.join(cwd, "under.txt"))); + }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), +); + it.effect.each([{ workspace: "." }, { workspace: "nested" }])( "checkpoint restore keeps untracked files over the size cap in workspace $workspace", ({ workspace }) => diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index f62f3f45f64e6..64b8bd0a40883 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -425,7 +425,7 @@ export class GitVcsDriver extends Context.Service< const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Checkpoints never capture untracked files over this size, and restore never cleans them away. // Hashing one into the object store can outlast the Git timeout and orphan a tmp_pack every turn. -export const CHECKPOINT_MAX_UNTRACKED_FILE_BYTES = 100 * 1024 * 1024; +const CHECKPOINT_MAX_UNTRACKED_FILE_BYTES = 100 * 1024 * 1024; const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64; const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds"; const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; @@ -816,19 +816,27 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "core.fsyncMethod=fsync", ] as const; - // Lists untracked files over the checkpoint cap, relative to cwd, or undefined when Git's - // listing was truncated. Pass the private index through env to classify against it. + // Lists files over the checkpoint cap that are untracked in the real index, as git status + // shows them, relative to cwd. Returns undefined when the listing is incomplete: Git failed + // (an unreadable index), output was truncated, or a path is not valid UTF-8 and so cannot be + // named back to Git or lstat. const listOversizedUntrackedFiles = Effect.fn( "GitVcsDriver.checkpoints.listOversizedUntrackedFiles", - )(function* (operation: string, cwd: string, env?: NodeJS.ProcessEnv) { + )(function* (operation: string, cwd: string) { const untracked = yield* execute({ operation, cwd, args: ["ls-files", "--others", "--exclude-standard", "-z", "--", "."], - ...(env !== undefined ? { env } : {}), + allowNonZeroExit: true, maxOutputBytes: WORKSPACE_FILES_MAX_OUTPUT_BYTES, }); - if (untracked.stdoutTruncated) return undefined; + if ( + untracked.exitCode !== 0 || + untracked.stdoutTruncated || + untracked.stdoutInvalidUtf8 === true + ) { + return undefined; + } return yield* Effect.filter( splitNullSeparatedGitStdoutPaths(untracked).filter((entry) => !entry.endsWith("/")), // lstat: Git stores a symlink, never its target, so a link is never oversized. @@ -995,9 +1003,9 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } } - // A truncated listing stages everything, as capture did before the size cap. + // An incomplete listing stages everything, as capture did before the size cap. const oversizedExclusions = ( - (yield* listOversizedUntrackedFiles(operation, input.cwd, commitEnv)) ?? [] + (yield* listOversizedUntrackedFiles(operation, input.cwd)) ?? [] ).map((entry) => `:(exclude,literal)${entry}`); const stageFiles = (exclusions: ReadonlyArray) => execute({ @@ -1148,7 +1156,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( command: "git ls-files", cwd: input.cwd, exitCode: 0, - detail: "Too many untracked files to restore the checkpoint without risking large files.", + detail: "Could not list every untracked file, so restore could delete large files.", }); } const keepOversized: Array = []; From 98bb73bee5aef14e56245bbd09c84516cfb59ac7 Mon Sep 17 00:00:00 2001 From: Tobi <155588579+spiky02plateau@users.noreply.github.com> Date: Sun, 4 Oct 2026 07:39:16 +0200 Subject: [PATCH 3/5] fix(server): checkpoint restore protects large files the restored .gitignore exposes Restore chose which oversized untracked files to protect using the current ignore rules. When git restore brought back a .gitignore that no longer ignores a large file, git clean deleted it. Restore now lists oversized untracked files before git restore, so an incomplete listing still fails before anything changes, and again after it, and protects the union of both. If the second listing is incomplete, restore fails without running git clean, so no untracked file is deleted. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/vcs/GitVcsDriver.test.ts | 28 +++++++++++ apps/server/src/vcs/GitVcsDriver.ts | 64 +++++++++++++----------- 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 7e27cba2b64b4..bb54320315f5f 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -319,6 +319,34 @@ it.effect.each([{ workspace: "." }, { workspace: "nested" }])( }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), ); +it.effect("checkpoint restore keeps an over-cap file the restored .gitignore exposes", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-cap-gitignore-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.writeFileString(path.join(cwd, ".gitignore"), "*.log\n"); + yield* fileSystem.writeFileString(path.join(cwd, "big.bin"), "x".repeat(1025)); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, ".gitignore"), "*.log\nbig.bin\n"); + + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, ".gitignore")), "*.log\n"); + assert.strictEqual( + yield* fileSystem.readFileString(path.join(cwd, "big.bin")), + "x".repeat(1025), + ); + }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), +); + it.effect("checkpoint recovery discovers nested HEAD independently of inherited GIT_DIR", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 64b8bd0a40883..8d3e4045c0de4 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -10,6 +10,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Predicate from "effect/Predicate"; import { ChildProcessSpawner } from "effect/process"; import { @@ -1146,34 +1147,23 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( return false; } - // Clean would delete files capture skipped. Exclude patterns, unlike pathspecs, also keep - // them inside an untracked directory; they are anchored at the top level. Listing before - // the restore covers everything clean can see afterwards. - const oversized = yield* listOversizedUntrackedFiles(operation, input.cwd); - if (oversized === undefined) { - return yield* new VcsProcessExitError({ - operation, - command: "git ls-files", - cwd: input.cwd, - exitCode: 0, - detail: "Could not list every untracked file, so restore could delete large files.", - }); - } - const keepOversized: Array = []; - if (oversized.length > 0) { - const prefix = yield* execute({ - operation, - cwd: input.cwd, - args: ["rev-parse", "--show-prefix"], - }); - for (const entry of oversized) { - const literal = `${prefix.stdout.replace(/\n$/, "")}${entry}`.replace( - /[\\*?[ ]/g, - "\\$&", - ); - keepOversized.push("-e", `/${literal}`); - } - } + // Clean would delete files capture skipped. List them before the restore, so an incomplete + // listing fails before anything changes, and again after it, because a restored .gitignore + // can expose files the current one hides. + const listOversizedOrFail = listOversizedUntrackedFiles(operation, input.cwd).pipe( + Effect.filterOrFail( + Predicate.isNotUndefined, + () => + new VcsProcessExitError({ + operation, + command: "git ls-files", + cwd: input.cwd, + exitCode: 0, + detail: "Could not list every untracked file, so restore could delete large files.", + }), + ), + ); + const oversizedBeforeRestore = yield* listOversizedOrFail; const tracked = yield* execute({ operation, @@ -1202,6 +1192,24 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( ), ); yield* recreateWorkspace; + const oversized = new Set([...oversizedBeforeRestore, ...(yield* listOversizedOrFail)]); + // Exclude patterns, unlike pathspecs, also keep files inside an untracked directory. They + // are anchored at the top level. + const keepOversized: Array = []; + if (oversized.size > 0) { + const prefix = yield* execute({ + operation, + cwd: input.cwd, + args: ["rev-parse", "--show-prefix"], + }); + for (const entry of oversized) { + const literal = `${prefix.stdout.replace(/\n$/, "")}${entry}`.replace( + /[\\*?[ ]/g, + "\\$&", + ); + keepOversized.push("-e", `/${literal}`); + } + } const cleaned = yield* execute({ operation, cwd: input.cwd, From 6e12d7a3fe6d7c17881f1a95005718612f8b9789 Mon Sep 17 00:00:00 2001 From: Tobi <155588579+spiky02plateau@users.noreply.github.com> Date: Sun, 4 Oct 2026 07:53:31 +0200 Subject: [PATCH 4/5] fix(server): checkpoint restore fails safe when a file cannot be classified The oversized-file helper treated any lstat failure as "not oversized", so restore's git clean could delete a file it never managed to classify. ENOENT still counts as not oversized: the file vanished after the listing, and clean has nothing to delete. Any other lstat failure now makes the listing incomplete, so capture stages everything as before and restore fails before git restore on the first listing, or without git clean on the second. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/vcs/GitVcsDriver.test.ts | 40 ++++++++++++++++++++++++ apps/server/src/vcs/GitVcsDriver.ts | 13 ++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index bb54320315f5f..3f8a1228dd216 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -15,6 +15,7 @@ import { ChildProcessSpawner } from "effect/process"; import { assert, it } from "@effect/vitest"; import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as ServerConfig from "../config.ts"; import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; import * as ProcessRunner from "../processRunner.ts"; @@ -319,6 +320,45 @@ it.effect.each([{ workspace: "." }, { workspace: "nested" }])( }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), ); +// Windows ignores POSIX directory modes and root bypasses them, so neither can deny the lstat. +it.effect.skipIf( + HostProcessPlatform.defaultValue() === "win32" || globalThis.process.getuid?.() === 0, +)("checkpoint restore refuses a file it cannot classify before changing files", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-cap-lstat-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "edited\n"); + const locked = path.join(cwd, "locked"); + yield* fileSystem.makeDirectory(locked); + yield* fileSystem.writeFileString(path.join(locked, "big.bin"), "x".repeat(1025)); + // Readable but not searchable: Git lists the file, lstat fails with EACCES. + yield* Effect.acquireRelease(fileSystem.chmod(locked, 0o600), () => + fileSystem.chmod(locked, 0o700).pipe(Effect.orDie), + ); + + const result = yield* Effect.result( + driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") assert.strictEqual(result.failure._tag, "VcsProcessExitError"); + assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, "file.txt")), "edited\n"); + yield* fileSystem.chmod(locked, 0o700); + assert.strictEqual( + yield* fileSystem.readFileString(path.join(locked, "big.bin")), + "x".repeat(1025), + ); + }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), +); + it.effect("checkpoint restore keeps an over-cap file the restored .gitignore exposes", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 8d3e4045c0de4..6b643ae555e15 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -819,8 +819,8 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( // Lists files over the checkpoint cap that are untracked in the real index, as git status // shows them, relative to cwd. Returns undefined when the listing is incomplete: Git failed - // (an unreadable index), output was truncated, or a path is not valid UTF-8 and so cannot be - // named back to Git or lstat. + // (an unreadable index), output was truncated, a path is not valid UTF-8 and so cannot be + // named back to Git or lstat, or lstat failed for a reason other than the file vanishing. const listOversizedUntrackedFiles = Effect.fn( "GitVcsDriver.checkpoints.listOversizedUntrackedFiles", )(function* (operation: string, cwd: string) { @@ -844,9 +844,16 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( (entry) => Effect.tryPromise(() => NodeFSP.lstat(path.join(cwd, entry))).pipe( Effect.map((stats) => stats.isFile() && stats.size > checkpointMaxUntrackedFileBytes), - Effect.orElseSucceed(() => false), + // A file that vanished after the listing leaves clean nothing to delete. + Effect.catchIf( + (error) => Predicate.hasProperty(error.cause, "code") && error.cause.code === "ENOENT", + () => Effect.succeed(false), + ), ), { concurrency: 16 }, + ).pipe( + // Any other lstat failure leaves a file unclassified, so the listing is incomplete. + Effect.orElseSucceed(() => undefined), ); }); From 35f532e0a1d74ff74b971dbb08a2b73281f07204 Mon Sep 17 00:00:00 2001 From: Tobi <155588579+spiky02plateau@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:58:10 +0200 Subject: [PATCH 5/5] test(server): checkpoint size cap tests follow upstream renames Upstream moved host process references into a HostProcess module and renamed test layers to the layerXyz form. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/vcs/GitVcsDriver.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 3f8a1228dd216..992730f470636 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -15,7 +15,7 @@ import { ChildProcessSpawner } from "effect/process"; import { assert, it } from "@effect/vitest"; import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts"; -import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "../config.ts"; import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; import * as ProcessRunner from "../processRunner.ts"; @@ -185,7 +185,7 @@ it.effect("checkpoint capture skips untracked files over the size cap", () => "at-cap.bin\nfile.txt\nlink.bin\n", ); assert.match((yield* git(["ls-tree", checkpointRef, "--", "link.bin"])).stdout, /^120000 /); - }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); // Marks every untracked listing incomplete, as Git output past the limit or with a non-UTF-8 path is. @@ -222,7 +222,7 @@ it.effect("checkpoint capture keeps an over-cap file staged in the real index", (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, "file.txt\nstaged.bin\n", ); - }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); it.effect("checkpoint capture stages everything when the untracked listing is truncated", () => @@ -243,7 +243,7 @@ it.effect("checkpoint capture stages everything when the untracked listing is tr (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, "file.txt\nover.bin\n", ); - }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); it.effect.each([ @@ -284,7 +284,7 @@ it.effect.each([ assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, "file.txt")), "edited\n"); assert.isTrue(yield* fileSystem.exists(path.join(cwd, "over.bin"))); assert.isTrue(yield* fileSystem.exists(path.join(cwd, "under.txt"))); - }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); it.effect.each([{ workspace: "." }, { workspace: "nested" }])( @@ -317,12 +317,12 @@ it.effect.each([{ workspace: "." }, { workspace: "nested" }])( assert.strictEqual(yield* fileSystem.readFileString(overCap), "x".repeat(1025)); assert.strictEqual(yield* fileSystem.readFileString(laterOverCap), "y".repeat(1025)); assert.isFalse(yield* fileSystem.exists(laterUnderCap)); - }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); // Windows ignores POSIX directory modes and root bypasses them, so neither can deny the lstat. it.effect.skipIf( - HostProcessPlatform.defaultValue() === "win32" || globalThis.process.getuid?.() === 0, + HostProcess.Platform.defaultValue() === "win32" || HostProcess.UserId.defaultValue() === 0, )("checkpoint restore refuses a file it cannot classify before changing files", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -356,7 +356,7 @@ it.effect.skipIf( yield* fileSystem.readFileString(path.join(locked, "big.bin")), "x".repeat(1025), ); - }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); it.effect("checkpoint restore keeps an over-cap file the restored .gitignore exposes", () => @@ -384,7 +384,7 @@ it.effect("checkpoint restore keeps an over-cap file the restored .gitignore exp yield* fileSystem.readFileString(path.join(cwd, "big.bin")), "x".repeat(1025), ); - }).pipe(Effect.scoped, Effect.provide(GitContractLayer)), + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); it.effect("checkpoint recovery discovers nested HEAD independently of inherited GIT_DIR", () =>