diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index d8dfaa27b626c..2144d3150dfb1 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -197,6 +197,229 @@ it.effect("checkpoint capture skips untracked nested repositories without a comm }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("checkpoint capture skips untracked files over the size cap", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-cap-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.makeDirectory(path.join(cwd, "models")); + yield* fileSystem.writeFileString(path.join(cwd, "models", "over [cap].bin"), "x".repeat(1025)); + yield* fileSystem.writeFileString(path.join(cwd, "at-cap.bin"), "x".repeat(1024)); + yield* fileSystem.symlink("models/over [cap].bin", path.join(cwd, "link.bin")); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + + assert.strictEqual( + (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, + "at-cap.bin\nfile.txt\nlink.bin\n", + ); + assert.match((yield* git(["ls-tree", checkpointRef, "--", "link.bin"])).stdout, /^120000 /); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +// Marks every untracked listing incomplete, as Git output past the limit or with a non-UTF-8 path is. +const markUntrackedListings = ( + liveProcess: VcsProcess.VcsProcess["Service"], + incomplete: Partial, +) => + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + liveProcess + .run(input) + .pipe( + Effect.map((result) => + input.args.includes("--others") ? { ...result, ...incomplete } : result, + ), + ), + }); + +it.effect("checkpoint capture keeps an over-cap file staged in the real index", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-staged-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.writeFileString(path.join(cwd, "staged.bin"), "x".repeat(1025)); + yield* git(["add", "staged.bin"]); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + + assert.strictEqual( + (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, + "file.txt\nstaged.bin\n", + ); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint capture stages everything when the untracked listing is truncated", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }).pipe(markUntrackedListings(liveProcess, { stdoutTruncated: true })); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-size-trunc-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.writeFileString(path.join(cwd, "over.bin"), "x".repeat(1025)); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + + assert.strictEqual( + (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout, + "file.txt\nover.bin\n", + ); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each([ + { incomplete: { stdoutTruncated: true }, reason: "truncated" }, + { incomplete: { stdoutInvalidUtf8: true }, reason: "non-UTF-8" }, +])( + "checkpoint restore refuses a $reason untracked listing before changing files", + ({ incomplete }) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-trunc-restore-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "edited\n"); + yield* fileSystem.writeFileString(path.join(cwd, "over.bin"), "x".repeat(1025)); + yield* fileSystem.writeFileString(path.join(cwd, "under.txt"), "later\n"); + const incompleteDriver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }).pipe(markUntrackedListings(liveProcess, incomplete)); + + const result = yield* Effect.result( + incompleteDriver.checkpoints.restoreCheckpoint({ + cwd, + checkpointRef, + fallbackToHead: false, + }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") assert.strictEqual(result.failure._tag, "VcsProcessExitError"); + assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, "file.txt")), "edited\n"); + assert.isTrue(yield* fileSystem.exists(path.join(cwd, "over.bin"))); + assert.isTrue(yield* fileSystem.exists(path.join(cwd, "under.txt"))); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each([{ workspace: "." }, { workspace: "nested" }])( + "checkpoint restore keeps untracked files over the size cap in workspace $workspace", + ({ workspace }) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const root = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-cap-restore-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, root); + const cwd = path.join(root, workspace); + const overCap = path.join(cwd, "models", "over [cap].bin"); + const laterOverCap = path.join(cwd, "models", "later over cap.bin"); + const laterUnderCap = path.join(cwd, "models", "later.txt"); + yield* fileSystem.makeDirectory(path.dirname(overCap), { recursive: true }); + yield* fileSystem.writeFileString(overCap, "x".repeat(1025)); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(laterOverCap, "y".repeat(1025)); + yield* fileSystem.writeFileString(laterUnderCap, "later\n"); + + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(yield* fileSystem.readFileString(overCap), "x".repeat(1025)); + assert.strictEqual(yield* fileSystem.readFileString(laterOverCap), "y".repeat(1025)); + assert.isFalse(yield* fileSystem.exists(laterUnderCap)); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +// Windows ignores POSIX directory modes and root bypasses them, so neither can deny the lstat. +it.effect.skipIf( + HostProcess.Platform.defaultValue() === "win32" || HostProcess.UserId.defaultValue() === 0, +)("checkpoint restore refuses a file it cannot classify before changing files", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-cap-lstat-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "edited\n"); + const locked = path.join(cwd, "locked"); + yield* fileSystem.makeDirectory(locked); + yield* fileSystem.writeFileString(path.join(locked, "big.bin"), "x".repeat(1025)); + // Readable but not searchable: Git lists the file, lstat fails with EACCES. + yield* Effect.acquireRelease(fileSystem.chmod(locked, 0o600), () => + fileSystem.chmod(locked, 0o700).pipe(Effect.orDie), + ); + + const result = yield* Effect.result( + driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") assert.strictEqual(result.failure._tag, "VcsProcessExitError"); + assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, "file.txt")), "edited\n"); + yield* fileSystem.chmod(locked, 0o700); + assert.strictEqual( + yield* fileSystem.readFileString(path.join(locked, "big.bin")), + "x".repeat(1025), + ); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint restore keeps an over-cap file the restored .gitignore exposes", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape({ + checkpointMaxUntrackedFileBytes: 1024, + }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-checkpoint-size-cap-gitignore-", + }); + const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.writeFileString(path.join(cwd, ".gitignore"), "*.log\n"); + yield* fileSystem.writeFileString(path.join(cwd, "big.bin"), "x".repeat(1025)); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, ".gitignore"), "*.log\nbig.bin\n"); + + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(yield* fileSystem.readFileString(path.join(cwd, ".gitignore")), "*.log\n"); + assert.strictEqual( + yield* fileSystem.readFileString(path.join(cwd, "big.bin")), + "x".repeat(1025), + ); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect.skipIf(!symlinksSupported)( "checkpoint capture writes its index into the real git dir when cwd is a symlinked subdirectory", () => @@ -509,17 +732,17 @@ it.effect.each([ const clock = yield* Clock.Clock; const privateIndexes = new Set(); let racedAttempts = 0; + let stagingCalls = 0; let discoveries = 0; let stageError: VcsProcessExitError | undefined; const captureProcess = yield* VcsProcess.make.pipe( Effect.provideService(ProcessRunner.ProcessRunner, { run: (input) => { - if (input.args.includes("--others")) discoveries += 1; + if (input.args.includes("add")) stagingCalls += 1; + // Every capture lists untracked files before staging; recovery lists them again after. + if (input.args.includes("--others") && stagingCalls > 0) discoveries += 1; if (input.env?.GIT_INDEX_FILE) privateIndexes.add(input.env.GIT_INDEX_FILE); - const initialStage = - phase === "add" && - nestedRecovery && - !input.args.some((arg) => arg.startsWith(":(exclude,literal)")); + const initialStage = phase === "add" && nestedRecovery && stagingCalls === 1; if (!input.args.includes(phase) || initialStage || ++racedAttempts !== 1) { return liveRunner.run(input); } @@ -616,12 +839,11 @@ it.effect.each(["discovery", "probe", "retry"] as const)( privateIndex = input.env?.GIT_INDEX_FILE; stagingAttempts += 1; } + const discovery = input.args.includes("--others") && stagingAttempts > 0; const block = - (blockedPhase === "discovery" && input.args.includes("--others")) || + (blockedPhase === "discovery" && discovery) || (blockedPhase === "probe" && input.cwd !== cwd && input.args.includes("rev-parse")) || - (blockedPhase === "retry" && - staging && - input.args.some((arg) => arg.startsWith(":(exclude,literal)"))); + (blockedPhase === "retry" && staging && stagingAttempts === 2); if (block) return ( blockedPhase === "retry" @@ -643,7 +865,7 @@ it.effect.each(["discovery", "probe", "retry"] as const)( ); return liveProcess.run(input).pipe( Effect.tap(() => - blockedPhase === "probe" && input.args.includes("--others") + blockedPhase === "probe" && discovery ? Deferred.succeed(discovered, undefined).pipe( Effect.andThen(Effect.sleep("3 seconds")), ) @@ -694,7 +916,7 @@ it.effect("checkpoint recovery preserves interruption and removes the private in run: (input) => { if (input.args.includes("add") && input.args.includes("-A")) privateIndex = input.env?.GIT_INDEX_FILE; - return input.args.includes("--others") + return input.args.includes("--others") && privateIndex !== undefined ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)) : liveProcess.run(input); }, diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 684285d608d5f..c701582ec2577 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -1,4 +1,6 @@ import * as NodeBuffer from "node:buffer"; +// @effect-diagnostics-next-line nodeBuiltinImport:off - FileSystem has no lstat +import * as NodeFSP from "node:fs/promises"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; @@ -8,6 +10,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Predicate from "effect/Predicate"; import { ChildProcessSpawner } from "effect/process"; import { @@ -421,6 +424,9 @@ export class GitVcsDriver extends Context.Service< >()("t3/vcs/GitVcsDriver") {} const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; +// Checkpoints never capture untracked files over this size, and restore never cleans them away. +// Hashing one into the object store can outlast the Git timeout and orphan a tmp_pack every turn. +const CHECKPOINT_MAX_UNTRACKED_FILE_BYTES = 100 * 1024 * 1024; const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64; const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds"; const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; @@ -537,7 +543,11 @@ const gitCommand = ( : {}), }); -export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () { +export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (options?: { + readonly checkpointMaxUntrackedFileBytes?: number; +}) { + const checkpointMaxUntrackedFileBytes = + options?.checkpointMaxUntrackedFileBytes ?? CHECKPOINT_MAX_UNTRACKED_FILE_BYTES; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const vcsProcess = yield* VcsProcess.VcsProcess; @@ -826,6 +836,46 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( // core.longpaths is disabled in the user's Git configuration. const longPaths = hostPlatform === "win32" ? ["-c", "core.longpaths=true"] : []; + // Lists files over the checkpoint cap that are untracked in the real index, as git status + // shows them, relative to cwd. Returns undefined when the listing is incomplete: Git failed + // (an unreadable index), output was truncated, a path is not valid UTF-8 and so cannot be + // named back to Git or lstat, or lstat failed for a reason other than the file vanishing. + const listOversizedUntrackedFiles = Effect.fn( + "GitVcsDriver.checkpoints.listOversizedUntrackedFiles", + )(function* (operation: string, cwd: string) { + const untracked = yield* execute({ + operation, + cwd, + args: [...longPaths, "ls-files", "--others", "--exclude-standard", "-z", "--", "."], + allowNonZeroExit: true, + maxOutputBytes: WORKSPACE_FILES_MAX_OUTPUT_BYTES, + }); + if ( + untracked.exitCode !== 0 || + untracked.stdoutTruncated || + untracked.stdoutInvalidUtf8 === true + ) { + return undefined; + } + return yield* Effect.filter( + splitNullSeparatedGitStdoutPaths(untracked).filter((entry) => !entry.endsWith("/")), + // lstat: Git stores a symlink, never its target, so a link is never oversized. + (entry) => + Effect.tryPromise(() => NodeFSP.lstat(path.join(cwd, entry))).pipe( + Effect.map((stats) => stats.isFile() && stats.size > checkpointMaxUntrackedFileBytes), + // A file that vanished after the listing leaves clean nothing to delete. + Effect.catchIf( + (error) => Predicate.hasProperty(error.cause, "code") && error.cause.code === "ENOENT", + () => Effect.succeed(false), + ), + ), + { concurrency: 16 }, + ).pipe( + // Any other lstat failure leaves a file unclassified, so the listing is incomplete. + Effect.orElseSucceed(() => undefined), + ); + }); + const checkpoints: VcsDriver.VcsCheckpointOps = { captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) { const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION; @@ -1039,6 +1089,10 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } } + // An incomplete listing stages everything, as capture did before the size cap. + const oversizedExclusions = ( + (yield* listOversizedUntrackedFiles(operation, input.cwd)) ?? [] + ).map((entry) => `:(exclude,literal)${entry}`); const stageFiles = (exclusions: ReadonlyArray) => execute({ operation, @@ -1057,7 +1111,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( ], env: commitEnv, }); - yield* stageFiles([]).pipe( + yield* stageFiles(oversizedExclusions).pipe( Effect.catchTags({ VcsProcessExitError: (error) => Effect.gen(function* () { @@ -1105,7 +1159,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } } if (exclusions.length === 0) return yield* error; - return yield* stageFiles(exclusions); + return yield* stageFiles([...oversizedExclusions, ...exclusions]); }).pipe( Effect.timeoutOrElse({ duration: CHECKPOINT_RECOVERY_TIMEOUT, @@ -1184,6 +1238,24 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( return false; } + // Clean would delete files capture skipped. List them before the restore, so an incomplete + // listing fails before anything changes, and again after it, because a restored .gitignore + // can expose files the current one hides. + const listOversizedOrFail = listOversizedUntrackedFiles(operation, input.cwd).pipe( + Effect.filterOrFail( + Predicate.isNotUndefined, + () => + new VcsProcessExitError({ + operation, + command: "git ls-files", + cwd: input.cwd, + exitCode: 0, + detail: "Could not list every untracked file, so restore could delete large files.", + }), + ), + ); + const oversizedBeforeRestore = yield* listOversizedOrFail; + const tracked = yield* execute({ operation, cwd: input.cwd, @@ -1220,10 +1292,28 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( ), ); yield* recreateWorkspace; + const oversized = new Set([...oversizedBeforeRestore, ...(yield* listOversizedOrFail)]); + // Exclude patterns, unlike pathspecs, also keep files inside an untracked directory. They + // are anchored at the top level. + const keepOversized: Array = []; + if (oversized.size > 0) { + const prefix = yield* execute({ + operation, + cwd: input.cwd, + args: ["rev-parse", "--show-prefix"], + }); + for (const entry of oversized) { + const literal = `${prefix.stdout.replace(/\n$/, "")}${entry}`.replace( + /[\\*?[ ]/g, + "\\$&", + ); + keepOversized.push("-e", `/${literal}`); + } + } const cleaned = yield* execute({ operation, cwd: input.cwd, - args: [...longPaths, "clean", "-fd", "--", "."], + args: [...longPaths, "clean", "-fd", ...keepOversized, "--", "."], allowNonZeroExit: true, }); if (cleaned.exitCode !== 0) {