diff --git a/apps/server/src/git/GitWorkflowService.ts b/apps/server/src/git/GitWorkflowService.ts index 794131444814e..3695fe7187eab 100644 --- a/apps/server/src/git/GitWorkflowService.ts +++ b/apps/server/src/git/GitWorkflowService.ts @@ -79,11 +79,6 @@ export class GitWorkflowService extends Context.Service< options?: GitVcsDriver.CreateWorktreeOptions, ) => Effect.Effect; readonly listLocalBranchNames: (cwd: string) => Effect.Effect; - readonly fetchRemote: (input: { - readonly cwd: string; - readonly remoteName: string; - readonly refName?: string; - }) => Effect.Effect; readonly remoteExists: (input: { readonly cwd: string; readonly remoteName: string; @@ -420,11 +415,6 @@ export const make = Effect.gen(function* () { Effect.andThen(git.listLocalBranchNames(cwd)), ); }), - fetchRemote: Effect.fn("GitWorkflowService.fetchRemote")(function (input) { - return ensureGitCommand("GitWorkflowService.fetchRemote", input.cwd).pipe( - Effect.andThen(git.fetchRemote(input)), - ); - }), remoteExists: Effect.fn("GitWorkflowService.remoteExists")(function (input) { return ensureGitCommand("GitWorkflowService.remoteExists", input.cwd).pipe( Effect.andThen(git.remoteExists(input)), diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index ff129690b7b24..aab41ac9150ef 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -250,12 +250,6 @@ export interface GitFetchRemoteTrackingBranchInput { remoteBranch: string; } -export interface GitFetchRemoteInput { - cwd: string; - remoteName: string; - refName?: string; -} - export interface GitRemoteExistsInput { cwd: string; remoteName: string; @@ -407,7 +401,6 @@ export class GitVcsDriver extends Context.Service< cwd: string, remoteName: string, ) => Effect.Effect; - readonly fetchRemote: (input: GitFetchRemoteInput) => Effect.Effect; readonly remoteExists: (input: GitRemoteExistsInput) => Effect.Effect; readonly remoteBranchExists: ( input: GitRemoteBranchExistsInput, diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index d9a5e7d2b9fde..e21491739e471 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -265,6 +265,8 @@ describe("Git output collection", () => { ); }); +const originMain = { remoteName: "origin", branchName: "main" } as const; + const makeTmpDir = ( prefix = "git-vcs-driver-test-", ): Effect.Effect => @@ -1497,7 +1499,14 @@ it.effect.each([ if (!ChildProcess.isStandardCommand(command)) return yield* Effect.die("expected Git command"); if (command.args[0] !== "fetch") return makeNonRepositoryHandle(); - assert.deepEqual(command.args, ["fetch", "--quiet", "--", "origin"]); + assert.deepEqual(command.args, [ + "fetch", + "--quiet", + "--no-tags", + "--", + "origin", + "+refs/heads/main:refs/remotes/origin/main", + ]); assert.equal(command.options.env?.LANGUAGE, "C"); assert.equal(command.options.env?.GIT_TERMINAL_PROMPT, "0"); yield* Ref.update(attempts, (count) => count + 1); @@ -1520,7 +1529,9 @@ it.effect.each([ Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ); const cwd = yield* makeTmpDir(); - const error = yield* driver.fetchRemote({ cwd, remoteName: "origin" }).pipe(Effect.flip); + const error = yield* driver + .fetchWorktreeBaseCommit({ cwd, refName: "main", remote: originMain }) + .pipe(Effect.flip); assert.include(error.detail, scenario.expected); assert.equal(error.exitCode, 128); assert.equal(error.stderrLength, stderr.length); @@ -1564,7 +1575,7 @@ it.effect("retries a fetch that lost a ref update to another Git process", () => Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ); const cwd = yield* makeTmpDir(); - yield* driver.fetchRemote({ cwd, remoteName: "origin", refName: "main" }); + yield* driver.fetchWorktreeBaseCommit({ cwd, refName: "main", remote: originMain }); assert.equal(yield* Ref.get(attempts), 2); }).pipe( Effect.provide( @@ -1605,6 +1616,67 @@ it.effect("does not fetch for status when the branch tracks a local branch", () ).pipe(Effect.provide(layerTest)), ); +it.effect("fetches for status only the default branch and checked-out upstreams and bases", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetches: Array> = []; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command) && command.args.includes("fetch")) { + fetches.push(command.args.slice(command.args.indexOf("fetch")).map(String)); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const worktree = NodePath.join(yield* makeTmpDir("git-vcs-driver-worktrees-"), "stale"); + yield* initRepoWithCommit(cwd); + yield* git(cwd, ["branch", "-M", "main"]); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + const branches = ["main", "feature", "parent", "idle-base", "gone", "idle", "unrelated"]; + yield* git(cwd, ["push", "origin", ...branches.map((name) => `HEAD:refs/heads/${name}`)]); + yield* git(cwd, ["remote", "set-head", "origin", "main"]); + yield* git(cwd, ["checkout", "-b", "feature", "--track", "origin/feature"]); + yield* git(cwd, ["branch", "--track", "idle", "origin/idle"]); + yield* git(cwd, ["config", "branch.feature.t3-merge-base", "origin/parent"]); + yield* git(cwd, ["config", "branch.idle.t3-merge-base", "origin/idle-base"]); + yield* git(cwd, ["worktree", "add", "-b", "stale", "--track", worktree, "origin/gone"]); + yield* git(remote, ["--git-dir=.", "update-ref", "-d", "refs/heads/gone"]); + // Pushing by path leaves the local remote-tracking refs behind the remote. + const moved = yield* git(cwd, ["commit-tree", "HEAD^{tree}", "-p", "HEAD", "-m", "moved"]); + const movedBranches = ["main", "feature", "parent", "idle-base", "idle", "unrelated"]; + yield* git(cwd, [ + "push", + remote, + ...movedBranches.map((name) => `${moved}:refs/heads/${name}`), + ]); + const before = yield* git(cwd, ["rev-parse", "origin/unrelated"]); + + const status = yield* driver.statusDetailsRemote(cwd); + + assert.deepStrictEqual(fetches, [ + [ + "fetch", + "--quiet", + "--no-tags", + "--no-auto-gc", + "--", + "origin", + "refs/heads/feature", + "refs/heads/main", + "refs/heads/parent", + ], + ]); + assert.equal(status.behindCount, 1); + assert.equal(yield* git(cwd, ["rev-parse", "origin/unrelated"]), before); + }), + ).pipe(Effect.provide(layerTest)), +); + it.effect("lets a worktree checkout that is still working run past five minutes", () => Effect.gen(function* () { const checkoutStarted = yield* Deferred.make(); @@ -4081,7 +4153,8 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* writeTextFile(cwd, "second.txt", "second\n"); yield* git(cwd, ["add", "second.txt"]); yield* git(cwd, ["commit", "-m", "second commit"]); - yield* git(cwd, ["push"]); + // Push by path so the status poll still has a moved branch to fetch. + yield* git(cwd, ["push", remote, `HEAD:refs/heads/${initialBranch}`]); yield* git(cwd, ["repack", "-d"]); yield* git(cwd, ["config", "gc.autoPackLimit", "1"]); yield* git(cwd, ["config", "gc.autoDetach", "false"]); @@ -5650,7 +5723,9 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const driver = yield* GitVcsDriver.GitVcsDriver; const missingRemote = `${cwd}/private-missing-remote`; yield* git(cwd, ["remote", "add", "origin", missingRemote]); - const error = yield* driver.fetchRemote({ cwd, remoteName: "origin" }).pipe(Effect.flip); + const error = yield* driver + .fetchWorktreeBaseCommit({ cwd, refName: "main", remote: originMain }) + .pipe(Effect.flip); assert.include(error.detail, "could not access the remote repository"); assert.equal(error.exitCode, 128); assert.isAbove(error.stderrLength ?? 0, 0); @@ -5679,7 +5754,11 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(yield* driver.resolvePrimaryRemoteName(cwd), remoteName); yield* driver.fetchRemoteTrackingBranch({ cwd, remoteName, remoteBranch: initialBranch }); - yield* driver.fetchRemote({ cwd, remoteName }); + yield* driver.fetchWorktreeBaseCommit({ + cwd, + refName: initialBranch, + remote: { remoteName, branchName: initialBranch }, + }); const head = yield* driver.fetchRefCommit({ cwd, ref: "refs/pull/7/head" }); assert.isFalse(yield* fileSystem.exists((yield* Path.Path).join(cwd, "t3-marker"))); @@ -6148,11 +6227,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(peer, ["push", "origin", "HEAD:refs/heads/unrelated"]); const driver = yield* GitVcsDriver.GitVcsDriver; - yield* driver.fetchRemote({ - cwd, - remoteName: "origin", - refName: `origin/${initialBranch}`, - }); + yield* driver.fetchWorktreeBaseCommit({ cwd, refName: initialBranch }); assert.isFalse( yield* driver.remoteBranchExists({ cwd, remoteName: "origin", refName: "unrelated" }), ); @@ -6222,8 +6297,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(status.aheadCount, 0); assert.equal(status.aheadOfDefaultCount, 0); - yield* driver.fetchRemote({ cwd, remoteName: "origin", refName: "local-only" }); - assert.isTrue( + // A missing branch must not fall back to fetching every branch. + assert.isNull( + yield* driver.fetchWorktreeBaseCommit({ + cwd, + refName: "local-only", + remote: { remoteName: "origin", branchName: "local-only" }, + }), + ); + assert.isFalse( yield* driver.remoteBranchExists({ cwd, remoteName: "origin", refName: "unrelated" }), ); }), diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 4dabf3b493ebe..3ef21db70e452 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -101,6 +101,8 @@ const STATUS_UPSTREAM_FETCH_TIMEOUT = Duration.minutes(5); const STATUS_UPSTREAM_REFRESH_FAILURE_BASE_COOLDOWN = Duration.seconds(30); const STATUS_UPSTREAM_REFRESH_FAILURE_MAX_COOLDOWN = Duration.minutes(15); const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048; +// Keeps each fetch command line well under Windows' 32 KiB limit. +const STATUS_FETCH_REFSPEC_BATCH = 100; const REPOSITORY_PATHS_CACHE_CAPACITY = 2_048; const REPOSITORY_PATHS_CACHE_TTL = Duration.minutes(10); const REPOSITORY_PATHS_REFRESH_COALESCE_TTL = Duration.seconds(5); @@ -532,6 +534,66 @@ function branchNameFromHeadRef(value: string): string | null { : null; } +const STATUS_FETCH_REF_FORMAT = + "%(refname)%00%(symref)%00%(worktreepath)%00%(upstream:remotename)%00%(upstream:remoteref)"; + +const STATUS_FETCH_BASE_CONFIG_PATTERN = String.raw`^branch\..*\.(t3-merge-base-ref|t3-merge-base|gh-merge-base)$`; + +// Status reads only the remote's default branch, plus the upstreams and recorded bases of +// checked-out branches. Fetching every branch takes tens of seconds on large remotes. +function statusFetchRefs( + forEachRefStdout: string, + baseConfigStdout: string, + lsRemoteStdout: string, + remoteName: string, +): string[] { + const trackingPrefix = `refs/remotes/${remoteName}/`; + const checkedOut = new Set(); + const wanted = new Set(); + let defaultBranch: string | null = null; + for (const line of forEachRefStdout.split("\n")) { + const [refName = "", symref = "", worktreePath, upstreamRemote, upstreamRef = ""] = + line.split("\0"); + if (refName === `${trackingPrefix}HEAD`) { + defaultBranch = parseDefaultBranchFromRemoteHeadRef(symref, remoteName); + } else if (worktreePath && refName.startsWith("refs/heads/")) { + checkedOut.add(refName.slice("refs/heads/".length)); + if (upstreamRemote === remoteName && upstreamRef.startsWith("refs/heads/")) { + wanted.add(upstreamRef.slice("refs/heads/".length)); + } + } + } + // `/HEAD` goes stale when the remote renames its default branch; status falls back to these. + for (const branch of [defaultBranch, ...DEFAULT_BASE_BRANCH_CANDIDATES]) { + if (branch !== null) wanted.add(branch); + } + // `git config --null --get-regexp` prints `\n\0`. + for (const entry of baseConfigStdout.split("\0")) { + const separator = entry.indexOf("\n"); + const key = entry.slice(0, separator); + const branch = key.slice("branch.".length, key.lastIndexOf(".")); + if (separator < 0 || !checkedOut.has(branch)) continue; + const base = entry.slice(separator + 1).trim(); + const name = base.startsWith(trackingPrefix) + ? base.slice(trackingPrefix.length) + : base.startsWith(`${remoteName}/`) + ? base.slice(remoteName.length + 1) + : base.startsWith("refs/") + ? "" + : base; + if (name.length > 0) wanted.add(name); + } + // A branch deleted on the remote fails the whole fetch, so only request advertised ones. + const advertised: string[] = []; + for (const line of lsRemoteStdout.split("\n")) { + const refName = line.split("\t")[1] ?? ""; + if (refName.startsWith("refs/heads/") && wanted.has(refName.slice("refs/heads/".length))) { + advertised.push(refName); + } + } + return advertised; +} + function parseDefaultBranchFromRemoteHeadRef(value: string, remoteName: string): string | null { const trimmed = value.trim(); const prefix = `refs/remotes/${remoteName}/`; @@ -1356,16 +1418,62 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* // this poll from starting `git gc --auto`. When that gc fails, for example on a repository // with missing objects, Git retries it on every fetch and leaves a full-size `tmp_pack_*` // behind each time, so a background poll could fill the disk. - const args = [ - "--git-dir", - gitCommonDir, - "fetch", - "--quiet", - "--no-tags", - "--no-auto-gc", - "--", - remoteName, - ]; + const gitDirArgs = ["--git-dir", gitCommonDir]; + const runStatusGit = ( + args: readonly string[], + fallbackErrorDetail: string, + allowNonZeroExit = false, + ) => + executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, [...gitDirArgs, ...args], { + allowNonZeroExit, + env: STATUS_UPSTREAM_REFRESH_ENV, + fallbackErrorDetail, + timeoutMs: Duration.toMillis(STATUS_UPSTREAM_FETCH_TIMEOUT), + forceKillAfterMs: 1_000, + maxOutputBytes: 16 * 1024 * 1024, + }).pipe(Effect.map((result) => result.stdout)); + const refresh = Effect.gen(function* () { + const [localRefs, baseConfig, remoteHeads] = yield* Effect.all( + [ + runStatusGit( + [ + "for-each-ref", + `--format=${STATUS_FETCH_REF_FORMAT}`, + "refs/heads/", + `refs/remotes/${remoteName}/HEAD`, + ], + "Git could not list local branches.", + ), + // Exits 1 when no branch records a base. + runStatusGit( + ["config", "--null", "--get-regexp", STATUS_FETCH_BASE_CONFIG_PATTERN], + "Git could not read branch bases.", + true, + ), + runStatusGit( + ["ls-remote", "--heads", "--", remoteName], + "Background Git ls-remote exited with a non-zero status.", + ), + ], + { concurrency: "unbounded" }, + ); + // Source-only refspecs let Git apply the remote's configured fetch mapping. + const refs = statusFetchRefs(localRefs, baseConfig, remoteHeads, remoteName); + for (let start = 0; start < refs.length; start += STATUS_FETCH_REFSPEC_BATCH) { + yield* runStatusGit( + [ + "fetch", + "--quiet", + "--no-tags", + "--no-auto-gc", + "--", + remoteName, + ...refs.slice(start, start + STATUS_FETCH_REFSPEC_BATCH), + ], + "Background Git fetch exited with a non-zero status.", + ); + } + }); const key = `${gitCommonDir}\0${remoteName}`; const failure = statusRemoteRefreshFailures.get(key); if (failure && (yield* Clock.currentTimeMillis) < failure.retryAt) return; @@ -1376,13 +1484,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* // Frequent poll deadlines and disconnected clients must not interrupt a // healthy writer. A separate safety limit prevents a hung transport from // occupying its worker slot indefinitely. - const started = yield* executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { - env: STATUS_UPSTREAM_REFRESH_ENV, - fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", - timeoutMs: Duration.toMillis(STATUS_UPSTREAM_FETCH_TIMEOUT), - forceKillAfterMs: 1_000, - }).pipe( - Effect.asVoid, + const started = yield* refresh.pipe( Effect.tap(() => Effect.sync(() => { statusRemoteRefreshFailures.delete(key); @@ -4579,8 +4681,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return { headCommit: input.targetCommit, moved: true, onTarget: true }; }); - const fetchRemote = Effect.fn("fetchRemote")(function* (input: GitVcsDriver.GitFetchRemoteInput) { - const args = ["fetch", "--quiet", "--", input.remoteName]; + // Fetches one branch: fetching a whole remote takes tens of seconds on hosts with thousands of branches. + const fetchRemote = Effect.fn("fetchRemote")(function* (input: { + readonly cwd: string; + readonly remoteName: string; + readonly refName: string; + }) { const options = { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: `git fetch ${input.remoteName} failed`, @@ -4605,28 +4711,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), ); }; - const fetchAll = runFetch(args).pipe( - Effect.flatMap((result) => - result.exitCode === 0 - ? Effect.void - : Effect.fail( - new GitCommandError({ - ...gitCommandContext({ - operation: "GitVcsDriver.fetchRemote", - cwd: input.cwd, - args, - }), - detail: fetchFailureDetail(result.stderr) ?? options.fallbackErrorDetail, - ...(result.exitCode === null ? {} : { exitCode: result.exitCode }), - stdoutLength: result.stdout.length, - stderrLength: result.stderr.length, - }), - ), - ), - ); - if (input.refName === undefined) { - return yield* fetchAll.pipe(Effect.as(true)); - } const branch = parseRemoteRefWithRemoteNames(input.refName, [input.remoteName])?.branchName ?? input.refName; const scopedArgs = [ @@ -5164,18 +5248,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }), resolvePrimaryRemoteName, resolveDefaultBranchName, - fetchRemote: Effect.fnUntraced(function* (input) { - return yield* withListRefsInvalidation( - input.cwd, - fetchRemote(input).pipe( - Effect.flatMap((fetched) => - fetched - ? Effect.void - : fetchRemote({ cwd: input.cwd, remoteName: input.remoteName }).pipe(Effect.asVoid), - ), - ), - ); - }), remoteExists, remoteBranchExists, resolveRemoteTrackingCommit,