From 8ce92997c0e4a25de3f92cf570525e8014f4e218 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 09:20:42 -0700 Subject: [PATCH 01/13] F3: add durable trusted rebase foundation --- agents/process-group.ts | 19 +- docs/implementation/pre-merge-rebase.md | 31 ++ docs/implementation/runner-lifecycle.md | 4 +- runner/production.ts | 43 ++- runner/rebase.ts | 293 ++++++++++++++++++ runner/recovery.ts | 88 +++++- runner/store.ts | 132 +++++++- runner/verify-checkout.ts | 106 +++++++ test/agent-process-group.test.ts | 8 + test/runner-lifecycle-store.test.ts | 115 +++++++ test/runner-production.test.ts | 9 + test/runner-rebase.test.ts | 391 ++++++++++++++++++++++++ test/runner-recovery.test.ts | 106 ++++++- 13 files changed, 1307 insertions(+), 38 deletions(-) create mode 100644 docs/implementation/pre-merge-rebase.md create mode 100644 runner/rebase.ts create mode 100644 runner/verify-checkout.ts create mode 100644 test/runner-rebase.test.ts diff --git a/agents/process-group.ts b/agents/process-group.ts index d93da450..7ec6072a 100644 --- a/agents/process-group.ts +++ b/agents/process-group.ts @@ -20,6 +20,8 @@ export interface ProcessGroupOptions { readonly graceMs?: number; /** Bytes kept of each of stdout and stderr; more stops the group. Default 16 MiB. */ readonly maxBuffer?: number; + /** Keep only the bounded prefix instead of stopping on excess output. Use only when exit status/state decides meaning. */ + readonly discardExcessOutput?: boolean; /** * Written to the leader's stdin, which is then closed; without it stdin is not connected. A leader that exits before * reading it all is not an error: what it did is in its status and output. @@ -114,15 +116,16 @@ export function runInProcessGroup(file: string, args: readonly string[], let unrecorded: unknown; try { options.onProcessGroup?.(Object.freeze({ pgid, startedAt: Date.now() })); } catch (error) { unrecorded = error; } - // Output past the limit stops a running group; once the leader has exited it only drops the excess. Either way the - // output is incomplete, so the call settles as ENOBUFS, never as a success. + // Normally output past the limit stops the group and settles as ENOBUFS. A caller that classifies from durable state + // may explicitly keep a bounded prefix and let the process finish instead. let truncated = false; - const collect = (chunks: Buffer[], add: (bytes: number) => number) => (chunk: Buffer) => { - if (add(chunk.length) > maxBuffer) { truncated = true; stop('output-limit'); } - else chunks.push(chunk); + const collect = (chunks: Buffer[], used: () => number, add: (bytes: number) => number) => (chunk: Buffer) => { + const room = Math.max(0, maxBuffer - used()); + if (room > 0) chunks.push(chunk.subarray(0, room)); + if (add(chunk.length) > maxBuffer) { truncated = true; if (!options.discardExcessOutput) stop('output-limit'); } }; - child.stdout!.on('data', collect(out, bytes => (outBytes += bytes))); - child.stderr!.on('data', collect(err, bytes => (errBytes += bytes))); + child.stdout!.on('data', collect(out, () => outBytes, bytes => (outBytes += bytes))); + child.stderr!.on('data', collect(err, () => errBytes, bytes => (errBytes += bytes))); const onAbort = () => stop('cancelled'); options.signal?.addEventListener('abort', onAbort, { once: true }); if (unrecorded !== undefined) { stopped = 'unrecorded'; signalGroup(pgid, 'SIGKILL'); } @@ -178,7 +181,7 @@ export function runInProcessGroup(file: string, args: readonly string[], { name: 'AbortError', code: 'ABORT_ERR' }) }); return; } - if (truncated && !stopped) { + if (truncated && !stopped && !options.discardExcessOutput) { resolve({ status: null, stdout, stderr, error: Object.assign( new Error(`${file} ${args[0] ?? ''} exceeded its output limit.`), { code: 'ENOBUFS' }) }); return; diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md new file mode 100644 index 00000000..ccad7080 --- /dev/null +++ b/docs/implementation/pre-merge-rebase.md @@ -0,0 +1,31 @@ +# Pre-merge rebase foundation + +**Issue and lane:** #22, lane F3. This is the trusted local rewrite and recovery boundary. It does not yet push the rewritten head, run plan `cmd:` checks, wait for GitHub checks, or hand a pair to the merge coordinator; those remain F4-F6 work in #22. + +## Ownership and lifecycle + +Before the first Git process, `Store.beginRebase` records one marker in `tasks.rebase_in_progress`: + +```text +{ attemptId, oldBase, oldHead, onto, startedAt, processGroup } +``` + +Admission checks the current plan revision, snapshot, review version, task state version, mergeable task status, cancellation state, active runner attempt, active merge, existing rebase, and both ends of the reviewed snapshot. Recording the marker advances the task state version. While it exists, another rebase, the legacy rewrite path, merge admission, runner admission, reassignment, and ordinary status changes fail closed. Cancellation remains pending until the owned rebase has settled and exact-attempt cleanup clears the marker; that cleanup then closes the task. + +`processGroup` is null only while no Git call owns work. Immediately before spawn it becomes `spawning`; the synchronous spawn callback replaces that with `{ pgid, startedAt }`. After that exact group is confirmed settled, a compare-and-swap clears it. A group that cannot be drained remains recorded, and a descendant that escapes the group leaves `unsettled`; restart recovery fails closed on either unknown state. A duplicate spawn or late settlement therefore cannot replace or erase another live process owner. + +`GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, and descended from the recorded base. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. + +`Store.finishRebase` accepts that result only from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. + +Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures settle the Git process group before cleanup. Cleanup removes only the validated UUID workspace. A partially created but unregistered worktree is removed only after Git's own worktree list proves it is not registered. + +## Restart recovery + +Startup validates every stored marker before touching its resource. If its recorded process group is still alive with the same start time, recovery terminates and awaits it first. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner. The production recovery adapter accepts only the configured plan key, then removes the matching worktree and unfinished result ref from that plan's runner repository. Only after that succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. + +This ordering is intentionally local-only. The later push integration must extend the durable lifecycle before its first external write and reconcile an ambiguous push outcome; it must not reuse this local completion point as proof that GitHub moved. + +## Regression evidence + +`test/runner-rebase.test.ts` covers clean one-to-one replay, the already-current no-op, process-group ownership, the whole-operation deadline, conflicts, cancellation before and during Git, exact cleanup, retained refs, and a read-only source checkout. `test/runner-lifecycle-store.test.ts` covers admission, durable process ownership, and compare-and-swap races plus owned/foreign provenance. `test/runner-recovery.test.ts` covers process termination and restart ordering plus fail-closed absence of a recovery implementation. `test/runner-production.test.ts` covers exact-plan recovery routing. diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 14a8b30b..9fe2fc81 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -267,7 +267,7 @@ codeboost never writes the user's repository. The commits it makes live in a **r | Holder | Owner | Rule | |---|---|---| | Partial output of a stopped writable attempt | F, using a D export | Done in D by #51 item 6: `exportTaskDiff(storage, { base, imageId, maxBytes, signal, onProcessGroup, timeoutMs, metadataBaseline })` (`agents/container/storage.ts`) takes the storage value or a recovery handle and `base`, the full ID of the commit it was seeded from, and returns `{ diff, truncated }`: at most `maxBytes` (1 MiB at most) of raw diff bytes. It checks both volumes' labels, runs Git in a read-only container with no network that writes nothing to either volume, covers edits and new untracked files (agents cannot commit or stage: the metadata is read-only to them, and the export first checks it against the seeder's baseline, refusing if it changed; a recovery handle needs the `metadataBaseline` F recorded), and cuts the output inside the container, so the Docker work stops at the limit. A changed file over 8 MiB, an untracked nested repository, or an untracked path Git will not add (a part named `.git`, or a symlink named `.gitmodules`) is named in a `codeboost:` notice line instead of diffed, so one large file cannot exhaust the export's memory or deadline; untracked symlinks are diffed as links. What Git skips without a word is named in a notice line too: an entry named `.git` (and, if it holds a repository, the new files beside it), a fifo or socket, an ignored untracked path, and anything in a submodule directory. A changed file whose `ident` or `working-tree-encoding` attribute changes what the diff shows is named too. Anything it cannot read fails the export, and a Git failure's reason includes Git's last two error lines. On abort or at its deadline it stops the client and removes the export container before it settles (an `AbortError`, or an `AggregateError` if that removal did not settle); recovery removes a leftover one. The task filesystems are Docker volumes behind a keeper container. F allocates them with `prepareTaskFilesystems` and removes them with `removeTaskFilesystems`; D's supervisor does not remove them at settlement. But F cannot read a volume from the host. So the order is: `settled` → a **D-provided bounded export** (a new D operation, for example `exportTaskDiff(filesystems, maxBytes)`, which runs `git diff` against the commit the storage was seeded from in a read-only container and returns at most 1 MiB) → F saves it → the terminal write stores `diagnostic_ref` → `removeTaskFilesystems` → the slot is freed. **The export has an overall deadline** (60 seconds, passed to D as an `AbortSignal`), because `maxBytes` bounds only the returned data, not the Docker work. On timeout or any export failure, F records "Partial output could not be exported: " as the diagnostic and continues with the terminal write and removal; shutdown and recovery never wait on an export past its deadline. F saves the export to a runner-owned diagnostics directory, with a total byte cap. **Retention never deletes a file an attempt row still references.** When the cap is exceeded, retention first deletes unreferenced files, oldest first. If that is not enough, it takes the oldest referenced file and, in one transaction, sets that row's `diagnostic_ref` to null and appends "(partial output removed by retention)" to its diagnostic. Only after that commit does it delete the file. No row is ever left pointing at a missing file. The attempt row references it as `diagnostic_ref`. If the capture fails, the row records that it failed. The task filesystem is never reused. | -| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldHead, onto, startedAt }`. F3 sets it by compare-and-swap from null, and clears it by compare-and-swap on the same `attemptId`. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3. | +| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, startedAt, processGroup }`. F3 sets it by compare-and-swap from null, tracks each Git process group until settlement, and clears it by compare-and-swap on the same `attemptId`. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3. | | Workspace rebuild | I3 | I3 never rebuilds before both rows above are resolved for the task. | **Process shutdown is a hard stop.** When the process stops, the running task does not finish. Its attempt ends `cancelled` with the reason "Stopped by shutdown". Restart recovery (lane I3) puts the task back in the queue. This is different from "Stop the queue" (lane I), which lets the running task finish. @@ -444,7 +444,7 @@ A merged v5 task also gets its `task-closed` event, keyed by the merge attempt I `tasks.status` holds the product states from the design (queued, running, needs human, needs amendment, needs approval, possibly already fixed, in review, approved but merge blocked, merged, cancelled). **Closed** means `merged` or `cancelled` (by the user's cancel task); a closed status never changes again. **Reject with feedback is not a closed status.** Following the design's step 8, it creates the next plan revision, marks the affected items to run again, and moves the task from `in review` to `queued`, in one transaction with its `reject` event. The design's learning section (L1) agrees: a task closes only when it is merged or cancelled, and reject rounds stay with the task. J therefore distills a task's feedback, including every reject round, when it closes by merge or cancel. **Human-gated** means `needs human`, `needs amendment`, `needs approval` or `possibly already fixed`. F1 defines the list and its invariants. F2 adds the per-item transitions. I1 adds queue admission and scheduling. -**Where the current context lives.** The `Current` check reads two rows in one transaction: `plans` (plan revision, snapshot ID) and `tasks` (assignment ID, referenced-code hash, context generation). The plan ID comes from the plan key. Every `Store` method that changes any of these fields increases `tasks.context_generation` **in the same transaction**. That covers `importRevision`, `applySuggestion`, `recordHistory`, `recordRebase`, and any new method that reassigns work or changes the referenced code. A regression test lists these methods and fails if one of them changes a context field without increasing the generation. An assignment change that keeps the same plan revision and snapshot therefore still makes old attempts non-current. +**Where the current context lives.** The `Current` check reads two rows in one transaction: `plans` (plan revision, snapshot ID) and `tasks` (assignment ID, referenced-code hash, context generation). The plan ID comes from the plan key. Every `Store` method that changes any of these fields increases `tasks.context_generation` **in the same transaction**. That covers `importRevision`, `applySuggestion`, `recordHistory`, `finishRebase`, and any new method that reassigns work or changes the referenced code. A regression test lists these methods and fails if one of them changes a context field without increasing the generation. An assignment change that keeps the same plan revision and snapshot therefore still makes old attempts non-current. **Where results live.** `attempts.result` holds the validated structured result of a `completed` attempt: for example review findings, a question answer or check outcomes. It is limited to 1 MiB, the same as the plan-document limit after extraction. Larger or already-owned artifacts stay with their existing owners, and `result` references them by ID. Commits go in the ledger and snapshots (`recordHistory`), and continuation evidence goes in checkpoints. Raw stdout and stderr are never kept beyond the bounded `diagnostic`. diff --git a/runner/production.ts b/runner/production.ts index 287d3d62..7a2212df 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -18,7 +18,8 @@ import { recoverStartup, removalCommand, type RecoveryDeps, type RecoveryReport, import { GitBranchPusher, pushUrl } from './branch-push.ts'; import { PullRequestPublisher } from './publish.ts'; import type { ReviewService } from './review.ts'; -import { openRunnerRepository, ownerOnlyDirectory } from './runner-repository.ts'; +import { openRunnerRepository, ownerOnlyDirectory, type RunnerRepository } from './runner-repository.ts'; +import { GitRebaser } from './rebase.ts'; import { createTaskWorkspace, workspaceFilesystems } from './workspace.ts'; /** @@ -88,7 +89,7 @@ export function parseRunnerConfig(value: unknown): RunnerConfig { * storage during a long build, and before any export's deadline starts: the build blocks, and a timer armed before it * would fire as soon as it returned. */ -export function dRecoveryDeps(image: () => string): RecoveryDeps { +export function dRecoveryDeps(image: () => string, rebaser?: Pick, rebasePlanKey?: string): RecoveryDeps { return { async recoverLeftovers(runnerOwner) { const report = await recoverLeftovers(runnerOwner, 120_000); @@ -104,6 +105,13 @@ export function dRecoveryDeps(image: () => string): RecoveryDeps { exportTaskDiff: (handle, input, maxBytes, signal) => exportTaskDiff(handle as Parameters[0], { base: input.base, metadataBaseline: input.metadataBaseline, imageId: image(), maxBytes, signal }), removeTaskFilesystems: handle => removeTaskFilesystemsAsync(handle as Parameters[0]), + ...(rebaser ? { abortRebase: async (planKey: string, marker: unknown) => { + if (!rebasePlanKey || planKey !== rebasePlanKey) + throw new Error('The interrupted rebase belongs to another configured plan; start that plan to recover it.'); + const attemptId = (marker as { attemptId?: unknown } | null)?.attemptId; + if (typeof attemptId !== 'string') throw new Error('The interrupted rebase has no attempt ID.'); + await rebaser.abort(attemptId); + } } : {}), }; } @@ -184,15 +192,28 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd o.onSlowStart?.(); let built: string | undefined; const image = () => built ??= (o.buildImage ?? buildAgentImage)(); + const identity = review.identity; + const rebasePlanKey = identityKey(identity); + let repositoryPromise: Promise | undefined, rebaserPromise: Promise | undefined; + const getRepository = () => repositoryPromise ??= openRunnerRepository({ runnerRoot: config.root, runnerOwner, + repositoryId: identity.repositoryId, source: review.repository }).then(repository => { + // The review and rebase recovery read the same runner-owned repository that execution writes. + if (review.runnerRepository !== undefined && review.runnerRepository !== repository.path) + throw new Error(`runnerRepository (${review.runnerRepository}) is not the runner's repository (${repository.path}); remove it from the configuration.`); + review.runnerRepository = repository.path; + return repository; + }); + const getRebaser = () => rebaserPromise ??= getRepository().then(repository => new GitRebaser({ repository, + runnerRoot: config.root, runnerOwner, committer: config.committer, + onProcessStarting: attemptId => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, null, 'spawning'), + onProcessGroup: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, 'spawning', group), + onProcessGroupSettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, null), + onProcessUnsettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, 'unsettled') })); const recovery = await recoverStartup({ store: service.store, runnerOwner, runnerRoot: config.root, diagnosticsDir, - diagnosticsCapBytes: config.diagnosticsCapBytes, deps: (o.recovery ?? dRecoveryDeps)(image) }); + diagnosticsCapBytes: config.diagnosticsCapBytes, + deps: o.recovery ? o.recovery(image) : dRecoveryDeps(image, { abort: async attemptId => (await getRebaser()).abort(attemptId) }, rebasePlanKey) }); + const repository = await getRepository(); const imageId = image(); - const identity = review.identity; - const repository = await openRunnerRepository({ runnerRoot: config.root, runnerOwner, repositoryId: identity.repositoryId, source: review.repository }); - // The review reads runner commits from the same repository the runner writes; a configured path must agree. - if (review.runnerRepository !== undefined && review.runnerRepository !== repository.path) - throw new Error(`runnerRepository (${review.runnerRepository}) is not the runner's repository (${repository.path}); remove it from the configuration.`); - review.runnerRepository = repository.path; const workspace = createTaskWorkspace({ store: service.store, runnerRoot: config.root, runnerOwner, repository, imageId, limits: { ...EXECUTE_STORAGE, ...config.limits }, committer: config.committer }); const issues = new GhIssueGateway(review.github.repository); @@ -225,8 +246,8 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd { diagnostics: { directory: diagnosticsDir, capBytes: config.diagnosticsCapBytes ?? DEFAULT_DIAGNOSTICS_CAP_BYTES } }); const github = review.github; // Never a `url` here (#101 review, finding 6): the push goes to the configured repository on GH_HOST, from the runner's - // own repository. Only commits the ledger records as codeboost's may be overwritten; before #22 adds rebasing, the - // ledger has no other kind of commit codeboost pushes (only recordRebase writes foreign entries). + // own repository. Only commits the ledger records as codeboost's may be overwritten. The F3 rebase foundation can + // write mapped foreign entries, but #22 does not wire their external push until its later durable-action slice. const pusher = new GitBranchPusher({ repository, repositoryId: identity.repositoryId, remote: github.repository, env: o.env as NodeJS.ProcessEnv, ownedCommits: requested => service.store.getLedger(requested).filter(entry => entry.origin === 'owned').map(entry => entry.sha) }); // The same environment as the push, so the PR calls go to the same GH_HOST with the same credentials. diff --git a/runner/rebase.ts b/runner/rebase.ts new file mode 100644 index 00000000..d8150649 --- /dev/null +++ b/runner/rebase.ts @@ -0,0 +1,293 @@ +import { lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { runInProcessGroup, type ProcessGroup } from '../agents/process-group.ts'; +import type { DockerOutcome } from '../agents/docker.ts'; +import { GIT_OPTIONS, gitEnvironment } from '../git/clone.ts'; +import { isUuidV4 } from './lifecycle.ts'; +import { ownerOnlyDirectory, type RunnerRepository } from './runner-repository.ts'; + +const COMMIT_ID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; +const MAX_REBASE_COMMITS = 500; +// runInProcessGroup can spend 1 s delivering SIGTERM, 10 s draining descendants, and 1 s draining held pipes. +const PROCESS_SETTLEMENT_RESERVE_MS = 13_000; +// Cleanup receives an operation-wide 30 s: 13 s to settle a cleanup process group and 17 s for Git and follow-up calls. +const CLEANUP_RESERVE_MS = 30_000; +const MIN_REBASE_TIMEOUT_MS = CLEANUP_RESERVE_MS + PROCESS_SETTLEMENT_RESERVE_MS + 1; + +export interface RebaseMapping { oldSha: string; newSha: string } +export interface RebaseResult { oldHead: string; base: string; head: string; mappings: RebaseMapping[] } +export interface GitRebaserOptions { + repository: RunnerRepository; + runnerRoot: string; + runnerOwner: string; + committer: { name: string; email: string }; + timeoutMs?: number; + onProcessStarting: (attemptId: string) => void; + onProcessGroup: (attemptId: string, group: ProcessGroup) => void; + onProcessGroupSettled: (attemptId: string, group: ProcessGroup | 'spawning') => void; + onProcessUnsettled: (attemptId: string, group: ProcessGroup) => void; +} +interface CallScope { attemptId: string; signal?: AbortSignal; deadline: number; workDeadline: number } + +/** A conflict is a review outcome, not an infrastructure failure. No rewritten ref is retained. */ +export class RebaseConflict extends Error {} + +/** The durable ref that keeps a completed rewrite reachable in the runner-owned bare repository. */ +export function rebaseRef(attemptId: string): string { + if (!isUuidV4(attemptId)) throw new Error('Rebase attempt ID must be a UUID v4.'); + return `refs/codeboost/rebases/${attemptId}`; +} + +/** + * Replays one bounded linear task history in an isolated worktree owned by the runner. The caller claims the matching + * Store marker before calling run(). A successful rewrite keeps its head under rebaseRef(attemptId); abort() removes + * only an unfinished attempt's worktree and ref. + */ +export class GitRebaser { + readonly #options: GitRebaserOptions; + readonly #root: string; + + constructor(options: GitRebaserOptions) { + if (!/^[0-9a-f]{32}$/.test(options.runnerOwner)) throw new Error('Invalid runner owner token.'); + if (!options.committer.name || !options.committer.email || /[<>\n\r\0]/.test(options.committer.name + options.committer.email)) + throw new Error('A valid rebase committer is required.'); + if (options.timeoutMs !== undefined && (!Number.isSafeInteger(options.timeoutMs) || + options.timeoutMs < MIN_REBASE_TIMEOUT_MS || options.timeoutMs > 120_000)) + throw new Error('Invalid rebase deadline.'); + this.#options = options; + this.#root = ownerOnlyDirectory(options.runnerRoot, options.runnerOwner, 'rebases'); + } + + async run(input: { attemptId: string; oldBase: string; oldHead: string; onto: string; signal?: AbortSignal }): Promise { + const { attemptId, oldBase, oldHead, onto, signal } = input; + rebaseRef(attemptId); + for (const value of [oldBase, oldHead, onto]) if (!COMMIT_ID.test(value)) throw new Error('A full commit ID is required for rebasing.'); + signal?.throwIfAborted(); + const scope = this.#scope(attemptId, signal); + const old = await this.#history(this.#options.repository.path, oldBase, oldHead, scope); + if (onto === oldBase) return { oldHead, base: onto, head: oldHead, mappings: old.map(sha => ({ oldSha: sha, newSha: sha })) }; + + const path = this.#path(attemptId); + if (lstatSync(path, { throwIfNoEntry: false })) throw new Error('The rebase workspace already exists.'); + let added = false, retained = false, result: RebaseResult | undefined, primary: unknown; + try { + await this.#git(this.#options.repository.path, ['worktree', 'add', '--detach', '--', path, oldHead], scope); + added = true; + // Git can report a successful checkout even when the host filesystem cannot represent the tree (for example, + // case-colliding or normalization-colliding names on default macOS volumes). Hash the files into the index and + // compare trees; status/stat-cache metadata is not proof that the contents are unchanged. + await this.#assertCheckout(path, oldHead, 'reviewed head', scope); + const outcome = await this.#call(path, ['rebase', '--quiet', '--reapply-cherry-picks', '--keep-empty', '--empty=keep', '--onto', onto, oldBase, oldHead], scope, false, true); + this.#throwIfCancelled(outcome, scope); + if (outcome.status !== 0) { + if (outcome.status === null || outcome.error) throw this.#failure('rebase', outcome); + // Rebase diagnostics can grow with valid commit messages and path counts. The index, not bounded text, proves a conflict. + const conflicted = await this.#call(path, ['diff', '--quiet', '--ignore-submodules=all', '--diff-filter=U', '--'], scope); + this.#throwIfCancelled(conflicted, scope); + if (conflicted.status === 1) throw new RebaseConflict('The rebase has conflicts and needs review.'); + if (conflicted.status !== 0) throw this.#failure('diff', conflicted); + throw this.#failure('rebase', outcome); + } + const head = await this.#git(path, ['rev-parse', '--verify', 'HEAD^{commit}'], scope); + if (!COMMIT_ID.test(head)) throw new Error('Git returned an invalid rebased head.'); + await this.#assertCheckout(path, head, 'rebased head', scope); + const next = await this.#history(path, onto, head, scope); + if (next.length !== old.length) throw new Error('The rebase did not preserve one commit for every task commit.'); + await this.#git(this.#options.repository.path, ['update-ref', rebaseRef(attemptId), head], scope); + retained = true; + result = { oldHead, base: onto, head, mappings: old.map((oldSha, index) => ({ oldSha, newSha: next[index]! })) }; + } catch (error) { primary = error; } + const cleanupFailures: unknown[] = []; + if (added || lstatSync(path, { throwIfNoEntry: false })) { + try { await this.#remove(path, scope); } catch (error) { cleanupFailures.push(error); } + } + // Cleanup is deliberately uninterruptible, but an abort that arrived while it ran still cancels the operation. + // Drop a completed rewrite before reporting that cancellation so no caller can apply a cancelled result. + if (!primary && signal?.aborted) { + primary = signal.reason; + if (retained) { + try { await this.#git(this.#options.repository.path, ['update-ref', '-d', rebaseRef(attemptId)], scope, true); } + catch (error) { cleanupFailures.push(error); } + } + } + if (!retained) { + try { await this.#git(this.#options.repository.path, ['update-ref', '-d', rebaseRef(attemptId)], scope, true); } + catch (error) { cleanupFailures.push(error); } + } + const cleanup = cleanupFailures.length > 1 ? new AggregateError(cleanupFailures, 'Rebase cleanup failed.') : cleanupFailures[0]; + if (primary && cleanup) throw new AggregateError([primary, cleanup], (primary as Error)?.message ?? 'Rebase and cleanup failed.', { cause: primary }); + if (primary) throw primary; + if (cleanup) throw cleanup; + return result!; + } + + /** Startup/live recovery for an attempt whose Store marker still exists. */ + async abort(attemptId: string): Promise { + const ref = rebaseRef(attemptId), path = this.#path(attemptId), stat = lstatSync(path, { throwIfNoEntry: false }); + const scope = this.#scope(attemptId); + if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) throw new Error('The rebase workspace is not a plain directory.'); + await this.#remove(path, scope); + await this.#git(this.#options.repository.path, ['update-ref', '-d', ref], scope, true); + } + + async #history(repository: string, base: string, head: string, scope: CallScope): Promise { + const text = await this.#git(repository, ['rev-list', '--reverse', '--parents', `--max-count=${MAX_REBASE_COMMITS + 1}`, `${base}..${head}`], scope); + const rows = text ? text.split('\n') : []; + if (rows.length > MAX_REBASE_COMMITS) throw new Error(`Rebase history exceeds ${MAX_REBASE_COMMITS} commits.`); + let parent = base; + const commits: string[] = []; + for (const row of rows) { + const fields = row.split(' '); + if (fields.length !== 2 || !fields.every(value => COMMIT_ID.test(value)) || fields[1] !== parent) + throw new Error('Rebase history must be complete, linear, and descend from its recorded base.'); + commits.push(fields[0]!); parent = fields[0]!; + } + if (parent !== head) throw new Error('The reviewed head does not descend linearly from its recorded base.'); + return commits; + } + + async #assertCheckout(repository: string, commit: string, label: string, scope: CallScope): Promise { + const verifier = fileURLToPath(new URL('./verify-checkout.ts', import.meta.url)); + const verified = await this.#process(process.execPath, [verifier], repository, {}, scope, false, 64 * 1024); + this.#throwIfCancelled(verified, scope); + if (verified.status !== 0) throw new Error(`The host filesystem cannot faithfully check out the ${label}.`, + { cause: this.#failure('checkout verification', verified, 'node') }); + const actual = await this.#git(repository, ['write-tree'], scope); + const expected = await this.#git(repository, ['rev-parse', '--verify', `${commit}^{tree}`], scope); + if (actual !== expected) throw new Error(`The host filesystem cannot faithfully check out the ${label}.`); + } + + #path(attemptId: string): string { rebaseRef(attemptId); return join(this.#root, attemptId); } + + async #remove(path: string, scope: CallScope): Promise { + // `worktree remove` also unregisters the checkout. It is scoped to the validated UUID path under the owner-only root. + const outcome = await this.#call(this.#options.repository.path, ['worktree', 'remove', '--force', '--', path], scope, true); + if (outcome.status !== 0) { + // A timeout, cancellation, output failure, unsettled group, or callback failure is not evidence that Git merely + // did not know this worktree. Preserve that actionable outcome and its durable process ownership. + if (outcome.status === null || outcome.error) throw this.#failure('worktree remove', outcome); + // An interrupted `worktree add` can create the directory before it registers it. Remove only that exact plain + // directory after Git proves it is not a registered worktree; never turn a registered cleanup failure into success. + const listing = await this.#git(this.#options.repository.path, ['worktree', 'list', '--porcelain', '-z'], scope, true); + const registered = listing.split('\0').some(record => record === `worktree ${path}`); + const stat = lstatSync(path, { throwIfNoEntry: false }); + if (registered && stat) throw this.#failure('worktree remove', outcome); + if (registered) { + await this.#git(this.#options.repository.path, ['worktree', 'prune'], scope, true); + const after = await this.#git(this.#options.repository.path, ['worktree', 'list', '--porcelain', '-z'], scope, true); + if (after.split('\0').some(record => record === `worktree ${path}`)) throw this.#failure('worktree remove', outcome); + } + if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) throw new Error('The rebase workspace is not a plain directory.'); + if (stat) await this.#deleteDirectory(path, scope); + } + if (lstatSync(path, { throwIfNoEntry: false })) throw this.#failure('worktree remove', outcome); + await this.#git(this.#options.repository.path, ['worktree', 'prune'], scope, true); + } + + async #git(repository: string, args: readonly string[], scope: CallScope, cleanup = false): Promise { + const outcome = await this.#call(repository, args, scope, cleanup); + if (!cleanup) this.#throwIfCancelled(outcome, scope); + if (outcome.status !== 0) throw this.#failure(args[0] ?? 'git', outcome); + return outcome.stdout.trim(); + } + + #throwIfCancelled(outcome: DockerOutcome, scope: CallScope): void { + if (!scope.signal?.aborted) return; + // A settlement callback failure is evidence too; #process has already combined it with the original signal reason. + if (outcome.error instanceof AggregateError) throw outcome.error; + scope.signal.throwIfAborted(); + } + + async #call(repository: string, args: readonly string[], scope: CallScope, cleanup = false, discardExcessOutput = false, + input?: Buffer) { + const env = { + ...gitEnvironment(), + GIT_COMMITTER_NAME: this.#options.committer.name, + GIT_COMMITTER_EMAIL: this.#options.committer.email, + GIT_EDITOR: 'true', + GIT_SEQUENCE_EDITOR: 'true', + }; + return this.#process('git', [...GIT_OPTIONS, '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', ...args], repository, + env, scope, cleanup, discardExcessOutput ? 64 * 1024 : 8 * 1024 * 1024, input, discardExcessOutput); + } + + async #deleteDirectory(path: string, scope: CallScope): Promise { + const script = "const fs=require('node:fs');fs.rmSync(fs.readFileSync(0,'utf8'),{recursive:true})"; + const outcome = await this.#process(process.execPath, ['-e', script], this.#root, {}, scope, true, 64 * 1024, Buffer.from(path)); + if (outcome.status !== 0) throw this.#failure('workspace cleanup', outcome, 'node'); + } + + async #process(file: string, args: readonly string[], cwd: string, env: NodeJS.ProcessEnv, scope: CallScope, + cleanup: boolean, maxBuffer: number, input?: Buffer, discardExcessOutput = false): Promise { + // Persist the spawning owner first. The operation deadline already exists, and the remaining subprocess budget is + // calculated only after this synchronous caller hook returns, so a blocking hook cannot extend the deadline. + try { this.#options.onProcessStarting(scope.attemptId); } + catch (error) { + const starting = error instanceof Error ? error : new Error(String(error)); + try { this.#options.onProcessGroupSettled(scope.attemptId, 'spawning'); } + catch (settlementError) { + const settlement = settlementError instanceof Error ? settlementError : new Error(String(settlementError)); + throw new AggregateError([starting, settlement], starting.message, { cause: starting }); + } + throw starting; + } + const limit = cleanup ? scope.deadline : scope.workDeadline; + const timeoutMs = Math.max(0, Math.ceil(limit - performance.now() - PROCESS_SETTLEMENT_RESERVE_MS)); + if (timeoutMs < 1) { + const deadline = Object.assign(new Error('The rebase deadline expired.'), { code: 'ETIMEDOUT' }); + try { this.#options.onProcessGroupSettled(scope.attemptId, 'spawning'); } + catch (error) { + const callback = error instanceof Error ? error : new Error(String(error)); + return { status: null, stdout: '', stderr: '', + error: new AggregateError([deadline, callback], deadline.message, { cause: deadline }) }; + } + return { status: null, stdout: '', stderr: '', error: deadline }; + } + let group: ProcessGroup | undefined; + const outcome = await runInProcessGroup(file, args, { + cwd, env, timeoutMs, graceMs: 1_000, signal: cleanup ? undefined : scope.signal, input, + onProcessGroup: value => { this.#options.onProcessGroup(scope.attemptId, value); group = value; }, maxBuffer, discardExcessOutput, + }); + const code = (outcome.error as NodeJS.ErrnoException | undefined)?.code; + let callbackFailure: unknown; + try { + if (group && code === 'EGROUPALIVE') { /* Its group still exists; retain the exact durable owner. */ } + else if (group && code === 'ESTDIOHELD') this.#options.onProcessUnsettled(scope.attemptId, group); + else if (group) this.#options.onProcessGroupSettled(scope.attemptId, group); + else this.#options.onProcessGroupSettled(scope.attemptId, 'spawning'); + } catch (error) { callbackFailure = error; } + const deadlineFailure = performance.now() >= limit + ? Object.assign(new Error('The rebase deadline expired while recording process settlement.'), { code: 'ETIMEDOUT' }) + : undefined; + const childFailure = outcome.error ?? (outcome.status === 0 ? undefined : + new Error(outcome.stderr.trim() || `${file} exited with status ${outcome.status}.`)); + if (callbackFailure !== undefined) { + const callback = callbackFailure instanceof Error ? callbackFailure : new Error(String(callbackFailure)); + const signalled = !cleanup && code === 'ABORT_ERR' && scope.signal?.aborted ? scope.signal.reason : undefined; + const original = signalled instanceof Error ? signalled : childFailure; + if (original || deadlineFailure) { + const primary = original ?? deadlineFailure!; + return { ...outcome, status: null, error: new AggregateError( + [primary, ...(deadlineFailure && deadlineFailure !== primary ? [deadlineFailure] : []), callback], primary.message, { cause: primary }) }; + } + return { ...outcome, status: null, error: callback }; + } + if (deadlineFailure && childFailure) return { ...outcome, status: null, + error: new AggregateError([childFailure, deadlineFailure], childFailure.message, { cause: childFailure }) }; + if (deadlineFailure) return { ...outcome, status: null, error: deadlineFailure }; + return outcome; + } + + #scope(attemptId: string, signal?: AbortSignal): CallScope { + const timeout = this.#options.timeoutMs ?? 120_000, deadline = performance.now() + timeout; + return { attemptId, signal, deadline, workDeadline: deadline - CLEANUP_RESERVE_MS }; + } + + #failure(command: string, outcome: DockerOutcome, program = 'git'): Error { + if (outcome.status === null && outcome.error) return outcome.error; + const detail = (outcome.status === null && outcome.error ? outcome.error.message : + outcome.stderr.trim() || outcome.stdout.trim() || outcome.error?.message || 'unknown failure').slice(0, 400); + return new Error(`${program} ${command} failed${outcome.status === null ? '' : ` (exit ${outcome.status})`}: ${JSON.stringify(detail)}`); + } +} diff --git a/runner/recovery.ts b/runner/recovery.ts index b6b77b7d..b1d23917 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -1,13 +1,15 @@ -import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, realpathSync, rmSync, statSync, statfsSync } from 'node:fs'; +import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, opendirSync, readdirSync, realpathSync, rmSync, statSync, statfsSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { execFileSync } from 'node:child_process'; import { createRequire } from 'node:module'; import type { DatabaseSync } from 'node:sqlite'; -import type { Store } from './store.ts'; +import type { RebaseMarker, Store } from './store.ts'; import { WRITABLE_KINDS, isUuidV4 } from './lifecycle.ts'; import { partialOutput, saveDiagnostic } from './diagnostics.ts'; import { ownerOnlyDirectory } from './runner-repository.ts'; +import { runInProcessGroup } from '../agents/process-group.ts'; +import type { DockerOutcome } from '../agents/docker.ts'; /** * Startup recovery and the single-runner lock. See docs/implementation/runner-lifecycle.md, @@ -138,7 +140,7 @@ export interface RecoveryDeps { } export interface RecoveryOptions { store: Store; runnerOwner: string; runnerRoot: string; diagnosticsDir: string; deps: RecoveryDeps; - now?: () => number; exportDeadlineMs?: number; graceMs?: number; + now?: () => number; exportDeadlineMs?: number; graceMs?: number; openFiles?: (dir: string) => string[] | Promise; /** The diagnostics directory's total byte cap; retention as on the live path (`saveDiagnostic`). */ diagnosticsCapBytes?: number; } @@ -198,7 +200,30 @@ export async function recoverStartup(o: RecoveryOptions): Promise | null; + const processGroup = marker?.processGroup; + if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldBase) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldHead) || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.onto) || + !Number.isSafeInteger(marker.startedAt) || marker.startedAt! < 0 || + (processGroup !== null && (!processGroup || typeof processGroup === 'string' || !Number.isSafeInteger(processGroup.pgid) || processGroup.pgid <= 1 || + !Number.isSafeInteger(processGroup.startedAt) || processGroup.startedAt < 0))) + throw new RecoveryBlocked('An interrupted rebase has an invalid recovery marker', [rebase.planKey]); + if (processGroup) { + if (processes.isAlive(processGroup.pgid, processGroup.startedAt)) await processes.terminate(processGroup.pgid, o.graceMs ?? 5_000); + // A descendant can escape the recorded process group before it is terminated. Whether the group was initially + // alive or dead, prove that no process still uses the workspace before releasing its durable owner. + const workspace = join(o.runnerRoot, o.runnerOwner, 'rebases', marker.attemptId); + const stat = lstatSync(workspace, { throwIfNoEntry: false }); + if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) + throw new RecoveryBlocked('An interrupted rebase workspace is not a plain directory', [workspace]); + const users = stat ? await (o.openFiles ? o.openFiles(workspace) : hostOpenFilesBounded(workspace)) : []; + if (users.length) throw new RecoveryBlocked('A process still uses an interrupted rebase workspace', users); + o.store.setRebaseProcessGroup(rebase.planKey, marker.attemptId, processGroup, null); + } + await o.deps.abortRebase(rebase.planKey, { ...marker, processGroup: null }); + if (!o.store.abortRebase(rebase.planKey, marker.attemptId)) + throw new RecoveryBlocked('An interrupted rebase changed while recovery aborted it', [rebase.planKey]); } for (const storage of matched) await o.deps.removeTaskFilesystems(storage.handle); // Step 7's set, read after finalization so it also holds preparations an earlier startup already finalized. @@ -261,3 +286,58 @@ export function hostOpenFiles(dir: string): string[] { throw new Error('Could not check which processes use the attempt directory. Refusing to release it.'); } } + +const OPEN_FILES_DEADLINE_MS = 15_000; +const MAX_PROC_ENTRIES = 100_000; +const MAX_FD_ENTRIES = 1_000_000; +type OpenFilesRun = (file: string, args: readonly string[], options: Parameters[2]) => Promise; +/** Recovery's bounded variant. Its lsof process and pipes settle inside the overall deadline before startup proceeds. */ +export async function hostOpenFilesBounded(dir: string, options: { timeoutMs?: number; platform?: NodeJS.Platform; + now?: () => number; run?: OpenFilesRun } = {}): Promise { + dir = realpathSync(dir); + const timeoutMs = options.timeoutMs ?? OPEN_FILES_DEADLINE_MS, now = options.now ?? performance.now.bind(performance); + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 12_001) throw new Error('Invalid open-file probe deadline.'); + const deadline = now() + timeoutMs; + if ((options.platform ?? process.platform) === 'linux') { + const users: string[] = [], disappeared = (error: unknown) => ['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code ?? ''); + let processes = 0, descriptors = 0; + const proc = opendirSync('/proc'); + try { + for (let entry = proc.readSync(); entry; entry = proc.readSync()) { + if (!/^\d+$/.test(entry.name)) continue; + if (++processes > MAX_PROC_ENTRIES || now() >= deadline) + throw new Error('Open-file probe exceeded its bound. Refusing to recover the rebase workspace.'); + const pid = entry.name, links: string[] = []; + try { if (process.getuid && lstatSync(`/proc/${pid}`).uid !== process.getuid()) continue; } + catch (error) { if (disappeared(error)) continue; throw error; } + try { links.push(realpathSync(`/proc/${pid}/cwd`)); } + catch (error) { if (!disappeared(error)) throw new Error(`Could not inspect process ${pid}. Refusing to recover the rebase workspace.`, { cause: error }); } + let fds: ReturnType | undefined; + try { fds = opendirSync(`/proc/${pid}/fd`); } + catch (error) { + if (disappeared(error)) continue; + throw new Error(`Could not inspect process ${pid}. Refusing to recover the rebase workspace.`, { cause: error }); + } + try { + for (let fd = fds.readSync(); fd; fd = fds.readSync()) { + if (++descriptors > MAX_FD_ENTRIES || now() >= deadline) + throw new Error('Open-file probe exceeded its bound. Refusing to recover the rebase workspace.'); + try { links.push(realpathSync(`/proc/${pid}/fd/${fd.name}`)); } + catch (error) { if (!disappeared(error)) throw new Error(`Could not inspect process ${pid}. Refusing to recover the rebase workspace.`, { cause: error }); } + } + } finally { fds.closeSync(); } + if (links.some(link => link === dir || link.startsWith(`${dir}/`))) users.push(pid); + } + } finally { proc.closeSync(); } + if (now() >= deadline) throw new Error('Open-file probe timed out. Refusing to recover the rebase workspace.'); + return users; + } + // runInProcessGroup may spend 1 s on SIGTERM, 10 s draining the group, and 1 s draining pipes after its timer. + const childTimeoutMs = timeoutMs - 12_000; + const outcome = await (options.run ?? runInProcessGroup)('/usr/sbin/lsof', ['-t', '+D', dir], + { env: { LC_ALL: 'C' }, timeoutMs: childTimeoutMs, graceMs: 1_000, maxBuffer: 1024 * 1024 }); + if (now() >= deadline) throw new Error('Open-file probe timed out. Refusing to recover the rebase workspace.'); + if (outcome.status === 0 && !outcome.stderr.trim()) return outcome.stdout.split('\n').filter(Boolean); + if (outcome.status === 1 && !outcome.stdout && !outcome.stderr.trim() && !outcome.error) return []; + throw new Error('Could not check which processes use the rebase workspace. Refusing to recover it.'); +} diff --git a/runner/store.ts b/runner/store.ts index 8e88a500..ecbf8cee 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -83,6 +83,14 @@ export interface TaskRecord { currentAttemptId: string | null; requeuePending: boolean; cancelRequested: string | null; rebaseInProgress: unknown; budgetDeadline: number | null; createdAt: string; updatedAt: string; } +export interface RebaseMarker { + attemptId: string; + oldBase: string; + oldHead: string; + onto: string; + startedAt: number; + processGroup: { pgid: number; startedAt: number } | 'spawning' | 'unsettled' | null; +} export interface AttemptRecord { id: string; kind: AttemptKind; phase: string; item: string | null; state: AttemptState; context: InvocationContext; deadline: number; firstReason: FirstReason | null; stopReason: StopReason | null; exitCode: number | null; signal: string | null; result: unknown; @@ -479,6 +487,7 @@ export class Store { const task = this.#task(key); if (this.#closed(task.status as TaskStatus)) throw new GuardRefusal(`The task is ${task.status}; it cannot be merged.`); if (task.cancel_requested !== null) throw new GuardRefusal('The task is being cancelled; it cannot be merged.'); + if (task.rebase_in_progress !== null) throw new GuardRefusal('A rebase is in progress for this task.'); if (!MERGEABLE_STATUSES.includes(task.status as TaskStatus)) throw new GuardRefusal(`The task is ${task.status}; merge it from review.`); if (this.#activeAttempt(key)) throw new GuardRefusal('An attempt is still active for this task; it cannot be merged.'); if (expectedTaskStateVersion !== null && task.state_version !== expectedTaskStateVersion) throw new GuardRefusal('Stale task state. Reload before writing.'); @@ -696,21 +705,111 @@ export class Store { return this.#transaction(() => { this.#expect(key, expected); this.#assertContextWritable(key); - const ledger = new Map(this.getLedger(identity).map(entry => [entry.sha, entry])); - const snapshot = this.#snapshot(key, base, head); - const destinations = new Set(); - for (const { oldSha, newSha } of mappings) { - sha(oldSha); sha(newSha); - if (destinations.has(newSha)) throw new Error('Rebase mappings must be one-to-one.'); - destinations.add(newSha); - const source = ledger.get(oldSha); - if (oldSha !== newSha) this.#entry(key, { sha: newSha, owner: source?.owner ?? null, origin: source?.origin ?? 'foreign', sourceSha: oldSha }); - else if (!source) this.#entry(key, { sha: newSha, owner: null, origin: 'foreign', sourceSha: null }); - this.#run('INSERT INTO rewrites VALUES (?,?,?,?)', key, snapshot.id, oldSha, newSha); - } + if (this.#task(key).rebase_in_progress !== null) throw new GuardRefusal('A claimed rebase is in progress; wait for it to settle.'); + return this.#applyRebase(identity, key, base, head, mappings); + }); + } + /** Claim the pre-merge rebase before its first Git process starts. */ + beginRebase(identity: PlanIdentity, expected: ReviewState & { reviewVersion: number }, expectedTaskStateVersion: number, + input: { oldBase: string; oldHead: string; onto: string; attemptId?: string; startedAt?: number }): RebaseMarker { + sha(input.oldBase); sha(input.oldHead); sha(input.onto); + if (!Number.isSafeInteger(expected.reviewVersion) || expected.reviewVersion < 0) throw new Error('A current review version is required for rebasing.'); + if (!Number.isSafeInteger(expectedTaskStateVersion) || expectedTaskStateVersion < 0) throw new Error('Invalid expected task state version.'); + const attemptId = input.attemptId ?? randomUUID(), startedAt = input.startedAt ?? Date.now(); + assertUuidV4(attemptId, 'Rebase attempt ID'); + if (!Number.isSafeInteger(startedAt) || startedAt < 0) throw new Error('Invalid rebase start time.'); + const marker: RebaseMarker = { attemptId, oldBase: input.oldBase, oldHead: input.oldHead, onto: input.onto, startedAt, processGroup: null }; + const key = identityKey(identity); + return this.#transaction(() => { + this.#expect(key, expected); + const task = this.#task(key); + if (task.state_version !== expectedTaskStateVersion) throw new GuardRefusal('Stale task state. Reload before writing.'); + if (this.#closed(task.status as TaskStatus)) throw new GuardRefusal(`The task is ${task.status}; it cannot be rebased.`); + if (!MERGEABLE_STATUSES.includes(task.status as TaskStatus)) throw new GuardRefusal(`The task is ${task.status}; rebase it from review.`); + if (task.cancel_requested !== null) throw new GuardRefusal('The task is being cancelled; it cannot be rebased.'); + if (this.#activeAttempt(key)) throw new GuardRefusal('An attempt is still active for this task; it cannot be rebased.'); + if (this.#activeMerge(key)) throw new GuardRefusal('A merge is in progress; wait for its outcome.'); + if (task.rebase_in_progress !== null) throw new GuardRefusal('A rebase is already in progress for this task.'); + const snapshot = this.getSnapshot(identity); + if (snapshot.base !== input.oldBase || snapshot.head !== input.oldHead) + throw new GuardRefusal('The reviewed base or head changed before the rebase started.'); + this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=?', encode(marker), key); + this.#touch(key); + return marker; + }); + } + /** Record or clear the exact Git process group currently owned by a rebase. This does not advance task context. */ + setRebaseProcessGroup(planKey: string, attemptId: string, + expected: RebaseMarker['processGroup'], group: RebaseMarker['processGroup']): void { + assertUuidV4(attemptId, 'Rebase attempt ID'); + for (const value of [expected, group]) if (value && value !== 'spawning' && value !== 'unsettled' && + (!Number.isSafeInteger(value.pgid) || value.pgid <= 1 || !Number.isSafeInteger(value.startedAt) || value.startedAt < 0)) + throw new Error('Invalid rebase process group.'); + this.#transaction(() => { + const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); + if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); + if (stable(marker.processGroup) !== stable(expected)) throw new GuardRefusal('Another Git process owns this rebase.'); + if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', + encode({ ...marker, processGroup: group }), planKey, task.rebase_in_progress as string).changes !== 1) + throw new GuardRefusal('This rebase attempt no longer owns the task.'); + }); + } + /** Publish a rebase only while every review and task counter captured by its attempt is still current. */ + finishRebase(identity: PlanIdentity, expected: ReviewState & { reviewVersion: number }, expectedTaskStateVersion: number, + attemptId: string, base: string, head: string, mappings: readonly { oldSha: string; newSha: string }[]): Snapshot { + assertUuidV4(attemptId, 'Rebase attempt ID'); + if (!Number.isSafeInteger(expected.reviewVersion) || expected.reviewVersion < 0) throw new Error('A current review version is required for rebasing.'); + if (!Number.isSafeInteger(expectedTaskStateVersion) || expectedTaskStateVersion < 0) throw new Error('Invalid expected task state version.'); + const key = identityKey(identity); + return this.#transaction(() => { + this.#expect(key, expected); + const task = this.#task(key), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); + // beginRebase itself advances the task version once; nothing else may have advanced it before this result lands. + if (task.state_version !== expectedTaskStateVersion + 1) throw new GuardRefusal('The task changed during the rebase. Discard its result.'); + if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); + if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + const captured = this.getSnapshot(identity); + if (marker.oldBase !== captured.base || marker.oldHead !== captured.head || marker.onto !== base) + throw new GuardRefusal('The rebase result does not match its captured base and head.'); + this.#assertContextWritable(key); + const snapshot = this.#applyRebase(identity, key, base, head, mappings); + this.#run('UPDATE tasks SET rebase_in_progress=NULL WHERE plan_key=?', key); return snapshot; }); } + /** Clear only the matching rebase, after the live operation settles or recovery removes its resources. */ + abortRebase(planKey: string, attemptId: string): boolean { + assertUuidV4(attemptId, 'Rebase attempt ID'); + return this.#transaction(() => { + const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); + if (marker?.attemptId !== attemptId) return false; + if (marker.processGroup !== null) return false; + const encoded = task.rebase_in_progress as string; + if (this.#run('UPDATE tasks SET rebase_in_progress=NULL WHERE plan_key=? AND rebase_in_progress=?', planKey, encoded).changes !== 1) return false; + if (task.cancel_requested !== null && !this.#closed(task.status as TaskStatus)) this.#closeTask(planKey, 'cancelled', task.cancel_requested as string); + else this.#touch(planKey); + return true; + }); + } + #applyRebase(identity: PlanIdentity, key: string, base: string, head: string, + mappings: readonly { oldSha: string; newSha: string }[]): Snapshot { + const ledger = new Map(this.getLedger(identity).map(entry => [entry.sha, entry])); + const sources = new Set(), destinations = new Set(); + // Validate the complete response before the snapshot or ledger changes. + for (const { oldSha, newSha } of mappings) { + sha(oldSha); sha(newSha); + if (sources.has(oldSha) || destinations.has(newSha)) throw new Error('Rebase mappings must be one-to-one.'); + sources.add(oldSha); destinations.add(newSha); + } + const snapshot = this.#snapshot(key, base, head); + for (const { oldSha, newSha } of mappings) { + const source = ledger.get(oldSha); + if (oldSha !== newSha) this.#entry(key, { sha: newSha, owner: source?.owner ?? null, origin: source?.origin ?? 'foreign', sourceSha: oldSha }); + else if (!source) this.#entry(key, { sha: newSha, owner: null, origin: 'foreign', sourceSha: null }); + this.#run('INSERT INTO rewrites VALUES (?,?,?,?)', key, snapshot.id, oldSha, newSha); + } + return snapshot; + } getRewrites(identity: PlanIdentity, snapshotId: string): { oldSha: string; newSha: string }[] { this.getSnapshot(identity, snapshotId); return this.#db.prepare('SELECT old_sha,new_sha FROM rewrites WHERE key=? AND snapshot_id=? ORDER BY old_sha').all(identityKey(identity), snapshotId).map(row => ({ oldSha: row.old_sha as string, newSha: row.new_sha as string })); @@ -1270,6 +1369,7 @@ export class Store { if (task.state_version !== expectedStateVersion) throw new GuardRefusal('Stale task state. Reload before writing.'); if (this.#closed(task.status as TaskStatus)) throw new GuardRefusal('A closed task never changes.'); if (this.#activeMerge(key)) throw new GuardRefusal('A merge is in progress; wait for its outcome.'); + if (task.rebase_in_progress !== null) throw new GuardRefusal('A rebase is in progress for this task.'); this.#run('UPDATE tasks SET assignment_id=?, referenced_code_hash=? WHERE plan_key=?', assignmentId, referencedCodeHash, key); this.#bumpContext(key); }); @@ -1284,6 +1384,7 @@ export class Store { if (this.#closed(task.status)) throw new GuardRefusal('A closed task never changes.'); if (this.#activeAttempt(key)) throw new GuardRefusal('An attempt is still active for this task.'); if (this.#activeMerge(key)) throw new GuardRefusal('A merge is in progress; wait for its outcome.'); + if (task.rebase_in_progress !== null) throw new GuardRefusal('A rebase is in progress for this task.'); this.#run('UPDATE tasks SET status=? WHERE plan_key=?', to, key); this.#touch(key); }); @@ -1326,6 +1427,7 @@ export class Store { if (task.cancel_requested !== null) throw new GuardRefusal('The task is being cancelled.'); if (this.#activeAttempt(key)) throw new GuardRefusal('An attempt is already active for this task.'); if (this.#activeMerge(key)) throw new GuardRefusal('A merge is in progress; wait for its outcome.'); + if (task.rebase_in_progress !== null) throw new GuardRefusal('A rebase is in progress for this task.'); // The whole-task budget (null until it starts) ends admission: the task waits for a person (time-limit mapping). if (task.budget_deadline !== null && (task.budget_deadline as number) <= now) throw new RefusalWithEffect('The task time budget has run out; it needs a person.', expire); @@ -1430,6 +1532,12 @@ export class Store { if (merge && (merge.state === 'submitting' || merge.state === 'queued')) throw new GuardRefusal('A merge is in progress. Cancel the task after it finishes or fails.'); const active = this.#activeAttempt(key); + if (task.rebase_in_progress !== null) { + if (task.cancel_requested !== null) throw new GuardRefusal('The task is already being cancelled.'); + this.#run('UPDATE tasks SET cancel_requested=? WHERE plan_key=?', actionId, key); + this.#touch(key); + return 'stopping'; + } if (!active) { this.#closeTask(key, 'cancelled', actionId); return 'closed'; } if (task.cancel_requested !== null) throw new GuardRefusal('The task is already being cancelled.'); this.#run(`UPDATE attempts SET first_reason='cancelled' WHERE id=? AND first_reason IS NULL`, active.id!); diff --git a/runner/verify-checkout.ts b/runner/verify-checkout.ts new file mode 100644 index 00000000..16c23374 --- /dev/null +++ b/runner/verify-checkout.ts @@ -0,0 +1,106 @@ +import { createHash } from 'node:crypto'; +import { lstatSync, opendirSync, readlinkSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; +import { GIT_OPTIONS, gitEnvironment } from '../git/clone.ts'; + +const MAX_INDEX_BYTES = 32 * 1024 * 1024; +const MAX_DIRECTORY_ENTRIES = 262_144; +// A C-quoted path can expand every input byte to four bytes. Keep transport above every accepted index listing; +// accepted input is then rejected by our explicit bounds, never by spawnSync's smaller implementation limit. +const MAX_GIT_OUTPUT_BYTES = MAX_INDEX_BYTES * 4 + 1024; +const git = (args: readonly string[], input?: Buffer): Buffer => { + const result = spawnSync('git', [...GIT_OPTIONS, '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', ...args], + { env: gitEnvironment(), input, maxBuffer: MAX_GIT_OUTPUT_BYTES, stdio: [input ? 'pipe' : 'ignore', 'pipe', 'pipe'] }); + if (result.status !== 0 || result.error) throw result.error ?? new Error(`git ${args[0]} failed while verifying the checkout.`); + return result.stdout; +}; +const split0 = (input: Buffer): Buffer[] => { + const values: Buffer[] = []; + let start = 0; + for (let i = 0; i < input.length; i++) if (input[i] === 0) { values.push(input.subarray(start, i)); start = i + 1; } + if (start !== input.length) throw new Error('Git returned an unterminated index record.'); + return values; +}; +const splitPath = (path: Buffer): Buffer[] => { + const values: Buffer[] = []; + let start = 0; + for (let i = 0; i <= path.length; i++) if (i === path.length || path[i] === 0x2f) { + const part = path.subarray(start, i); + if (!part.length || part.equals(Buffer.from('.')) || part.equals(Buffer.from('..'))) throw new Error('Git returned an unsafe index path.'); + values.push(part); start = i + 1; + } + return values; +}; +const objectId = (algorithm: 'sha1' | 'sha256', bytes: Buffer): string => + createHash(algorithm).update(Buffer.from(`blob ${bytes.length}\0`)).update(bytes).digest('hex'); + +/** Byte-exact, content-based proof that the disposable worktree represents its index. Gitlinks stay index-only. */ +export function verifyCheckout(root = process.cwd()): void { + const repositoryGit = (args: readonly string[], input?: Buffer) => git(['-C', root, ...args], input); + const rawOutput = repositoryGit(['ls-files', '--stage', '-z']); + if (rawOutput.length > MAX_INDEX_BYTES) throw new Error('Checkout index exceeds its byte bound.'); + const raw = split0(rawOutput); + if (raw.length > MAX_DIRECTORY_ENTRIES) throw new Error('Checkout index exceeds its entry bound.'); + const quotedOutput = repositoryGit(['ls-files', '--stage']).toString('utf8'); + if (quotedOutput && !quotedOutput.endsWith('\n')) throw new Error('Git returned an unterminated quoted index listing.'); + const quoted = quotedOutput ? quotedOutput.slice(0, -1).split('\n') : []; + if (quoted.length !== raw.length) throw new Error('Git returned inconsistent index listings.'); + const format = repositoryGit(['rev-parse', '--show-object-format']).toString('ascii').trim(); + if (format !== 'sha1' && format !== 'sha256') throw new Error('Git returned an unsupported object format.'); + const rootBytes = Buffer.from(root), regular: { oid: string; quoted: string }[] = []; + const rootStat = lstatSync(rootBytes); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('Checkout root is not a plain directory.'); + let entries = 0; + const directories = new Map>(); + const hasEntry = (directory: Buffer, expected: Buffer): boolean => { + const key = directory.toString('hex'), cached = directories.get(key); + if (cached) return cached.has(expected.toString('hex')); + // Node supports Buffer directory entries at runtime, but its opendirSync overload still omits the documented + // `buffer` encoding that readdirSync exposes. Preserve raw names rather than decoding hostile bytes as UTF-8. + const opened = opendirSync(directory, { encoding: 'buffer' as BufferEncoding }); + const names = new Set(); + try { + for (let entry = opened.readSync(); entry; entry = opened.readSync()) { + if (++entries > MAX_DIRECTORY_ENTRIES) throw new Error('Checkout verification exceeded its directory-entry bound.'); + names.add((entry.name as unknown as Buffer).toString('hex')); + } + } finally { opened.closeSync(); } + directories.set(key, names); + return names.has(expected.toString('hex')); + }; + for (let index = 0; index < raw.length; index++) { + const record = raw[index]!, tab = record.indexOf(0x09), header = record.subarray(0, tab).toString('ascii').split(' '); + if (tab < 0 || header.length !== 3 || header[2] !== '0') throw new Error('Git returned an invalid index record.'); + const [mode, oid] = header, path = record.subarray(tab + 1), parts = splitPath(path); + if (mode === '160000') continue; + let parent = rootBytes; + for (let partIndex = 0; partIndex < parts.length; partIndex++) { + const part = parts[partIndex]!; + if (!hasEntry(parent, part)) throw new Error(`The host filesystem changed indexed path component ${part.toString('hex')}.`); + parent = Buffer.concat([parent, Buffer.from('/'), part]); + if (partIndex < parts.length - 1) { + const directory = lstatSync(parent); + if (!directory.isDirectory() || directory.isSymbolicLink()) throw new Error('An indexed path traverses a non-directory.'); + } + } + const stat = lstatSync(parent); + if (mode === '120000') { + if (!stat.isSymbolicLink() || objectId(format, readlinkSync(parent, { encoding: 'buffer' })) !== oid) + throw new Error('An indexed symlink does not match its blob.'); + } else if (mode === '100644' || mode === '100755') { + if (!stat.isFile() || ((stat.mode & 0o111) !== 0) !== (mode === '100755')) throw new Error('An indexed file mode changed.'); + const text = quoted[index]!, quotedTab = text.indexOf('\t'); + if (quotedTab < 0 || !text.startsWith(`${mode} ${oid} 0\t`)) throw new Error('Git returned inconsistent index records.'); + regular.push({ oid: oid!, quoted: text.slice(quotedTab + 1) }); + } else throw new Error('The index contains an unsupported file mode.'); + } + if (regular.length) { + const hashes = repositoryGit(['hash-object', '--stdin-paths'], Buffer.from(`${regular.map(entry => entry.quoted).join('\n')}\n`)) + .toString('ascii').trim().split('\n'); + if (hashes.length !== regular.length || hashes.some((hash, index) => hash !== regular[index]!.oid)) + throw new Error('An indexed file does not match its blob.'); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) verifyCheckout(); diff --git a/test/agent-process-group.test.ts b/test/agent-process-group.test.ts index eb686c50..4fa7ea7b 100644 --- a/test/agent-process-group.test.ts +++ b/test/agent-process-group.test.ts @@ -138,6 +138,14 @@ describe('runInProcessGroup', () => { expect(outcome.stdout.length).toBeLessThanOrEqual(1024); }); + it('can retain a bounded prefix without stopping when callers classify from exit status or durable state', async () => { + const outcome = await runInProcessGroup(process.execPath, ['-e', "process.stdout.write('x'.repeat(4096))"], + { env, timeoutMs: 5_000, maxBuffer: 1024, discardExcessOutput: true }); + expect(outcome.status).toBe(0); + expect(outcome.error).toBeUndefined(); + expect(outcome.stdout).toBe('x'.repeat(1024)); + }); + it('resolves, never rejects, when spawn itself throws', async () => { let reported = false; // A NUL byte in an argument makes spawn throw synchronously, as ENOMEM or E2BIG from the spawn itself would. diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index 5c7ed89e..583c36ad 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -532,6 +532,121 @@ describe('user actions', () => { }); }); +describe('pre-merge rebase ownership', () => { + const reviewed = (store: Store) => ({ revision: 1, snapshotId: store.getSnapshot(identity).id, reviewVersion: store.reviewVersion(identity) }); + + it('claims one exact attempt and blocks merge and runner admission until it settles', () => { + const { store } = fixture(), expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; + store.transitionTask(identity, taskVersion, 'approved but merge blocked'); + const readyVersion = store.getTask(identity).stateVersion; + expect(() => store.beginRebase(identity, { revision: expected.revision, snapshotId: expected.snapshotId } as typeof expected, + readyVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3) })).toThrow(/current review version/); + const marker = store.beginRebase(identity, expected, readyVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3), startedAt: 123 }); + expect(store.getTask(identity).rebaseInProgress).toEqual({ attemptId: marker.attemptId, oldBase: oid(1), oldHead: oid(2), onto: oid(3), startedAt: 123, + processGroup: null }); + const group = { pgid: 4242, startedAt: 456 }; + expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, + { pgid: 1, startedAt: 456 })).toThrow(/Invalid rebase process group/); + store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, 'spawning'); + expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, group)).toThrow(/Another Git process/); + store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, 'spawning', group); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: { pgid: 4242, startedAt: 456 } }); + expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, + { pgid: 5252, startedAt: 567 })).toThrow(/Another Git process/); + expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, + { pgid: 3131, startedAt: 345 }, null)).toThrow(/Another Git process/); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: group }); + expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, randomUUID(), null, null)).toThrow(/no longer owns/); + store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, group, null); + expect(() => store.beginRebase(identity, expected, store.getTask(identity).stateVersion, + { oldBase: oid(1), oldHead: oid(2), onto: oid(3) })).toThrow(/already in progress/); + expect(() => store.beginMergeAttempt(identity, expected, oid(2), null, 'direct')).toThrow(/rebase is in progress/); + expect(() => store.transitionTask(identity, store.getTask(identity).stateVersion, 'queued')).toThrow(/rebase is in progress/); + expect(() => store.setAssignment(identity, store.getTask(identity).stateVersion, 'other', 'other-head')).toThrow(/rebase is in progress/); + }); + + it('publishes only a current attempt and preserves owned and foreign ledger provenance', () => { + const { store } = fixture(); + store.recordHistory(identity, { revision: 1, snapshotId: store.getSnapshot(identity).id }, oid(1), oid(4), [ + { sha: oid(3), owner: 'P1', origin: 'owned', sourceSha: null }, + { sha: oid(4), owner: null, origin: 'foreign', sourceSha: null }, + ]); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; + const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(4), onto: oid(5) }); + expect(() => store.finishRebase(identity, { revision: expected.revision, snapshotId: expected.snapshotId } as typeof expected, + taskVersion, marker.attemptId, oid(5), oid(7), [ + { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, + ])).toThrow(/current review version/); + expect(() => store.finishRebase(identity, expected, taskVersion, randomUUID(), oid(5), oid(7), [ + { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, + ])).toThrow(/no longer owns/); + const group = { pgid: 4242, startedAt: 456 }; + store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, 'spawning'); + store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, 'spawning', group); + expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ + { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, + ])).toThrow(/has not settled/); + store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, group, null); + const snapshot = store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ + { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, + ]); + expect(snapshot).toMatchObject({ base: oid(5), head: oid(7) }); + expect(store.getTask(identity).rebaseInProgress).toBeNull(); + expect(store.getLedger(identity)).toEqual(expect.arrayContaining([ + { sha: oid(6), owner: 'P1', origin: 'owned', sourceSha: oid(3) }, + { sha: oid(7), owner: null, origin: 'foreign', sourceSha: oid(4) }, + ])); + expect(store.getRewrites(identity, snapshot.id)).toEqual([ + { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, + ]); + }); + + it('fails closed when review state changes, and clears only by the exact recovery handle', () => { + const { store } = fixture(); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; + const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3) }); + store.addReviewNote(identity, expected, 'P1', 'change', 'Review changed while Git was running.'); + expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(3), oid(4), [ + { oldSha: oid(2), newSha: oid(4) }, + ])).toThrow(/Stale review state/); + expect(store.abortRebase(store.getTask(identity).planKey, randomUUID())).toBe(false); + expect(store.getTask(identity).rebaseInProgress).not.toBeNull(); + expect(store.abortRebase(store.getTask(identity).planKey, marker.attemptId)).toBe(true); + expect(store.abortRebase(store.getTask(identity).planKey, marker.attemptId)).toBe(false); + expect(store.getTask(identity).rebaseInProgress).toBeNull(); + }); + + it('rejects a result after another durable task change without erasing its recovery marker', () => { + const { store } = fixture(); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; + const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3) }); + expect(store.cancelTask(identity, store.getTask(identity).stateVersion, randomUUID())).toBe('stopping'); + expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(3), oid(4), [ + { oldSha: oid(2), newSha: oid(4) }, + ])).toThrow(/task changed during the rebase/); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ attemptId: marker.attemptId }); + expect(store.abortRebase(store.getTask(identity).planKey, marker.attemptId)).toBe(true); + expect(store.getTask(identity)).toMatchObject({ status: 'cancelled', rebaseInProgress: null }); + }); + + it('binds a claim to both ends of the reviewed snapshot and blocks the legacy rewrite path', () => { + const { store } = fixture(); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; + expect(() => store.beginRebase(identity, expected, taskVersion, + { oldBase: oid(9), oldHead: oid(2), onto: oid(3) })).toThrow(/base or head changed/); + const marker = store.beginRebase(identity, expected, taskVersion, + { oldBase: oid(1), oldHead: oid(2), onto: oid(3) }); + expect(() => store.recordRebase(identity, expected, oid(3), oid(4), [ + { oldSha: oid(2), newSha: oid(4) }, + ])).toThrow(/claimed rebase is in progress/); + expect(store.abortRebase(store.getTask(identity).planKey, marker.attemptId)).toBe(true); + }); +}); + describe('runner commits and preparation groups (#87)', () => { it('knows a task has a runner commit only from a completed writable attempt whose result made one', () => { const { store } = queued(); diff --git a/test/runner-production.test.ts b/test/runner-production.test.ts index 79a4745a..4320c5d4 100644 --- a/test/runner-production.test.ts +++ b/test/runner-production.test.ts @@ -15,6 +15,7 @@ import { recoverLeftovers } from '../agents/recovery.ts'; import { exportTaskDiff, removeTaskFilesystemsAsync } from '../agents/container/storage.ts'; import { RunnerCoordinator, type RunnerDeps } from '../runner/coordinator.ts'; import { SafetyFindings, type ExecutionSources } from '../runner/execution.ts'; +import type { GitRebaser } from '../runner/rebase.ts'; vi.mock('../agents/recovery.ts', async original => ({ ...await original(), recoverLeftovers: vi.fn() })); const issueReads: number[] = []; @@ -226,6 +227,14 @@ describe('recovery warnings', () => { }); describe('D adapters', () => { + it('routes an interrupted rebase marker to the production rebaser before Store recovery clears it', async () => { + const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID() }; + const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); + await deps.abortRebase!('plan-key', marker); + expect(abort).toHaveBeenCalledWith(marker.attemptId); + await expect(deps.abortRebase!('another-plan', marker)).rejects.toThrow(/another configured plan/); + expect(abort).toHaveBeenCalledTimes(1); + }); it('passes D\'s own handles through, and turns every unowned object into the command that removes it', async () => { const handle = Object.freeze({ runnerOwner: OWNER, attemptId: randomUUID(), allocationId: randomUUID() }); const spy = vi.mocked(recoverLeftovers).mockResolvedValue({ removed: [], storage: [handle], unowned: [ diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts new file mode 100644 index 00000000..c98b5c2c --- /dev/null +++ b/test/runner-rebase.test.ts @@ -0,0 +1,391 @@ +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { fixtureGit as git } from './fixtures/git.ts'; +import { GitRebaser, RebaseConflict, rebaseRef } from '../runner/rebase.ts'; +import { ensureCommit, openRunnerRepository } from '../runner/runner-repository.ts'; +import { verifyCheckout } from '../runner/verify-checkout.ts'; + +const OWNER = '0123456789abcdef0123456789abcdef'; +const roots: string[] = []; +vi.setConfig({ testTimeout: 20_000 }); +afterEach(() => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function commit(repository: string, path: string, text: string, message: string): string { + writeFileSync(join(repository, path), text); + git(repository, 'add', '--', path); + git(repository, 'commit', '-qm', message); + return git(repository, 'rev-parse', 'HEAD'); +} + +async function setup(conflict = false) { + const root = mkdtempSync(join(tmpdir(), 'codeboost-rebase-')); roots.push(root); + const source = join(root, 'source'); + git(root, 'init', '-q', '-b', 'main', source); + git(source, 'config', 'user.name', 'Source'); git(source, 'config', 'user.email', 'source@example.invalid'); + const base = commit(source, 'a.txt', 'base\n', 'base'); + git(source, 'switch', '-qc', 'feature'); + const owned = commit(source, 'a.txt', 'feature\n', 'owned'); + const foreign = commit(source, 'feature.txt', 'collaborator\n', 'foreign'); + git(source, 'switch', '-q', 'main'); + const onto = commit(source, conflict ? 'a.txt' : 'base.txt', conflict ? 'main\n' : 'main moved\n', 'move base'); + const runnerRoot = join(root, 'runner'); + const repository = await openRunnerRepository({ runnerRoot, runnerOwner: OWNER, repositoryId: 'repo', source }); + await ensureCommit(repository, foreign); await ensureCommit(repository, onto); + const rebaser = new GitRebaser({ repository, runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + return { root, source, runnerRoot, repository, rebaser, base, owned, foreign, onto }; +} + +describe('trusted pre-merge rebase', () => { + it('replays a complete linear history one-for-one and retains only its immutable result ref', async () => { + const s = await setup(), attemptId = randomUUID(); + const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + expect(result).toMatchObject({ oldHead: s.foreign, base: s.onto }); + expect(result.mappings.map(entry => entry.oldSha)).toEqual([s.owned, s.foreign]); + expect(new Set(result.mappings.map(entry => entry.newSha)).size).toBe(2); + expect(result.mappings.every(entry => entry.oldSha !== entry.newSha)).toBe(true); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); + expect(git(s.repository.path, 'rev-list', '--reverse', `${s.onto}..${result.head}`).split('\n')).toEqual(result.mappings.map(entry => entry.newSha)); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + // The source checkout is read-only to the rebaser. + expect(git(s.source, 'rev-parse', 'feature')).toBe(s.foreign); + }); + + it('returns identity mappings without creating mutable state when the base is already current', async () => { + const s = await setup(), attemptId = randomUUID(); + const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base }); + expect(result).toEqual({ oldHead: s.foreign, base: s.base, head: s.foreign, + mappings: [{ oldSha: s.owned, newSha: s.owned }, { oldSha: s.foreign, newSha: s.foreign }] }); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + }); + + it('accepts tracked files that a later commit adds to gitignore', async () => { + const s = await setup(), attemptId = randomUUID(); + git(s.source, 'switch', '-q', '-C', 'ignored-feature', s.base); + commit(s.source, 'kept.log', 'tracked\n', 'add tracked file'); + const head = commit(s.source, '.gitignore', '*.log\n', 'ignore later logs'); + await ensureCommit(s.repository, head); + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: head, onto: s.onto })) + .resolves.toMatchObject({ oldHead: head, base: s.onto }); + }); + + it('preserves an index-only gitlink whose byte-exact name ends in whitespace', async () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-gitlink-')); roots.push(root); + const template = join(root, 'empty-template'); mkdirSync(template); + git(root, 'init', '-q', `--template=${template}`, '-b', 'main', 'source'); + const source = join(root, 'source'); + git(source, 'config', 'user.name', 'Source'); git(source, 'config', 'user.email', 'source@example.invalid'); + const base = commit(source, 'base.txt', 'base\n', 'base'); + git(source, 'update-index', '--add', '--cacheinfo', `160000,${base},sub `); + git(source, 'commit', '-qm', 'gitlink'); + expect(() => verifyCheckout(source)).not.toThrow(); + }); + + it('verifies regular tracked files whose byte-exact names end in whitespace', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-space-path-')); roots.push(root); + const template = join(root, 'empty-template'); mkdirSync(template); + git(root, 'init', '-q', `--template=${template}`, '-b', 'main', 'source'); + const source = join(root, 'source'), name = 'kept '; + git(source, 'config', 'user.name', 'Source'); git(source, 'config', 'user.email', 'source@example.invalid'); + writeFileSync(join(source, name), 'tracked\n'); + git(source, 'add', '--', name); + expect(() => verifyCheckout(source)).not.toThrow(); + }); + + it('walks each checkout directory once instead of rejecting a normal large directory quadratically', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-many-paths-')); roots.push(root); + const template = join(root, 'empty-template'); mkdirSync(template); + git(root, 'init', '-q', `--template=${template}`, '-b', 'main', 'source'); + const source = join(root, 'source'), many = join(source, 'many'); mkdirSync(many); + for (let index = 0; index < 2_500; index++) writeFileSync(join(many, `f${String(index).padStart(4, '0')}.txt`), 'x'); + git(source, 'add', '--', 'many'); + expect(() => verifyCheckout(source)).not.toThrow(); + }); + + it('reports every spawned process group as owned until that exact process settles', async () => { + const s = await setup(), attemptId = randomUUID(), active = new Map(), seen: number[] = []; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, + onProcessGroup: (ownedAttempt, group) => { + expect(ownedAttempt).toBe(attemptId); expect(active.size).toBe(0); active.set(group.pgid, group.startedAt); seen.push(group.pgid); + }, + onProcessGroupSettled: (ownedAttempt, group) => { + expect(ownedAttempt).toBe(attemptId); + if (group !== 'spawning') { expect(active.get(group.pgid)).toBe(group.startedAt); active.delete(group.pgid); } + }, onProcessUnsettled: () => {} }); + await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + expect(seen.length).toBeGreaterThan(1); + expect(active.size).toBe(0); + }); + + it('refuses an operation deadline too short to include process-group settlement', async () => { + const s = await setup(); + expect(() => new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_000, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} })).toThrow(/Invalid rebase deadline/); + }); + + it('does not start Git when a synchronous ownership hook exhausts the work deadline', async () => { + const s = await setup(), attemptId = randomUUID(); + let starts = 0, settled = 0; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_001, + onProcessStarting: () => { starts++; const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, + onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, + onProcessGroupSettled: (_attempt, group) => { if (group === 'spawning') settled++; }, onProcessUnsettled: () => {} }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })).rejects.toThrow(/deadline expired/); + expect({ starts, settled }).toEqual({ starts: 1, settled: 1 }); + }); + + it('clears a possibly committed spawning marker when its ownership hook throws before spawn', async () => { + const s = await setup(), attemptId = randomUUID(); + let owner: 'none' | 'spawning' | 'group' = 'none', starts = 0, groups = 0, firstSettlement = true; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => { owner = 'spawning'; if (++starts === 1) throw new Error('ownership commit outcome unknown'); }, + onProcessGroup: () => { expect(owner).toBe('spawning'); owner = 'group'; groups++; }, + onProcessGroupSettled: () => { + if (firstSettlement) { expect(groups).toBe(0); firstSettlement = false; } + expect(owner).not.toBe('none'); owner = 'none'; + }, onProcessUnsettled: () => {} }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })) + .rejects.toThrow('ownership commit outcome unknown'); + expect(owner).toBe('none'); + expect(groups).toBe(0); + }); + + it('preserves a pre-spawn deadline and its failed settlement write together', async () => { + const s = await setup(), attemptId = randomUUID(); + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_001, + onProcessStarting: () => { const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, + onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, + onProcessGroupSettled: () => { throw new Error('settlement write failed'); }, onProcessUnsettled: () => {} }); + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) + .then(() => null, error => error as AggregateError); + expect(failure).toBeInstanceOf(AggregateError); + if (!failure) throw new Error('Expected the deadline and settlement write to fail.'); + expect(failure.errors.map(error => (error as Error).message)).toEqual([ + 'The rebase deadline expired.', 'settlement write failed', + ]); + }); + + it('rejects a successful Git call when its settlement hook exhausts the work deadline', async () => { + const s = await setup(), attemptId = randomUUID(); + let overran = false; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, + onProcessGroupSettled: () => { + if (!overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } + }, + onProcessUnsettled: () => {} }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) + .rejects.toThrow(/deadline expired while recording process settlement/); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + }); + + it('preserves a child failure when settlement also overruns the deadline', async () => { + const s = await setup(true), attemptId = randomUUID(); + let processes = 0, overran = false; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, + onProcessGroup: () => { processes++; }, + onProcessGroupSettled: () => { + if (processes === 6 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } + }, onProcessUnsettled: () => {} }); + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) + .then(() => null, error => error as AggregateError); + expect(failure).toBeInstanceOf(AggregateError); + if (!failure) throw new Error('Expected the child failure and deadline to be retained.'); + const primary = failure.cause as AggregateError; + expect(primary).toBeInstanceOf(AggregateError); + expect((primary.errors[0] as Error).message).toMatch(/CONFLICT|could not apply/i); + expect((primary.errors[1] as Error).message).toMatch(/deadline expired while recording process settlement/); + }); + + it('reports a conflict, removes the mutable checkout, and retains no rewritten ref', async () => { + const s = await setup(true), attemptId = randomUUID(); + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })).rejects.toBeInstanceOf(RebaseConflict); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + expect(git(s.source, 'rev-parse', 'feature')).toBe(s.foreign); + }); + + it('refuses a case-colliding reviewed tree when the host cannot check it out faithfully', async () => { + const s = await setup(), attemptId = randomUUID(); + const probe = join(s.root, 'case-probe'); mkdirSync(probe); + writeFileSync(join(probe, 'A'), 'upper'); writeFileSync(join(probe, 'a'), 'lower'); + const caseSensitive = readdirSync(probe).sort().join('\0') === 'A\0a'; + writeFileSync(join(s.source, 'upper-blob'), 'upper\n'); writeFileSync(join(s.source, 'lower-blob'), 'lower\n'); + const upper = git(s.source, 'hash-object', '-w', 'upper-blob'), lower = git(s.source, 'hash-object', '-w', 'lower-blob'); + git(s.source, 'read-tree', '--empty'); + git(s.source, 'update-index', '--add', '--cacheinfo', `100644,${upper},A`); + git(s.source, 'update-index', '--add', '--cacheinfo', `100644,${lower},a`); + const tree = git(s.source, 'write-tree'); + const collidingHead = git(s.source, 'commit-tree', tree, '-p', s.base, '-m', 'case-colliding feature'); + await ensureCommit(s.repository, collidingHead); + const operation = s.rebaser.run({ attemptId, oldBase: s.base, oldHead: collidingHead, onto: s.onto }); + if (caseSensitive) await expect(operation).resolves.toMatchObject({ oldHead: collidingHead, base: s.onto }); + else await expect(operation).rejects.toThrow(/cannot faithfully check out/); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + if (!caseSensitive) expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + }); + + it('refuses a case collision introduced by the target base when the host cannot represent the rebased tree', async () => { + const s = await setup(), attemptId = randomUUID(); + const probe = join(s.root, 'target-case-probe'); mkdirSync(probe); + writeFileSync(join(probe, 'Case'), 'upper'); writeFileSync(join(probe, 'case'), 'lower'); + const caseSensitive = readdirSync(probe).sort().join('\0') === 'Case\0case'; + git(s.source, 'switch', '-q', '-C', 'collision-feature', s.base); + const collidingHead = commit(s.source, 'Case', 'feature\n', 'add upper-case path'); + writeFileSync(join(s.source, 'lower-blob'), 'base\n'); + const lower = git(s.source, 'hash-object', '-w', 'lower-blob'); + git(s.source, 'read-tree', s.base); + git(s.source, 'update-index', '--add', '--cacheinfo', `100644,${lower},case`); + const tree = git(s.source, 'write-tree'); + const collidingBase = git(s.source, 'commit-tree', tree, '-p', s.base, '-m', 'add lower-case path'); + await ensureCommit(s.repository, collidingHead); await ensureCommit(s.repository, collidingBase); + const operation = s.rebaser.run({ attemptId, oldBase: s.base, oldHead: collidingHead, onto: collidingBase }); + if (caseSensitive) await expect(operation).resolves.toMatchObject({ oldHead: collidingHead, base: collidingBase }); + else await expect(operation).rejects.toThrow(/cannot faithfully check out the rebased head/); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + if (!caseSensitive) expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + }); + + it('honours an already-aborted signal before the first Git write', async () => { + const s = await setup(), attemptId = randomUUID(), stop = new AbortController(); + stop.abort(new Error('cancelled before rebase')); + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + .rejects.toThrow('cancelled before rebase'); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + }); + + it('settles a cancellation during rebase and removes the partially rewritten worktree', async () => { + const s = await setup(), attemptId = randomUUID(), stop = new AbortController(); + let processes = 0; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, + onProcessGroup: () => { if (++processes === 3) stop.abort(new Error('cancelled during rebase')); }, + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + .rejects.toThrow('cancelled during rebase'); + expect(processes).toBeGreaterThanOrEqual(4); // history, worktree add, stopped rebase, cleanup + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + }); + + it('preserves cancellation while classifying a failed rebase conflict', async () => { + const s = await setup(true), attemptId = randomUUID(), stop = new AbortController(); + let processes = 0; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, + onProcessGroup: () => { if (++processes === 5) stop.abort(new Error('cancelled during conflict classification')); }, + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + .rejects.toThrow('cancelled during conflict classification'); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + }); + + it('preserves cancellation and a settlement callback failure together', async () => { + const s = await setup(), attemptId = randomUUID(), stop = new AbortController(); + let failed = false; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => stop.abort(new Error('cancelled by caller')), + onProcessGroupSettled: () => { if (!failed) { failed = true; throw new Error('settlement write failed'); } }, + onProcessUnsettled: () => {} }); + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal }) + .then(() => null, error => error as AggregateError); + expect(failure).toBeInstanceOf(AggregateError); + if (!failure) throw new Error('Expected cancellation and settlement recording to fail.'); + expect(failure.errors.map(error => (error as Error).message)).toEqual([ + 'cancelled by caller', 'settlement write failed', + ]); + }); + + it('preserves the original cancellation when cleanup also fails', async () => { + const s = await setup(), attemptId = randomUUID(), stop = new AbortController(); + let starts = 0, processes = 0; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => { if (++starts === 4) throw new Error('cleanup record failed'); }, + onProcessGroup: () => { if (++processes === 3) stop.abort(new Error('original cancellation')); }, + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal }) + .then(() => null, error => error as AggregateError); + expect(failure).toBeInstanceOf(AggregateError); + if (!failure) throw new Error('Expected the rebase and cleanup to fail.'); + expect(failure.message).toContain('original cancellation'); + expect(failure.errors.map(error => (error as Error).message)).toEqual(expect.arrayContaining([ + 'original cancellation', 'cleanup record failed', + ])); + }); + + it('discards a completed rewrite when cancellation arrives during uninterruptible cleanup', async () => { + const s = await setup(), attemptId = randomUUID(), stop = new AbortController(); + let processes = 0; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, + onProcessGroup: () => { if (++processes === 8) stop.abort(new Error('cancelled during cleanup')); }, + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + .rejects.toThrow('cancelled during cleanup'); + expect(processes).toBeGreaterThanOrEqual(10); // result ref, worktree cleanup, prune, then result-ref deletion + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + }); + + it('reports a ref-deletion failure after the result ref write has an ambiguous callback failure', async () => { + const s = await setup(), attemptId = randomUUID(); + let settled = 0, starts = 0; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => { if (++starts === 9) throw new Error('ref deletion record failed'); }, + onProcessGroup: () => {}, + onProcessGroupSettled: () => { if (++settled === 6) throw new Error('result ref outcome unknown'); }, + onProcessUnsettled: () => {} }); + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) + .then(() => null, error => error as AggregateError); + expect(failure).toBeInstanceOf(AggregateError); + if (!failure) throw new Error('Expected the ref write and its cleanup to fail.'); + expect(failure.errors.map(error => (error as Error).message)).toEqual(expect.arrayContaining([ + 'result ref outcome unknown', 'ref deletion record failed', + ])); + }); + + it('aborts only the validated attempt workspace and ref', async () => { + const s = await setup(), attemptId = randomUUID(); + const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); + await s.rebaser.abort(attemptId); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + await expect(s.rebaser.abort('../other')).rejects.toThrow(/UUID v4/); + }); + + it('prunes an exact stale worktree registration even when its directory is already gone', async () => { + const s = await setup(), attemptId = randomUUID(), path = join(s.runnerRoot, OWNER, 'rebases', attemptId); + git(s.repository.path, 'worktree', 'add', '--detach', '--', path, s.foreign); + rmSync(path, { recursive: true, force: true }); + expect(git(s.repository.path, 'worktree', 'list', '--porcelain')).toContain(path); + await s.rebaser.abort(attemptId); + expect(git(s.repository.path, 'worktree', 'list', '--porcelain')).not.toContain(path); + }); + + it('removes an unregistered interrupted workspace through the bounded subprocess path', async () => { + const s = await setup(), attemptId = randomUUID(), path = join(s.runnerRoot, OWNER, 'rebases', attemptId); + mkdirSync(path, { recursive: true }); writeFileSync(join(path, 'partial'), 'left by interrupted worktree add'); + await s.rebaser.abort(attemptId); + expect(existsSync(path)).toBe(false); + }); +}); diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index c5fba3c8..22becb8a 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url'; import { DatabaseSync } from 'node:sqlite'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { Store } from '../runner/store.ts'; -import { LockHeld, RecoveryBlocked, acquireRunnerLock, hostOpenFiles, recoverStartup, releasePreparation, startTimeMatches, type RecoveryDeps, type RunnerLock } from '../runner/recovery.ts'; +import { LockHeld, RecoveryBlocked, acquireRunnerLock, hostOpenFiles, hostOpenFilesBounded, recoverStartup, releasePreparation, startTimeMatches, type RecoveryDeps, type RunnerLock } from '../runner/recovery.ts'; import type { PlanIdentity } from '../core/identity.ts'; import type { Plan, PlanContext } from '../core/plan.ts'; @@ -238,6 +238,97 @@ describe('startup recovery sequence', () => { expect(store.getAttempt(id(1), attempt.id).state).toBe('running'); } }); + it('aborts each exact interrupted rebase before clearing its durable ownership marker', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)); + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123 }); + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', { pgid: 5151, startedAt: 456 }); + const owned = store.getTask(id(1)).rebaseInProgress; + const cleared = { ...(owned as object), processGroup: null }; + const seen: unknown[] = []; + const { d, calls } = deps({ abortRebase: async (planKey, value) => { + calls.push('abort-rebase'); seen.push({ planKey, value, stillOwned: store.getTask(id(1)).rebaseInProgress }); + } }); + await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: d }); + expect(seen).toEqual([{ planKey: store.getTask(id(1)).planKey, value: cleared, stillOwned: cleared }]); + expect(calls).toEqual(['recover', 'terminate:5151', 'abort-rebase']); + expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); + }); + it('retains a dead-group marker while another process still uses its rebase workspace', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)); + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); + const group = { pgid: 5151, startedAt: 456 }; + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', group); + mkdirSync(join(root, 'r', token, 'rebases', marker.attemptId), { recursive: true }); + const abortRebase = vi.fn(async () => undefined); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase, processes: { isAlive: () => false, terminate: async () => undefined } }).d, + openFiles: () => ['escaped-child'] })).rejects.toThrow(/still uses/); + expect(abortRebase).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: group }); + }); + it('retains a terminated-group marker while an escaped descendant still uses its rebase workspace', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)); + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); + const group = { pgid: 5151, startedAt: 456 }; + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', group); + mkdirSync(join(root, 'r', token, 'rebases', marker.attemptId), { recursive: true }); + const terminate = vi.fn(async () => undefined), abortRebase = vi.fn(async () => undefined); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase, processes: { isAlive: () => true, terminate } }).d, + openFiles: () => ['escaped-child'] })).rejects.toThrow(/still uses/); + expect(terminate).toHaveBeenCalledWith(group.pgid, 5_000); + expect(abortRebase).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: group }); + }); + it('fails closed and retains an interrupted rebase when no recovery implementation is present', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)); + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps().d })) + .rejects.toThrow(/needs F3 to abort/); + expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); + }); + it('fails closed without cleanup when a crash left the spawned process identity unknown', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)); + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); + const abortRebase = vi.fn(async () => undefined); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase }).d })).rejects.toThrow(/invalid recovery marker/); + expect(abortRebase).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'spawning' }); + }); + it('rejects process-group ID 1 without sending any recovery signal', async () => { + const { d: root, store, raw } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), attemptId = randomUUID(), planKey = store.getTask(id(1)).planKey; + const marker = { attemptId, oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123, + processGroup: { pgid: 1, startedAt: 456 } }; + raw(`UPDATE tasks SET rebase_in_progress='${JSON.stringify(marker)}' WHERE plan_key='${planKey}'`); + const terminate = vi.fn(async () => undefined), abortRebase = vi.fn(async () => undefined); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase, processes: { isAlive: () => true, terminate } }).d })).rejects.toThrow(/invalid recovery marker/); + expect(terminate).not.toHaveBeenCalled(); + expect(abortRebase).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); + }); it('records an export timeout as a diagnostic and still finalizes and removes the storage', async () => { const { d: root, store, admit, allocate } = fixture(); const attempt = admit(id(1)); const h = allocate(id(1), attempt.id, 'h'); store.markRunning(id(1), attempt.id); @@ -362,4 +453,17 @@ describe('host process checks', () => { await once(child, 'spawn'); expect(hostOpenFiles(join(root, 'link', 'attempt'))).toContain(String(child.pid)); }); + it('fails closed when the bounded recovery probe exceeds its overall deadline', async () => { + const root = dir(); + let at = 0; + await expect(hostOpenFilesBounded(root, { platform: 'darwin', now: () => at, + run: async () => { at = 15_000; return { status: 0, stdout: '', stderr: '' }; } })) + .rejects.toThrow(/timed out/); + }); + it('does not treat an incomplete lsof scan as proving that the workspace is unused', async () => { + const root = dir(); + await expect(hostOpenFilesBounded(root, { platform: 'darwin', + run: async () => ({ status: 1, stdout: '', stderr: 'lsof: WARNING: cannot stat() path' }) })) + .rejects.toThrow(/Could not check/); + }); }); From ce56295c98fe33e15e54c0515b9b7da10e3d222b Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 09:43:11 -0700 Subject: [PATCH 02/13] Harden retained rebase results --- docs/implementation/pre-merge-rebase.md | 4 +-- runner/rebase.ts | 28 +++++++++++++------ runner/store.ts | 5 ++++ test/runner-lifecycle-store.test.ts | 16 +++++++++++ test/runner-rebase.test.ts | 37 ++++++++++++++++++------- 5 files changed, 69 insertions(+), 21 deletions(-) diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md index ccad7080..6072a2ee 100644 --- a/docs/implementation/pre-merge-rebase.md +++ b/docs/implementation/pre-merge-rebase.md @@ -16,9 +16,9 @@ Admission checks the current plan revision, snapshot, review version, task state `GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, and descended from the recorded base. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. -`Store.finishRebase` accepts that result only from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. +`Store.finishRebase` accepts that result only from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. -Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures settle the Git process group before cleanup. Cleanup removes only the validated UUID workspace. A partially created but unregistered worktree is removed only after Git's own worktree list proves it is not registered. +Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures settle the Git process group before cleanup. Cleanup removes only the validated UUID workspace. A partially created but unregistered worktree is removed only after Git's own worktree list proves it is not registered. Result refs are created atomically from the absent state; a reused attempt ID cannot overwrite or delete a previously retained result, while an ambiguous create is reconciled against the exact attempted value. ## Restart recovery diff --git a/runner/rebase.ts b/runner/rebase.ts index d8150649..cc2ca4d4 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -70,7 +70,13 @@ export class GitRebaser { const path = this.#path(attemptId); if (lstatSync(path, { throwIfNoEntry: false })) throw new Error('The rebase workspace already exists.'); - let added = false, retained = false, result: RebaseResult | undefined, primary: unknown; + const ref = rebaseRef(attemptId); + const existing = await this.#call(this.#options.repository.path, ['show-ref', '--verify', '--quiet', ref], scope); + this.#throwIfCancelled(existing, scope); + if (existing.status === 0) throw new Error('This rebase attempt already has a retained result.'); + if (existing.status !== 1) throw this.#failure('show-ref', existing); + let added = false, retained = false, uncertainRef = false, attemptedRef: string | undefined; + let result: RebaseResult | undefined, primary: unknown; try { await this.#git(this.#options.repository.path, ['worktree', 'add', '--detach', '--', path, oldHead], scope); added = true; @@ -94,8 +100,16 @@ export class GitRebaser { await this.#assertCheckout(path, head, 'rebased head', scope); const next = await this.#history(path, onto, head, scope); if (next.length !== old.length) throw new Error('The rebase did not preserve one commit for every task commit.'); - await this.#git(this.#options.repository.path, ['update-ref', rebaseRef(attemptId), head], scope); - retained = true; + attemptedRef = head; + const created = await this.#call(this.#options.repository.path, ['update-ref', ref, head, '0'.repeat(head.length)], scope); + if (created.status === 0) retained = true; + else if (created.status === null) uncertainRef = true; + this.#throwIfCancelled(created, scope); + if (created.status !== 0) { + // A null status can mean the ref write landed but its settlement record did not. Cleanup reconciles that exact + // attempted value. A normal nonzero exit proves atomic creation refused and must preserve the winning ref. + throw this.#failure('update-ref', created); + } result = { oldHead, base: onto, head, mappings: old.map((oldSha, index) => ({ oldSha, newSha: next[index]! })) }; } catch (error) { primary = error; } const cleanupFailures: unknown[] = []; @@ -106,13 +120,9 @@ export class GitRebaser { // Drop a completed rewrite before reporting that cancellation so no caller can apply a cancelled result. if (!primary && signal?.aborted) { primary = signal.reason; - if (retained) { - try { await this.#git(this.#options.repository.path, ['update-ref', '-d', rebaseRef(attemptId)], scope, true); } - catch (error) { cleanupFailures.push(error); } - } } - if (!retained) { - try { await this.#git(this.#options.repository.path, ['update-ref', '-d', rebaseRef(attemptId)], scope, true); } + if ((retained && primary && signal?.aborted) || (!retained && uncertainRef)) { + try { await this.#git(this.#options.repository.path, ['update-ref', '-d', ref, attemptedRef!], scope, true); retained = false; } catch (error) { cleanupFailures.push(error); } } const cleanup = cleanupFailures.length > 1 ? new AggregateError(cleanupFailures, 'Rebase cleanup failed.') : cleanupFailures[0]; diff --git a/runner/store.ts b/runner/store.ts index ecbf8cee..cfeed700 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -771,6 +771,11 @@ export class Store { const captured = this.getSnapshot(identity); if (marker.oldBase !== captured.base || marker.oldHead !== captured.head || marker.onto !== base) throw new GuardRefusal('The rebase result does not match its captured base and head.'); + const endpoint = mappings.at(-1); + if (marker.oldHead === marker.oldBase + ? mappings.length !== 0 || head !== base + : !endpoint || endpoint.oldSha !== marker.oldHead || endpoint.newSha !== head) + throw new GuardRefusal('The rebase mapping does not end at the captured and rewritten heads.'); this.#assertContextWritable(key); const snapshot = this.#applyRebase(identity, key, base, head, mappings); this.#run('UPDATE tasks SET rebase_in_progress=NULL WHERE plan_key=?', key); diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index 583c36ad..f32911ad 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -581,6 +581,9 @@ describe('pre-merge rebase ownership', () => { expect(() => store.finishRebase(identity, expected, taskVersion, randomUUID(), oid(5), oid(7), [ { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, ])).toThrow(/no longer owns/); + expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ + { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(8) }, + ])).toThrow(/does not end/); const group = { pgid: 4242, startedAt: 456 }; store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, 'spawning'); store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, 'spawning', group); @@ -618,6 +621,19 @@ describe('pre-merge rebase ownership', () => { expect(store.getTask(identity).rebaseInProgress).toBeNull(); }); + it('accepts an empty mapping only when the captured history and rewritten result are both empty', () => { + const { store } = fixture(); + const current = store.getSnapshot(identity); + store.recordHistory(identity, { revision: 1, snapshotId: current.id }, oid(1), oid(1), []); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; + const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(1), onto: oid(5) }); + expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(6), [])) + .toThrow(/does not end/); + expect(store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(5), [])) + .toMatchObject({ base: oid(5), head: oid(5) }); + }); + it('rejects a result after another durable task change without erasing its recovery marker', () => { const { store } = fixture(); store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index c98b5c2c..e177a18d 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -67,6 +67,17 @@ describe('trusted pre-merge rebase', () => { expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); }); + it('never overwrites or deletes a retained result when an attempt ID is reused', async () => { + const s = await setup(), attemptId = randomUUID(); + const first = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + git(s.source, 'switch', '-q', 'main'); + const nextBase = commit(s.source, 'later.txt', 'later\n', 'move base again'); + await ensureCommit(s.repository, nextBase); + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: nextBase })) + .rejects.toThrow(/already has a retained result/); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(first.head); + }); + it('accepts tracked files that a later commit adds to gitignore', async () => { const s = await setup(), attemptId = randomUUID(); git(s.source, 'switch', '-q', '-C', 'ignored-feature', s.base); @@ -201,7 +212,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { processes++; }, onProcessGroupSettled: () => { - if (processes === 6 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } + if (processes === 7 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } }, onProcessUnsettled: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) .then(() => null, error => error as AggregateError); @@ -276,11 +287,11 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, - onProcessGroup: () => { if (++processes === 3) stop.abort(new Error('cancelled during rebase')); }, + onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('cancelled during rebase')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during rebase'); - expect(processes).toBeGreaterThanOrEqual(4); // history, worktree add, stopped rebase, cleanup + expect(processes).toBeGreaterThanOrEqual(9); // stopped rebase, worktree removal, then pruning expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); }); @@ -290,7 +301,7 @@ describe('trusted pre-merge rebase', () => { let processes = 0; const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, - onProcessGroup: () => { if (++processes === 5) stop.abort(new Error('cancelled during conflict classification')); }, + onProcessGroup: () => { if (++processes === 8) stop.abort(new Error('cancelled during conflict classification')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during conflict classification'); @@ -319,8 +330,8 @@ describe('trusted pre-merge rebase', () => { let starts = 0, processes = 0; const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, - onProcessStarting: () => { if (++starts === 4) throw new Error('cleanup record failed'); }, - onProcessGroup: () => { if (++processes === 3) stop.abort(new Error('original cancellation')); }, + onProcessStarting: () => { if (++starts === 8) throw new Error('cleanup record failed'); }, + onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('original cancellation')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); @@ -337,11 +348,11 @@ describe('trusted pre-merge rebase', () => { let processes = 0; const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, - onProcessGroup: () => { if (++processes === 8) stop.abort(new Error('cancelled during cleanup')); }, + onProcessGroup: () => { if (++processes === 14) stop.abort(new Error('cancelled during cleanup')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during cleanup'); - expect(processes).toBeGreaterThanOrEqual(10); // result ref, worktree cleanup, prune, then result-ref deletion + expect(processes).toBeGreaterThanOrEqual(16); // result ref, worktree cleanup, prune, then result-ref deletion expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); }); @@ -351,9 +362,14 @@ describe('trusted pre-merge rebase', () => { let settled = 0, starts = 0; const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, - onProcessStarting: () => { if (++starts === 9) throw new Error('ref deletion record failed'); }, + onProcessStarting: () => { if (++starts === 16) throw new Error('ref deletion record failed'); }, onProcessGroup: () => {}, - onProcessGroupSettled: () => { if (++settled === 6) throw new Error('result ref outcome unknown'); }, + onProcessGroupSettled: () => { + if (++settled === 13) { + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toMatch(/^[0-9a-f]{40,64}$/); + throw new Error('result ref outcome unknown'); + } + }, onProcessUnsettled: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) .then(() => null, error => error as AggregateError); @@ -362,6 +378,7 @@ describe('trusted pre-merge rebase', () => { expect(failure.errors.map(error => (error as Error).message)).toEqual(expect.arrayContaining([ 'result ref outcome unknown', 'ref deletion record failed', ])); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).not.toThrow(); }); it('aborts only the validated attempt workspace and ref', async () => { From 73bdfba0e7298458450a8833703f279d6dcb667b Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 10:13:25 -0700 Subject: [PATCH 03/13] Cover retained ref fast path --- test/runner-rebase.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index e177a18d..4785fed9 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -75,6 +75,8 @@ describe('trusted pre-merge rebase', () => { await ensureCommit(s.repository, nextBase); await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: nextBase })) .rejects.toThrow(/already has a retained result/); + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) + .rejects.toThrow(/already has a retained result/); expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(first.head); }); @@ -192,12 +194,12 @@ describe('trusted pre-merge rebase', () => { it('rejects a successful Git call when its settlement hook exhausts the work deadline', async () => { const s = await setup(), attemptId = randomUUID(); - let overran = false; + let settled = 0, overran = false; const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => { - if (!overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } + if (++settled === 2 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } }, onProcessUnsettled: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) From 28473fa72e9bddba7077be7c3e24c06ee57beaa2 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 10:09:27 -0700 Subject: [PATCH 04/13] Bind recovery to exact rebase results --- docs/implementation/pre-merge-rebase.md | 6 +- docs/implementation/runner-lifecycle.md | 2 +- runner/production.ts | 9 ++- runner/rebase.ts | 37 ++++++--- runner/recovery.ts | 100 +++++++++++++++--------- runner/store.ts | 36 ++++++++- test/runner-lifecycle-store.test.ts | 11 ++- test/runner-production.test.ts | 10 ++- test/runner-rebase.test.ts | 60 ++++++++++---- test/runner-recovery.test.ts | 38 +++++++++ 10 files changed, 237 insertions(+), 72 deletions(-) diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md index 6072a2ee..9c0c2ade 100644 --- a/docs/implementation/pre-merge-rebase.md +++ b/docs/implementation/pre-merge-rebase.md @@ -7,7 +7,7 @@ Before the first Git process, `Store.beginRebase` records one marker in `tasks.rebase_in_progress`: ```text -{ attemptId, oldBase, oldHead, onto, startedAt, processGroup } +{ attemptId, oldBase, oldHead, onto, startedAt, resultHead, resultMappings, processGroup } ``` Admission checks the current plan revision, snapshot, review version, task state version, mergeable task status, cancellation state, active runner attempt, active merge, existing rebase, and both ends of the reviewed snapshot. Recording the marker advances the task state version. While it exists, another rebase, the legacy rewrite path, merge admission, runner admission, reassignment, and ordinary status changes fail closed. Cancellation remains pending until the owned rebase has settled and exact-attempt cleanup clears the marker; that cleanup then closes the task. @@ -16,13 +16,13 @@ Admission checks the current plan revision, snapshot, review version, task state `GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, and descended from the recorded base. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. -`Store.finishRebase` accepts that result only from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. +After replay settles, the Store binds the complete ordered mapping and exact retained ref value to that attempt before the result can return. `Store.finishRebase` accepts only that byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures settle the Git process group before cleanup. Cleanup removes only the validated UUID workspace. A partially created but unregistered worktree is removed only after Git's own worktree list proves it is not registered. Result refs are created atomically from the absent state; a reused attempt ID cannot overwrite or delete a previously retained result, while an ambiguous create is reconciled against the exact attempted value. ## Restart recovery -Startup validates every stored marker before touching its resource. If its recorded process group is still alive with the same start time, recovery terminates and awaits it first. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner. The production recovery adapter accepts only the configured plan key, then removes the matching worktree and unfinished result ref from that plan's runner repository. Only after that succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. +Startup validates every stored marker before touching its resource. If its recorded process group is still alive with the same start time, recovery terminates and awaits it first. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner; Linux performs that potentially blocking `/proc` walk in an owned, killable subprocess whose settlement is inside the overall deadline. The production recovery adapter accepts only the configured plan key, then removes the matching worktree. It deletes a result ref only with Git's exact-old-value guard using the durably recorded `resultHead`; a legacy, pre-result, or reused-attempt marker without that value preserves any existing ref. Only after cleanup succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. This ordering is intentionally local-only. The later push integration must extend the durable lifecycle before its first external write and reconcile an ambiguous push outcome; it must not reuse this local completion point as proof that GitHub moved. diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 9fe2fc81..722a1b6f 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -267,7 +267,7 @@ codeboost never writes the user's repository. The commits it makes live in a **r | Holder | Owner | Rule | |---|---|---| | Partial output of a stopped writable attempt | F, using a D export | Done in D by #51 item 6: `exportTaskDiff(storage, { base, imageId, maxBytes, signal, onProcessGroup, timeoutMs, metadataBaseline })` (`agents/container/storage.ts`) takes the storage value or a recovery handle and `base`, the full ID of the commit it was seeded from, and returns `{ diff, truncated }`: at most `maxBytes` (1 MiB at most) of raw diff bytes. It checks both volumes' labels, runs Git in a read-only container with no network that writes nothing to either volume, covers edits and new untracked files (agents cannot commit or stage: the metadata is read-only to them, and the export first checks it against the seeder's baseline, refusing if it changed; a recovery handle needs the `metadataBaseline` F recorded), and cuts the output inside the container, so the Docker work stops at the limit. A changed file over 8 MiB, an untracked nested repository, or an untracked path Git will not add (a part named `.git`, or a symlink named `.gitmodules`) is named in a `codeboost:` notice line instead of diffed, so one large file cannot exhaust the export's memory or deadline; untracked symlinks are diffed as links. What Git skips without a word is named in a notice line too: an entry named `.git` (and, if it holds a repository, the new files beside it), a fifo or socket, an ignored untracked path, and anything in a submodule directory. A changed file whose `ident` or `working-tree-encoding` attribute changes what the diff shows is named too. Anything it cannot read fails the export, and a Git failure's reason includes Git's last two error lines. On abort or at its deadline it stops the client and removes the export container before it settles (an `AbortError`, or an `AggregateError` if that removal did not settle); recovery removes a leftover one. The task filesystems are Docker volumes behind a keeper container. F allocates them with `prepareTaskFilesystems` and removes them with `removeTaskFilesystems`; D's supervisor does not remove them at settlement. But F cannot read a volume from the host. So the order is: `settled` → a **D-provided bounded export** (a new D operation, for example `exportTaskDiff(filesystems, maxBytes)`, which runs `git diff` against the commit the storage was seeded from in a read-only container and returns at most 1 MiB) → F saves it → the terminal write stores `diagnostic_ref` → `removeTaskFilesystems` → the slot is freed. **The export has an overall deadline** (60 seconds, passed to D as an `AbortSignal`), because `maxBytes` bounds only the returned data, not the Docker work. On timeout or any export failure, F records "Partial output could not be exported: " as the diagnostic and continues with the terminal write and removal; shutdown and recovery never wait on an export past its deadline. F saves the export to a runner-owned diagnostics directory, with a total byte cap. **Retention never deletes a file an attempt row still references.** When the cap is exceeded, retention first deletes unreferenced files, oldest first. If that is not enough, it takes the oldest referenced file and, in one transaction, sets that row's `diagnostic_ref` to null and appends "(partial output removed by retention)" to its diagnostic. Only after that commit does it delete the file. No row is ever left pointing at a missing file. The attempt row references it as `diagnostic_ref`. If the capture fails, the row records that it failed. The task filesystem is never reused. | -| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, startedAt, processGroup }`. F3 sets it by compare-and-swap from null, tracks each Git process group until settlement, and clears it by compare-and-swap on the same `attemptId`. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3. | +| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, startedAt, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null, tracks each Git process group until settlement, and records the complete ordered mapping plus the exact retained result-ref value before publishing it. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3, but deletes a retained ref only when its current value equals `resultHead`; a legacy or pre-result marker without that ownership value preserves the ref. | | Workspace rebuild | I3 | I3 never rebuilds before both rows above are resolved for the task. | **Process shutdown is a hard stop.** When the process stops, the running task does not finish. Its attempt ends `cancelled` with the reason "Stopped by shutdown". Restart recovery (lane I3) puts the task back in the queue. This is different from "Stop the queue" (lane I), which lets the running task finish. diff --git a/runner/production.ts b/runner/production.ts index 7a2212df..0b71fb9a 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -109,8 +109,9 @@ export function dRecoveryDeps(image: () => string, rebaser?: Pick service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, null, 'spawning'), onProcessGroup: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, 'spawning', group), onProcessGroupSettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, null), - onProcessUnsettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, 'unsettled') })); + onProcessUnsettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, 'unsettled'), + onResult: (attemptId, head, mappings) => service.store.setRebaseResult(rebasePlanKey, attemptId, head, mappings) })); const recovery = await recoverStartup({ store: service.store, runnerOwner, runnerRoot: config.root, diagnosticsDir, diagnosticsCapBytes: config.diagnosticsCapBytes, - deps: o.recovery ? o.recovery(image) : dRecoveryDeps(image, { abort: async attemptId => (await getRebaser()).abort(attemptId) }, rebasePlanKey) }); + deps: o.recovery ? o.recovery(image) : dRecoveryDeps(image, + { abort: async (attemptId, resultHead) => (await getRebaser()).abort(attemptId, resultHead) }, rebasePlanKey) }); const repository = await getRepository(); const imageId = image(); const workspace = createTaskWorkspace({ store: service.store, runnerRoot: config.root, runnerOwner, repository, imageId, diff --git a/runner/rebase.ts b/runner/rebase.ts index cc2ca4d4..1fa2d3f2 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -27,6 +27,7 @@ export interface GitRebaserOptions { onProcessGroup: (attemptId: string, group: ProcessGroup) => void; onProcessGroupSettled: (attemptId: string, group: ProcessGroup | 'spawning') => void; onProcessUnsettled: (attemptId: string, group: ProcessGroup) => void; + onResult: (attemptId: string, retainedHead: string | null, mappings: readonly { oldSha: string; newSha: string }[]) => void; } interface CallScope { attemptId: string; signal?: AbortSignal; deadline: number; workDeadline: number } @@ -65,16 +66,22 @@ export class GitRebaser { for (const value of [oldBase, oldHead, onto]) if (!COMMIT_ID.test(value)) throw new Error('A full commit ID is required for rebasing.'); signal?.throwIfAborted(); const scope = this.#scope(attemptId, signal); - const old = await this.#history(this.#options.repository.path, oldBase, oldHead, scope); - if (onto === oldBase) return { oldHead, base: onto, head: oldHead, mappings: old.map(sha => ({ oldSha: sha, newSha: sha })) }; - - const path = this.#path(attemptId); - if (lstatSync(path, { throwIfNoEntry: false })) throw new Error('The rebase workspace already exists.'); const ref = rebaseRef(attemptId); const existing = await this.#call(this.#options.repository.path, ['show-ref', '--verify', '--quiet', ref], scope); this.#throwIfCancelled(existing, scope); if (existing.status === 0) throw new Error('This rebase attempt already has a retained result.'); if (existing.status !== 1) throw this.#failure('show-ref', existing); + const old = await this.#history(this.#options.repository.path, oldBase, oldHead, scope); + if (onto === oldBase) { + const mappings = old.map(sha => ({ oldSha: sha, newSha: sha })); + this.#assertResultCurrent(scope); + this.#options.onResult(attemptId, null, mappings); + this.#assertResultCurrent(scope); + return { oldHead, base: onto, head: oldHead, mappings }; + } + + const path = this.#path(attemptId); + if (lstatSync(path, { throwIfNoEntry: false })) throw new Error('The rebase workspace already exists.'); let added = false, retained = false, uncertainRef = false, attemptedRef: string | undefined; let result: RebaseResult | undefined, primary: unknown; try { @@ -110,7 +117,11 @@ export class GitRebaser { // attempted value. A normal nonzero exit proves atomic creation refused and must preserve the winning ref. throw this.#failure('update-ref', created); } - result = { oldHead, base: onto, head, mappings: old.map((oldSha, index) => ({ oldSha, newSha: next[index]! })) }; + const mappings = old.map((oldSha, index) => ({ oldSha, newSha: next[index]! })); + this.#assertResultCurrent(scope); + this.#options.onResult(attemptId, head, mappings); + this.#assertResultCurrent(scope); + result = { oldHead, base: onto, head, mappings }; } catch (error) { primary = error; } const cleanupFailures: unknown[] = []; if (added || lstatSync(path, { throwIfNoEntry: false })) { @@ -121,7 +132,7 @@ export class GitRebaser { if (!primary && signal?.aborted) { primary = signal.reason; } - if ((retained && primary && signal?.aborted) || (!retained && uncertainRef)) { + if ((retained && primary) || (!retained && uncertainRef)) { try { await this.#git(this.#options.repository.path, ['update-ref', '-d', ref, attemptedRef!], scope, true); retained = false; } catch (error) { cleanupFailures.push(error); } } @@ -133,12 +144,14 @@ export class GitRebaser { } /** Startup/live recovery for an attempt whose Store marker still exists. */ - async abort(attemptId: string): Promise { + async abort(attemptId: string, resultHead?: string): Promise { const ref = rebaseRef(attemptId), path = this.#path(attemptId), stat = lstatSync(path, { throwIfNoEntry: false }); + if (resultHead !== undefined && !COMMIT_ID.test(resultHead)) throw new Error('A full retained result ID is required.'); const scope = this.#scope(attemptId); if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) throw new Error('The rebase workspace is not a plain directory.'); await this.#remove(path, scope); - await this.#git(this.#options.repository.path, ['update-ref', '-d', ref], scope, true); + if (resultHead !== undefined) + await this.#git(this.#options.repository.path, ['update-ref', '-d', ref, resultHead], scope, true); } async #history(repository: string, base: string, head: string, scope: CallScope): Promise { @@ -294,6 +307,12 @@ export class GitRebaser { return { attemptId, signal, deadline, workDeadline: deadline - CLEANUP_RESERVE_MS }; } + #assertResultCurrent(scope: CallScope): void { + scope.signal?.throwIfAborted(); + if (performance.now() >= scope.workDeadline) + throw Object.assign(new Error('The rebase deadline expired while recording its result.'), { code: 'ETIMEDOUT' }); + } + #failure(command: string, outcome: DockerOutcome, program = 'git'): Error { if (outcome.status === null && outcome.error) return outcome.error; const detail = (outcome.status === null && outcome.error ? outcome.error.message : diff --git a/runner/recovery.ts b/runner/recovery.ts index b1d23917..6c01c2c3 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -1,8 +1,9 @@ -import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, opendirSync, readdirSync, realpathSync, rmSync, statSync, statfsSync } from 'node:fs'; +import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, opendirSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, statfsSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { execFileSync } from 'node:child_process'; import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; import type { DatabaseSync } from 'node:sqlite'; import type { RebaseMarker, Store } from './store.ts'; import { WRITABLE_KINDS, isUuidV4 } from './lifecycle.ts'; @@ -148,6 +149,24 @@ export interface RecoveryReport { finalized: { attemptId: string; planKey: string; state: string; requeued: boolean }[]; requeue: string[]; removedDirectories: string[]; unknownEntries: string[]; unmatchedStorage: string[]; repairedMerges: string[]; } +function validRebaseResult(marker: Partial, head: unknown, mappings: unknown): boolean { + if (mappings === null) return head === null; + if (!Array.isArray(mappings) || mappings.length > 500 || + (marker.onto === marker.oldBase) !== (head === null)) return false; + const sources = new Set(), destinations = new Set(); + for (const mapping of mappings as unknown[]) { + if (!mapping || typeof mapping !== 'object') return false; + const { oldSha, newSha } = mapping as { oldSha?: unknown; newSha?: unknown }; + if (typeof oldSha !== 'string' || typeof newSha !== 'string' || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(oldSha) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(newSha) || + sources.has(oldSha) || destinations.has(newSha)) return false; + sources.add(oldSha); destinations.add(newSha); + } + if (marker.oldHead === marker.oldBase) return mappings.length === 0; + const endpoint = mappings.at(-1) as { oldSha: string; newSha: string } | undefined; + return !!endpoint && endpoint.oldSha === marker.oldHead && + (head === null ? endpoint.newSha === marker.oldHead && mappings.every(mapping => mapping.oldSha === mapping.newSha) : endpoint.newSha === head); +} const EXPORT_LIMIT = 1024 * 1024; /** @@ -201,11 +220,13 @@ export async function recoverStartup(o: RecoveryOptions): Promise | null; - const processGroup = marker?.processGroup; + const processGroup = marker?.processGroup, resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null; if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldBase) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldHead) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.onto) || !Number.isSafeInteger(marker.startedAt) || marker.startedAt! < 0 || + (resultHead !== null && (typeof resultHead !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(resultHead))) || + !validRebaseResult(marker, resultHead, resultMappings) || (processGroup !== null && (!processGroup || typeof processGroup === 'string' || !Number.isSafeInteger(processGroup.pgid) || processGroup.pgid <= 1 || !Number.isSafeInteger(processGroup.startedAt) || processGroup.startedAt < 0))) throw new RecoveryBlocked('An interrupted rebase has an invalid recovery marker', [rebase.planKey]); @@ -221,7 +242,7 @@ export async function recoverStartup(o: RecoveryOptions): Promise[2]) => Promise; +function linuxOpenFiles(dir: string): string[] { + dir = realpathSync(dir); + const users: string[] = [], disappeared = (error: unknown) => ['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code ?? ''); + let processes = 0, descriptors = 0; + const proc = opendirSync('/proc'); + try { + for (let entry = proc.readSync(); entry; entry = proc.readSync()) { + if (!/^\d+$/.test(entry.name)) continue; + if (++processes > MAX_PROC_ENTRIES) throw new Error('Open-file probe exceeded its process bound.'); + const pid = entry.name, links: string[] = []; + try { if (process.getuid && lstatSync(`/proc/${pid}`).uid !== process.getuid()) continue; } + catch (error) { if (disappeared(error)) continue; throw error; } + try { links.push(realpathSync(`/proc/${pid}/cwd`)); } + catch (error) { if (!disappeared(error)) throw error; } + let fds: ReturnType | undefined; + try { fds = opendirSync(`/proc/${pid}/fd`); } + catch (error) { if (disappeared(error)) continue; throw error; } + try { + for (let fd = fds.readSync(); fd; fd = fds.readSync()) { + if (++descriptors > MAX_FD_ENTRIES) throw new Error('Open-file probe exceeded its descriptor bound.'); + try { links.push(realpathSync(`/proc/${pid}/fd/${fd.name}`)); } + catch (error) { if (!disappeared(error)) throw error; } + } + } finally { fds.closeSync(); } + if (links.some(link => link === dir || link.startsWith(`${dir}/`))) users.push(pid); + } + } finally { proc.closeSync(); } + return users; +} /** Recovery's bounded variant. Its lsof process and pipes settle inside the overall deadline before startup proceeds. */ export async function hostOpenFilesBounded(dir: string, options: { timeoutMs?: number; platform?: NodeJS.Platform; now?: () => number; run?: OpenFilesRun } = {}): Promise { - dir = realpathSync(dir); const timeoutMs = options.timeoutMs ?? OPEN_FILES_DEADLINE_MS, now = options.now ?? performance.now.bind(performance); if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 12_001) throw new Error('Invalid open-file probe deadline.'); const deadline = now() + timeoutMs; if ((options.platform ?? process.platform) === 'linux') { - const users: string[] = [], disappeared = (error: unknown) => ['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code ?? ''); - let processes = 0, descriptors = 0; - const proc = opendirSync('/proc'); - try { - for (let entry = proc.readSync(); entry; entry = proc.readSync()) { - if (!/^\d+$/.test(entry.name)) continue; - if (++processes > MAX_PROC_ENTRIES || now() >= deadline) - throw new Error('Open-file probe exceeded its bound. Refusing to recover the rebase workspace.'); - const pid = entry.name, links: string[] = []; - try { if (process.getuid && lstatSync(`/proc/${pid}`).uid !== process.getuid()) continue; } - catch (error) { if (disappeared(error)) continue; throw error; } - try { links.push(realpathSync(`/proc/${pid}/cwd`)); } - catch (error) { if (!disappeared(error)) throw new Error(`Could not inspect process ${pid}. Refusing to recover the rebase workspace.`, { cause: error }); } - let fds: ReturnType | undefined; - try { fds = opendirSync(`/proc/${pid}/fd`); } - catch (error) { - if (disappeared(error)) continue; - throw new Error(`Could not inspect process ${pid}. Refusing to recover the rebase workspace.`, { cause: error }); - } - try { - for (let fd = fds.readSync(); fd; fd = fds.readSync()) { - if (++descriptors > MAX_FD_ENTRIES || now() >= deadline) - throw new Error('Open-file probe exceeded its bound. Refusing to recover the rebase workspace.'); - try { links.push(realpathSync(`/proc/${pid}/fd/${fd.name}`)); } - catch (error) { if (!disappeared(error)) throw new Error(`Could not inspect process ${pid}. Refusing to recover the rebase workspace.`, { cause: error }); } - } - } finally { fds.closeSync(); } - if (links.some(link => link === dir || link.startsWith(`${dir}/`))) users.push(pid); - } - } finally { proc.closeSync(); } + const childTimeoutMs = timeoutMs - 12_000; + const outcome = await (options.run ?? runInProcessGroup)(process.execPath, + [fileURLToPath(import.meta.url), '--scan-open-files'], { env: { PATH: process.env.PATH ?? '' }, input: Buffer.from(dir), + timeoutMs: childTimeoutMs, graceMs: 1_000, maxBuffer: 1024 * 1024 }); if (now() >= deadline) throw new Error('Open-file probe timed out. Refusing to recover the rebase workspace.'); - return users; + if (outcome.status === 0 && !outcome.stderr.trim()) { + const users = outcome.stdout.split('\n').filter(Boolean); + if (users.every(pid => /^\d+$/.test(pid))) return users; + } + throw new Error('Could not check which processes use the rebase workspace. Refusing to recover it.'); } // runInProcessGroup may spend 1 s on SIGTERM, 10 s draining the group, and 1 s draining pipes after its timer. const childTimeoutMs = timeoutMs - 12_000; @@ -341,3 +368,6 @@ export async function hostOpenFilesBounded(dir: string, options: { timeoutMs?: n if (outcome.status === 1 && !outcome.stdout && !outcome.stderr.trim() && !outcome.error) return []; throw new Error('Could not check which processes use the rebase workspace. Refusing to recover it.'); } + +if (process.argv[1] === fileURLToPath(import.meta.url) && process.argv[2] === '--scan-open-files') + process.stdout.write(`${linuxOpenFiles(readFileSync(0, 'utf8')).join('\n')}\n`); diff --git a/runner/store.ts b/runner/store.ts index cfeed700..5ce60f7b 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -89,6 +89,8 @@ export interface RebaseMarker { oldHead: string; onto: string; startedAt: number; + resultHead: string | null; + resultMappings: { oldSha: string; newSha: string }[] | null; processGroup: { pgid: number; startedAt: number } | 'spawning' | 'unsettled' | null; } export interface AttemptRecord { @@ -718,7 +720,8 @@ export class Store { const attemptId = input.attemptId ?? randomUUID(), startedAt = input.startedAt ?? Date.now(); assertUuidV4(attemptId, 'Rebase attempt ID'); if (!Number.isSafeInteger(startedAt) || startedAt < 0) throw new Error('Invalid rebase start time.'); - const marker: RebaseMarker = { attemptId, oldBase: input.oldBase, oldHead: input.oldHead, onto: input.onto, startedAt, processGroup: null }; + const marker: RebaseMarker = { attemptId, oldBase: input.oldBase, oldHead: input.oldHead, onto: input.onto, startedAt, + resultHead: null, resultMappings: null, processGroup: null }; const key = identityKey(identity); return this.#transaction(() => { this.#expect(key, expected); @@ -754,6 +757,33 @@ export class Store { throw new GuardRefusal('This rebase attempt no longer owns the task.'); }); } + /** Bind the complete result and its exact retained ref value before the rebaser can return it to its caller. */ + setRebaseResult(planKey: string, attemptId: string, head: string | null, + mappings: readonly { oldSha: string; newSha: string }[]): void { + assertUuidV4(attemptId, 'Rebase attempt ID'); if (head !== null) sha(head); + if (mappings.length > 500) throw new Error('Rebase history exceeds 500 commits.'); + const sources = new Set(), destinations = new Set(); + for (const mapping of mappings) { + sha(mapping.oldSha); sha(mapping.newSha); + if (sources.has(mapping.oldSha) || destinations.has(mapping.newSha)) throw new Error('Rebase mappings must be one-to-one.'); + sources.add(mapping.oldSha); destinations.add(mapping.newSha); + } + this.#transaction(() => { + const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); + if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); + if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if ((marker.onto === marker.oldBase) !== (head === null)) throw new GuardRefusal('The retained rebase result does not match its target.'); + const endpoint = mappings.at(-1); + if (marker.oldHead === marker.oldBase ? mappings.length !== 0 : !endpoint || endpoint.oldSha !== marker.oldHead || + (head === null ? endpoint.newSha !== marker.oldHead || mappings.some(mapping => mapping.oldSha !== mapping.newSha) : endpoint.newSha !== head)) + throw new GuardRefusal('The retained rebase history does not end at its captured head.'); + if (marker.resultMappings !== null && (marker.resultHead !== head || stable(marker.resultMappings) !== stable(mappings))) + throw new GuardRefusal('Another result owns this rebase.'); + if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', + encode({ ...marker, resultHead: head, resultMappings: [...mappings] }), planKey, task.rebase_in_progress as string).changes !== 1) + throw new GuardRefusal('This rebase attempt no longer owns the task.'); + }); + } /** Publish a rebase only while every review and task counter captured by its attempt is still current. */ finishRebase(identity: PlanIdentity, expected: ReviewState & { reviewVersion: number }, expectedTaskStateVersion: number, attemptId: string, base: string, head: string, mappings: readonly { oldSha: string; newSha: string }[]): Snapshot { @@ -768,13 +798,15 @@ export class Store { if (task.state_version !== expectedTaskStateVersion + 1) throw new GuardRefusal('The task changed during the rebase. Discard its result.'); if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if (marker.resultMappings === null || (marker.onto === marker.oldBase ? marker.resultHead !== null : marker.resultHead !== head)) + throw new GuardRefusal('The rebase result does not own its retained ref.'); const captured = this.getSnapshot(identity); if (marker.oldBase !== captured.base || marker.oldHead !== captured.head || marker.onto !== base) throw new GuardRefusal('The rebase result does not match its captured base and head.'); const endpoint = mappings.at(-1); if (marker.oldHead === marker.oldBase ? mappings.length !== 0 || head !== base - : !endpoint || endpoint.oldSha !== marker.oldHead || endpoint.newSha !== head) + : !endpoint || endpoint.oldSha !== marker.oldHead || endpoint.newSha !== head || stable(mappings) !== stable(marker.resultMappings)) throw new GuardRefusal('The rebase mapping does not end at the captured and rewritten heads.'); this.#assertContextWritable(key); const snapshot = this.#applyRebase(identity, key, base, head, mappings); diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index f32911ad..49a852e9 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -543,7 +543,7 @@ describe('pre-merge rebase ownership', () => { readyVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3) })).toThrow(/current review version/); const marker = store.beginRebase(identity, expected, readyVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3), startedAt: 123 }); expect(store.getTask(identity).rebaseInProgress).toEqual({ attemptId: marker.attemptId, oldBase: oid(1), oldHead: oid(2), onto: oid(3), startedAt: 123, - processGroup: null }); + resultHead: null, resultMappings: null, processGroup: null }); const group = { pgid: 4242, startedAt: 456 }; expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, { pgid: 1, startedAt: 456 })).toThrow(/Invalid rebase process group/); @@ -581,6 +581,12 @@ describe('pre-merge rebase ownership', () => { expect(() => store.finishRebase(identity, expected, taskVersion, randomUUID(), oid(5), oid(7), [ { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, ])).toThrow(/no longer owns/); + const resultMappings = [{ oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }]; + store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), resultMappings); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ resultMappings }); + expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ + { oldSha: oid(4), newSha: oid(7) }, + ])).toThrow(/does not end/); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(8) }, ])).toThrow(/does not end/); @@ -629,7 +635,8 @@ describe('pre-merge rebase ownership', () => { const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(1), onto: oid(5) }); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(6), [])) - .toThrow(/does not end/); + .toThrow(/does not own/); + store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(5), []); expect(store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(5), [])) .toMatchObject({ base: oid(5), head: oid(5) }); }); diff --git a/test/runner-production.test.ts b/test/runner-production.test.ts index 4320c5d4..0879cb95 100644 --- a/test/runner-production.test.ts +++ b/test/runner-production.test.ts @@ -228,13 +228,19 @@ describe('recovery warnings', () => { describe('D adapters', () => { it('routes an interrupted rebase marker to the production rebaser before Store recovery clears it', async () => { - const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID() }; + const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultHead: '7'.repeat(40) }; const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); await deps.abortRebase!('plan-key', marker); - expect(abort).toHaveBeenCalledWith(marker.attemptId); + expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead); await expect(deps.abortRebase!('another-plan', marker)).rejects.toThrow(/another configured plan/); expect(abort).toHaveBeenCalledTimes(1); }); + it('does not grant a reused or legacy recovery marker ownership of an existing result ref', async () => { + const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultHead: null }; + const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); + await deps.abortRebase!('plan-key', marker); + expect(abort).toHaveBeenCalledWith(marker.attemptId, undefined); + }); it('passes D\'s own handles through, and turns every unowned object into the command that removes it', async () => { const handle = Object.freeze({ runnerOwner: OWNER, attemptId: randomUUID(), allocationId: randomUUID() }); const spy = vi.mocked(recoverLeftovers).mockResolvedValue({ removed: [], storage: [handle], unowned: [ diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index 4785fed9..bbc9d9c3 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -39,7 +39,8 @@ async function setup(conflict = false) { await ensureCommit(repository, foreign); await ensureCommit(repository, onto); const rebaser = new GitRebaser({ repository, runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, - onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResult: () => {} }); return { root, source, runnerRoot, repository, rebaser, base, owned, foreign, onto }; } @@ -67,6 +68,17 @@ describe('trusted pre-merge rebase', () => { expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); }); + it('rejects a result callback that blocks past the operation deadline', async () => { + const s = await setup(), attemptId = randomUUID(); + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResult: () => vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER) }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) + .rejects.toThrow(/deadline expired while recording its result/); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + }); + it('never overwrites or deletes a retained result when an attempt ID is reused', async () => { const s = await setup(), attemptId = randomUUID(); const first = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); @@ -77,6 +89,7 @@ describe('trusted pre-merge rebase', () => { .rejects.toThrow(/already has a retained result/); await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) .rejects.toThrow(/already has a retained result/); + await s.rebaser.abort(attemptId); // A reused live/startup marker has no ownership value for the earlier result. expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(first.head); }); @@ -134,7 +147,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: (ownedAttempt, group) => { expect(ownedAttempt).toBe(attemptId); if (group !== 'spawning') { expect(active.get(group.pgid)).toBe(group.startedAt); active.delete(group.pgid); } - }, onProcessUnsettled: () => {} }); + }, onProcessUnsettled: () => {}, onResult: () => {} }); await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); expect(seen.length).toBeGreaterThan(1); expect(active.size).toBe(0); @@ -144,7 +157,8 @@ describe('trusted pre-merge rebase', () => { const s = await setup(); expect(() => new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_000, - onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} })).toThrow(/Invalid rebase deadline/); + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResult: () => {} })).toThrow(/Invalid rebase deadline/); }); it('does not start Git when a synchronous ownership hook exhausts the work deadline', async () => { @@ -154,7 +168,8 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_001, onProcessStarting: () => { starts++; const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, - onProcessGroupSettled: (_attempt, group) => { if (group === 'spawning') settled++; }, onProcessUnsettled: () => {} }); + onProcessGroupSettled: (_attempt, group) => { if (group === 'spawning') settled++; }, onProcessUnsettled: () => {}, + onResult: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })).rejects.toThrow(/deadline expired/); expect({ starts, settled }).toEqual({ starts: 1, settled: 1 }); }); @@ -169,7 +184,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: () => { if (firstSettlement) { expect(groups).toBe(0); firstSettlement = false; } expect(owner).not.toBe('none'); owner = 'none'; - }, onProcessUnsettled: () => {} }); + }, onProcessUnsettled: () => {}, onResult: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })) .rejects.toThrow('ownership commit outcome unknown'); expect(owner).toBe('none'); @@ -182,7 +197,8 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_001, onProcessStarting: () => { const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, - onProcessGroupSettled: () => { throw new Error('settlement write failed'); }, onProcessUnsettled: () => {} }); + onProcessGroupSettled: () => { throw new Error('settlement write failed'); }, onProcessUnsettled: () => {}, + onResult: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -201,7 +217,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: () => { if (++settled === 2 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } }, - onProcessUnsettled: () => {} }); + onProcessUnsettled: () => {}, onResult: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) .rejects.toThrow(/deadline expired while recording process settlement/); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -215,7 +231,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroup: () => { processes++; }, onProcessGroupSettled: () => { if (processes === 7 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } - }, onProcessUnsettled: () => {} }); + }, onProcessUnsettled: () => {}, onResult: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -290,7 +306,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('cancelled during rebase')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during rebase'); expect(processes).toBeGreaterThanOrEqual(9); // stopped rebase, worktree removal, then pruning @@ -304,7 +320,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 8) stop.abort(new Error('cancelled during conflict classification')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during conflict classification'); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -317,7 +333,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => stop.abort(new Error('cancelled by caller')), onProcessGroupSettled: () => { if (!failed) { failed = true; throw new Error('settlement write failed'); } }, - onProcessUnsettled: () => {} }); + onProcessUnsettled: () => {}, onResult: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -334,7 +350,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => { if (++starts === 8) throw new Error('cleanup record failed'); }, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('original cancellation')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -351,7 +367,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 14) stop.abort(new Error('cancelled during cleanup')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during cleanup'); expect(processes).toBeGreaterThanOrEqual(16); // result ref, worktree cleanup, prune, then result-ref deletion @@ -372,7 +388,7 @@ describe('trusted pre-merge rebase', () => { throw new Error('result ref outcome unknown'); } }, - onProcessUnsettled: () => {} }); + onProcessUnsettled: () => {}, onResult: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -383,15 +399,29 @@ describe('trusted pre-merge rebase', () => { expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).not.toThrow(); }); - it('aborts only the validated attempt workspace and ref', async () => { + it('aborts only a retained ref whose exact value is durably owned by the attempt', async () => { const s = await setup(), attemptId = randomUUID(); const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); await s.rebaser.abort(attemptId); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); + await s.rebaser.abort(attemptId, result.head); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); await expect(s.rebaser.abort('../other')).rejects.toThrow(/UUID v4/); }); + it('removes its retained ref when durable result ownership cannot be recorded', async () => { + const s = await setup(), attemptId = randomUUID(); + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResult: () => { throw new Error('result ownership write failed'); } }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })) + .rejects.toThrow('result ownership write failed'); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + }); + it('prunes an exact stale worktree registration even when its directory is already gone', async () => { const s = await setup(), attemptId = randomUUID(), path = join(s.runnerRoot, OWNER, 'rebases', attemptId); git(s.repository.path, 'worktree', 'add', '--detach', '--', path, s.foreign); diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index 22becb8a..0236f6e0 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -257,6 +257,34 @@ describe('startup recovery sequence', () => { expect(calls).toEqual(['recover', 'terminate:5151', 'abort-rebase']); expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); }); + it('passes only the exact durably owned result ref to rebase recovery', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)); + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123 }); + store.setRebaseResult(store.getTask(id(1)).planKey, marker.attemptId, oid(4), + [{ oldSha: snapshot.head, newSha: oid(4) }]); + const abortRebase = vi.fn(async () => undefined); + await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase }).d }); + expect(abortRebase).toHaveBeenCalledWith(store.getTask(id(1)).planKey, + expect.objectContaining({ attemptId: marker.attemptId, resultHead: oid(4), + resultMappings: [{ oldSha: snapshot.head, newSha: oid(4) }], processGroup: null })); + expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); + }); + it('normalizes a legacy rebase marker without result ownership and preserves its ref during recovery', async () => { + const { d: root, store, raw } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), attemptId = randomUUID(), planKey = store.getTask(id(1)).planKey; + const marker = { attemptId, oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123, processGroup: null }; + raw(`UPDATE tasks SET rebase_in_progress='${JSON.stringify(marker)}' WHERE plan_key='${planKey}'`); + const abortRebase = vi.fn(async () => undefined); + await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase }).d }); + expect(abortRebase).toHaveBeenCalledWith(planKey, { ...marker, resultHead: null, resultMappings: null }); + expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); + }); it('retains a dead-group marker while another process still uses its rebase workspace', async () => { const { d: root, store } = fixture(); store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); @@ -460,6 +488,16 @@ describe('host process checks', () => { run: async () => { at = 15_000; return { status: 0, stdout: '', stderr: '' }; } })) .rejects.toThrow(/timed out/); }); + it('runs the Linux proc walk in the owned subprocess whose settlement is inside the deadline', async () => { + const root = dir(); let at = 0, seen: { input?: Buffer; timeoutMs: number } | undefined; + await expect(hostOpenFilesBounded(root, { platform: 'linux', now: () => at, + run: async (file, args, options) => { + expect(file).toBe(process.execPath); expect(args).toContain('--scan-open-files'); seen = options; + at = 15_000; return { status: 0, stdout: '123\n', stderr: '' }; + } })).rejects.toThrow(/timed out/); + expect(seen?.input?.toString()).toBe(root); + expect(seen?.timeoutMs).toBe(3_000); + }); it('does not treat an incomplete lsof scan as proving that the workspace is unused', async () => { const root = dir(); await expect(hostOpenFilesBounded(root, { platform: 'darwin', From ebcd20f04b0cd24fc25b23931955fd780eed434f Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 10:24:21 -0700 Subject: [PATCH 05/13] Bind rebase results to reviewed history --- docs/implementation/pre-merge-rebase.md | 8 ++-- docs/implementation/runner-lifecycle.md | 2 +- runner/production.ts | 2 +- runner/rebase.ts | 12 ++--- runner/recovery.ts | 19 ++++++-- runner/store.ts | 33 ++++++++------ test/runner-lifecycle-store.test.ts | 56 ++++++++++++++++++----- test/runner-rebase.test.ts | 60 ++++++++++++++----------- test/runner-recovery.test.ts | 17 ++++--- 9 files changed, 134 insertions(+), 75 deletions(-) diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md index 9c0c2ade..dc61e224 100644 --- a/docs/implementation/pre-merge-rebase.md +++ b/docs/implementation/pre-merge-rebase.md @@ -7,16 +7,16 @@ Before the first Git process, `Store.beginRebase` records one marker in `tasks.rebase_in_progress`: ```text -{ attemptId, oldBase, oldHead, onto, startedAt, resultHead, resultMappings, processGroup } +{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultHead, resultMappings, processGroup } ``` -Admission checks the current plan revision, snapshot, review version, task state version, mergeable task status, cancellation state, active runner attempt, active merge, existing rebase, and both ends of the reviewed snapshot. Recording the marker advances the task state version. While it exists, another rebase, the legacy rewrite path, merge admission, runner admission, reassignment, and ordinary status changes fail closed. Cancellation remains pending until the owned rebase has settled and exact-attempt cleanup clears the marker; that cleanup then closes the task. +Admission checks the current plan revision, snapshot, review version, task state version, mergeable task status, cancellation state, active runner attempt, active merge, existing rebase, both ends of the reviewed snapshot, and the complete ordered old history captured by review. Recording the marker advances the task state version. While it exists, another rebase, the legacy rewrite path, merge admission, runner admission, reassignment, and ordinary status changes fail closed. Cancellation remains pending until the owned rebase has settled and exact-attempt cleanup clears the marker; that cleanup then closes the task. `processGroup` is null only while no Git call owns work. Immediately before spawn it becomes `spawning`; the synchronous spawn callback replaces that with `{ pgid, startedAt }`. After that exact group is confirmed settled, a compare-and-swap clears it. A group that cannot be drained remains recorded, and a descendant that escapes the group leaves `unsettled`; restart recovery fails closed on either unknown state. A duplicate spawn or late settlement therefore cannot replace or erase another live process owner. -`GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, and descended from the recorded base. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. +`GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, descended from the recorded base, and byte-for-byte equal to the ordered history in the durable marker. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. -After replay settles, the Store binds the complete ordered mapping and exact retained ref value to that attempt before the result can return. `Store.finishRebase` accepts only that byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. +After replay settles, the rebaser supplies the Store's result hook only the ordered rewritten history. The Store zips that history by position with the independently captured `oldHistory`, then binds the resulting complete mapping and exact retained ref value before the result can return. `Store.finishRebase` accepts only that byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures settle the Git process group before cleanup. Cleanup removes only the validated UUID workspace. A partially created but unregistered worktree is removed only after Git's own worktree list proves it is not registered. Result refs are created atomically from the absent state; a reused attempt ID cannot overwrite or delete a previously retained result, while an ambiguous create is reconciled against the exact attempted value. diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 722a1b6f..265fa88d 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -267,7 +267,7 @@ codeboost never writes the user's repository. The commits it makes live in a **r | Holder | Owner | Rule | |---|---|---| | Partial output of a stopped writable attempt | F, using a D export | Done in D by #51 item 6: `exportTaskDiff(storage, { base, imageId, maxBytes, signal, onProcessGroup, timeoutMs, metadataBaseline })` (`agents/container/storage.ts`) takes the storage value or a recovery handle and `base`, the full ID of the commit it was seeded from, and returns `{ diff, truncated }`: at most `maxBytes` (1 MiB at most) of raw diff bytes. It checks both volumes' labels, runs Git in a read-only container with no network that writes nothing to either volume, covers edits and new untracked files (agents cannot commit or stage: the metadata is read-only to them, and the export first checks it against the seeder's baseline, refusing if it changed; a recovery handle needs the `metadataBaseline` F recorded), and cuts the output inside the container, so the Docker work stops at the limit. A changed file over 8 MiB, an untracked nested repository, or an untracked path Git will not add (a part named `.git`, or a symlink named `.gitmodules`) is named in a `codeboost:` notice line instead of diffed, so one large file cannot exhaust the export's memory or deadline; untracked symlinks are diffed as links. What Git skips without a word is named in a notice line too: an entry named `.git` (and, if it holds a repository, the new files beside it), a fifo or socket, an ignored untracked path, and anything in a submodule directory. A changed file whose `ident` or `working-tree-encoding` attribute changes what the diff shows is named too. Anything it cannot read fails the export, and a Git failure's reason includes Git's last two error lines. On abort or at its deadline it stops the client and removes the export container before it settles (an `AbortError`, or an `AggregateError` if that removal did not settle); recovery removes a leftover one. The task filesystems are Docker volumes behind a keeper container. F allocates them with `prepareTaskFilesystems` and removes them with `removeTaskFilesystems`; D's supervisor does not remove them at settlement. But F cannot read a volume from the host. So the order is: `settled` → a **D-provided bounded export** (a new D operation, for example `exportTaskDiff(filesystems, maxBytes)`, which runs `git diff` against the commit the storage was seeded from in a read-only container and returns at most 1 MiB) → F saves it → the terminal write stores `diagnostic_ref` → `removeTaskFilesystems` → the slot is freed. **The export has an overall deadline** (60 seconds, passed to D as an `AbortSignal`), because `maxBytes` bounds only the returned data, not the Docker work. On timeout or any export failure, F records "Partial output could not be exported: " as the diagnostic and continues with the terminal write and removal; shutdown and recovery never wait on an export past its deadline. F saves the export to a runner-owned diagnostics directory, with a total byte cap. **Retention never deletes a file an attempt row still references.** When the cap is exceeded, retention first deletes unreferenced files, oldest first. If that is not enough, it takes the oldest referenced file and, in one transaction, sets that row's `diagnostic_ref` to null and appends "(partial output removed by retention)" to its diagnostic. Only after that commit does it delete the file. No row is ever left pointing at a missing file. The attempt row references it as `diagnostic_ref`. If the capture fails, the row records that it failed. The task filesystem is never reused. | -| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, startedAt, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null, tracks each Git process group until settlement, and records the complete ordered mapping plus the exact retained result-ref value before publishing it. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3, but deletes a retained ref only when its current value equals `resultHead`; a legacy or pre-result marker without that ownership value preserves the ref. | +| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null with the complete ordered reviewed history, tracks each Git process group until settlement, and records the complete ordered mapping plus the exact retained result-ref value before publishing it. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3, but deletes a retained ref only when its current value equals `resultHead`; a legacy or pre-result marker without that ownership value preserves the ref. | | Workspace rebuild | I3 | I3 never rebuilds before both rows above are resolved for the task. | **Process shutdown is a hard stop.** When the process stops, the running task does not finish. Its attempt ends `cancelled` with the reason "Stopped by shutdown". Restart recovery (lane I3) puts the task back in the queue. This is different from "Stop the queue" (lane I), which lets the running task finish. diff --git a/runner/production.ts b/runner/production.ts index 0b71fb9a..09883a3b 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -210,7 +210,7 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd onProcessGroup: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, 'spawning', group), onProcessGroupSettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, null), onProcessUnsettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, 'unsettled'), - onResult: (attemptId, head, mappings) => service.store.setRebaseResult(rebasePlanKey, attemptId, head, mappings) })); + onResult: (attemptId, head, history) => service.store.setRebaseResult(rebasePlanKey, attemptId, head, history) })); const recovery = await recoverStartup({ store: service.store, runnerOwner, runnerRoot: config.root, diagnosticsDir, diagnosticsCapBytes: config.diagnosticsCapBytes, deps: o.recovery ? o.recovery(image) : dRecoveryDeps(image, diff --git a/runner/rebase.ts b/runner/rebase.ts index 1fa2d3f2..66f6638a 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -27,7 +27,7 @@ export interface GitRebaserOptions { onProcessGroup: (attemptId: string, group: ProcessGroup) => void; onProcessGroupSettled: (attemptId: string, group: ProcessGroup | 'spawning') => void; onProcessUnsettled: (attemptId: string, group: ProcessGroup) => void; - onResult: (attemptId: string, retainedHead: string | null, mappings: readonly { oldSha: string; newSha: string }[]) => void; + onResult: (attemptId: string, retainedHead: string | null, rewrittenHistory: readonly string[]) => void; } interface CallScope { attemptId: string; signal?: AbortSignal; deadline: number; workDeadline: number } @@ -60,8 +60,8 @@ export class GitRebaser { this.#root = ownerOnlyDirectory(options.runnerRoot, options.runnerOwner, 'rebases'); } - async run(input: { attemptId: string; oldBase: string; oldHead: string; onto: string; signal?: AbortSignal }): Promise { - const { attemptId, oldBase, oldHead, onto, signal } = input; + async run(input: { attemptId: string; oldBase: string; oldHead: string; oldHistory: readonly string[]; onto: string; signal?: AbortSignal }): Promise { + const { attemptId, oldBase, oldHead, oldHistory, onto, signal } = input; rebaseRef(attemptId); for (const value of [oldBase, oldHead, onto]) if (!COMMIT_ID.test(value)) throw new Error('A full commit ID is required for rebasing.'); signal?.throwIfAborted(); @@ -72,10 +72,12 @@ export class GitRebaser { if (existing.status === 0) throw new Error('This rebase attempt already has a retained result.'); if (existing.status !== 1) throw this.#failure('show-ref', existing); const old = await this.#history(this.#options.repository.path, oldBase, oldHead, scope); + if (old.length !== oldHistory.length || old.some((sha, index) => sha !== oldHistory[index])) + throw new Error('The repository history no longer matches the durably captured review history.'); if (onto === oldBase) { const mappings = old.map(sha => ({ oldSha: sha, newSha: sha })); this.#assertResultCurrent(scope); - this.#options.onResult(attemptId, null, mappings); + this.#options.onResult(attemptId, null, old); this.#assertResultCurrent(scope); return { oldHead, base: onto, head: oldHead, mappings }; } @@ -119,7 +121,7 @@ export class GitRebaser { } const mappings = old.map((oldSha, index) => ({ oldSha, newSha: next[index]! })); this.#assertResultCurrent(scope); - this.#options.onResult(attemptId, head, mappings); + this.#options.onResult(attemptId, head, next); this.#assertResultCurrent(scope); result = { oldHead, base: onto, head, mappings }; } catch (error) { primary = error; } diff --git a/runner/recovery.ts b/runner/recovery.ts index 6c01c2c3..bf4c4832 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -149,7 +149,14 @@ export interface RecoveryReport { finalized: { attemptId: string; planKey: string; state: string; requeued: boolean }[]; requeue: string[]; removedDirectories: string[]; unknownEntries: string[]; unmatchedStorage: string[]; repairedMerges: string[]; } -function validRebaseResult(marker: Partial, head: unknown, mappings: unknown): boolean { +function validOldHistory(marker: Partial, history: unknown): boolean { + if (history === null) return true; // A pre-field marker owns no result and is cleanup-only. + if (!Array.isArray(history) || history.length > 500 || + history.some(value => typeof value !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(value)) || + new Set(history).size !== history.length) return false; + return marker.oldHead === marker.oldBase ? history.length === 0 : history.at(-1) === marker.oldHead; +} +function validRebaseResult(marker: Partial, history: unknown, head: unknown, mappings: unknown): boolean { if (mappings === null) return head === null; if (!Array.isArray(mappings) || mappings.length > 500 || (marker.onto === marker.oldBase) !== (head === null)) return false; @@ -162,6 +169,8 @@ function validRebaseResult(marker: Partial, head: unknown, mapping sources.has(oldSha) || destinations.has(newSha)) return false; sources.add(oldSha); destinations.add(newSha); } + if (Array.isArray(history) && (history.length !== mappings.length || + history.some((source, index) => mappings[index]?.oldSha !== source))) return false; if (marker.oldHead === marker.oldBase) return mappings.length === 0; const endpoint = mappings.at(-1) as { oldSha: string; newSha: string } | undefined; return !!endpoint && endpoint.oldSha === marker.oldHead && @@ -220,13 +229,15 @@ export async function recoverStartup(o: RecoveryOptions): Promise | null; - const processGroup = marker?.processGroup, resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null; + const processGroup = marker?.processGroup, oldHistory = marker?.oldHistory ?? null, + resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null; if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldBase) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldHead) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.onto) || !Number.isSafeInteger(marker.startedAt) || marker.startedAt! < 0 || + !validOldHistory(marker, oldHistory) || (resultHead !== null && (typeof resultHead !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(resultHead))) || - !validRebaseResult(marker, resultHead, resultMappings) || + !validRebaseResult(marker, oldHistory, resultHead, resultMappings) || (processGroup !== null && (!processGroup || typeof processGroup === 'string' || !Number.isSafeInteger(processGroup.pgid) || processGroup.pgid <= 1 || !Number.isSafeInteger(processGroup.startedAt) || processGroup.startedAt < 0))) throw new RecoveryBlocked('An interrupted rebase has an invalid recovery marker', [rebase.planKey]); @@ -242,7 +253,7 @@ export async function recoverStartup(o: RecoveryOptions): Promise 500) throw new Error('Rebase history exceeds 500 commits.'); + const oldHistory = [...input.oldHistory]; + for (const value of oldHistory) sha(value); + if (new Set(oldHistory).size !== oldHistory.length || + (input.oldHead === input.oldBase ? oldHistory.length !== 0 : oldHistory.at(-1) !== input.oldHead)) + throw new Error('The rebase history must be complete, ordered, and end at the captured head.'); if (!Number.isSafeInteger(expected.reviewVersion) || expected.reviewVersion < 0) throw new Error('A current review version is required for rebasing.'); if (!Number.isSafeInteger(expectedTaskStateVersion) || expectedTaskStateVersion < 0) throw new Error('Invalid expected task state version.'); const attemptId = input.attemptId ?? randomUUID(), startedAt = input.startedAt ?? Date.now(); assertUuidV4(attemptId, 'Rebase attempt ID'); if (!Number.isSafeInteger(startedAt) || startedAt < 0) throw new Error('Invalid rebase start time.'); - const marker: RebaseMarker = { attemptId, oldBase: input.oldBase, oldHead: input.oldHead, onto: input.onto, startedAt, + const marker: RebaseMarker = { attemptId, oldBase: input.oldBase, oldHead: input.oldHead, onto: input.onto, oldHistory, startedAt, resultHead: null, resultMappings: null, processGroup: null }; const key = identityKey(identity); return this.#transaction(() => { @@ -758,25 +765,23 @@ export class Store { }); } /** Bind the complete result and its exact retained ref value before the rebaser can return it to its caller. */ - setRebaseResult(planKey: string, attemptId: string, head: string | null, - mappings: readonly { oldSha: string; newSha: string }[]): void { + setRebaseResult(planKey: string, attemptId: string, head: string | null, rewrittenHistory: readonly string[]): void { assertUuidV4(attemptId, 'Rebase attempt ID'); if (head !== null) sha(head); - if (mappings.length > 500) throw new Error('Rebase history exceeds 500 commits.'); - const sources = new Set(), destinations = new Set(); - for (const mapping of mappings) { - sha(mapping.oldSha); sha(mapping.newSha); - if (sources.has(mapping.oldSha) || destinations.has(mapping.newSha)) throw new Error('Rebase mappings must be one-to-one.'); - sources.add(mapping.oldSha); destinations.add(mapping.newSha); - } + if (rewrittenHistory.length > 500) throw new Error('Rebase history exceeds 500 commits.'); + for (const value of rewrittenHistory) sha(value); + if (new Set(rewrittenHistory).size !== rewrittenHistory.length) throw new Error('Rebase history must not repeat a commit.'); this.#transaction(() => { const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if (marker.oldHistory === null || rewrittenHistory.length !== marker.oldHistory.length) + throw new GuardRefusal('The rebase result does not cover the complete captured history.'); if ((marker.onto === marker.oldBase) !== (head === null)) throw new GuardRefusal('The retained rebase result does not match its target.'); - const endpoint = mappings.at(-1); - if (marker.oldHead === marker.oldBase ? mappings.length !== 0 : !endpoint || endpoint.oldSha !== marker.oldHead || - (head === null ? endpoint.newSha !== marker.oldHead || mappings.some(mapping => mapping.oldSha !== mapping.newSha) : endpoint.newSha !== head)) + const rewrittenHead = rewrittenHistory.at(-1); + if (marker.oldHead === marker.oldBase ? rewrittenHistory.length !== 0 : + (head === null ? stable(rewrittenHistory) !== stable(marker.oldHistory) : rewrittenHead !== head)) throw new GuardRefusal('The retained rebase history does not end at its captured head.'); + const mappings = marker.oldHistory.map((oldSha, index) => ({ oldSha, newSha: rewrittenHistory[index]! })); if (marker.resultMappings !== null && (marker.resultHead !== head || stable(marker.resultMappings) !== stable(mappings))) throw new GuardRefusal('Another result owns this rebase.'); if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index 49a852e9..2259873f 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -540,9 +540,11 @@ describe('pre-merge rebase ownership', () => { store.transitionTask(identity, taskVersion, 'approved but merge blocked'); const readyVersion = store.getTask(identity).stateVersion; expect(() => store.beginRebase(identity, { revision: expected.revision, snapshotId: expected.snapshotId } as typeof expected, - readyVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3) })).toThrow(/current review version/); - const marker = store.beginRebase(identity, expected, readyVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3), startedAt: 123 }); - expect(store.getTask(identity).rebaseInProgress).toEqual({ attemptId: marker.attemptId, oldBase: oid(1), oldHead: oid(2), onto: oid(3), startedAt: 123, + readyVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] })).toThrow(/current review version/); + const marker = store.beginRebase(identity, expected, readyVersion, + { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)], startedAt: 123 }); + expect(store.getTask(identity).rebaseInProgress).toEqual({ attemptId: marker.attemptId, oldBase: oid(1), oldHead: oid(2), onto: oid(3), + oldHistory: [oid(2)], startedAt: 123, resultHead: null, resultMappings: null, processGroup: null }); const group = { pgid: 4242, startedAt: 456 }; expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, @@ -559,7 +561,7 @@ describe('pre-merge rebase ownership', () => { expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, randomUUID(), null, null)).toThrow(/no longer owns/); store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, group, null); expect(() => store.beginRebase(identity, expected, store.getTask(identity).stateVersion, - { oldBase: oid(1), oldHead: oid(2), onto: oid(3) })).toThrow(/already in progress/); + { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] })).toThrow(/already in progress/); expect(() => store.beginMergeAttempt(identity, expected, oid(2), null, 'direct')).toThrow(/rebase is in progress/); expect(() => store.transitionTask(identity, store.getTask(identity).stateVersion, 'queued')).toThrow(/rebase is in progress/); expect(() => store.setAssignment(identity, store.getTask(identity).stateVersion, 'other', 'other-head')).toThrow(/rebase is in progress/); @@ -573,7 +575,8 @@ describe('pre-merge rebase ownership', () => { ]); store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; - const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(4), onto: oid(5) }); + const marker = store.beginRebase(identity, expected, taskVersion, + { oldBase: oid(1), oldHead: oid(4), onto: oid(5), oldHistory: [oid(3), oid(4)] }); expect(() => store.finishRebase(identity, { revision: expected.revision, snapshotId: expected.snapshotId } as typeof expected, taskVersion, marker.attemptId, oid(5), oid(7), [ { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, @@ -581,8 +584,13 @@ describe('pre-merge rebase ownership', () => { expect(() => store.finishRebase(identity, expected, taskVersion, randomUUID(), oid(5), oid(7), [ { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, ])).toThrow(/no longer owns/); + const beforeResult = { snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }; + expect(() => store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), [oid(7)])) + .toThrow(/complete captured history/); + expect({ snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }) + .toEqual(beforeResult); const resultMappings = [{ oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }]; - store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), resultMappings); + store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), [oid(6), oid(7)]); expect(store.getTask(identity).rebaseInProgress).toMatchObject({ resultMappings }); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ { oldSha: oid(4), newSha: oid(7) }, @@ -611,11 +619,35 @@ describe('pre-merge rebase ownership', () => { ]); }); + it('refuses a truncated result and constructs the only ordered mapping from the pre-replay and rewritten histories', () => { + const { store } = fixture(); + store.recordHistory(identity, { revision: 1, snapshotId: store.getSnapshot(identity).id }, oid(1), oid(5), [ + { sha: oid(3), owner: 'P1', origin: 'owned', sourceSha: null }, + { sha: oid(4), owner: null, origin: 'foreign', sourceSha: null }, + { sha: oid(5), owner: null, origin: 'foreign', sourceSha: null }, + ]); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; + const marker = store.beginRebase(identity, expected, taskVersion, + { oldBase: oid(1), oldHead: oid(5), onto: oid(6), oldHistory: [oid(3), oid(4), oid(5)] }); + const before = { snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }; + expect(() => store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(9), [oid(9)])) + .toThrow(/complete captured history/); + expect({ snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }) + .toEqual(before); + store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(9), [oid(7), oid(8), oid(9)]); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ resultMappings: [ + { oldSha: oid(3), newSha: oid(7) }, { oldSha: oid(4), newSha: oid(8) }, { oldSha: oid(5), newSha: oid(9) }, + ] }); + expect(store.abortRebase(store.getTask(identity).planKey, marker.attemptId)).toBe(true); + }); + it('fails closed when review state changes, and clears only by the exact recovery handle', () => { const { store } = fixture(); store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; - const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3) }); + const marker = store.beginRebase(identity, expected, taskVersion, + { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] }); store.addReviewNote(identity, expected, 'P1', 'change', 'Review changed while Git was running.'); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(3), oid(4), [ { oldSha: oid(2), newSha: oid(4) }, @@ -633,7 +665,8 @@ describe('pre-merge rebase ownership', () => { store.recordHistory(identity, { revision: 1, snapshotId: current.id }, oid(1), oid(1), []); store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; - const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(1), onto: oid(5) }); + const marker = store.beginRebase(identity, expected, taskVersion, + { oldBase: oid(1), oldHead: oid(1), onto: oid(5), oldHistory: [] }); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(6), [])) .toThrow(/does not own/); store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(5), []); @@ -645,7 +678,8 @@ describe('pre-merge rebase ownership', () => { const { store } = fixture(); store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; - const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(2), onto: oid(3) }); + const marker = store.beginRebase(identity, expected, taskVersion, + { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] }); expect(store.cancelTask(identity, store.getTask(identity).stateVersion, randomUUID())).toBe('stopping'); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(3), oid(4), [ { oldSha: oid(2), newSha: oid(4) }, @@ -660,9 +694,9 @@ describe('pre-merge rebase ownership', () => { store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); const expected = reviewed(store), taskVersion = store.getTask(identity).stateVersion; expect(() => store.beginRebase(identity, expected, taskVersion, - { oldBase: oid(9), oldHead: oid(2), onto: oid(3) })).toThrow(/base or head changed/); + { oldBase: oid(9), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] })).toThrow(/base or head changed/); const marker = store.beginRebase(identity, expected, taskVersion, - { oldBase: oid(1), oldHead: oid(2), onto: oid(3) }); + { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] }); expect(() => store.recordRebase(identity, expected, oid(3), oid(4), [ { oldSha: oid(2), newSha: oid(4) }, ])).toThrow(/claimed rebase is in progress/); diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index bbc9d9c3..e378b184 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -41,13 +41,13 @@ async function setup(conflict = false) { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); - return { root, source, runnerRoot, repository, rebaser, base, owned, foreign, onto }; + return { root, source, runnerRoot, repository, rebaser, base, owned, foreign, history: [owned, foreign], onto }; } describe('trusted pre-merge rebase', () => { it('replays a complete linear history one-for-one and retains only its immutable result ref', async () => { const s = await setup(), attemptId = randomUUID(); - const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); expect(result).toMatchObject({ oldHead: s.foreign, base: s.onto }); expect(result.mappings.map(entry => entry.oldSha)).toEqual([s.owned, s.foreign]); expect(new Set(result.mappings.map(entry => entry.newSha)).size).toBe(2); @@ -61,7 +61,7 @@ describe('trusted pre-merge rebase', () => { it('returns identity mappings without creating mutable state when the base is already current', async () => { const s = await setup(), attemptId = randomUUID(); - const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base }); + const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base }); expect(result).toEqual({ oldHead: s.foreign, base: s.base, head: s.foreign, mappings: [{ oldSha: s.owned, newSha: s.owned }, { oldSha: s.foreign, newSha: s.foreign }] }); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -74,20 +74,27 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER) }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base })) .rejects.toThrow(/deadline expired while recording its result/); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); }); + it('refuses when Git history no longer matches the durably captured ordered history', async () => { + const s = await setup(), attemptId = randomUUID(); + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, + oldHistory: [s.foreign], onto: s.onto })).rejects.toThrow(/no longer matches/); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); + }); + it('never overwrites or deletes a retained result when an attempt ID is reused', async () => { const s = await setup(), attemptId = randomUUID(); - const first = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + const first = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); git(s.source, 'switch', '-q', 'main'); const nextBase = commit(s.source, 'later.txt', 'later\n', 'move base again'); await ensureCommit(s.repository, nextBase); - await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: nextBase })) + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: nextBase })) .rejects.toThrow(/already has a retained result/); - await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base })) .rejects.toThrow(/already has a retained result/); await s.rebaser.abort(attemptId); // A reused live/startup marker has no ownership value for the earlier result. expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(first.head); @@ -99,7 +106,8 @@ describe('trusted pre-merge rebase', () => { commit(s.source, 'kept.log', 'tracked\n', 'add tracked file'); const head = commit(s.source, '.gitignore', '*.log\n', 'ignore later logs'); await ensureCommit(s.repository, head); - await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: head, onto: s.onto })) + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: head, + oldHistory: git(s.repository.path, 'rev-list', '--reverse', `${s.base}..${head}`).split('\n'), onto: s.onto })) .resolves.toMatchObject({ oldHead: head, base: s.onto }); }); @@ -148,7 +156,7 @@ describe('trusted pre-merge rebase', () => { expect(ownedAttempt).toBe(attemptId); if (group !== 'spawning') { expect(active.get(group.pgid)).toBe(group.startedAt); active.delete(group.pgid); } }, onProcessUnsettled: () => {}, onResult: () => {} }); - await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); expect(seen.length).toBeGreaterThan(1); expect(active.size).toBe(0); }); @@ -170,7 +178,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, onProcessGroupSettled: (_attempt, group) => { if (group === 'spawning') settled++; }, onProcessUnsettled: () => {}, onResult: () => {} }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })).rejects.toThrow(/deadline expired/); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })).rejects.toThrow(/deadline expired/); expect({ starts, settled }).toEqual({ starts: 1, settled: 1 }); }); @@ -185,7 +193,7 @@ describe('trusted pre-merge rebase', () => { if (firstSettlement) { expect(groups).toBe(0); firstSettlement = false; } expect(owner).not.toBe('none'); owner = 'none'; }, onProcessUnsettled: () => {}, onResult: () => {} }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })) + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) .rejects.toThrow('ownership commit outcome unknown'); expect(owner).toBe('none'); expect(groups).toBe(0); @@ -199,7 +207,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, onProcessGroupSettled: () => { throw new Error('settlement write failed'); }, onProcessUnsettled: () => {}, onResult: () => {} }); - const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); if (!failure) throw new Error('Expected the deadline and settlement write to fail.'); @@ -218,7 +226,7 @@ describe('trusted pre-merge rebase', () => { if (++settled === 2 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } }, onProcessUnsettled: () => {}, onResult: () => {} }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.base })) + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base })) .rejects.toThrow(/deadline expired while recording process settlement/); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); }); @@ -232,7 +240,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: () => { if (processes === 7 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } }, onProcessUnsettled: () => {}, onResult: () => {} }); - const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); if (!failure) throw new Error('Expected the child failure and deadline to be retained.'); @@ -244,7 +252,7 @@ describe('trusted pre-merge rebase', () => { it('reports a conflict, removes the mutable checkout, and retains no rewritten ref', async () => { const s = await setup(true), attemptId = randomUUID(); - await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })).rejects.toBeInstanceOf(RebaseConflict); + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })).rejects.toBeInstanceOf(RebaseConflict); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); expect(git(s.source, 'rev-parse', 'feature')).toBe(s.foreign); @@ -263,7 +271,7 @@ describe('trusted pre-merge rebase', () => { const tree = git(s.source, 'write-tree'); const collidingHead = git(s.source, 'commit-tree', tree, '-p', s.base, '-m', 'case-colliding feature'); await ensureCommit(s.repository, collidingHead); - const operation = s.rebaser.run({ attemptId, oldBase: s.base, oldHead: collidingHead, onto: s.onto }); + const operation = s.rebaser.run({ attemptId, oldBase: s.base, oldHead: collidingHead, oldHistory: [collidingHead], onto: s.onto }); if (caseSensitive) await expect(operation).resolves.toMatchObject({ oldHead: collidingHead, base: s.onto }); else await expect(operation).rejects.toThrow(/cannot faithfully check out/); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -284,7 +292,7 @@ describe('trusted pre-merge rebase', () => { const tree = git(s.source, 'write-tree'); const collidingBase = git(s.source, 'commit-tree', tree, '-p', s.base, '-m', 'add lower-case path'); await ensureCommit(s.repository, collidingHead); await ensureCommit(s.repository, collidingBase); - const operation = s.rebaser.run({ attemptId, oldBase: s.base, oldHead: collidingHead, onto: collidingBase }); + const operation = s.rebaser.run({ attemptId, oldBase: s.base, oldHead: collidingHead, oldHistory: [collidingHead], onto: collidingBase }); if (caseSensitive) await expect(operation).resolves.toMatchObject({ oldHead: collidingHead, base: collidingBase }); else await expect(operation).rejects.toThrow(/cannot faithfully check out the rebased head/); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -294,7 +302,7 @@ describe('trusted pre-merge rebase', () => { it('honours an already-aborted signal before the first Git write', async () => { const s = await setup(), attemptId = randomUUID(), stop = new AbortController(); stop.abort(new Error('cancelled before rebase')); - await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + await expect(s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled before rebase'); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); }); @@ -307,7 +315,7 @@ describe('trusted pre-merge rebase', () => { onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('cancelled during rebase')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during rebase'); expect(processes).toBeGreaterThanOrEqual(9); // stopped rebase, worktree removal, then pruning expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -321,7 +329,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 8) stop.abort(new Error('cancelled during conflict classification')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during conflict classification'); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); }); @@ -334,7 +342,7 @@ describe('trusted pre-merge rebase', () => { onProcessStarting: () => {}, onProcessGroup: () => stop.abort(new Error('cancelled by caller')), onProcessGroupSettled: () => { if (!failed) { failed = true; throw new Error('settlement write failed'); } }, onProcessUnsettled: () => {}, onResult: () => {} }); - const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal }) + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); if (!failure) throw new Error('Expected cancellation and settlement recording to fail.'); @@ -351,7 +359,7 @@ describe('trusted pre-merge rebase', () => { onProcessStarting: () => { if (++starts === 8) throw new Error('cleanup record failed'); }, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('original cancellation')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); - const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal }) + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); if (!failure) throw new Error('Expected the rebase and cleanup to fail.'); @@ -368,7 +376,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 14) stop.abort(new Error('cancelled during cleanup')); }, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto, signal: stop.signal })) + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during cleanup'); expect(processes).toBeGreaterThanOrEqual(16); // result ref, worktree cleanup, prune, then result-ref deletion expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -389,7 +397,7 @@ describe('trusted pre-merge rebase', () => { } }, onProcessUnsettled: () => {}, onResult: () => {} }); - const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }) + const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); if (!failure) throw new Error('Expected the ref write and its cleanup to fail.'); @@ -401,7 +409,7 @@ describe('trusted pre-merge rebase', () => { it('aborts only a retained ref whose exact value is durably owned by the attempt', async () => { const s = await setup(), attemptId = randomUUID(); - const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto }); + const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); await s.rebaser.abort(attemptId); expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); @@ -416,7 +424,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => { throw new Error('result ownership write failed'); } }); - await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, onto: s.onto })) + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) .rejects.toThrow('result ownership write failed'); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index 0236f6e0..2db9b27c 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -243,7 +243,7 @@ describe('startup recovery sequence', () => { store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, - store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123 }); + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head], startedAt: 123 }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', { pgid: 5151, startedAt: 456 }); const owned = store.getTask(id(1)).rebaseInProgress; @@ -262,9 +262,8 @@ describe('startup recovery sequence', () => { store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, - store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123 }); - store.setRebaseResult(store.getTask(id(1)).planKey, marker.attemptId, oid(4), - [{ oldSha: snapshot.head, newSha: oid(4) }]); + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head], startedAt: 123 }); + store.setRebaseResult(store.getTask(id(1)).planKey, marker.attemptId, oid(4), [oid(4)]); const abortRebase = vi.fn(async () => undefined); await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase }).d }); @@ -282,7 +281,7 @@ describe('startup recovery sequence', () => { const abortRebase = vi.fn(async () => undefined); await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase }).d }); - expect(abortRebase).toHaveBeenCalledWith(planKey, { ...marker, resultHead: null, resultMappings: null }); + expect(abortRebase).toHaveBeenCalledWith(planKey, { ...marker, oldHistory: null, resultHead: null, resultMappings: null }); expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); }); it('retains a dead-group marker while another process still uses its rebase workspace', async () => { @@ -290,7 +289,7 @@ describe('startup recovery sequence', () => { store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, - store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); const group = { pgid: 5151, startedAt: 456 }; store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', group); @@ -307,7 +306,7 @@ describe('startup recovery sequence', () => { store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, - store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); const group = { pgid: 5151, startedAt: 456 }; store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', group); @@ -325,7 +324,7 @@ describe('startup recovery sequence', () => { store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, - store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps().d })) .rejects.toThrow(/needs F3 to abort/); expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); @@ -335,7 +334,7 @@ describe('startup recovery sequence', () => { store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, - store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3) }); + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); const abortRebase = vi.fn(async () => undefined); await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), From 2ecae1045406a8891edb09bc4c7044b5f464b681 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 10:38:07 -0700 Subject: [PATCH 06/13] Persist intended rebase ref before creation --- docs/implementation/pre-merge-rebase.md | 6 +-- docs/implementation/runner-lifecycle.md | 2 +- runner/production.ts | 8 ++-- runner/rebase.ts | 43 ++++++++++++++--- runner/recovery.ts | 7 ++- runner/store.ts | 35 +++++++++++--- test/runner-lifecycle-store.test.ts | 16 ++++--- test/runner-production.test.ts | 10 +++- test/runner-rebase.test.ts | 61 ++++++++++++++++++------- test/runner-recovery.test.ts | 8 ++-- 10 files changed, 145 insertions(+), 51 deletions(-) diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md index dc61e224..2a0d1587 100644 --- a/docs/implementation/pre-merge-rebase.md +++ b/docs/implementation/pre-merge-rebase.md @@ -7,7 +7,7 @@ Before the first Git process, `Store.beginRebase` records one marker in `tasks.rebase_in_progress`: ```text -{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultHead, resultMappings, processGroup } +{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, processGroup } ``` Admission checks the current plan revision, snapshot, review version, task state version, mergeable task status, cancellation state, active runner attempt, active merge, existing rebase, both ends of the reviewed snapshot, and the complete ordered old history captured by review. Recording the marker advances the task state version. While it exists, another rebase, the legacy rewrite path, merge admission, runner admission, reassignment, and ordinary status changes fail closed. Cancellation remains pending until the owned rebase has settled and exact-attempt cleanup clears the marker; that cleanup then closes the task. @@ -16,13 +16,13 @@ Admission checks the current plan revision, snapshot, review version, task state `GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, descended from the recorded base, and byte-for-byte equal to the ordered history in the durable marker. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. -After replay settles, the rebaser supplies the Store's result hook only the ordered rewritten history. The Store zips that history by position with the independently captured `oldHistory`, then binds the resulting complete mapping and exact retained ref value before the result can return. `Store.finishRebase` accepts only that byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. +After replay settles and before its first result-ref write, the rebaser supplies the Store's preparation hook only the ordered rewritten history. The Store zips that history by position with the independently captured `oldHistory`, then durably binds the resulting complete mapping and exact intended ref value as `prepared`. A successful atomic ref create advances it to `ready`; an already-current no-op uses the same two phases without a ref write. `Store.finishRebase` accepts only a `ready`, byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures settle the Git process group before cleanup. Cleanup removes only the validated UUID workspace. A partially created but unregistered worktree is removed only after Git's own worktree list proves it is not registered. Result refs are created atomically from the absent state; a reused attempt ID cannot overwrite or delete a previously retained result, while an ambiguous create is reconciled against the exact attempted value. ## Restart recovery -Startup validates every stored marker before touching its resource. If its recorded process group is still alive with the same start time, recovery terminates and awaits it first. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner; Linux performs that potentially blocking `/proc` walk in an owned, killable subprocess whose settlement is inside the overall deadline. The production recovery adapter accepts only the configured plan key, then removes the matching worktree. It deletes a result ref only with Git's exact-old-value guard using the durably recorded `resultHead`; a legacy, pre-result, or reused-attempt marker without that value preserves any existing ref. Only after cleanup succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. +Startup validates every stored marker before touching its resource. If its recorded process group is still alive with the same start time, recovery terminates and awaits it first. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner; Linux performs that potentially blocking `/proc` walk in an owned, killable subprocess whose settlement is inside the overall deadline. The production recovery adapter accepts only the configured plan key, then removes the matching worktree. It deletes a result ref only with Git's exact-old-value guard using the durably recorded `resultHead`; absence is already-clean success. A `prepared` marker preserves a different pre-existing value, while a `ready` marker fails closed if its ref changed. A legacy, pre-result, or reused-attempt marker without an intended value preserves any existing ref. Only after cleanup succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. This ordering is intentionally local-only. The later push integration must extend the durable lifecycle before its first external write and reconcile an ambiguous push outcome; it must not reuse this local completion point as proof that GitHub moved. diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 265fa88d..5f3cb752 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -267,7 +267,7 @@ codeboost never writes the user's repository. The commits it makes live in a **r | Holder | Owner | Rule | |---|---|---| | Partial output of a stopped writable attempt | F, using a D export | Done in D by #51 item 6: `exportTaskDiff(storage, { base, imageId, maxBytes, signal, onProcessGroup, timeoutMs, metadataBaseline })` (`agents/container/storage.ts`) takes the storage value or a recovery handle and `base`, the full ID of the commit it was seeded from, and returns `{ diff, truncated }`: at most `maxBytes` (1 MiB at most) of raw diff bytes. It checks both volumes' labels, runs Git in a read-only container with no network that writes nothing to either volume, covers edits and new untracked files (agents cannot commit or stage: the metadata is read-only to them, and the export first checks it against the seeder's baseline, refusing if it changed; a recovery handle needs the `metadataBaseline` F recorded), and cuts the output inside the container, so the Docker work stops at the limit. A changed file over 8 MiB, an untracked nested repository, or an untracked path Git will not add (a part named `.git`, or a symlink named `.gitmodules`) is named in a `codeboost:` notice line instead of diffed, so one large file cannot exhaust the export's memory or deadline; untracked symlinks are diffed as links. What Git skips without a word is named in a notice line too: an entry named `.git` (and, if it holds a repository, the new files beside it), a fifo or socket, an ignored untracked path, and anything in a submodule directory. A changed file whose `ident` or `working-tree-encoding` attribute changes what the diff shows is named too. Anything it cannot read fails the export, and a Git failure's reason includes Git's last two error lines. On abort or at its deadline it stops the client and removes the export container before it settles (an `AbortError`, or an `AggregateError` if that removal did not settle); recovery removes a leftover one. The task filesystems are Docker volumes behind a keeper container. F allocates them with `prepareTaskFilesystems` and removes them with `removeTaskFilesystems`; D's supervisor does not remove them at settlement. But F cannot read a volume from the host. So the order is: `settled` → a **D-provided bounded export** (a new D operation, for example `exportTaskDiff(filesystems, maxBytes)`, which runs `git diff` against the commit the storage was seeded from in a read-only container and returns at most 1 MiB) → F saves it → the terminal write stores `diagnostic_ref` → `removeTaskFilesystems` → the slot is freed. **The export has an overall deadline** (60 seconds, passed to D as an `AbortSignal`), because `maxBytes` bounds only the returned data, not the Docker work. On timeout or any export failure, F records "Partial output could not be exported: " as the diagnostic and continues with the terminal write and removal; shutdown and recovery never wait on an export past its deadline. F saves the export to a runner-owned diagnostics directory, with a total byte cap. **Retention never deletes a file an attempt row still references.** When the cap is exceeded, retention first deletes unreferenced files, oldest first. If that is not enough, it takes the oldest referenced file and, in one transaction, sets that row's `diagnostic_ref` to null and appends "(partial output removed by retention)" to its diagnostic. Only after that commit does it delete the file. No row is ever left pointing at a missing file. The attempt row references it as `diagnostic_ref`. If the capture fails, the row records that it failed. The task filesystem is never reused. | -| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null with the complete ordered reviewed history, tracks each Git process group until settlement, and records the complete ordered mapping plus the exact retained result-ref value before publishing it. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3, but deletes a retained ref only when its current value equals `resultHead`; a legacy or pre-result marker without that ownership value preserves the ref. | +| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null with the complete ordered reviewed history, tracks each Git process group until settlement, records the complete ordered mapping plus the intended result-ref value as `prepared` before the first ref write, and advances it to `ready` only after successful atomic creation. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3. Exact owned deletion treats absence as success; a `prepared` marker preserves another value, while `ready` fails closed if its ref changed. A legacy or pre-result marker without an ownership value preserves the ref. | | Workspace rebuild | I3 | I3 never rebuilds before both rows above are resolved for the task. | **Process shutdown is a hard stop.** When the process stops, the running task does not finish. Its attempt ends `cancelled` with the reason "Stopped by shutdown". Restart recovery (lane I3) puts the task back in the queue. This is different from "Stop the queue" (lane I), which lets the running task finish. diff --git a/runner/production.ts b/runner/production.ts index 09883a3b..ecaf9946 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -110,8 +110,9 @@ export function dRecoveryDeps(image: () => string, rebaser?: Pick service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, 'spawning', group), onProcessGroupSettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, null), onProcessUnsettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, 'unsettled'), - onResult: (attemptId, head, history) => service.store.setRebaseResult(rebasePlanKey, attemptId, head, history) })); + onResultPrepared: (attemptId, head, history) => service.store.prepareRebaseResult(rebasePlanKey, attemptId, head, history), + onResultReady: attemptId => service.store.completeRebaseResult(rebasePlanKey, attemptId) })); const recovery = await recoverStartup({ store: service.store, runnerOwner, runnerRoot: config.root, diagnosticsDir, diagnosticsCapBytes: config.diagnosticsCapBytes, deps: o.recovery ? o.recovery(image) : dRecoveryDeps(image, - { abort: async (attemptId, resultHead) => (await getRebaser()).abort(attemptId, resultHead) }, rebasePlanKey) }); + { abort: async (attemptId, resultHead, prepared) => (await getRebaser()).abort(attemptId, resultHead, prepared) }, rebasePlanKey) }); const repository = await getRepository(); const imageId = image(); const workspace = createTaskWorkspace({ store: service.store, runnerRoot: config.root, runnerOwner, repository, imageId, diff --git a/runner/rebase.ts b/runner/rebase.ts index 66f6638a..d8a940dd 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -27,7 +27,8 @@ export interface GitRebaserOptions { onProcessGroup: (attemptId: string, group: ProcessGroup) => void; onProcessGroupSettled: (attemptId: string, group: ProcessGroup | 'spawning') => void; onProcessUnsettled: (attemptId: string, group: ProcessGroup) => void; - onResult: (attemptId: string, retainedHead: string | null, rewrittenHistory: readonly string[]) => void; + onResultPrepared: (attemptId: string, intendedHead: string | null, rewrittenHistory: readonly string[]) => void; + onResultReady: (attemptId: string) => void; } interface CallScope { attemptId: string; signal?: AbortSignal; deadline: number; workDeadline: number } @@ -77,7 +78,9 @@ export class GitRebaser { if (onto === oldBase) { const mappings = old.map(sha => ({ oldSha: sha, newSha: sha })); this.#assertResultCurrent(scope); - this.#options.onResult(attemptId, null, old); + this.#options.onResultPrepared(attemptId, null, old); + this.#assertResultCurrent(scope); + this.#options.onResultReady(attemptId); this.#assertResultCurrent(scope); return { oldHead, base: onto, head: oldHead, mappings }; } @@ -109,7 +112,11 @@ export class GitRebaser { await this.#assertCheckout(path, head, 'rebased head', scope); const next = await this.#history(path, onto, head, scope); if (next.length !== old.length) throw new Error('The rebase did not preserve one commit for every task commit.'); + const mappings = old.map((oldSha, index) => ({ oldSha, newSha: next[index]! })); attemptedRef = head; + this.#assertResultCurrent(scope); + this.#options.onResultPrepared(attemptId, head, next); + this.#assertResultCurrent(scope); const created = await this.#call(this.#options.repository.path, ['update-ref', ref, head, '0'.repeat(head.length)], scope); if (created.status === 0) retained = true; else if (created.status === null) uncertainRef = true; @@ -119,9 +126,8 @@ export class GitRebaser { // attempted value. A normal nonzero exit proves atomic creation refused and must preserve the winning ref. throw this.#failure('update-ref', created); } - const mappings = old.map((oldSha, index) => ({ oldSha, newSha: next[index]! })); this.#assertResultCurrent(scope); - this.#options.onResult(attemptId, head, next); + this.#options.onResultReady(attemptId); this.#assertResultCurrent(scope); result = { oldHead, base: onto, head, mappings }; } catch (error) { primary = error; } @@ -146,14 +152,37 @@ export class GitRebaser { } /** Startup/live recovery for an attempt whose Store marker still exists. */ - async abort(attemptId: string, resultHead?: string): Promise { + async abort(attemptId: string, resultHead?: string, prepared = false): Promise { const ref = rebaseRef(attemptId), path = this.#path(attemptId), stat = lstatSync(path, { throwIfNoEntry: false }); if (resultHead !== undefined && !COMMIT_ID.test(resultHead)) throw new Error('A full retained result ID is required.'); const scope = this.#scope(attemptId); if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) throw new Error('The rebase workspace is not a plain directory.'); await this.#remove(path, scope); - if (resultHead !== undefined) - await this.#git(this.#options.repository.path, ['update-ref', '-d', ref, resultHead], scope, true); + if (resultHead !== undefined) { + const exists = await this.#call(this.#options.repository.path, ['show-ref', '--verify', '--quiet', ref], scope, true); + if (exists.status === 1) return; + if (exists.status !== 0) throw this.#failure('show-ref', exists); + const current = await this.#call(this.#options.repository.path, ['rev-parse', '--verify', ref], scope, true); + if (current.status !== 0) { + const after = await this.#call(this.#options.repository.path, ['show-ref', '--verify', '--quiet', ref], scope, true); + if (after.status === 1) return; + throw this.#failure('rev-parse', current); + } + if (current.stdout.trim() !== resultHead) { + if (prepared) return; + throw new Error('Another retained result owns this rebase ref.'); + } + try { await this.#git(this.#options.repository.path, ['update-ref', '-d', ref, resultHead], scope, true); } + catch (error) { + const after = await this.#call(this.#options.repository.path, ['show-ref', '--verify', '--quiet', ref], scope, true); + if (after.status === 1) return; + if (prepared && after.status === 0) { + const value = await this.#call(this.#options.repository.path, ['rev-parse', '--verify', ref], scope, true); + if (value.status === 0 && value.stdout.trim() !== resultHead) return; + } + throw error; + } + } } async #history(repository: string, base: string, head: string, scope: CallScope): Promise { diff --git a/runner/recovery.ts b/runner/recovery.ts index bf4c4832..74b06659 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -230,13 +230,16 @@ export async function recoverStartup(o: RecoveryOptions): Promise | null; const processGroup = marker?.processGroup, oldHistory = marker?.oldHistory ?? null, - resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null; + resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null, + resultState = marker?.resultState ?? (resultMappings === null ? 'none' : 'ready'); if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldBase) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldHead) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.onto) || !Number.isSafeInteger(marker.startedAt) || marker.startedAt! < 0 || !validOldHistory(marker, oldHistory) || (resultHead !== null && (typeof resultHead !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(resultHead))) || + !['none', 'prepared', 'ready'].includes(resultState) || + (resultState === 'none' ? resultHead !== null || resultMappings !== null : resultMappings === null) || !validRebaseResult(marker, oldHistory, resultHead, resultMappings) || (processGroup !== null && (!processGroup || typeof processGroup === 'string' || !Number.isSafeInteger(processGroup.pgid) || processGroup.pgid <= 1 || !Number.isSafeInteger(processGroup.startedAt) || processGroup.startedAt < 0))) @@ -253,7 +256,7 @@ export async function recoverStartup(o: RecoveryOptions): Promise { this.#expect(key, expected); @@ -764,8 +765,8 @@ export class Store { throw new GuardRefusal('This rebase attempt no longer owns the task.'); }); } - /** Bind the complete result and its exact retained ref value before the rebaser can return it to its caller. */ - setRebaseResult(planKey: string, attemptId: string, head: string | null, rewrittenHistory: readonly string[]): void { + /** Persist the complete intended result before the rebaser's first ref write. */ + prepareRebaseResult(planKey: string, attemptId: string, head: string | null, rewrittenHistory: readonly string[]): void { assertUuidV4(attemptId, 'Rebase attempt ID'); if (head !== null) sha(head); if (rewrittenHistory.length > 500) throw new Error('Rebase history exceeds 500 commits.'); for (const value of rewrittenHistory) sha(value); @@ -782,10 +783,29 @@ export class Store { (head === null ? stable(rewrittenHistory) !== stable(marker.oldHistory) : rewrittenHead !== head)) throw new GuardRefusal('The retained rebase history does not end at its captured head.'); const mappings = marker.oldHistory.map((oldSha, index) => ({ oldSha, newSha: rewrittenHistory[index]! })); - if (marker.resultMappings !== null && (marker.resultHead !== head || stable(marker.resultMappings) !== stable(mappings))) - throw new GuardRefusal('Another result owns this rebase.'); + if (marker.resultState !== 'none') { + if (marker.resultHead !== head || stable(marker.resultMappings) !== stable(mappings)) + throw new GuardRefusal('Another result owns this rebase.'); + return; + } + if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', + encode({ ...marker, resultState: 'prepared', resultHead: head, resultMappings: [...mappings] }), + planKey, task.rebase_in_progress as string).changes !== 1) + throw new GuardRefusal('This rebase attempt no longer owns the task.'); + }); + } + /** Confirm that the prepared result's ref write succeeded (or that the no-op needs no ref). */ + completeRebaseResult(planKey: string, attemptId: string): void { + assertUuidV4(attemptId, 'Rebase attempt ID'); + this.#transaction(() => { + const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); + if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); + if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if (marker.resultState === 'ready') return; + if (marker.resultState !== 'prepared' || marker.resultMappings === null) + throw new GuardRefusal('The rebase result was not prepared before its ref write.'); if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', - encode({ ...marker, resultHead: head, resultMappings: [...mappings] }), planKey, task.rebase_in_progress as string).changes !== 1) + encode({ ...marker, resultState: 'ready' }), planKey, task.rebase_in_progress as string).changes !== 1) throw new GuardRefusal('This rebase attempt no longer owns the task.'); }); } @@ -803,7 +823,8 @@ export class Store { if (task.state_version !== expectedTaskStateVersion + 1) throw new GuardRefusal('The task changed during the rebase. Discard its result.'); if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); - if (marker.resultMappings === null || (marker.onto === marker.oldBase ? marker.resultHead !== null : marker.resultHead !== head)) + if (marker.resultState !== 'ready' || marker.resultMappings === null || + (marker.onto === marker.oldBase ? marker.resultHead !== null : marker.resultHead !== head)) throw new GuardRefusal('The rebase result does not own its retained ref.'); const captured = this.getSnapshot(identity); if (marker.oldBase !== captured.base || marker.oldHead !== captured.head || marker.onto !== base) diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index 2259873f..9b08fd09 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -545,7 +545,7 @@ describe('pre-merge rebase ownership', () => { { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)], startedAt: 123 }); expect(store.getTask(identity).rebaseInProgress).toEqual({ attemptId: marker.attemptId, oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)], startedAt: 123, - resultHead: null, resultMappings: null, processGroup: null }); + resultState: 'none', resultHead: null, resultMappings: null, processGroup: null }); const group = { pgid: 4242, startedAt: 456 }; expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, { pgid: 1, startedAt: 456 })).toThrow(/Invalid rebase process group/); @@ -585,13 +585,14 @@ describe('pre-merge rebase ownership', () => { { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }, ])).toThrow(/no longer owns/); const beforeResult = { snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }; - expect(() => store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), [oid(7)])) + expect(() => store.prepareRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), [oid(7)])) .toThrow(/complete captured history/); expect({ snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }) .toEqual(beforeResult); const resultMappings = [{ oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(7) }]; - store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), [oid(6), oid(7)]); - expect(store.getTask(identity).rebaseInProgress).toMatchObject({ resultMappings }); + store.prepareRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(7), [oid(6), oid(7)]); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ resultState: 'prepared', resultMappings }); + store.completeRebaseResult(store.getTask(identity).planKey, marker.attemptId); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ { oldSha: oid(4), newSha: oid(7) }, ])).toThrow(/does not end/); @@ -631,11 +632,11 @@ describe('pre-merge rebase ownership', () => { const marker = store.beginRebase(identity, expected, taskVersion, { oldBase: oid(1), oldHead: oid(5), onto: oid(6), oldHistory: [oid(3), oid(4), oid(5)] }); const before = { snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }; - expect(() => store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(9), [oid(9)])) + expect(() => store.prepareRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(9), [oid(9)])) .toThrow(/complete captured history/); expect({ snapshot: store.getSnapshot(identity), ledger: store.getLedger(identity), marker: store.getTask(identity).rebaseInProgress }) .toEqual(before); - store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(9), [oid(7), oid(8), oid(9)]); + store.prepareRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(9), [oid(7), oid(8), oid(9)]); expect(store.getTask(identity).rebaseInProgress).toMatchObject({ resultMappings: [ { oldSha: oid(3), newSha: oid(7) }, { oldSha: oid(4), newSha: oid(8) }, { oldSha: oid(5), newSha: oid(9) }, ] }); @@ -669,7 +670,8 @@ describe('pre-merge rebase ownership', () => { { oldBase: oid(1), oldHead: oid(1), onto: oid(5), oldHistory: [] }); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(6), [])) .toThrow(/does not own/); - store.setRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(5), []); + store.prepareRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(5), []); + store.completeRebaseResult(store.getTask(identity).planKey, marker.attemptId); expect(store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(5), [])) .toMatchObject({ base: oid(5), head: oid(5) }); }); diff --git a/test/runner-production.test.ts b/test/runner-production.test.ts index 0879cb95..3e138add 100644 --- a/test/runner-production.test.ts +++ b/test/runner-production.test.ts @@ -231,7 +231,7 @@ describe('D adapters', () => { const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultHead: '7'.repeat(40) }; const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); await deps.abortRebase!('plan-key', marker); - expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead); + expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead, false); await expect(deps.abortRebase!('another-plan', marker)).rejects.toThrow(/another configured plan/); expect(abort).toHaveBeenCalledTimes(1); }); @@ -239,7 +239,13 @@ describe('D adapters', () => { const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultHead: null }; const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); await deps.abortRebase!('plan-key', marker); - expect(abort).toHaveBeenCalledWith(marker.attemptId, undefined); + expect(abort).toHaveBeenCalledWith(marker.attemptId, undefined, false); + }); + it('lets prepared recovery preserve a different pre-existing ref while clearing its marker', async () => { + const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultState: 'prepared', resultHead: '7'.repeat(40) }; + const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); + await deps.abortRebase!('plan-key', marker); + expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead, true); }); it('passes D\'s own handles through, and turns every unowned object into the command that removes it', async () => { const handle = Object.freeze({ runnerOwner: OWNER, attemptId: randomUUID(), allocationId: randomUUID() }); diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index e378b184..9068f0d7 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -40,7 +40,7 @@ async function setup(conflict = false) { const rebaser = new GitRebaser({ repository, runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResult: () => {} }); + onResultPrepared: () => {}, onResultReady: () => {} }); return { root, source, runnerRoot, repository, rebaser, base, owned, foreign, history: [owned, foreign], onto }; } @@ -73,7 +73,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResult: () => vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER) }); + onResultPrepared: () => vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER), onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base })) .rejects.toThrow(/deadline expired while recording its result/); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); @@ -155,7 +155,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: (ownedAttempt, group) => { expect(ownedAttempt).toBe(attemptId); if (group !== 'spawning') { expect(active.get(group.pgid)).toBe(group.startedAt); active.delete(group.pgid); } - }, onProcessUnsettled: () => {}, onResult: () => {} }); + }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); expect(seen.length).toBeGreaterThan(1); expect(active.size).toBe(0); @@ -166,7 +166,7 @@ describe('trusted pre-merge rebase', () => { expect(() => new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_000, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResult: () => {} })).toThrow(/Invalid rebase deadline/); + onResultPrepared: () => {}, onResultReady: () => {} })).toThrow(/Invalid rebase deadline/); }); it('does not start Git when a synchronous ownership hook exhausts the work deadline', async () => { @@ -177,7 +177,7 @@ describe('trusted pre-merge rebase', () => { onProcessStarting: () => { starts++; const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, onProcessGroupSettled: (_attempt, group) => { if (group === 'spawning') settled++; }, onProcessUnsettled: () => {}, - onResult: () => {} }); + onResultPrepared: () => {}, onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })).rejects.toThrow(/deadline expired/); expect({ starts, settled }).toEqual({ starts: 1, settled: 1 }); }); @@ -192,7 +192,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: () => { if (firstSettlement) { expect(groups).toBe(0); firstSettlement = false; } expect(owner).not.toBe('none'); owner = 'none'; - }, onProcessUnsettled: () => {}, onResult: () => {} }); + }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) .rejects.toThrow('ownership commit outcome unknown'); expect(owner).toBe('none'); @@ -206,7 +206,7 @@ describe('trusted pre-merge rebase', () => { onProcessStarting: () => { const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, onProcessGroupSettled: () => { throw new Error('settlement write failed'); }, onProcessUnsettled: () => {}, - onResult: () => {} }); + onResultPrepared: () => {}, onResultReady: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -225,7 +225,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: () => { if (++settled === 2 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } }, - onProcessUnsettled: () => {}, onResult: () => {} }); + onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base })) .rejects.toThrow(/deadline expired while recording process settlement/); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -239,7 +239,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroup: () => { processes++; }, onProcessGroupSettled: () => { if (processes === 7 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } - }, onProcessUnsettled: () => {}, onResult: () => {} }); + }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -314,7 +314,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('cancelled during rebase')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during rebase'); expect(processes).toBeGreaterThanOrEqual(9); // stopped rebase, worktree removal, then pruning @@ -328,7 +328,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 8) stop.abort(new Error('cancelled during conflict classification')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during conflict classification'); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -341,7 +341,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => stop.abort(new Error('cancelled by caller')), onProcessGroupSettled: () => { if (!failed) { failed = true; throw new Error('settlement write failed'); } }, - onProcessUnsettled: () => {}, onResult: () => {} }); + onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -358,7 +358,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => { if (++starts === 8) throw new Error('cleanup record failed'); }, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('original cancellation')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -375,7 +375,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 14) stop.abort(new Error('cancelled during cleanup')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResult: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during cleanup'); expect(processes).toBeGreaterThanOrEqual(16); // result ref, worktree cleanup, prune, then result-ref deletion @@ -396,7 +396,7 @@ describe('trusted pre-merge rebase', () => { throw new Error('result ref outcome unknown'); } }, - onProcessUnsettled: () => {}, onResult: () => {} }); + onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -415,21 +415,50 @@ describe('trusted pre-merge rebase', () => { expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); await s.rebaser.abort(attemptId, result.head); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + await expect(s.rebaser.abort(attemptId, result.head)).resolves.toBeUndefined(); await expect(s.rebaser.abort('../other')).rejects.toThrow(/UUID v4/); }); + it('fails closed when an attempt ref exists at a value other than the durably owned result', async () => { + const s = await setup(), attemptId = randomUUID(); + const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); + git(s.repository.path, 'update-ref', rebaseRef(attemptId), s.foreign, result.head); + await expect(s.rebaser.abort(attemptId, result.head)).rejects.toThrow(/Another retained result/); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(s.foreign); + await expect(s.rebaser.abort(attemptId, result.head, true)).resolves.toBeUndefined(); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(s.foreign); + }); + it('removes its retained ref when durable result ownership cannot be recorded', async () => { const s = await setup(), attemptId = randomUUID(); const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResult: () => { throw new Error('result ownership write failed'); } }); + onResultPrepared: () => { throw new Error('result ownership write failed'); }, onResultReady: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) .rejects.toThrow('result ownership write failed'); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); }); + it('keeps the intended head recoverable when readiness recording fails after ref creation', async () => { + const s = await setup(), attemptId = randomUUID(); + let preparedHead: string | undefined; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResultPrepared: (_attempt, head) => { + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + preparedHead = head ?? undefined; + }, + onResultReady: () => { throw new Error('result readiness write failed'); } }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) + .rejects.toThrow('result readiness write failed'); + expect(preparedHead).toMatch(/^[0-9a-f]{40,64}$/); + expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); + await expect(rebaser.abort(attemptId, preparedHead, true)).resolves.toBeUndefined(); + }); + it('prunes an exact stale worktree registration even when its directory is already gone', async () => { const s = await setup(), attemptId = randomUUID(), path = join(s.runnerRoot, OWNER, 'rebases', attemptId); git(s.repository.path, 'worktree', 'add', '--detach', '--', path, s.foreign); diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index 2db9b27c..a7330394 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -263,12 +263,13 @@ describe('startup recovery sequence', () => { const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head], startedAt: 123 }); - store.setRebaseResult(store.getTask(id(1)).planKey, marker.attemptId, oid(4), [oid(4)]); + store.prepareRebaseResult(store.getTask(id(1)).planKey, marker.attemptId, oid(4), [oid(4)]); + store.completeRebaseResult(store.getTask(id(1)).planKey, marker.attemptId); const abortRebase = vi.fn(async () => undefined); await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase }).d }); expect(abortRebase).toHaveBeenCalledWith(store.getTask(id(1)).planKey, - expect.objectContaining({ attemptId: marker.attemptId, resultHead: oid(4), + expect.objectContaining({ attemptId: marker.attemptId, resultState: 'ready', resultHead: oid(4), resultMappings: [{ oldSha: snapshot.head, newSha: oid(4) }], processGroup: null })); expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); }); @@ -281,7 +282,8 @@ describe('startup recovery sequence', () => { const abortRebase = vi.fn(async () => undefined); await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase }).d }); - expect(abortRebase).toHaveBeenCalledWith(planKey, { ...marker, oldHistory: null, resultHead: null, resultMappings: null }); + expect(abortRebase).toHaveBeenCalledWith(planKey, { ...marker, oldHistory: null, resultState: 'none', + resultHead: null, resultMappings: null }); expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); }); it('retains a dead-group marker while another process still uses its rebase workspace', async () => { From bc21a8183a04c805edbe44b4b847cb8e1770c5c0 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 10:47:19 -0700 Subject: [PATCH 07/13] Distinguish prepared rebase ref outcomes --- docs/implementation/pre-merge-rebase.md | 2 +- docs/implementation/runner-lifecycle.md | 2 +- runner/production.ts | 10 +++-- runner/rebase.ts | 20 +++++---- runner/recovery.ts | 5 ++- runner/store.ts | 16 ++++--- test/runner-lifecycle-store.test.ts | 3 ++ test/runner-production.test.ts | 6 +-- test/runner-rebase.test.ts | 55 ++++++++++++++++--------- test/runner-recovery.test.ts | 13 ++++++ 10 files changed, 91 insertions(+), 41 deletions(-) diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md index 2a0d1587..5c6643d3 100644 --- a/docs/implementation/pre-merge-rebase.md +++ b/docs/implementation/pre-merge-rebase.md @@ -16,7 +16,7 @@ Admission checks the current plan revision, snapshot, review version, task state `GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, descended from the recorded base, and byte-for-byte equal to the ordered history in the durable marker. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. -After replay settles and before its first result-ref write, the rebaser supplies the Store's preparation hook only the ordered rewritten history. The Store zips that history by position with the independently captured `oldHistory`, then durably binds the resulting complete mapping and exact intended ref value as `prepared`. A successful atomic ref create advances it to `ready`; an already-current no-op uses the same two phases without a ref write. `Store.finishRebase` accepts only a `ready`, byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. +After replay settles and before its first result-ref write, the rebaser supplies the Store's preparation hook only the ordered rewritten history. The Store zips that history by position with the independently captured `oldHistory`, then durably binds the resulting complete mapping and exact intended ref value as `prepared`. The atomic create outcome advances it to `ready`, `uncertain`, or `refused`; an already-current no-op uses `prepared` then `ready` without a ref write. `Store.finishRebase` accepts only a `ready`, byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures settle the Git process group before cleanup. Cleanup removes only the validated UUID workspace. A partially created but unregistered worktree is removed only after Git's own worktree list proves it is not registered. Result refs are created atomically from the absent state; a reused attempt ID cannot overwrite or delete a previously retained result, while an ambiguous create is reconciled against the exact attempted value. diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 5f3cb752..13827060 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -267,7 +267,7 @@ codeboost never writes the user's repository. The commits it makes live in a **r | Holder | Owner | Rule | |---|---|---| | Partial output of a stopped writable attempt | F, using a D export | Done in D by #51 item 6: `exportTaskDiff(storage, { base, imageId, maxBytes, signal, onProcessGroup, timeoutMs, metadataBaseline })` (`agents/container/storage.ts`) takes the storage value or a recovery handle and `base`, the full ID of the commit it was seeded from, and returns `{ diff, truncated }`: at most `maxBytes` (1 MiB at most) of raw diff bytes. It checks both volumes' labels, runs Git in a read-only container with no network that writes nothing to either volume, covers edits and new untracked files (agents cannot commit or stage: the metadata is read-only to them, and the export first checks it against the seeder's baseline, refusing if it changed; a recovery handle needs the `metadataBaseline` F recorded), and cuts the output inside the container, so the Docker work stops at the limit. A changed file over 8 MiB, an untracked nested repository, or an untracked path Git will not add (a part named `.git`, or a symlink named `.gitmodules`) is named in a `codeboost:` notice line instead of diffed, so one large file cannot exhaust the export's memory or deadline; untracked symlinks are diffed as links. What Git skips without a word is named in a notice line too: an entry named `.git` (and, if it holds a repository, the new files beside it), a fifo or socket, an ignored untracked path, and anything in a submodule directory. A changed file whose `ident` or `working-tree-encoding` attribute changes what the diff shows is named too. Anything it cannot read fails the export, and a Git failure's reason includes Git's last two error lines. On abort or at its deadline it stops the client and removes the export container before it settles (an `AbortError`, or an `AggregateError` if that removal did not settle); recovery removes a leftover one. The task filesystems are Docker volumes behind a keeper container. F allocates them with `prepareTaskFilesystems` and removes them with `removeTaskFilesystems`; D's supervisor does not remove them at settlement. But F cannot read a volume from the host. So the order is: `settled` → a **D-provided bounded export** (a new D operation, for example `exportTaskDiff(filesystems, maxBytes)`, which runs `git diff` against the commit the storage was seeded from in a read-only container and returns at most 1 MiB) → F saves it → the terminal write stores `diagnostic_ref` → `removeTaskFilesystems` → the slot is freed. **The export has an overall deadline** (60 seconds, passed to D as an `AbortSignal`), because `maxBytes` bounds only the returned data, not the Docker work. On timeout or any export failure, F records "Partial output could not be exported: " as the diagnostic and continues with the terminal write and removal; shutdown and recovery never wait on an export past its deadline. F saves the export to a runner-owned diagnostics directory, with a total byte cap. **Retention never deletes a file an attempt row still references.** When the cap is exceeded, retention first deletes unreferenced files, oldest first. If that is not enough, it takes the oldest referenced file and, in one transaction, sets that row's `diagnostic_ref` to null and appends "(partial output removed by retention)" to its diagnostic. Only after that commit does it delete the file. No row is ever left pointing at a missing file. The attempt row references it as `diagnostic_ref`. If the capture fails, the row records that it failed. The task filesystem is never reused. | -| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null with the complete ordered reviewed history, tracks each Git process group until settlement, records the complete ordered mapping plus the intended result-ref value as `prepared` before the first ref write, and advances it to `ready` only after successful atomic creation. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3. Exact owned deletion treats absence as success; a `prepared` marker preserves another value, while `ready` fails closed if its ref changed. A legacy or pre-result marker without an ownership value preserves the ref. | +| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null with the complete ordered reviewed history, tracks each Git process group until settlement, records the complete ordered mapping plus the intended result-ref value as `prepared` before the first ref write, and records the atomic outcome as `ready`, `uncertain`, or `refused`. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3. Exact owned deletion treats absence as success; `prepared` blocks on a same-valued ref because intent is not ownership, `uncertain` deletes only the exact intended value, `refused` preserves the winner, and `ready` fails closed if its ref changed. A legacy or pre-result marker without an ownership value preserves the ref. | | Workspace rebuild | I3 | I3 never rebuilds before both rows above are resolved for the task. | **Process shutdown is a hard stop.** When the process stops, the running task does not finish. Its attempt ends `cancelled` with the reason "Stopped by shutdown". Restart recovery (lane I3) puts the task back in the queue. This is different from "Stop the queue" (lane I), which lets the running task finish. diff --git a/runner/production.ts b/runner/production.ts index ecaf9946..899c131a 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -112,7 +112,9 @@ export function dRecoveryDeps(image: () => string, rebaser?: Pick service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, null), onProcessUnsettled: (attemptId, group) => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, group, 'unsettled'), onResultPrepared: (attemptId, head, history) => service.store.prepareRebaseResult(rebasePlanKey, attemptId, head, history), - onResultReady: attemptId => service.store.completeRebaseResult(rebasePlanKey, attemptId) })); + onResultState: (attemptId, state) => state === 'ready' + ? service.store.completeRebaseResult(rebasePlanKey, attemptId) + : service.store.setRebaseResultState(rebasePlanKey, attemptId, state) })); const recovery = await recoverStartup({ store: service.store, runnerOwner, runnerRoot: config.root, diagnosticsDir, diagnosticsCapBytes: config.diagnosticsCapBytes, deps: o.recovery ? o.recovery(image) : dRecoveryDeps(image, - { abort: async (attemptId, resultHead, prepared) => (await getRebaser()).abort(attemptId, resultHead, prepared) }, rebasePlanKey) }); + { abort: async (attemptId, resultHead, state) => (await getRebaser()).abort(attemptId, resultHead, state) }, rebasePlanKey) }); const repository = await getRepository(); const imageId = image(); const workspace = createTaskWorkspace({ store: service.store, runnerRoot: config.root, runnerOwner, repository, imageId, diff --git a/runner/rebase.ts b/runner/rebase.ts index d8a940dd..579e13f0 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -28,7 +28,7 @@ export interface GitRebaserOptions { onProcessGroupSettled: (attemptId: string, group: ProcessGroup | 'spawning') => void; onProcessUnsettled: (attemptId: string, group: ProcessGroup) => void; onResultPrepared: (attemptId: string, intendedHead: string | null, rewrittenHistory: readonly string[]) => void; - onResultReady: (attemptId: string) => void; + onResultState: (attemptId: string, state: 'uncertain' | 'refused' | 'ready') => void; } interface CallScope { attemptId: string; signal?: AbortSignal; deadline: number; workDeadline: number } @@ -80,7 +80,7 @@ export class GitRebaser { this.#assertResultCurrent(scope); this.#options.onResultPrepared(attemptId, null, old); this.#assertResultCurrent(scope); - this.#options.onResultReady(attemptId); + this.#options.onResultState(attemptId, 'ready'); this.#assertResultCurrent(scope); return { oldHead, base: onto, head: oldHead, mappings }; } @@ -119,7 +119,10 @@ export class GitRebaser { this.#assertResultCurrent(scope); const created = await this.#call(this.#options.repository.path, ['update-ref', ref, head, '0'.repeat(head.length)], scope); if (created.status === 0) retained = true; - else if (created.status === null) uncertainRef = true; + else if (created.status === null) { + uncertainRef = true; + this.#options.onResultState(attemptId, 'uncertain'); + } else this.#options.onResultState(attemptId, 'refused'); this.#throwIfCancelled(created, scope); if (created.status !== 0) { // A null status can mean the ref write landed but its settlement record did not. Cleanup reconciles that exact @@ -127,7 +130,7 @@ export class GitRebaser { throw this.#failure('update-ref', created); } this.#assertResultCurrent(scope); - this.#options.onResultReady(attemptId); + this.#options.onResultState(attemptId, 'ready'); this.#assertResultCurrent(scope); result = { oldHead, base: onto, head, mappings }; } catch (error) { primary = error; } @@ -152,12 +155,14 @@ export class GitRebaser { } /** Startup/live recovery for an attempt whose Store marker still exists. */ - async abort(attemptId: string, resultHead?: string, prepared = false): Promise { + async abort(attemptId: string, resultHead?: string, + resultState: 'none' | 'prepared' | 'uncertain' | 'refused' | 'ready' = 'ready'): Promise { const ref = rebaseRef(attemptId), path = this.#path(attemptId), stat = lstatSync(path, { throwIfNoEntry: false }); if (resultHead !== undefined && !COMMIT_ID.test(resultHead)) throw new Error('A full retained result ID is required.'); const scope = this.#scope(attemptId); if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) throw new Error('The rebase workspace is not a plain directory.'); await this.#remove(path, scope); + if (resultState === 'refused') return; if (resultHead !== undefined) { const exists = await this.#call(this.#options.repository.path, ['show-ref', '--verify', '--quiet', ref], scope, true); if (exists.status === 1) return; @@ -169,14 +174,15 @@ export class GitRebaser { throw this.#failure('rev-parse', current); } if (current.stdout.trim() !== resultHead) { - if (prepared) return; + if (resultState !== 'ready') return; throw new Error('Another retained result owns this rebase ref.'); } + if (resultState === 'prepared') throw new Error('The prepared rebase ref outcome is ambiguous; refusing recovery.'); try { await this.#git(this.#options.repository.path, ['update-ref', '-d', ref, resultHead], scope, true); } catch (error) { const after = await this.#call(this.#options.repository.path, ['show-ref', '--verify', '--quiet', ref], scope, true); if (after.status === 1) return; - if (prepared && after.status === 0) { + if (resultState !== 'ready' && after.status === 0) { const value = await this.#call(this.#options.repository.path, ['rev-parse', '--verify', ref], scope, true); if (value.status === 0 && value.stdout.trim() !== resultHead) return; } diff --git a/runner/recovery.ts b/runner/recovery.ts index 74b06659..b52a787b 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -171,7 +171,8 @@ function validRebaseResult(marker: Partial, history: unknown, head } if (Array.isArray(history) && (history.length !== mappings.length || history.some((source, index) => mappings[index]?.oldSha !== source))) return false; - if (marker.oldHead === marker.oldBase) return mappings.length === 0; + if (marker.oldHead === marker.oldBase) + return mappings.length === 0 && head === (marker.onto === marker.oldBase ? null : marker.onto); const endpoint = mappings.at(-1) as { oldSha: string; newSha: string } | undefined; return !!endpoint && endpoint.oldSha === marker.oldHead && (head === null ? endpoint.newSha === marker.oldHead && mappings.every(mapping => mapping.oldSha === mapping.newSha) : endpoint.newSha === head); @@ -238,7 +239,7 @@ export async function recoverStartup(o: RecoveryOptions): Promise ({ oldSha, newSha: rewrittenHistory[index]! })); if (marker.resultState !== 'none') { if (marker.resultHead !== head || stable(marker.resultMappings) !== stable(mappings)) @@ -794,21 +796,25 @@ export class Store { throw new GuardRefusal('This rebase attempt no longer owns the task.'); }); } - /** Confirm that the prepared result's ref write succeeded (or that the no-op needs no ref). */ - completeRebaseResult(planKey: string, attemptId: string): void { + /** Record the external ref-write outcome before live cleanup or recovery can interpret the prepared intent. */ + setRebaseResultState(planKey: string, attemptId: string, state: 'uncertain' | 'refused' | 'ready'): void { assertUuidV4(attemptId, 'Rebase attempt ID'); this.#transaction(() => { const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); - if (marker.resultState === 'ready') return; + if (marker.resultState === state) return; if (marker.resultState !== 'prepared' || marker.resultMappings === null) throw new GuardRefusal('The rebase result was not prepared before its ref write.'); if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', - encode({ ...marker, resultState: 'ready' }), planKey, task.rebase_in_progress as string).changes !== 1) + encode({ ...marker, resultState: state }), planKey, task.rebase_in_progress as string).changes !== 1) throw new GuardRefusal('This rebase attempt no longer owns the task.'); }); } + /** Confirm that the prepared result's ref write succeeded (or that the no-op needs no ref). */ + completeRebaseResult(planKey: string, attemptId: string): void { + this.setRebaseResultState(planKey, attemptId, 'ready'); + } /** Publish a rebase only while every review and task counter captured by its attempt is still current. */ finishRebase(identity: PlanIdentity, expected: ReviewState & { reviewVersion: number }, expectedTaskStateVersion: number, attemptId: string, base: string, head: string, mappings: readonly { oldSha: string; newSha: string }[]): Snapshot { diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index 9b08fd09..2cd69495 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -670,6 +670,9 @@ describe('pre-merge rebase ownership', () => { { oldBase: oid(1), oldHead: oid(1), onto: oid(5), oldHistory: [] }); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(6), [])) .toThrow(/does not own/); + expect(() => store.prepareRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(6), [])) + .toThrow(/target base/); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ resultState: 'none', resultHead: null, resultMappings: null }); store.prepareRebaseResult(store.getTask(identity).planKey, marker.attemptId, oid(5), []); store.completeRebaseResult(store.getTask(identity).planKey, marker.attemptId); expect(store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(5), [])) diff --git a/test/runner-production.test.ts b/test/runner-production.test.ts index 3e138add..529c694f 100644 --- a/test/runner-production.test.ts +++ b/test/runner-production.test.ts @@ -231,7 +231,7 @@ describe('D adapters', () => { const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultHead: '7'.repeat(40) }; const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); await deps.abortRebase!('plan-key', marker); - expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead, false); + expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead, 'none'); await expect(deps.abortRebase!('another-plan', marker)).rejects.toThrow(/another configured plan/); expect(abort).toHaveBeenCalledTimes(1); }); @@ -239,13 +239,13 @@ describe('D adapters', () => { const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultHead: null }; const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); await deps.abortRebase!('plan-key', marker); - expect(abort).toHaveBeenCalledWith(marker.attemptId, undefined, false); + expect(abort).toHaveBeenCalledWith(marker.attemptId, undefined, 'none'); }); it('lets prepared recovery preserve a different pre-existing ref while clearing its marker', async () => { const abort = vi.fn(async () => undefined), marker = { attemptId: randomUUID(), resultState: 'prepared', resultHead: '7'.repeat(40) }; const deps = dRecoveryDeps(() => 'sha256:x', { abort } as unknown as GitRebaser, 'plan-key'); await deps.abortRebase!('plan-key', marker); - expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead, true); + expect(abort).toHaveBeenCalledWith(marker.attemptId, marker.resultHead, 'prepared'); }); it('passes D\'s own handles through, and turns every unowned object into the command that removes it', async () => { const handle = Object.freeze({ runnerOwner: OWNER, attemptId: randomUUID(), allocationId: randomUUID() }); diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index 9068f0d7..bbd24265 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -40,7 +40,7 @@ async function setup(conflict = false) { const rebaser = new GitRebaser({ repository, runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResultPrepared: () => {}, onResultReady: () => {} }); + onResultPrepared: () => {}, onResultState: () => {} }); return { root, source, runnerRoot, repository, rebaser, base, owned, foreign, history: [owned, foreign], onto }; } @@ -73,7 +73,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResultPrepared: () => vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER), onResultReady: () => {} }); + onResultPrepared: () => vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER), onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base })) .rejects.toThrow(/deadline expired while recording its result/); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); @@ -155,7 +155,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: (ownedAttempt, group) => { expect(ownedAttempt).toBe(attemptId); if (group !== 'spawning') { expect(active.get(group.pgid)).toBe(group.startedAt); active.delete(group.pgid); } - }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); expect(seen.length).toBeGreaterThan(1); expect(active.size).toBe(0); @@ -166,7 +166,7 @@ describe('trusted pre-merge rebase', () => { expect(() => new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, timeoutMs: 43_000, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResultPrepared: () => {}, onResultReady: () => {} })).toThrow(/Invalid rebase deadline/); + onResultPrepared: () => {}, onResultState: () => {} })).toThrow(/Invalid rebase deadline/); }); it('does not start Git when a synchronous ownership hook exhausts the work deadline', async () => { @@ -177,7 +177,7 @@ describe('trusted pre-merge rebase', () => { onProcessStarting: () => { starts++; const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, onProcessGroupSettled: (_attempt, group) => { if (group === 'spawning') settled++; }, onProcessUnsettled: () => {}, - onResultPrepared: () => {}, onResultReady: () => {} }); + onResultPrepared: () => {}, onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })).rejects.toThrow(/deadline expired/); expect({ starts, settled }).toEqual({ starts: 1, settled: 1 }); }); @@ -192,7 +192,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: () => { if (firstSettlement) { expect(groups).toBe(0); firstSettlement = false; } expect(owner).not.toBe('none'); owner = 'none'; - }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) .rejects.toThrow('ownership commit outcome unknown'); expect(owner).toBe('none'); @@ -206,7 +206,7 @@ describe('trusted pre-merge rebase', () => { onProcessStarting: () => { const until = performance.now() + 2; while (performance.now() < until) { /* block */ } }, onProcessGroup: () => { throw new Error('Git must not spawn after the deadline.'); }, onProcessGroupSettled: () => { throw new Error('settlement write failed'); }, onProcessUnsettled: () => {}, - onResultPrepared: () => {}, onResultReady: () => {} }); + onResultPrepared: () => {}, onResultState: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -225,7 +225,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroupSettled: () => { if (++settled === 2 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } }, - onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.base })) .rejects.toThrow(/deadline expired while recording process settlement/); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -239,7 +239,7 @@ describe('trusted pre-merge rebase', () => { onProcessGroup: () => { processes++; }, onProcessGroupSettled: () => { if (processes === 7 && !overran) { overran = true; vi.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER); } - }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + }, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -314,7 +314,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('cancelled during rebase')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during rebase'); expect(processes).toBeGreaterThanOrEqual(9); // stopped rebase, worktree removal, then pruning @@ -328,7 +328,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 8) stop.abort(new Error('cancelled during conflict classification')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during conflict classification'); expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(false); @@ -341,7 +341,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => stop.abort(new Error('cancelled by caller')), onProcessGroupSettled: () => { if (!failed) { failed = true; throw new Error('settlement write failed'); } }, - onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -358,7 +358,7 @@ describe('trusted pre-merge rebase', () => { committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => { if (++starts === 8) throw new Error('cleanup record failed'); }, onProcessGroup: () => { if (++processes === 7) stop.abort(new Error('original cancellation')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -375,7 +375,7 @@ describe('trusted pre-merge rebase', () => { const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => { if (++processes === 14) stop.abort(new Error('cancelled during cleanup')); }, - onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, signal: stop.signal })) .rejects.toThrow('cancelled during cleanup'); expect(processes).toBeGreaterThanOrEqual(16); // result ref, worktree cleanup, prune, then result-ref deletion @@ -396,7 +396,7 @@ describe('trusted pre-merge rebase', () => { throw new Error('result ref outcome unknown'); } }, - onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultReady: () => {} }); + onProcessUnsettled: () => {}, onResultPrepared: () => {}, onResultState: () => {} }); const failure = await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }) .then(() => null, error => error as AggregateError); expect(failure).toBeInstanceOf(AggregateError); @@ -413,6 +413,8 @@ describe('trusted pre-merge rebase', () => { expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); await s.rebaser.abort(attemptId); expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); + await expect(s.rebaser.abort(attemptId, result.head, 'prepared')).rejects.toThrow(/outcome is ambiguous/); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(result.head); await s.rebaser.abort(attemptId, result.head); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); await expect(s.rebaser.abort(attemptId, result.head)).resolves.toBeUndefined(); @@ -425,16 +427,31 @@ describe('trusted pre-merge rebase', () => { git(s.repository.path, 'update-ref', rebaseRef(attemptId), s.foreign, result.head); await expect(s.rebaser.abort(attemptId, result.head)).rejects.toThrow(/Another retained result/); expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(s.foreign); - await expect(s.rebaser.abort(attemptId, result.head, true)).resolves.toBeUndefined(); + await expect(s.rebaser.abort(attemptId, result.head, 'prepared')).resolves.toBeUndefined(); expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(s.foreign); }); + it('records a definite same-head create refusal and preserves the winning ref during recovery', async () => { + const s = await setup(), attemptId = randomUUID(), states: string[] = []; + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResultPrepared: (_attempt, head) => git(s.repository.path, 'update-ref', rebaseRef(attemptId), head!), + onResultState: (_attempt, state) => { states.push(state); } }); + await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) + .rejects.toThrow(/update-ref failed/); + expect(states).toEqual(['refused']); + const winner = git(s.repository.path, 'rev-parse', rebaseRef(attemptId)); + await expect(rebaser.abort(attemptId, winner, 'refused')).resolves.toBeUndefined(); + expect(git(s.repository.path, 'rev-parse', rebaseRef(attemptId))).toBe(winner); + }); + it('removes its retained ref when durable result ownership cannot be recorded', async () => { const s = await setup(), attemptId = randomUUID(); const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, - onResultPrepared: () => { throw new Error('result ownership write failed'); }, onResultReady: () => {} }); + onResultPrepared: () => { throw new Error('result ownership write failed'); }, onResultState: () => {} }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) .rejects.toThrow('result ownership write failed'); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); @@ -451,12 +468,12 @@ describe('trusted pre-merge rebase', () => { expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); preparedHead = head ?? undefined; }, - onResultReady: () => { throw new Error('result readiness write failed'); } }); + onResultState: () => { throw new Error('result readiness write failed'); } }); await expect(rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto })) .rejects.toThrow('result readiness write failed'); expect(preparedHead).toMatch(/^[0-9a-f]{40,64}$/); expect(() => git(s.repository.path, 'show-ref', '--verify', rebaseRef(attemptId))).toThrow(); - await expect(rebaser.abort(attemptId, preparedHead, true)).resolves.toBeUndefined(); + await expect(rebaser.abort(attemptId, preparedHead, 'prepared')).resolves.toBeUndefined(); }); it('prunes an exact stale worktree registration even when its directory is already gone', async () => { diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index a7330394..c1e985a4 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -358,6 +358,19 @@ describe('startup recovery sequence', () => { expect(abortRebase).not.toHaveBeenCalled(); expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); }); + it('rejects an empty-history result bound to a head other than its target before cleanup', async () => { + const { d: root, store, raw } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), attemptId = randomUUID(), planKey = store.getTask(id(1)).planKey; + const marker = { attemptId, oldBase: snapshot.base, oldHead: snapshot.base, onto: oid(5), oldHistory: [], startedAt: 123, + resultState: 'ready', resultHead: oid(6), resultMappings: [], processGroup: null }; + raw(`UPDATE tasks SET rebase_in_progress='${JSON.stringify(marker)}' WHERE plan_key='${planKey}'`); + const abortRebase = vi.fn(async () => undefined); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase }).d })).rejects.toThrow(/invalid recovery marker/); + expect(abortRebase).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); + }); it('records an export timeout as a diagnostic and still finalizes and removes the storage', async () => { const { d: root, store, admit, allocate } = fixture(); const attempt = admit(id(1)); const h = allocate(id(1), attempt.id, 'h'); store.markRunning(id(1), attempt.id); From d7340563186b763ef76391316277a0ba557deb06 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 11:07:58 -0700 Subject: [PATCH 08/13] Preserve Git path for checkout verification --- runner/rebase.ts | 2 +- test/runner-rebase.test.ts | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/runner/rebase.ts b/runner/rebase.ts index 579e13f0..b32134d4 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -209,7 +209,7 @@ export class GitRebaser { async #assertCheckout(repository: string, commit: string, label: string, scope: CallScope): Promise { const verifier = fileURLToPath(new URL('./verify-checkout.ts', import.meta.url)); - const verified = await this.#process(process.execPath, [verifier], repository, {}, scope, false, 64 * 1024); + const verified = await this.#process(process.execPath, [verifier], repository, gitEnvironment(), scope, false, 64 * 1024); this.#throwIfCancelled(verified, scope); if (verified.status !== 0) throw new Error(`The host filesystem cannot faithfully check out the ${label}.`, { cause: this.#failure('checkout verification', verified, 'node') }); diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index bbd24265..a5ccd3ff 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -1,4 +1,5 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { randomUUID } from 'node:crypto'; @@ -45,6 +46,26 @@ async function setup(conflict = false) { } describe('trusted pre-merge rebase', () => { + it('uses the configured Git PATH inside the checkout verifier subprocess', async () => { + const s = await setup(), attemptId = randomUUID(), bin = join(s.root, 'git-bin'), marker = join(s.root, 'verified'); + mkdirSync(bin); + const realGit = execFileSync('/bin/sh', ['-c', 'command -v git'], { encoding: 'utf8' }).trim(); + const quote = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`; + writeFileSync(join(bin, 'git'), [ + '#!/bin/sh', + `for arg do if [ "$arg" = ls-files ]; then : > ${quote(marker)}; fi; done`, + `exec ${quote(realGit)} "$@"`, + ].join('\n'), { mode: 0o755 }); + const path = process.env.PATH; + process.env.PATH = `${bin}:${path ?? ''}`; + try { + await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); + expect(existsSync(marker)).toBe(true); + } finally { + if (path === undefined) delete process.env.PATH; else process.env.PATH = path; + } + }); + it('replays a complete linear history one-for-one and retains only its immutable result ref', async () => { const s = await setup(), attemptId = randomUUID(); const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); From 0a1da818541c99f0acb7766811a78172bebdaa4b Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 11:23:38 -0700 Subject: [PATCH 09/13] Pin Git before entering rebase worktrees --- runner/rebase.ts | 34 +++++++++++++++++++++++++++++----- runner/verify-checkout.ts | 13 ++++++++----- test/runner-rebase.test.ts | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/runner/rebase.ts b/runner/rebase.ts index b32134d4..34bb4897 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -1,5 +1,5 @@ -import { lstatSync } from 'node:fs'; -import { join } from 'node:path'; +import { accessSync, constants, lstatSync, realpathSync, statSync } from 'node:fs'; +import { delimiter, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { runInProcessGroup, type ProcessGroup } from '../agents/process-group.ts'; import type { DockerOutcome } from '../agents/docker.ts'; @@ -32,6 +32,24 @@ export interface GitRebaserOptions { } interface CallScope { attemptId: string; signal?: AbortSignal; deadline: number; workDeadline: number } +const configuredGit = (): { executable: string; environment: NodeJS.ProcessEnv } => { + const source = process.env.PATH; + if (!source) throw new Error('Git requires a configured PATH.'); + // Resolve relative and empty entries while still in the runner's trusted working directory. Carrying them into an + // agent-controlled worktree would let that checkout supply the `git` executable used to verify itself. + const path = source.split(delimiter).map(entry => resolve(process.cwd(), entry || '.')).join(delimiter); + for (const directory of path.split(delimiter)) { + const candidate = join(directory, 'git'); + try { + const executable = realpathSync(candidate); + if (!statSync(executable).isFile()) continue; + accessSync(executable, constants.X_OK); + return { executable, environment: { ...gitEnvironment(), PATH: path } }; + } catch { /* This PATH entry does not provide an executable Git. */ } + } + throw new Error('Git is not executable on the configured PATH.'); +}; + /** A conflict is a review outcome, not an infrastructure failure. No rewritten ref is retained. */ export class RebaseConflict extends Error {} @@ -49,6 +67,8 @@ export function rebaseRef(attemptId: string): string { export class GitRebaser { readonly #options: GitRebaserOptions; readonly #root: string; + readonly #gitExecutable: string; + readonly #gitEnvironment: NodeJS.ProcessEnv; constructor(options: GitRebaserOptions) { if (!/^[0-9a-f]{32}$/.test(options.runnerOwner)) throw new Error('Invalid runner owner token.'); @@ -57,7 +77,10 @@ export class GitRebaser { if (options.timeoutMs !== undefined && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < MIN_REBASE_TIMEOUT_MS || options.timeoutMs > 120_000)) throw new Error('Invalid rebase deadline.'); + const git = configuredGit(); this.#options = options; + this.#gitExecutable = git.executable; + this.#gitEnvironment = git.environment; this.#root = ownerOnlyDirectory(options.runnerRoot, options.runnerOwner, 'rebases'); } @@ -209,7 +232,8 @@ export class GitRebaser { async #assertCheckout(repository: string, commit: string, label: string, scope: CallScope): Promise { const verifier = fileURLToPath(new URL('./verify-checkout.ts', import.meta.url)); - const verified = await this.#process(process.execPath, [verifier], repository, gitEnvironment(), scope, false, 64 * 1024); + const verified = await this.#process(process.execPath, [verifier, this.#gitExecutable], repository, + this.#gitEnvironment, scope, false, 64 * 1024); this.#throwIfCancelled(verified, scope); if (verified.status !== 0) throw new Error(`The host filesystem cannot faithfully check out the ${label}.`, { cause: this.#failure('checkout verification', verified, 'node') }); @@ -262,13 +286,13 @@ export class GitRebaser { async #call(repository: string, args: readonly string[], scope: CallScope, cleanup = false, discardExcessOutput = false, input?: Buffer) { const env = { - ...gitEnvironment(), + ...this.#gitEnvironment, GIT_COMMITTER_NAME: this.#options.committer.name, GIT_COMMITTER_EMAIL: this.#options.committer.email, GIT_EDITOR: 'true', GIT_SEQUENCE_EDITOR: 'true', }; - return this.#process('git', [...GIT_OPTIONS, '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', ...args], repository, + return this.#process(this.#gitExecutable, [...GIT_OPTIONS, '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', ...args], repository, env, scope, cleanup, discardExcessOutput ? 64 * 1024 : 8 * 1024 * 1024, input, discardExcessOutput); } diff --git a/runner/verify-checkout.ts b/runner/verify-checkout.ts index 16c23374..02fbdf45 100644 --- a/runner/verify-checkout.ts +++ b/runner/verify-checkout.ts @@ -9,8 +9,8 @@ const MAX_DIRECTORY_ENTRIES = 262_144; // A C-quoted path can expand every input byte to four bytes. Keep transport above every accepted index listing; // accepted input is then rejected by our explicit bounds, never by spawnSync's smaller implementation limit. const MAX_GIT_OUTPUT_BYTES = MAX_INDEX_BYTES * 4 + 1024; -const git = (args: readonly string[], input?: Buffer): Buffer => { - const result = spawnSync('git', [...GIT_OPTIONS, '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', ...args], +const git = (executable: string, args: readonly string[], input?: Buffer): Buffer => { + const result = spawnSync(executable, [...GIT_OPTIONS, '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', ...args], { env: gitEnvironment(), input, maxBuffer: MAX_GIT_OUTPUT_BYTES, stdio: [input ? 'pipe' : 'ignore', 'pipe', 'pipe'] }); if (result.status !== 0 || result.error) throw result.error ?? new Error(`git ${args[0]} failed while verifying the checkout.`); return result.stdout; @@ -36,8 +36,8 @@ const objectId = (algorithm: 'sha1' | 'sha256', bytes: Buffer): string => createHash(algorithm).update(Buffer.from(`blob ${bytes.length}\0`)).update(bytes).digest('hex'); /** Byte-exact, content-based proof that the disposable worktree represents its index. Gitlinks stay index-only. */ -export function verifyCheckout(root = process.cwd()): void { - const repositoryGit = (args: readonly string[], input?: Buffer) => git(['-C', root, ...args], input); +export function verifyCheckout(root = process.cwd(), executable = 'git'): void { + const repositoryGit = (args: readonly string[], input?: Buffer) => git(executable, ['-C', root, ...args], input); const rawOutput = repositoryGit(['ls-files', '--stage', '-z']); if (rawOutput.length > MAX_INDEX_BYTES) throw new Error('Checkout index exceeds its byte bound.'); const raw = split0(rawOutput); @@ -103,4 +103,7 @@ export function verifyCheckout(root = process.cwd()): void { } } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) verifyCheckout(); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (!process.argv[2]) throw new Error('Checkout verification requires a pinned Git executable.'); + verifyCheckout(process.cwd(), process.argv[2]); +} diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index a5ccd3ff..33bd2c98 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -59,13 +59,46 @@ describe('trusted pre-merge rebase', () => { const path = process.env.PATH; process.env.PATH = `${bin}:${path ?? ''}`; try { - await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResultPrepared: () => {}, onResultState: () => {} }); + await rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); expect(existsSync(marker)).toBe(true); } finally { if (path === undefined) delete process.env.PATH; else process.env.PATH = path; } }); + it('never resolves Git from a relative PATH component inside the untrusted checkout', async () => { + const s = await setup(), attemptId = randomUUID(), marker = join(s.root, 'untrusted-git-ran'); + const realGit = execFileSync('/bin/sh', ['-c', 'command -v git'], { encoding: 'utf8' }).trim(); + const quote = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`; + git(s.source, 'switch', '-q', 'feature'); + mkdirSync(join(s.source, 'bin')); + writeFileSync(join(s.source, 'bin', 'git'), [ + '#!/bin/sh', + `: > ${quote(marker)}`, + `exec ${quote(realGit)} "$@"`, + ].join('\n'), { mode: 0o755 }); + git(s.source, 'add', '--', 'bin/git'); + git(s.source, 'commit', '-qm', 'add untrusted git'); + const head = git(s.source, 'rev-parse', 'HEAD'); + await ensureCommit(s.repository, head); + const path = process.env.PATH; + process.env.PATH = `bin:${path ?? ''}`; + try { + const rebaser = new GitRebaser({ repository: s.repository, runnerRoot: s.runnerRoot, runnerOwner: OWNER, + committer: { name: 'Codeboost', email: 'codeboost@example.invalid' }, + onProcessStarting: () => {}, onProcessGroup: () => {}, onProcessGroupSettled: () => {}, onProcessUnsettled: () => {}, + onResultPrepared: () => {}, onResultState: () => {} }); + await rebaser.run({ attemptId, oldBase: s.base, oldHead: head, oldHistory: [...s.history, head], onto: s.onto }); + expect(existsSync(marker)).toBe(false); + } finally { + if (path === undefined) delete process.env.PATH; else process.env.PATH = path; + } + }); + it('replays a complete linear history one-for-one and retains only its immutable result ref', async () => { const s = await setup(), attemptId = randomUUID(); const result = await s.rebaser.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto }); From 7f5c16c845bd7dcd8ed60b84635b4652f9cd42e6 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 11:44:27 -0700 Subject: [PATCH 10/13] Harden interrupted rebase recovery ownership --- runner/recovery.ts | 48 +++++++++++++++++++++++++++--------- test/runner-recovery.test.ts | 43 ++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/runner/recovery.ts b/runner/recovery.ts index b52a787b..0c9c037e 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -100,22 +100,44 @@ export function acquireRunnerLock(databasePath: string, options: { lockRoot?: st export interface ProcessControl { /** The group leader is alive and started at the recorded time (guards against PID reuse). */ isAlive(pgid: number, startedAt: number): boolean; - /** SIGTERM the group, SIGKILL after the grace period, and resolve only once it has exited. */ - terminate(pgid: number, graceMs: number): Promise; + /** Revalidate its recorded start, SIGTERM, SIGKILL after grace, and resolve only once the group has exited. */ + terminate(pgid: number, startedAt: number, graceMs: number): Promise; } +const groupAlive = (pgid: number) => { + try { process.kill(-pgid, 0); return true; } + catch (error) { return (error as NodeJS.ErrnoException).code !== 'ESRCH'; } +}; +const ownsGroupAtSignal = (pgid: number, startedAt: number): boolean => { + if (!groupAlive(pgid)) return false; + try { + const output = execFileSync('ps', ['-o', 'lstart=', '-p', String(pgid)], + { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C' } }); + const started = Date.parse(output.trim()); + if (!Number.isFinite(started) || Math.abs(started - startedAt) >= 2_000) + throw new Error('The recorded process group identity changed before recovery could signal it.'); + } catch (error) { + throw new Error('Could not prove the recorded process group still owns its ID; refusing to signal it.', + { cause: error instanceof Error ? error : undefined }); + } + return true; +}; export const hostProcesses: ProcessControl = { isAlive(pgid, startedAt) { - try { process.kill(-pgid, 0); } catch { return false; } + if (!groupAlive(pgid)) return false; try { return startTimeMatches(execFileSync('ps', ['-o', 'lstart=', '-p', String(pgid)], { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C' } }), startedAt); } catch { return true; } // Alive but unreadable: treat as ours and stop it (fail closed). }, - async terminate(pgid, graceMs) { - const alive = () => { try { process.kill(-pgid, 0); return true; } catch { return false; } }; - try { process.kill(-pgid, 'SIGTERM'); } catch { return; } + async terminate(pgid, startedAt, graceMs) { + if (!ownsGroupAtSignal(pgid, startedAt)) return; + try { process.kill(-pgid, 'SIGTERM'); } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ESRCH') return; throw error; } const until = Date.now() + graceMs; - while (alive() && Date.now() < until) await new Promise(resolve => setTimeout(resolve, 50)); - if (alive()) { try { process.kill(-pgid, 'SIGKILL'); } catch {} } - while (alive()) await new Promise(resolve => setTimeout(resolve, 50)); + while (groupAlive(pgid) && Date.now() < until) await new Promise(resolve => setTimeout(resolve, 50)); + if (ownsGroupAtSignal(pgid, startedAt)) { + try { process.kill(-pgid, 'SIGKILL'); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; } + } + while (groupAlive(pgid)) await new Promise(resolve => setTimeout(resolve, 50)); }, }; @@ -191,7 +213,7 @@ export async function recoverStartup(o: RecoveryOptions): Promise a.preparationStartedAt !== null && a.preparationPgid === null).map(a => a.id); // 2b. Stop leftover preparation before D's recovery or any storage work. for (const a of interrupted) if (a.preparationPgid !== null && a.preparationStartedAt !== null && processes.isAlive(a.preparationPgid, a.preparationStartedAt)) - await processes.terminate(a.preparationPgid, o.graceMs ?? 5_000); + await processes.terminate(a.preparationPgid, a.preparationStartedAt, o.graceMs ?? 5_000); // 2c/2d. D's recovery; a rejection propagates and stops startup. const recovered = await o.deps.recoverLeftovers(o.runnerOwner); if (recovered.unowned.length) throw new RecoveryBlocked('Docker holds codeboost objects this runner will not remove itself: objects without a runner label may belong to an older build that is still running, and objects of this runner it cannot identify are not ones it made. Stop every older codeboost process, check and run these commands, then start again', recovered.unowned); @@ -232,12 +254,13 @@ export async function recoverStartup(o: RecoveryOptions): Promise | null; const processGroup = marker?.processGroup, oldHistory = marker?.oldHistory ?? null, resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null, - resultState = marker?.resultState ?? (resultMappings === null ? 'none' : 'ready'); + resultState = marker?.resultState ?? 'none'; if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldBase) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldHead) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.onto) || !Number.isSafeInteger(marker.startedAt) || marker.startedAt! < 0 || !validOldHistory(marker, oldHistory) || + (oldHistory === null && (resultState !== 'none' || resultHead !== null || resultMappings !== null)) || (resultHead !== null && (typeof resultHead !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(resultHead))) || !['none', 'prepared', 'uncertain', 'refused', 'ready'].includes(resultState) || (resultState === 'none' ? resultHead !== null || resultMappings !== null : resultMappings === null) || @@ -246,7 +269,8 @@ export async function recoverStartup(o: RecoveryOptions): Promise { resultHead: null, resultMappings: null }); expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); }); + it('rejects incomplete and cleanup-only markers that claim retained-result ownership', async () => { + for (const shape of ['missing-state', 'missing-history'] as const) { + const { d: root, store, raw } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), attemptId = randomUUID(), planKey = store.getTask(id(1)).planKey; + const result = { resultHead: oid(4), resultMappings: [{ oldSha: snapshot.head, newSha: oid(4) }] }; + const marker = { attemptId, oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123, + ...(shape === 'missing-state' ? { oldHistory: [snapshot.head], ...result } : { resultState: 'ready', ...result }), + processGroup: null }; + raw(`UPDATE tasks SET rebase_in_progress='${JSON.stringify(marker)}' WHERE plan_key='${planKey}'`); + const abortRebase = vi.fn(async () => undefined); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase }).d })).rejects.toThrow(/invalid recovery marker/); + expect(abortRebase).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); + } + }); it('retains a dead-group marker while another process still uses its rebase workspace', async () => { const { d: root, store } = fixture(); store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); @@ -317,7 +334,7 @@ describe('startup recovery sequence', () => { await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase, processes: { isAlive: () => true, terminate } }).d, openFiles: () => ['escaped-child'] })).rejects.toThrow(/still uses/); - expect(terminate).toHaveBeenCalledWith(group.pgid, 5_000); + expect(terminate).toHaveBeenCalledWith(group.pgid, group.startedAt, 5_000); expect(abortRebase).not.toHaveBeenCalled(); expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: group }); }); @@ -481,6 +498,28 @@ describe('startup recovery sequence', () => { }); describe('host process checks', () => { + it('treats a permission-denied liveness probe as alive instead of releasing ownership', () => { + vi.spyOn(process, 'kill').mockImplementation(() => { throw Object.assign(new Error('denied'), { code: 'EPERM' }); }); + expect(hostProcesses.isAlive(4242, 0)).toBe(true); + }); + + it('revalidates the recorded start identity before signalling a process group', async () => { + const child = spawn('/bin/sh', ['-c', 'exec sleep 30'], { detached: true, stdio: 'ignore' }); + children.push(child); + const startedAt = Date.now(); + await once(child, 'spawn'); + const pgid = child.pid!; + try { + expect(hostProcesses.isAlive(pgid, startedAt)).toBe(true); + await expect(hostProcesses.terminate(pgid, startedAt + 60_000, 10)).rejects.toThrow(/identity|recorded process group/); + expect(() => process.kill(pgid, 0)).not.toThrow(); + } finally { + try { process.kill(-pgid, 'SIGKILL'); } catch {} + await once(child, 'exit'); + children.splice(children.indexOf(child), 1); + } + }); + it('treats a start time it cannot read as a match, so a live group is still stopped', () => { const at = Date.parse('Sat Sep 26 02:01:04 2026'); expect(startTimeMatches('Sat Sep 26 02:01:04 2026\n', at)).toBe(true); From b69745a9459c5060706a604ad3f8ea9491b956bd Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 12:27:17 -0700 Subject: [PATCH 11/13] Harden recovered process group ownership --- agents/process-group.ts | 20 ++++++- docs/architecture.md | 2 +- docs/implementation/runner-lifecycle.md | 8 +-- runner/recovery.ts | 62 +++++++++------------- runner/store.ts | 33 ++++++++---- runner/workspace.ts | 5 +- test/agent-process-group.test.ts | 2 + test/planning-drafts.test.ts | 4 +- test/publish.test.ts | 2 +- test/runner-lifecycle-store.test.ts | 42 ++++++++++----- test/runner-recovery.test.ts | 69 ++++++++++++++++++------- 11 files changed, 158 insertions(+), 91 deletions(-) diff --git a/agents/process-group.ts b/agents/process-group.ts index 7ec6072a..dd7f712e 100644 --- a/agents/process-group.ts +++ b/agents/process-group.ts @@ -1,12 +1,28 @@ import { spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; import { NOT_STARTED, type DockerOutcome } from './docker.ts'; /** A subprocess's process group, reported as soon as it exists so the caller can record it durably. */ export interface ProcessGroup { /** The group ID, which is the leader's PID. */ readonly pgid: number; - /** `Date.now()` right after the spawn; with the ID, it tells this group from a later one that reuses the ID. */ + /** `Date.now()` right after the spawn, retained for lifecycle timing and diagnostics. */ readonly startedAt: number; + /** Linux boot ID plus kernel start ticks. Null means crash recovery must not signal this group automatically. */ + readonly identity: string | null; +} + +/** A kernel-backed identity that changes across both PID reuse and host reboot. */ +export function processIdentity(pid: number): string | null { + if (process.platform !== 'linux') return null; + try { + const boot = readFileSync('/proc/sys/kernel/random/boot_id', 'ascii').trim(); + const stat = readFileSync(`/proc/${pid}/stat`, 'ascii'), close = stat.lastIndexOf(')'); + const fields = close < 0 ? [] : stat.slice(close + 2).trim().split(/ +/); + const ticks = fields[19]; // field 22 overall; fields starts with field 3 (`state`). + if (!/^[0-9a-f-]{36}$/.test(boot) || !ticks || !/^\d+$/.test(ticks)) return null; + return `linux:${boot}:${ticks}`; + } catch { return null; } } export interface ProcessGroupOptions { readonly env: NodeJS.ProcessEnv; @@ -114,7 +130,7 @@ export function runInProcessGroup(file: string, args: readonly string[], // If the caller cannot record the group, the child must not outlive this call: it is killed at once below, and // the call still settles only after the group has exited, with the caller's error. let unrecorded: unknown; - try { options.onProcessGroup?.(Object.freeze({ pgid, startedAt: Date.now() })); } + try { options.onProcessGroup?.(Object.freeze({ pgid, startedAt: Date.now(), identity: processIdentity(pgid) })); } catch (error) { unrecorded = error; } // Normally output past the limit stops the group and settles as ENOBUFS. A caller that classifies from durable state // may explicitly keep a bounded prefix and let the process finish instead. diff --git a/docs/architecture.md b/docs/architecture.md index 10f156cf..40546caa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -356,7 +356,7 @@ The diagram shows the designed transitions. With the `runner` block, running, ne ### Stored data -All state is in one SQLite file, opened with WAL and full synchronization. Each write takes an immediate write lock. The schema version is `PRAGMA user_version`, and migrations run in explicit steps (version 15 today); an unknown version fails. +All state is in one SQLite file, opened with WAL and full synchronization. Each write takes an immediate write lock. The schema version is `PRAGMA user_version`, and migrations run in explicit steps (version 16 today); an unknown version fails. | Group | Tables | Notes | |---|---|---| diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 13827060..5a12e7b5 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -126,7 +126,7 @@ A `pending` attempt has no `settled` promise, so the slot rules need a separate - Preparation receives an `AbortSignal` from the job. Stopping a pending attempt aborts that signal with the first reason. - **Preparation files have a durable owner.** The attempt ID is used as a directory name only after it passes the UUID check under "Terms used", so it cannot contain `/` or `..`. Before preparation creates anything, F uses the fixed attempt directory `//attempts//`, and passes it as the parent to the clone helper. The helper names its clone with `mkdtemp`, so the fixed parent is what makes the clone findable after a crash. **Today's `createTaskClone` (`git/clone.ts`) runs git with synchronous `execFileSync`.** It cannot be aborted and would block the event loop, so F1 cannot use it for preparation as it is. D, which owns `git/clone.ts`, must add an asynchronous variant that takes an `AbortSignal`, spawns git in its own process group, and exposes that group's ID (third prerequisite under "Shutdown"). -- **Preparation processes have a durable owner.** Right after spawning a preparation child, in the same synchronous turn, F saves the process group ID and the child's start time on the attempt row (`preparation_pgid`, `preparation_started_at`). A process group outlives the runner if the runner crashes, so startup recovery uses these fields to stop it (step 4). To close the gap between spawn and that write, F first saves `preparation_started_at` with a null `preparation_pgid` (**preparation starting**), then spawns, then saves the group ID. If recovery finds an attempt that is marked as starting but has no group ID, a child may exist that nobody can identify. Recovery then **fails closed**: it does not open admission, it leaves that attempt directory in place, and the CLI names the attempt and its directory. The user must stop the unknown process and then confirm with `--release-preparation `. A quiet directory proves nothing, because a blocked process may resume later, so codeboost does not rely on it. Before removing anything, it checks that no process has a file open or a working directory inside the attempt directory, using `lsof +D` on macOS or `/proc/*/fd` and `/proc/*/cwd` on Linux. If a process is found, or the check cannot run, it refuses and keeps admission closed. The path comes only from IDs that are already saved, so a crash cannot leave an unknown path. The job removes the directory when the attempt ends. Startup recovery handles the rest (step 4). +- **Preparation processes have a durable owner.** Right after spawning a preparation child, in the same synchronous turn, F saves the process group ID, observed start time, and kernel identity on the attempt row (`preparation_pgid`, `preparation_started_at`, `preparation_identity`). On Linux the identity combines the boot ID with the leader's kernel start ticks, so recovery can distinguish PID reuse and a reboot without consulting a PATH-selected subprocess. A process group outlives the runner if the runner crashes, so startup recovery revalidates this identity immediately before each signal and stops the group (step 4). Where an exact identity is unavailable, recovery fails closed instead of signalling a numeric ID. To close the gap between spawn and that write, F first saves `preparation_started_at` with a null `preparation_pgid` (**preparation starting**), then spawns, then saves the group ID and identity. If recovery finds an attempt that is marked as starting but has no group ID, a child may exist that nobody can identify. Recovery then **fails closed**: it does not open admission, it leaves that attempt directory in place, and the CLI names the attempt and its directory. The user must stop the unknown process and then confirm with `--release-preparation `. A quiet directory proves nothing, because a blocked process may resume later, so codeboost does not rely on it. Before removing anything, it checks that no process has a file open or a working directory inside the attempt directory, using `lsof +D` on macOS or `/proc/*/fd` and `/proc/*/cwd` on Linux. If a process is found, or the check cannot run, it refuses and keeps admission closed. The path comes only from IDs that are already saved, so a crash cannot leave an unknown path. The job removes the directory when the attempt ends. Startup recovery handles the rest (step 4). - An `AbortSignal` does not stop a child process by itself. Every preparation subprocess (for example `git clone`) is started in its own process group, owned by the job. On abort, F sends `SIGTERM` to the group, waits a fixed grace period (5 seconds), then sends `SIGKILL`, and awaits the `close` event. Preparation settles only after that. A child that ignores `SIGTERM` is therefore still bounded by the kill. Preparation never leaves work running after it settles. - Preparation counts against two limits, so it cannot hold a slot forever. The task budget is already running during preparation: the admission transaction of the task's first attempt moves the task to `running` and sets `budget_deadline` (see "Proposed storage additions"), before preparation starts. If the **task time budget** runs out first, the job records the first reason `time-limit`, then aborts the signal; the pending row ends `cancelled` and the task goes to `needs human`, as for a running attempt. If the **attempt deadline** passes first, the job aborts the signal with `timeout` and records no first reason; the row ends `failed` "Timed out while preparing". - Shutdown aborts every preparation signal in step 4. @@ -290,7 +290,7 @@ This belongs to D (D5 or a D follow-up), not F. It changes `agents/contract.ts`, **Also before F1 merges: asynchronous launch.** Done in #51 item 2 (see "Launch setup does not block the server"). -**Third prerequisite before F1 merges: abortable preparation helpers.** Done in #51 item 5: `createTaskCloneAsync` and `prepareTaskFilesystemsAsync` take `signal` and `onProcessGroup`. Each subprocess runs as the leader of a new process group (`agents/process-group.ts`); `onProcessGroup` receives `{ pgid, startedAt }` synchronously after the spawn, so F can save it as `preparation_pgid` and `preparation_started_at` before anything else runs. Every Git or Docker call is a separate group, so F updates the saved group on each call. On abort or at the deadline the group gets `SIGTERM`, then `SIGKILL` after 5 seconds; the call settles only after the leader and every other member of its group have exited. Two bounds apply: a member still alive 10 seconds after the leader exits (repeated `SIGKILL` included) settles the call with `EGROUPALIVE`, which means the group may still be running and F must treat the outcome as unknown; and an output pipe still open 1 second after the group is gone (held by a process that left the group) is closed, so settlement never waits on it, and the call settles with `ESTDIOHELD`: its output may be incomplete, so it is never reported as a success. A spawn that fails outright (for example `ENOMEM`) settles as not started, and the helper never rejects. An aborted clone removes its partial directory; an aborted storage allocation removes what it created, with cleanup that is not cancelled. The synchronous helpers remain for existing callers and share the same steps. What follows is the original requirement. Both preparation helpers were synchronous. `createTaskClone` runs git with `execFileSync`, and `prepareTaskFilesystems` runs `docker` with `execFileSync` and `spawnSync` (`agents/container/storage.ts`). Either would block the event loop, and cancel or shutdown could not abort it. D must add asynchronous variants of both that take an `AbortSignal`, spawn their subprocesses in their own process group, expose the group ID, and settle only after those processes have exited. Storage allocation must also label its volumes with the runner owner and attempt ID before it returns (see "Recovered storage handles"), so that storage allocated just before a crash is found by recovery. +**Third prerequisite before F1 merges: abortable preparation helpers.** Done in #51 item 5: `createTaskCloneAsync` and `prepareTaskFilesystemsAsync` take `signal` and `onProcessGroup`. Each subprocess runs as the leader of a new process group (`agents/process-group.ts`); `onProcessGroup` receives `{ pgid, startedAt, identity }` synchronously after the spawn, so F can save it as `preparation_pgid`, `preparation_started_at`, and `preparation_identity` before anything else runs. Every Git or Docker call is a separate group, so F updates the saved group on each call. On abort or at the deadline the group gets `SIGTERM`, then `SIGKILL` after 5 seconds; the call settles only after the leader and every other member of its group have exited. Two bounds apply: a member still alive 10 seconds after the leader exits (repeated `SIGKILL` included) settles the call with `EGROUPALIVE`, which means the group may still be running and F must treat the outcome as unknown; and an output pipe still open 1 second after the group is gone (held by a process that left the group) is closed, so settlement never waits on it, and the call settles with `ESTDIOHELD`: its output may be incomplete, so it is never reported as a success. A spawn that fails outright (for example `ENOMEM`) settles as not started, and the helper never rejects. An aborted clone removes its partial directory; an aborted storage allocation removes what it created, with cleanup that is not cancelled. The synchronous helpers remain for existing callers and share the same steps. What follows is the original requirement. Both preparation helpers were synchronous. `createTaskClone` runs git with `execFileSync`, and `prepareTaskFilesystems` runs `docker` with `execFileSync` and `spawnSync` (`agents/container/storage.ts`). Either would block the event loop, and cancel or shutdown could not abort it. D must add asynchronous variants of both that take an `AbortSignal`, spawn their subprocesses in their own process group, expose the group ID, and settle only after those processes have exited. Storage allocation must also label its volumes with the runner owner and attempt ID before it returns (see "Recovered storage handles"), so that storage allocated just before a crash is found by recovery. **Prerequisite before F2's writable attempts: a bounded task-volume export.** D must add a bounded diff export over the task volumes (see "Partial output and interrupted rebases"). F1 does not need it: F1 runs only read-only attempts, and their task storage is removed without an export. @@ -303,7 +303,7 @@ This runs before the coordinator opens. 1. Take the single-runner lock for this database (decision 1), **before opening the `Store`**, because opening runs migrations. If another process holds it, exit with "Another codeboost runner is using this database." The OS lock cannot name the holder, so the message does not claim to. Do not serve the review screen: it is not read-only, because `ReviewService.load()` records history when HEAD moves (`runner/review.ts`) and `act()` writes review actions. A true read-only mode would need `Store`-level write refusal and is out of scope for F1. 2. After the lock, in this exact order: - **2a. Open the `Store`.** Migrations run here, under the lock. Read the runner owner token, or create it (see the table under "Shutdown"). - - **2b. Stop leftover preparation.** For every attempt row with a saved `preparation_pgid`: if a process group with that ID and start time is still alive, send `SIGTERM`, then `SIGKILL` after the grace period, and wait until it has exited. A crashed clone or storage allocator could otherwise still be creating or writing resources while D's recovery scans them, or while F exports and removes them. + - **2b. Stop leftover preparation.** For every attempt row with a saved `preparation_pgid`: if that numeric group is still alive, prove its saved kernel identity still matches immediately before `SIGTERM` and again before `SIGKILL`, then wait only through a bounded post-kill settlement period. An absent or changed identity and a group that does not settle both keep durable ownership and stop startup. A crashed clone or storage allocator could otherwise still be creating or writing resources while D's recovery scans them, or while F exports and removes them. - **2c. Call D's startup recovery** with the token, and await it. - **2d. If D's recovery rejects, startup stops here**, before step 3 finalizes any row or step 6 hands anything to I3: close the `Store`, release the lock, and exit with D's diagnostic. The next start retries. 3. **Export phase, outside any transaction.** Every attempt kind gets task storage from D, including read-only ones, so recovery can find handles for any kind. **Export only for writable attempts** (`execute`, `fix`, `rebase-fix`) whose row is not `completed`. The partial-output rule applies only to stopped writable attempts. For each matching handle, run the bounded partial-output export. Write it to the diagnostics directory under the fixed name `.diff`, so a retry overwrites instead of duplicating. Each export has the same 60-second deadline as above; a timeout counts as an export failure. An export failure is kept in memory as that attempt's diagnostic ("Partial output could not be exported: …"), with no file. This phase makes no database writes. Before F2 there are no writable attempts, so this phase exports nothing, but read-only attempts can still leave handles to remove in step 4. Then finalize the unclean leftovers. These are `pending` or `running` rows left by a crash, or by a shutdown whose terminal write failed. **Finalization phase: one transaction** that only finalizes rows and stores each `diagnostic_ref` (or the export-failure diagnostic). It does no D or file work. If it fails, for example because of a storage error, startup stops there: it does not run step 4, does not open admission, closes the `Store`, releases the lock, and exits with the error. The next start runs recovery again from step 1, and its export phase rewrites the same fixed-name files. A diagnostics file whose attempt row never got a `diagnostic_ref` is removed by the diagnostics directory's normal retention. Finalize each one, storing its `diagnostic_ref` from the export in the same transaction, using the first-reason table in "Rules for the running state", rule 2: @@ -438,7 +438,7 @@ A merged v5 task also gets its `task-closed` event, keyed by the merge attempt I | Table | Key columns | |---|---| | `tasks` | `plan_key` (primary key, references `plans(key)`), `status`, `requeue_pending` (boolean), `rebase_in_progress` (JSON or null; see "Partial output and interrupted rebases"), `budget_deadline` (absolute time the task budget ends; null until the budget starts: the admission transaction of the task's first attempt moves the task to `running` and sets it to that time plus the configured budget, so the budget covers preparation), `cancel_requested` (action ID or null), `state_version`, `context_generation`, `assignment_id`, `referenced_code_hash`, `current_attempt_id` (nullable; the composite foreign key `(plan_key, current_attempt_id)` references `attempts(plan_key, id)`, so a task can only point at its own attempt), `created_at`, `updated_at` | -| `attempts` | `id` (**primary key**; never reused), `deadline` (the attempt's absolute deadline, saved at admission), `plan_key` (references `tasks(plan_key)`; `(plan_key, id)` is also unique, for the composite key), `kind`, `phase`, `item`, `state`, `context` (JSON), `first_reason`, `stop_reason`, `exit_code`, `signal`, `result` (JSON, only for `completed`, 1 MiB or less), `diagnostic` (bounded), `diagnostic_ref` (partial-output file, or null), `preparation_pgid` and `preparation_started_at` (or null), `allocation_id` (task storage, or null), `created_at`, `started_at`, `settled_at` | +| `attempts` | `id` (**primary key**; never reused), `deadline` (the attempt's absolute deadline, saved at admission), `plan_key` (references `tasks(plan_key)`; `(plan_key, id)` is also unique, for the composite key), `kind`, `phase`, `item`, `state`, `context` (JSON), `first_reason`, `stop_reason`, `exit_code`, `signal`, `result` (JSON, only for `completed`, 1 MiB or less), `diagnostic` (bounded), `diagnostic_ref` (partial-output file, or null), `preparation_pgid`, `preparation_started_at`, and `preparation_identity` (or null), `allocation_id` (task storage, or null), `created_at`, `started_at`, `settled_at` | | `user_actions` | `plan_key`, `action_id` (together the primary key), `kind`, `request_hash`, `response` (JSON, bounded), `created_at` | | `feedback_events` | the fields listed above, with `id` as primary key, with a unique index on `(plan_key, kind, action_id)` | diff --git a/runner/recovery.ts b/runner/recovery.ts index 0c9c037e..b3a3e481 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -9,7 +9,7 @@ import type { RebaseMarker, Store } from './store.ts'; import { WRITABLE_KINDS, isUuidV4 } from './lifecycle.ts'; import { partialOutput, saveDiagnostic } from './diagnostics.ts'; import { ownerOnlyDirectory } from './runner-repository.ts'; -import { runInProcessGroup } from '../agents/process-group.ts'; +import { processIdentity, runInProcessGroup } from '../agents/process-group.ts'; import type { DockerOutcome } from '../agents/docker.ts'; /** @@ -98,55 +98,41 @@ export function acquireRunnerLock(databasePath: string, options: { lockRoot?: st } export interface ProcessControl { - /** The group leader is alive and started at the recorded time (guards against PID reuse). */ - isAlive(pgid: number, startedAt: number): boolean; - /** Revalidate its recorded start, SIGTERM, SIGKILL after grace, and resolve only once the group has exited. */ - terminate(pgid: number, startedAt: number, graceMs: number): Promise; + /** The numeric group still exists. Exact ownership is revalidated only at the signal boundary. */ + isAlive(pgid: number): boolean; + /** Revalidate its kernel identity, SIGTERM, SIGKILL after grace, and resolve only once the group has exited. */ + terminate(pgid: number, identity: string | null, graceMs: number): Promise; } const groupAlive = (pgid: number) => { try { process.kill(-pgid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code !== 'ESRCH'; } }; -const ownsGroupAtSignal = (pgid: number, startedAt: number): boolean => { +const ownsGroupAtSignal = (pgid: number, identity: string | null): boolean => { if (!groupAlive(pgid)) return false; - try { - const output = execFileSync('ps', ['-o', 'lstart=', '-p', String(pgid)], - { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C' } }); - const started = Date.parse(output.trim()); - if (!Number.isFinite(started) || Math.abs(started - startedAt) >= 2_000) - throw new Error('The recorded process group identity changed before recovery could signal it.'); - } catch (error) { - throw new Error('Could not prove the recorded process group still owns its ID; refusing to signal it.', - { cause: error instanceof Error ? error : undefined }); - } + const current = processIdentity(pgid); + if (identity === null || current === null) + throw new Error('Could not prove the recorded process group still owns its ID; refusing to signal it.'); + if (current !== identity) throw new Error('The recorded process group identity changed before recovery could signal it.'); return true; }; export const hostProcesses: ProcessControl = { - isAlive(pgid, startedAt) { - if (!groupAlive(pgid)) return false; - try { return startTimeMatches(execFileSync('ps', ['-o', 'lstart=', '-p', String(pgid)], { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C' } }), startedAt); } - catch { return true; } // Alive but unreadable: treat as ours and stop it (fail closed). - }, - async terminate(pgid, startedAt, graceMs) { - if (!ownsGroupAtSignal(pgid, startedAt)) return; + isAlive(pgid) { return groupAlive(pgid); }, + async terminate(pgid, identity, graceMs) { + if (!ownsGroupAtSignal(pgid, identity)) return; try { process.kill(-pgid, 'SIGTERM'); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ESRCH') return; throw error; } const until = Date.now() + graceMs; while (groupAlive(pgid) && Date.now() < until) await new Promise(resolve => setTimeout(resolve, 50)); - if (ownsGroupAtSignal(pgid, startedAt)) { + if (ownsGroupAtSignal(pgid, identity)) { try { process.kill(-pgid, 'SIGKILL'); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; } } - while (groupAlive(pgid)) await new Promise(resolve => setTimeout(resolve, 50)); + const settlement = Date.now() + graceMs; + while (groupAlive(pgid) && Date.now() < settlement) await new Promise(resolve => setTimeout(resolve, 50)); + if (groupAlive(pgid)) throw new Error('The recorded process group did not exit after SIGKILL; retaining its ownership.'); }, }; -/** Whether `ps -o lstart=` output names the recorded start time. An unreadable time counts as a match (fail closed). */ -export function startTimeMatches(lstart: string, startedAt: number): boolean { - const started = Date.parse(lstart.trim()); - return !Number.isFinite(started) || Math.abs(started - startedAt) < 2_000; -} - export interface RecoveredStorage { readonly attemptId: string; readonly allocationId: string; readonly handle: unknown } export interface RecoveryDeps { /** D (#51): stop leftover agent containers, proxies and networks for this owner; keep task storage and return authenticated handles. */ @@ -212,8 +198,8 @@ export async function recoverStartup(o: RecoveryOptions): Promise a.preparationStartedAt !== null && a.preparationPgid === null).map(a => a.id); // 2b. Stop leftover preparation before D's recovery or any storage work. - for (const a of interrupted) if (a.preparationPgid !== null && a.preparationStartedAt !== null && processes.isAlive(a.preparationPgid, a.preparationStartedAt)) - await processes.terminate(a.preparationPgid, a.preparationStartedAt, o.graceMs ?? 5_000); + for (const a of interrupted) if (a.preparationPgid !== null && a.preparationStartedAt !== null && processes.isAlive(a.preparationPgid)) + await processes.terminate(a.preparationPgid, a.preparationIdentity, o.graceMs ?? 5_000); // 2c/2d. D's recovery; a rejection propagates and stops startup. const recovered = await o.deps.recoverLeftovers(o.runnerOwner); if (recovered.unowned.length) throw new RecoveryBlocked('Docker holds codeboost objects this runner will not remove itself: objects without a runner label may belong to an older build that is still running, and objects of this runner it cannot identify are not ones it made. Stop every older codeboost process, check and run these commands, then start again', recovered.unowned); @@ -252,7 +238,8 @@ export async function recoverStartup(o: RecoveryOptions): Promise | null; - const processGroup = marker?.processGroup, oldHistory = marker?.oldHistory ?? null, + const processGroup = marker?.processGroup, processGroupIdentity = processGroup && typeof processGroup === 'object' + ? processGroup.identity ?? null : null, oldHistory = marker?.oldHistory ?? null, resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null, resultState = marker?.resultState ?? 'none'; if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || @@ -266,11 +253,12 @@ export async function recoverStartup(o: RecoveryOptions): Promise { const version = this.#get('PRAGMA user_version')!.user_version as number; - if (![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15].includes(version)) throw new Error('Unsupported store schema version.'); - if (version === 15) return; + if (![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(version)) throw new Error('Unsupported store schema version.'); + if (version === 16) return; if (version === 0) this.#db.exec(` CREATE TABLE plans (key TEXT PRIMARY KEY, issue INTEGER NOT NULL, revision INTEGER NOT NULL, snapshot_id TEXT); CREATE TABLE revisions (key TEXT NOT NULL REFERENCES plans(key), revision INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(key,revision)); @@ -264,6 +265,12 @@ export class Store { this.#db.exec('ALTER TABLE requests ADD COLUMN continuation TEXT'); this.#db.exec('PRAGMA user_version=15'); } + // A process group can be signalled after a crash only when its exact kernel identity still matches. + if (version < 16) { + if (!this.#db.prepare('PRAGMA table_info(attempts)').all().some(column => column.name === 'preparation_identity')) + this.#db.exec('ALTER TABLE attempts ADD COLUMN preparation_identity TEXT'); + this.#db.exec('PRAGMA user_version=16'); + } }); } catch (error) { this.#db.close(); throw error; } } @@ -754,7 +761,8 @@ export class Store { expected: RebaseMarker['processGroup'], group: RebaseMarker['processGroup']): void { assertUuidV4(attemptId, 'Rebase attempt ID'); for (const value of [expected, group]) if (value && value !== 'spawning' && value !== 'unsettled' && - (!Number.isSafeInteger(value.pgid) || value.pgid <= 1 || !Number.isSafeInteger(value.startedAt) || value.startedAt < 0)) + (!Number.isSafeInteger(value.pgid) || value.pgid <= 1 || !Number.isSafeInteger(value.startedAt) || value.startedAt < 0 || + (value.identity !== null && !/^linux:[0-9a-f-]{36}:\d+$/.test(value.identity)))) throw new Error('Invalid rebase process group.'); this.#transaction(() => { const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); @@ -2085,14 +2093,16 @@ export class Store { } /** * Saved in the same synchronous turn as the spawn. Preparation can run several subprocesses in turn: each one replaces - * the last, with its own start time when given, so startup recovery's start-time check (which guards against PID reuse) - * matches the group it finds. + * the last, with its own start time and kernel identity when given, so startup recovery can prove that the numeric ID + * still names the group it recorded. */ - recordPreparationGroup(identity: PlanIdentity, id: string, pgid: number, startedAt?: number): void { + recordPreparationGroup(identity: PlanIdentity, id: string, pgid: number, startedAt?: number, processIdentity: string | null = null): void { if (!Number.isSafeInteger(pgid) || pgid < 2) throw new GuardRefusal('Invalid process group.'); if (startedAt !== undefined && !Number.isSafeInteger(startedAt)) throw new GuardRefusal('Invalid process start time.'); - if (this.#run(`UPDATE attempts SET preparation_pgid=?, preparation_started_at=COALESCE(?, preparation_started_at) WHERE plan_key=? AND id=? AND preparation_started_at IS NOT NULL`, - pgid, startedAt ?? null, identityKey(identity), id).changes !== 1) + if (processIdentity !== null && !/^linux:[0-9a-f-]{36}:\d+$/.test(processIdentity)) + throw new GuardRefusal('Invalid process identity.'); + if (this.#run(`UPDATE attempts SET preparation_pgid=?, preparation_started_at=COALESCE(?, preparation_started_at), preparation_identity=? WHERE plan_key=? AND id=? AND preparation_started_at IS NOT NULL`, + pgid, startedAt ?? null, processIdentity, identityKey(identity), id).changes !== 1) throw new GuardRefusal('Preparation was not marked as starting.'); } /** F chooses the allocation ID and saves it before the asynchronous allocation starts. */ @@ -2117,7 +2127,8 @@ export class Store { interruptedAttempts(): InterruptedAttempt[] { return this.#db.prepare("SELECT * FROM attempts WHERE state IN ('pending','running') ORDER BY rowid").all().map(row => ({ ...this.#attemptRecord(row), planKey: row.plan_key as string, preparationPgid: row.preparation_pgid as number | null, - preparationStartedAt: row.preparation_started_at as number | null, allocationId: row.allocation_id as string | null, + preparationStartedAt: row.preparation_started_at as number | null, preparationIdentity: row.preparation_identity as string | null, + allocationId: row.allocation_id as string | null, metadataBaseline: row.metadata_baseline as string | null, storageBase: row.storage_base as string | null, })); } diff --git a/runner/workspace.ts b/runner/workspace.ts index 1b16f2f8..9f4e9e61 100644 --- a/runner/workspace.ts +++ b/runner/workspace.ts @@ -50,7 +50,10 @@ export function createTaskWorkspace(options: WorkspaceOptions): TaskWorkspace { // running. store.markPreparationStarting(identity, attempt.id, Date.now()); let recorded = false; - const record = (group: ProcessGroup) => { store.recordPreparationGroup(identity, attempt.id, group.pgid, group.startedAt); recorded = true; }; + const record = (group: ProcessGroup) => { + store.recordPreparationGroup(identity, attempt.id, group.pgid, group.startedAt, group.identity); + recorded = true; + }; try { await ensureCommit(repository, head, { signal, onProcessGroup: record }); ownerOnlyDirectory(options.runnerRoot, options.runnerOwner, 'attempts'); diff --git a/test/agent-process-group.test.ts b/test/agent-process-group.test.ts index 4fa7ea7b..729317f8 100644 --- a/test/agent-process-group.test.ts +++ b/test/agent-process-group.test.ts @@ -20,6 +20,8 @@ describe('runInProcessGroup', () => { // Synchronously, before anything else can run: the caller can record it before a crash could lose it. expect(reported?.pgid).toBeGreaterThan(1); expect(Math.abs(reported!.startedAt - Date.now())).toBeLessThan(5_000); + if (process.platform === 'linux') expect(reported!.identity).toMatch(/^linux:[0-9a-f-]{36}:\d+$/); + else expect(reported!.identity).toBeNull(); expect(await pending).toMatchObject({ status: 3, stdout: 'out\n', stderr: 'err\n' }); expect(groupAlive(reported!.pgid)).toBe(false); }); diff --git a/test/planning-drafts.test.ts b/test/planning-drafts.test.ts index 049222d7..9ef94ef6 100644 --- a/test/planning-drafts.test.ts +++ b/test/planning-drafts.test.ts @@ -170,7 +170,7 @@ it.each([9, 10])('migrates a v%i database to the current schema, reading its exi const migrated = open(); expect(migrated.getSuggestions(identity, id)).toMatchObject({ mode: 'suggest', state: 'pending' }); const db = new DatabaseSync(path); - try { expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); } finally { db.close(); } + try { expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); } finally { db.close(); } }); it('adds checkpoint bindings to v14 planning requests without losing their state', () => { const { store, path, open } = fixture(), id = store.beginSuggestions(identity, state(store), 'draft'); @@ -181,7 +181,7 @@ it('adds checkpoint bindings to v14 planning requests without losing their state expect(migrated.getDraft(identity, id).state).toBe('pending'); const db = new DatabaseSync(path); try { - expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); + expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); expect(db.prepare('PRAGMA table_info(requests)').all().some(column => column.name === 'continuation')).toBe(true); } finally { db.close(); } }); diff --git a/test/publish.test.ts b/test/publish.test.ts index f7646423..c93e1a35 100644 --- a/test/publish.test.ts +++ b/test/publish.test.ts @@ -337,7 +337,7 @@ describe('schema v7', () => { expect(upgraded.taskPullRequests(identity)).toEqual([]); expect(upgraded.latestAlreadyFixed(identity)).toBeNull(); const db = new DatabaseSync(path, { readOnly: true }); - expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); + expect(db.prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); const names = db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table','index') AND (name LIKE '%pull_requests%' OR name LIKE 'already_fixed_checks%') AND name NOT LIKE 'sqlite_autoindex%' ORDER BY name").all().map(row => row.name); expect(names).toEqual(['already_fixed_checks', 'already_fixed_checks_task', 'task_pull_requests', 'task_pull_requests_number', 'task_pull_requests_opening', 'task_pull_requests_task']); db.close(); diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index 2cd69495..b3a61a6a 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -105,7 +105,7 @@ describe('schema v12 review ordering', () => { migrated.saveReview(identity, { revision: 1, snapshotId: snapshot.id, reviewVersion: migratedVersion }, [{ item: 'P1', fingerprint: 'approved-after-upgrade' }], []); expect(migrated.unapprovedExecutionItems(identity, 1)).toEqual([]); - expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); }); }); @@ -546,17 +546,19 @@ describe('pre-merge rebase ownership', () => { expect(store.getTask(identity).rebaseInProgress).toEqual({ attemptId: marker.attemptId, oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)], startedAt: 123, resultState: 'none', resultHead: null, resultMappings: null, processGroup: null }); - const group = { pgid: 4242, startedAt: 456 }; + const group = { pgid: 4242, startedAt: 456, identity: 'linux:00000000-0000-0000-0000-000000000000:1' }; expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, - { pgid: 1, startedAt: 456 })).toThrow(/Invalid rebase process group/); + { pgid: 1, startedAt: 456, identity: null })).toThrow(/Invalid rebase process group/); + expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, + { pgid: 4242, startedAt: 456, identity: 'not-kernel-backed' })).toThrow(/Invalid rebase process group/); store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, 'spawning'); expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, group)).toThrow(/Another Git process/); store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, 'spawning', group); expect(store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: { pgid: 4242, startedAt: 456 } }); expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, - { pgid: 5252, startedAt: 567 })).toThrow(/Another Git process/); + { pgid: 5252, startedAt: 567, identity: 'linux:00000000-0000-0000-0000-000000000000:2' })).toThrow(/Another Git process/); expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, - { pgid: 3131, startedAt: 345 }, null)).toThrow(/Another Git process/); + { pgid: 3131, startedAt: 345, identity: null }, null)).toThrow(/Another Git process/); expect(store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: group }); expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, randomUUID(), null, null)).toThrow(/no longer owns/); store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, group, null); @@ -599,7 +601,7 @@ describe('pre-merge rebase ownership', () => { expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ { oldSha: oid(3), newSha: oid(6) }, { oldSha: oid(4), newSha: oid(8) }, ])).toThrow(/does not end/); - const group = { pgid: 4242, startedAt: 456 }; + const group = { pgid: 4242, startedAt: 456, identity: 'linux:00000000-0000-0000-0000-000000000000:1' }; store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, 'spawning'); store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, 'spawning', group); expect(() => store.finishRebase(identity, expected, taskVersion, marker.attemptId, oid(5), oid(7), [ @@ -728,12 +730,26 @@ describe('runner commits and preparation groups (#87)', () => { it('keeps each recorded preparation group paired with its own start time', () => { const { store } = queued(); const attempt = admit(store); store.markPreparationStarting(identity, attempt.id, 1_000); - store.recordPreparationGroup(identity, attempt.id, 4242, 41_000); - expect(store.interruptedAttempts()[0]).toMatchObject({ preparationPgid: 4242, preparationStartedAt: 41_000 }); + const kernelIdentity = 'linux:00000000-0000-0000-0000-000000000000:41'; + store.recordPreparationGroup(identity, attempt.id, 4242, 41_000, kernelIdentity); + expect(store.interruptedAttempts()[0]).toMatchObject({ preparationPgid: 4242, preparationStartedAt: 41_000, + preparationIdentity: kernelIdentity }); // Without a time, the marker's own stays. store.recordPreparationGroup(identity, attempt.id, 4343); - expect(store.interruptedAttempts()[0]).toMatchObject({ preparationPgid: 4343, preparationStartedAt: 41_000 }); + expect(store.interruptedAttempts()[0]).toMatchObject({ preparationPgid: 4343, preparationStartedAt: 41_000, + preparationIdentity: null }); expect(() => store.recordPreparationGroup(identity, attempt.id, 4444, 1.5)).toThrow('start time'); + expect(() => store.recordPreparationGroup(identity, attempt.id, 4444, 42_000, 'not-kernel-backed')).toThrow('process identity'); + }); + + it('adds preparation identity to a version 15 database', () => { + const { path, store } = queued(); const attempt = admit(store); + store.close(); stores.splice(stores.indexOf(store), 1); + const db = new DatabaseSync(path); + db.exec('ALTER TABLE attempts DROP COLUMN preparation_identity; PRAGMA user_version=15;'); db.close(); + const reopened = open(path); + expect(reopened.interruptedAttempts()).toEqual([expect.objectContaining({ id: attempt.id, preparationIdentity: null })]); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); }); }); @@ -867,7 +883,7 @@ describe('durable safety findings (#87 item 3)', () => { db.exec('ALTER TABLE attempts DROP COLUMN safety_finding; PRAGMA user_version=7;'); db.close(); const reopened = open(path); expect(reopened.getAttempt(identity, attempt.id).safetyFinding).toBeNull(); - expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); }); it('adds the durable owed marker to a version 12 database', () => { @@ -877,7 +893,7 @@ describe('durable safety findings (#87 item 3)', () => { db.exec('ALTER TABLE attempts DROP COLUMN safety_owed; PRAGMA user_version=12;'); db.close(); const reopened = open(path); expect(reopened.owedSafetyFindings()).toEqual([]); - expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); }); it('restores and clears an escalation owed behind a human gate across real store reopens', () => { @@ -917,7 +933,7 @@ describe('allocation baseline (#91)', () => { db.exec('ALTER TABLE attempts DROP COLUMN metadata_baseline; ALTER TABLE attempts DROP COLUMN storage_base; PRAGMA user_version=8;'); db.close(); const reopened = open(path); expect(reopened.interruptedAttempts()).toEqual([expect.objectContaining({ id: attempt.id, metadataBaseline: null, storageBase: null })]); - expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); }); }); describe('publish outcomes (#103)', () => { @@ -933,7 +949,7 @@ describe('publish outcomes (#103)', () => { reopened.recordPublish(identity, { outcome: 'opened', draft: false, message: 'Pull request #1 is open.', number: 1, url: 'https://github.com/o/r/pull/1' }); expect(reopened.lastPublish(identity)).toMatchObject({ outcome: 'opened', number: 1 }); expect(reopened.getTask(identity)).toEqual(before); - expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 15 }); + expect(new DatabaseSync(path).prepare('PRAGMA user_version').get()).toEqual({ user_version: 16 }); }); it('stamps an outcome with the version the publish saw, and refuses one it cannot have seen (#114)', () => { const { store } = queued(), version = store.getTask(identity).stateVersion; diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index 787aad4e..9f949296 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -8,7 +8,8 @@ import { fileURLToPath } from 'node:url'; import { DatabaseSync } from 'node:sqlite'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { Store } from '../runner/store.ts'; -import { LockHeld, RecoveryBlocked, acquireRunnerLock, hostOpenFiles, hostOpenFilesBounded, hostProcesses, recoverStartup, releasePreparation, startTimeMatches, type RecoveryDeps, type RunnerLock } from '../runner/recovery.ts'; +import { processIdentity } from '../agents/process-group.ts'; +import { LockHeld, RecoveryBlocked, acquireRunnerLock, hostOpenFiles, hostOpenFilesBounded, hostProcesses, recoverStartup, releasePreparation, type RecoveryDeps, type RunnerLock } from '../runner/recovery.ts'; import type { PlanIdentity } from '../core/identity.ts'; import type { Plan, PlanContext } from '../core/plan.ts'; @@ -213,15 +214,19 @@ describe('startup recovery sequence', () => { const { d: root, store, admit, allocate } = fixture(2); const writable = admit(id(1)); const w = allocate(id(1), writable.id, 'w'); store.markRunning(id(1), writable.id); const readOnly = admit(id(2), { kind: 'review' }); - store.markPreparationStarting(id(2), readOnly.id, Date.now()); store.recordPreparationGroup(id(2), readOnly.id, 4242); + const preparationIdentity = 'linux:00000000-0000-0000-0000-000000000000:42'; + store.markPreparationStarting(id(2), readOnly.id, Date.now()); + store.recordPreparationGroup(id(2), readOnly.id, 4242, Date.now(), preparationIdentity); const r = allocate(id(2), readOnly.id, 'r'); - const inputs: unknown[] = []; + const inputs: unknown[] = [], terminate = vi.fn(async (pgid: number) => { calls.push(`terminate:${pgid}`); }); const { d, calls } = deps({ recoverLeftovers: async () => { calls.push('recover'); return { storage: [w, r], unowned: [] }; }, exportTaskDiff: async (_h, input) => { inputs.push(input); calls.push(`export:${store.getAttempt(id(1), writable.id).state}`); return { diff: Buffer.from('partial'), truncated: false }; }, + processes: { isAlive: () => true, terminate }, }); const report = await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'runner'), diagnosticsDir: join(root, 'diag'), deps: d }); expect(calls).toEqual(['terminate:4242', 'recover', 'export:running', 'remove:w', 'remove:r']); + expect(terminate).toHaveBeenCalledWith(4242, preparationIdentity, 5_000); const finalized = store.getAttempt(id(1), writable.id); expect(finalized).toMatchObject({ state: 'failed', diagnosticRef: join(root, 'diag', `${writable.id}.diff`) }); expect(readFileSync(finalized.diagnosticRef!, 'utf8')).toBe('partial'); @@ -245,7 +250,8 @@ describe('startup recovery sequence', () => { const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head], startedAt: 123 }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); - store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', { pgid: 5151, startedAt: 456 }); + store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', + { pgid: 5151, startedAt: 456, identity: 'linux:00000000-0000-0000-0000-000000000000:1' }); const owned = store.getTask(id(1)).rebaseInProgress; const cleared = { ...(owned as object), processGroup: null }; const seen: unknown[] = []; @@ -310,7 +316,7 @@ describe('startup recovery sequence', () => { const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); - const group = { pgid: 5151, startedAt: 456 }; + const group = { pgid: 5151, startedAt: 456, identity: 'linux:00000000-0000-0000-0000-000000000000:1' }; store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', group); mkdirSync(join(root, 'r', token, 'rebases', marker.attemptId), { recursive: true }); const abortRebase = vi.fn(async () => undefined); @@ -327,14 +333,14 @@ describe('startup recovery sequence', () => { const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); - const group = { pgid: 5151, startedAt: 456 }; + const group = { pgid: 5151, startedAt: 456, identity: 'linux:00000000-0000-0000-0000-000000000000:1' }; store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, 'spawning', group); mkdirSync(join(root, 'r', token, 'rebases', marker.attemptId), { recursive: true }); const terminate = vi.fn(async () => undefined), abortRebase = vi.fn(async () => undefined); await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase, processes: { isAlive: () => true, terminate } }).d, openFiles: () => ['escaped-child'] })).rejects.toThrow(/still uses/); - expect(terminate).toHaveBeenCalledWith(group.pgid, group.startedAt, 5_000); + expect(terminate).toHaveBeenCalledWith(group.pgid, group.identity, 5_000); expect(abortRebase).not.toHaveBeenCalled(); expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: group }); }); @@ -366,7 +372,7 @@ describe('startup recovery sequence', () => { store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)), attemptId = randomUUID(), planKey = store.getTask(id(1)).planKey; const marker = { attemptId, oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), startedAt: 123, - processGroup: { pgid: 1, startedAt: 456 } }; + processGroup: { pgid: 1, startedAt: 456, identity: null } }; raw(`UPDATE tasks SET rebase_in_progress='${JSON.stringify(marker)}' WHERE plan_key='${planKey}'`); const terminate = vi.fn(async () => undefined), abortRebase = vi.fn(async () => undefined); await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), @@ -500,18 +506,34 @@ describe('startup recovery sequence', () => { describe('host process checks', () => { it('treats a permission-denied liveness probe as alive instead of releasing ownership', () => { vi.spyOn(process, 'kill').mockImplementation(() => { throw Object.assign(new Error('denied'), { code: 'EPERM' }); }); - expect(hostProcesses.isAlive(4242, 0)).toBe(true); + expect(hostProcesses.isAlive(4242)).toBe(true); }); - it('revalidates the recorded start identity before signalling a process group', async () => { + it('uses a kernel identity and never runs a PATH-selected ps with runner secrets', async () => { + const root = dir(), marker = join(root, 'ps-ran'), ps = join(root, 'ps'), path = process.env.PATH; + writeFileSync(ps, `#!/bin/sh\nprintf '%s' "$CODEBOOST_TEST_SECRET" > '${marker}'\n`, { mode: 0o755 }); + process.env.PATH = `${root}:${path ?? ''}`; process.env.CODEBOOST_TEST_SECRET = 'must-not-leak'; + try { + vi.spyOn(process, 'kill').mockImplementation(() => { throw Object.assign(new Error('denied'), { code: 'EPERM' }); }); + expect(hostProcesses.isAlive(4242)).toBe(true); + await expect(hostProcesses.terminate(4242, null, 10)).rejects.toThrow(/prove the recorded process group/); + expect(existsSync(marker)).toBe(false); + } finally { + if (path === undefined) delete process.env.PATH; else process.env.PATH = path; + delete process.env.CODEBOOST_TEST_SECRET; + } + }); + + it('revalidates the exact kernel identity before signalling a process group', async () => { const child = spawn('/bin/sh', ['-c', 'exec sleep 30'], { detached: true, stdio: 'ignore' }); children.push(child); - const startedAt = Date.now(); await once(child, 'spawn'); const pgid = child.pid!; try { - expect(hostProcesses.isAlive(pgid, startedAt)).toBe(true); - await expect(hostProcesses.terminate(pgid, startedAt + 60_000, 10)).rejects.toThrow(/identity|recorded process group/); + expect(hostProcesses.isAlive(pgid)).toBe(true); + const identity = processIdentity(pgid), wrong = identity === null ? 'linux:00000000-0000-0000-0000-000000000000:1' + : `${identity}0`; + await expect(hostProcesses.terminate(pgid, wrong, 10)).rejects.toThrow(/identity|recorded process group/); expect(() => process.kill(pgid, 0)).not.toThrow(); } finally { try { process.kill(-pgid, 'SIGKILL'); } catch {} @@ -520,12 +542,21 @@ describe('host process checks', () => { } }); - it('treats a start time it cannot read as a match, so a live group is still stopped', () => { - const at = Date.parse('Sat Sep 26 02:01:04 2026'); - expect(startTimeMatches('Sat Sep 26 02:01:04 2026\n', at)).toBe(true); - expect(startTimeMatches('Sat Sep 26 02:01:04 2026', at + 60_000)).toBe(false); - // procps under a Japanese locale; Date.parse cannot read it. - expect(startTimeMatches('土 9月 26 02:01:04 2026', at + 60_000)).toBe(true); + it.skipIf(process.platform !== 'linux')('bounds settlement after SIGKILL and retains ownership on failure', async () => { + const child = spawn('/bin/sh', ['-c', 'exec sleep 30'], { detached: true, stdio: 'ignore' }); + children.push(child); await once(child, 'spawn'); + const pgid = child.pid!, identity = processIdentity(pgid)!; + vi.useFakeTimers(); + const signal = vi.spyOn(process, 'kill').mockImplementation(() => true); + try { + const pending = hostProcesses.terminate(pgid, identity, 100); + await vi.advanceTimersByTimeAsync(500); + await expect(pending).rejects.toThrow(/did not exit after SIGKILL/); + } finally { + signal.mockRestore(); vi.useRealTimers(); + try { process.kill(-pgid, 'SIGKILL'); } catch {} + await once(child, 'exit'); children.splice(children.indexOf(child), 1); + } }); it('finds a process working in the attempt directory when the path goes through a symlink', async () => { const root = dir(), real = join(root, 'real'), attempt = join(real, 'attempt'); From 58ff11a418cb950adfa769292188910cfe8d19fe Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 12:41:00 -0700 Subject: [PATCH 12/13] Use monotonic recovery deadlines --- docs/implementation/pre-merge-rebase.md | 4 ++-- runner/recovery.ts | 8 ++++---- test/runner-recovery.test.ts | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md index 5c6643d3..fda2737e 100644 --- a/docs/implementation/pre-merge-rebase.md +++ b/docs/implementation/pre-merge-rebase.md @@ -12,7 +12,7 @@ Before the first Git process, `Store.beginRebase` records one marker in `tasks.r Admission checks the current plan revision, snapshot, review version, task state version, mergeable task status, cancellation state, active runner attempt, active merge, existing rebase, both ends of the reviewed snapshot, and the complete ordered old history captured by review. Recording the marker advances the task state version. While it exists, another rebase, the legacy rewrite path, merge admission, runner admission, reassignment, and ordinary status changes fail closed. Cancellation remains pending until the owned rebase has settled and exact-attempt cleanup clears the marker; that cleanup then closes the task. -`processGroup` is null only while no Git call owns work. Immediately before spawn it becomes `spawning`; the synchronous spawn callback replaces that with `{ pgid, startedAt }`. After that exact group is confirmed settled, a compare-and-swap clears it. A group that cannot be drained remains recorded, and a descendant that escapes the group leaves `unsettled`; restart recovery fails closed on either unknown state. A duplicate spawn or late settlement therefore cannot replace or erase another live process owner. +`processGroup` is null only while no Git call owns work. Immediately before spawn it becomes `spawning`; the synchronous spawn callback replaces that with `{ pgid, startedAt, identity }`. On Linux, `identity` combines the host boot ID with the leader's `/proc` start ticks. After that exact group is confirmed settled, a compare-and-swap clears it. A group that cannot be drained remains recorded, and a descendant that escapes the group leaves `unsettled`; restart recovery fails closed on either unknown state. A duplicate spawn or late settlement therefore cannot replace or erase another live process owner. `GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, descended from the recorded base, and byte-for-byte equal to the ordered history in the durable marker. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. @@ -22,7 +22,7 @@ Conflicts are a review outcome (`RebaseConflict`). Cancellation and failures set ## Restart recovery -Startup validates every stored marker before touching its resource. If its recorded process group is still alive with the same start time, recovery terminates and awaits it first. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner; Linux performs that potentially blocking `/proc` walk in an owned, killable subprocess whose settlement is inside the overall deadline. The production recovery adapter accepts only the configured plan key, then removes the matching worktree. It deletes a result ref only with Git's exact-old-value guard using the durably recorded `resultHead`; absence is already-clean success. A `prepared` marker preserves a different pre-existing value, while a `ready` marker fails closed if its ref changed. A legacy, pre-result, or reused-attempt marker without an intended value preserves any existing ref. Only after cleanup succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. +Startup validates every stored marker before touching its resource. If its recorded process group is still alive, recovery re-reads the kernel identity immediately before signalling and requires the boot ID and `/proc` start ticks to match; a legacy or current marker with a null identity fails closed instead of signalling a numeric ID. Signal grace and post-kill settlement use monotonic bounded deadlines. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner; Linux performs that potentially blocking `/proc` walk in an owned, killable subprocess whose settlement is inside the overall deadline. The production recovery adapter accepts only the configured plan key, then removes the matching worktree. It deletes a result ref only with Git's exact-old-value guard using the durably recorded `resultHead`; absence is already-clean success. A `prepared` marker preserves a different pre-existing value, while a `ready` marker fails closed if its ref changed. A legacy, pre-result, or reused-attempt marker without an intended value preserves any existing ref. Only after cleanup succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. This ordering is intentionally local-only. The later push integration must extend the durable lifecycle before its first external write and reconcile an ambiguous push outcome; it must not reuse this local completion point as proof that GitHub moved. diff --git a/runner/recovery.ts b/runner/recovery.ts index b3a3e481..a85c7993 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -121,14 +121,14 @@ export const hostProcesses: ProcessControl = { if (!ownsGroupAtSignal(pgid, identity)) return; try { process.kill(-pgid, 'SIGTERM'); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ESRCH') return; throw error; } - const until = Date.now() + graceMs; - while (groupAlive(pgid) && Date.now() < until) await new Promise(resolve => setTimeout(resolve, 50)); + const until = performance.now() + graceMs; + while (groupAlive(pgid) && performance.now() < until) await new Promise(resolve => setTimeout(resolve, 50)); if (ownsGroupAtSignal(pgid, identity)) { try { process.kill(-pgid, 'SIGKILL'); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; } } - const settlement = Date.now() + graceMs; - while (groupAlive(pgid) && Date.now() < settlement) await new Promise(resolve => setTimeout(resolve, 50)); + const settlement = performance.now() + graceMs; + while (groupAlive(pgid) && performance.now() < settlement) await new Promise(resolve => setTimeout(resolve, 50)); if (groupAlive(pgid)) throw new Error('The recorded process group did not exit after SIGKILL; retaining its ownership.'); }, }; diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index 9f949296..404b2f97 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -548,12 +548,13 @@ describe('host process checks', () => { const pgid = child.pid!, identity = processIdentity(pgid)!; vi.useFakeTimers(); const signal = vi.spyOn(process, 'kill').mockImplementation(() => true); + const wallClock = vi.spyOn(Date, 'now').mockImplementation(() => 1_000_000 - performance.now()); try { const pending = hostProcesses.terminate(pgid, identity, 100); await vi.advanceTimersByTimeAsync(500); await expect(pending).rejects.toThrow(/did not exit after SIGKILL/); } finally { - signal.mockRestore(); vi.useRealTimers(); + wallClock.mockRestore(); signal.mockRestore(); vi.useRealTimers(); try { process.kill(-pgid, 'SIGKILL'); } catch {} await once(child, 'exit'); children.splice(children.indexOf(child), 1); } From 9b286956d3ea2cc4785681d5616e32bed4ea7375 Mon Sep 17 00:00:00 2001 From: mchwang Date: Tue, 6 Oct 2026 12:51:19 -0700 Subject: [PATCH 13/13] Observe recovery rejection before timer advance --- test/runner-recovery.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index 404b2f97..6c654ab6 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -551,8 +551,9 @@ describe('host process checks', () => { const wallClock = vi.spyOn(Date, 'now').mockImplementation(() => 1_000_000 - performance.now()); try { const pending = hostProcesses.terminate(pgid, identity, 100); + const rejected = expect(pending).rejects.toThrow(/did not exit after SIGKILL/); await vi.advanceTimersByTimeAsync(500); - await expect(pending).rejects.toThrow(/did not exit after SIGKILL/); + await rejected; } finally { wallClock.mockRestore(); signal.mockRestore(); vi.useRealTimers(); try { process.kill(-pgid, 'SIGKILL'); } catch {}