diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts index fbac356b..37df0b29 100644 --- a/agents/adapters/claude.ts +++ b/agents/adapters/claude.ts @@ -4,7 +4,8 @@ import { readCapturedFile } from '../container/profile.ts'; import { createClaudeCommand, createPhasePolicy, MAX_COMMAND_SCHEMA_BYTES } from '../policy.ts'; import { launchInvocation } from './supervisor.ts'; import { setUpProfile } from './setup.ts'; -import { assertAdapterRequest, createAdapterInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; +import { assertAdapterRequest, capAdapterInvocationBudget, createAdapterInvocationBudget, + type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; /** * Read Claude's `--output-format json` envelope. A result whose `subtype` is not `success`, or whose `is_error` is @@ -46,7 +47,9 @@ export function startClaudeInvocation(request: AgentAdapterRequest, oauthToken: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!oauthToken || oauthToken.includes('\0')) throw new Error('Claude OAuth token is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = createAdapterInvocationBudget(request.invocation, options.timeoutMs); + const remaining = options.invocationBudget + ? capAdapterInvocationBudget(options.invocationBudget, options.timeoutMs) + : createAdapterInvocationBudget(request.invocation, options.timeoutMs); const structured = policy.phase === 'planning'; // Invalid input throws here, before anything is allocated; only Docker setup runs inside the handle. const command = createClaudeCommand(policy, request.prompt, @@ -55,5 +58,6 @@ export function startClaudeInvocation(request: AgentAdapterRequest, return launchInvocation(request.invocation, remaining, (signal, start) => setUpProfile(request, remaining, signal, network => ({ ...request, policy, network, command, claudeToken: oauthToken }), profile => start(profile, { ...options, secrets: { CLAUDE_CODE_OAUTH_TOKEN: oauthToken }, + processLifecycle: request.processLifecycle, invocationBudget: remaining, decode: (_profile, raw) => parseClaudeOutput(raw, structured) }))); } diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts index 5fe0e8bb..ecdd57ec 100644 --- a/agents/adapters/codex.ts +++ b/agents/adapters/codex.ts @@ -4,7 +4,8 @@ import { agentContainerId } from '../container/run.ts'; import { createCodexCommand, createPhasePolicy } from '../policy.ts'; import { launchInvocation, readBoundedContainerFile } from './supervisor.ts'; import { setUpProfile } from './setup.ts'; -import { assertAdapterRequest, createAdapterInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; +import { assertAdapterRequest, capAdapterInvocationBudget, createAdapterInvocationBudget, + type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; // Read the output from the container this invocation created, by ID; a same-named replacement must not answer. @@ -25,7 +26,9 @@ export function startCodexInvocation(request: AgentAdapterRequest, authFile: string, options: AgentAdapterOptions = {}): InvocationHandle { if (!authFile || authFile.includes('\0')) throw new Error('Codex auth path is malformed.'); const policy = createPhasePolicy(request.invocation); - const remaining = createAdapterInvocationBudget(request.invocation, options.timeoutMs); + const remaining = options.invocationBudget + ? capAdapterInvocationBudget(options.invocationBudget, options.timeoutMs) + : createAdapterInvocationBudget(request.invocation, options.timeoutMs); // Invalid input, including any phase Codex is refused (today every phase, #93), throws here, before anything is // allocated; only Docker setup runs inside the handle. const command = createCodexCommand(policy, request.prompt); diff --git a/agents/adapters/setup.ts b/agents/adapters/setup.ts index fc295031..cf91d50c 100644 --- a/agents/adapters/setup.ts +++ b/agents/adapters/setup.ts @@ -20,18 +20,19 @@ export async function setUpProfile(request: AgentAdapterRequest, remaining: () = profileOptions: (network: VendorNetwork) => Omit, start: (profile: ContainerProfile) => InvocationHandle): Promise { const network = await createVendorNetwork(request.invocation, request.imageId, request.networkAllocationId, - Math.min(60_000, remaining()), signal); + Math.min(60_000, remaining()), signal, request.processLifecycle, remaining); let profile: ContainerProfile; try { profile = await createContainerProfile({ ...profileOptions(network), timeoutMs: Math.min(60_000, remaining()), - signal }); + invocationBudget: remaining, signal }); } catch (error) { // Profile creation removes the network itself once it has claimed it; otherwise the network is still ours. if (error instanceof ProfileCreationCleanupError) throw error; - try { await removeVendorNetwork(network, CLEANUP_TIMEOUT_MS); } + try { await removeVendorNetwork(network, CLEANUP_TIMEOUT_MS, request.processLifecycle); } catch (cleanupError) { throw new AdapterSetupCleanupError(error, cleanupError, - (budgetMs = CLEANUP_TIMEOUT_MS) => removeVendorNetwork(network, budgetMs), () => vendorNetworkResources(network)); + (budgetMs = CLEANUP_TIMEOUT_MS) => removeVendorNetwork(network, budgetMs, request.processLifecycle), + () => vendorNetworkResources(network)); } throw error; } @@ -39,10 +40,11 @@ export async function setUpProfile(request: AgentAdapterRequest, remaining: () = // the launcher treats a failure without a handle as "nothing allocated", so release (or retain) them here. try { return start(profile); } catch (error) { - try { await disposeContainerProfile(profile, CLEANUP_TIMEOUT_MS); } + try { await disposeContainerProfile(profile, CLEANUP_TIMEOUT_MS, request.processLifecycle); } catch (cleanupError) { throw new AdapterSetupCleanupError(error, cleanupError, - (budgetMs = CLEANUP_TIMEOUT_MS) => disposeContainerProfile(profile, budgetMs), () => containerProfileResources(profile)); + (budgetMs = CLEANUP_TIMEOUT_MS) => disposeContainerProfile(profile, budgetMs, request.processLifecycle), + () => containerProfileResources(profile)); } throw error; } diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts index e5c3d72e..96560d0f 100644 --- a/agents/adapters/supervisor.ts +++ b/agents/adapters/supervisor.ts @@ -6,6 +6,8 @@ import { agentContainerId, agentContainerResources, createValidatedContainer, di isContainerProfileRetired, retireContainerProfile, validateContainer } from '../container/run.ts'; import { assertContainerProfileAuthenticity, containerProfileResources, disposeContainerProfile, isContainerProfileAuthentic, type ContainerProfile } from '../container/profile.ts'; +import { drainProcessGroup, processIdentity, type ProcessGroup } from '../process-group.ts'; +import type { ProcessGroupLifecycle, ProcessGroupOwner } from '../tracked-docker.ts'; export const OUTPUT_LIMITS = Object.freeze({ stdoutBytes: 16 * 1024 * 1024, @@ -16,6 +18,7 @@ const DEFAULT_TIMEOUT_MS = 10 * 60_000; const CAPTURE_ABORT_GRACE_MS = 1_000; const ACKNOWLEDGEMENT_RETRY_MS = 250; const DIAGNOSTIC_BYTES = 1024; +const ATTACH_PIPE_CLOSE_MS = 1_000; const active = new Map(); const activeProfiles = new WeakSet(); const cleanupRecoveries = new Set(); @@ -42,10 +45,67 @@ export interface SupervisorOptions { readonly invocationBudget?: () => number; readonly decode?: (profile: ContainerProfile, rawStdout: Buffer, maximumBytes: number, timeoutMs: number, signal: AbortSignal) => DecodedOutput | Promise; + /** Durable ownership for Docker clients that can create or start resources. */ + readonly processLifecycle?: ProcessGroupLifecycle; } export class OutputLimitError extends Error {} export class CaptureDeadlineError extends Error {} +/** Once the attach leader exits its numeric process-group ID may be reused, even while old pipes remain open. */ +export function createAttachExitGuard() { + let exited = false; + return Object.freeze({ markExited: () => { exited = true; }, canSignal: () => !exited }); +} + +/** Signal an attached client only while this process still owns its live leader. */ +export function signalAttachedChild(guard: ReturnType, child: ChildProcess, + signal: NodeJS.Signals): void { + if (!guard.canSignal() || child.pid === undefined) return; + try { process.kill(-child.pid, signal); } catch { child.kill(signal); } +} + +/** Start the streaming attach client without losing a synchronous cancellation raised by lifecycle hooks. */ +export function startOwnedAttachClient(options: { + readonly lifecycle: ProcessGroupLifecycle; + readonly stopped: () => boolean; + readonly spawn: () => ChildProcess; + readonly attach: (child: ChildProcess, owner: ProcessGroupOwner, group?: ProcessGroup) => void; + readonly terminate: () => void; + readonly recordFailure: (error: unknown) => void; +}): void { + let owner: ProcessGroupOwner = 'spawning'; + options.lifecycle.starting(); + if (options.stopped()) { + options.lifecycle.settled('spawning'); + throw new Error('Invocation stopped before the Docker attach client started.'); + } + let child: ChildProcess; + try { child = options.spawn(); } + catch (error) { options.lifecycle.settled('spawning'); throw error; } + let spawnedGroup: ProcessGroup | undefined; + if (child.pid !== undefined) { + const group = Object.freeze({ pgid: child.pid, startedAt: Date.now(), identity: processIdentity(child.pid) }); + spawnedGroup = group; + try { options.lifecycle.started(group); owner = group; } + catch (error) { + options.recordFailure(error); + try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); } + } + } + options.attach(child, owner, spawnedGroup); + if (options.stopped()) options.terminate(); +} + +/** Observe both normal exit and an error that Node does not promise will be followed by exit. */ +export function watchAttachedChild(child: ChildProcess, + finish: (code: number | null, signal: NodeJS.Signals | null) => void, + failed: (error: Error) => void): void { + child.once('exit', (code, signal) => { finish(code, signal); }); + // Finish first: it disables later PGID signalling and hands the recorded group to the drain path. The failure + // callback can synchronously cancel the invocation, and must not signal a numeric ID that drain may just release. + child.once('error', error => { finish(null, null); failed(error); }); +} + const dockerEnvironment = () => ({ PATH: process.env.PATH, DOCKER_HOST: process.env.DOCKER_HOST }); const positiveInteger = (value: number, name: string) => { if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be a positive integer.`); @@ -194,11 +254,13 @@ const profileResources = (profile: ContainerProfile) => * before this profile created any container. */ const rejectProfile = (profile: ContainerProfile, error: unknown, register = true, - cleanup: (profile: ContainerProfile, budgetMs: number) => Promise = disposeValidatedContainer, - reason?: StopReason): InvocationHandle => { + cleanup: (profile: ContainerProfile, budgetMs: number, processLifecycle?: ProcessGroupLifecycle) => Promise + = disposeValidatedContainer, + reason?: StopReason, processLifecycle?: ProcessGroupLifecycle): InvocationHandle => { const invocation = assertPhasePolicy(profile.policy); activeProfiles.add(profile); - return retainCleanup(invocation, budget => isContainerProfileAuthentic(profile) ? cleanup(profile, budget) : undefined, + return retainCleanup(invocation, budget => isContainerProfileAuthentic(profile) + ? cleanup(profile, budget, processLifecycle) : undefined, `Invocation was rejected: ${String(error)}`, () => profileResources(profile), register, unreleased => { activeProfiles.delete(profile); if (unreleased) retireContainerProfile(profile); @@ -401,7 +463,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super // This profile never created a container; the name belongs to the active invocation, so only // release this profile's own staging and network. return rejectProfile(profile, new Error('An invocation with this attempt ID is still active.'), false, - disposeContainerProfile); + disposeContainerProfile, undefined, options.processLifecycle); } // Rejections before container creation own no container, so they release (and retry) only the profile's own // staging and network; a name held by another invocation or a failing inspect cannot block that. @@ -416,14 +478,16 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super positiveInteger(carriedBudget, 'invocationBudget'); if (carriedBudget > DEFAULT_TIMEOUT_MS) throw new Error('invocationBudget cannot exceed the production ten-minute ceiling.'); + if (options.processLifecycle && profile.deferredOutput) + throw new Error('Durably tracked invocations cannot use concurrent deferred-output controls.'); } catch (error) { return rejectProfile(profile, error, true, disposeContainerProfile, - isDeadlineError(error) ? 'timeout' : undefined); + isDeadlineError(error) ? 'timeout' : undefined, options.processLifecycle); } - const wallRemaining = invocation.deadline - Date.now(); + const wallRemaining = options.invocationBudget ? carriedBudget : invocation.deadline - Date.now(); if (!Number.isSafeInteger(wallRemaining) || wallRemaining < 1) { return rejectProfile(profile, new Error('Invocation deadline has already expired.'), true, disposeContainerProfile, - 'timeout'); + 'timeout', options.processLifecycle); } const duration = Math.min(wallRemaining, configuredTimeout, carriedBudget), deadline = performance.now() + duration; const remaining = () => { @@ -438,6 +502,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super let closed = false, terminating = false, settlementComplete = false; let decodedOutput: DecodedOutput | undefined, decodePromise: Promise | undefined; let decodeAbort: AbortController | undefined; + const attachExit = createAttachExitGuard(); let protocolToken: string | undefined, protocolStarted = false, protocolReady = false; let readyStatus: number | undefined, acknowledgementFailures = 0; let protocolBuffer = Buffer.alloc(0); @@ -489,10 +554,17 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super }); }; const terminate = () => { - if (terminating || closed || !child) return; + if (terminating || closed || !child || !attachExit.canSignal()) return; terminating = true; const current = child; current.stdout?.resume(); current.stderr?.resume(); + if (options.processLifecycle) { + if (current.pid !== undefined) { + signalAttachedChild(attachExit, current, 'SIGTERM'); + later(() => { if (!closed) signalAttachedChild(attachExit, current, 'SIGKILL'); }, 1_000); + } + return; + } void runControl(['stop', '--signal=TERM', '--time=1', container]); later(() => { if (!closed) void runControl(['kill', '--signal=KILL', container]); }, 1_500); later(() => { @@ -683,7 +755,7 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super break; } try { - await disposeValidatedContainer(profile, budget); + await disposeValidatedContainer(profile, budget, options.processLifecycle); wakeCleanup = undefined; break; } catch (error) { @@ -729,15 +801,40 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super resolveSettled(result); }; - const attach = (started: ChildProcess) => { + const attach = (started: ChildProcess, processOwner?: ProcessGroupOwner, spawnedGroup?: ProcessGroup) => { child = started; + let finished = false, pipeClosed = false, resolvePipeClosed!: () => void; + const pipesClosed = new Promise(resolve => { resolvePipeClosed = resolve; }); started.stdout?.on('data', value => capture('stdout', value)); started.stderr?.on('data', captureStderr); started.stdout?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); started.stderr?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); - started.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); - started.once('close', async (code, signal) => { + const finish = async (code: number | null, signal: NodeJS.Signals | null) => { + if (finished) return; + finished = true; + // Disable every later deadline/cancellation signal before awaiting group drain or pipe closure. Once exit fires, + // the numeric PGID can be reused; drain owns the old group until it reports empty and must be the last signaller. + attachExit.markExited(); + let unsettled: 'group-alive' | 'stdio-held' | undefined; + if (spawnedGroup && options.processLifecycle + && !await drainProcessGroup(spawnedGroup)) unsettled = 'group-alive'; + let pipeTimer: ReturnType | undefined; + if (!pipeClosed) await Promise.race([pipesClosed, new Promise(resolve => { + pipeTimer = setTimeout(resolve, ATTACH_PIPE_CLOSE_MS); + })]); + clearTimeout(pipeTimer); + if (!pipeClosed) { + unsettled ??= 'stdio-held'; + started.stdout?.destroy(); started.stderr?.destroy(); + } closed = true; + if (processOwner && options.processLifecycle) try { + if (unsettled) options.processLifecycle.unsettled(processOwner, unsettled); + else options.processLifecycle.settled(processOwner); + } catch (error) { + stopReason ??= 'capture-failure'; failureDetail ??= + `Could not settle Docker client ownership: ${error instanceof Error ? error.message : String(error)}`; + } for (const timer of timers) clearTimeout(timer); timers.clear(); if (!stopReason && performance.now() >= deadline) stop('timeout'); @@ -764,15 +861,24 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super if (decodedOutput.providerFailed) exitCode = exitCode === 0 ? 1 : exitCode; } await settleWith(exitCode, signal, finalStdout, finalStderr); + }; + started.once('close', (code, signal) => { + pipeClosed = true; resolvePipeClosed(); + if (!processOwner) void finish(code, signal); }); + if (processOwner) watchAttachedChild(started, (code, signal) => { void finish(code, signal); }, error => { + failureDetail ??= error.message; stop('capture-failure'); + }); + else started.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); }; void (async () => { try { - await createValidatedContainer(profile, remaining(), options.secrets ?? {}, setupAbort.signal); + await createValidatedContainer(profile, remaining(), options.secrets ?? {}, setupAbort.signal, + options.processLifecycle); container = agentContainerId(profile) ?? ''; if (!container) throw new Error('Docker did not return the created agent container ID.'); - await validateContainer(container, profile, remaining(), setupAbort.signal); + await validateContainer(container, profile, remaining(), setupAbort.signal, options.processLifecycle); if (stopReason) throw new Error('Invocation was stopped during setup.'); } catch (error) { closed = true; @@ -784,9 +890,28 @@ export function startProfileInvocation(profile: ContainerProfile, options: Super await settleWith(null, null, Buffer.alloc(0), Buffer.alloc(0)); return; } - attach(spawn('docker', ['start', '--attach', container], { - env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], - })); + try { + const spawnAttach = () => spawn('docker', ['start', '--attach', container], { + env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], detached: options.processLifecycle !== undefined, + }); + if (options.processLifecycle) startOwnedAttachClient({ lifecycle: options.processLifecycle, + stopped: () => { + // Lifecycle hooks are synchronous and can keep the deadline timer from running. Recheck the monotonic + // deadline at the spawn boundary so an overdue invocation cannot create an attach client afterward. + if (!stopReason && performance.now() >= deadline) stop('timeout'); + return stopReason !== undefined; + }, spawn: spawnAttach, + attach: (started, owner, group) => attach(started, owner, group), terminate, + recordFailure: error => { + failureDetail ??= `Could not record Docker client ownership: ${error instanceof Error ? error.message : String(error)}`; + stopReason ??= 'capture-failure'; + } }); + else attach(spawnAttach()); + } catch (error) { + closed = true; stopReason ??= 'capture-failure'; + failureDetail ??= `Container start failed: ${error instanceof Error ? error.message : String(error)}`; + await settleWith(null, null, Buffer.alloc(0), Buffer.alloc(0)); + } })(); return handle; } diff --git a/agents/adapters/types.ts b/agents/adapters/types.ts index 62d204d3..e5127015 100644 --- a/agents/adapters/types.ts +++ b/agents/adapters/types.ts @@ -4,11 +4,16 @@ import type { TaskTreeCheck } from '../container/changes.ts'; import { assertBuiltAgentImage } from '../container/image.ts'; import { assertResourceOwner } from '../labels.ts'; import type { CaptureLimits } from './supervisor.ts'; +import type { ProcessGroupLifecycle } from '../tracked-docker.ts'; export interface AgentAdapterRequest { readonly invocation: InvocationInput; readonly filesystems: TaskFilesystems; readonly inputDirectory: string; + /** Optional owner-only root for profile snapshots whose lifecycle is durably owned by the caller. */ + readonly cleanupRoot?: string; + /** Optional durable ownership hooks for Docker clients that can create or start resources. */ + readonly processLifecycle?: ProcessGroupLifecycle; readonly imageId: string; readonly prompt: string; /** @@ -28,6 +33,8 @@ export function assertAdapterRequest(request: AgentAdapterRequest): void { export interface AgentAdapterOptions { readonly timeoutMs?: number; readonly limits?: Partial; + /** Trusted monotonic budget supplied by a caller that already converted its transported wall deadline. */ + readonly invocationBudget?: () => number; } const MAXIMUM_INVOCATION_MS = 10 * 60_000; @@ -56,3 +63,22 @@ export function createAdapterInvocationBudget(invocation: InvocationInput, throw new Error('timeoutMs cannot exceed the production ten-minute ceiling.'); return createInvocationBudget(invocation, timeoutMs); } + +/** Preserve the caller's monotonic deadline while applying this adapter's configured timeout as an additional cap. */ +export function capAdapterInvocationBudget(invocationBudget: () => number, + timeoutMs = MAXIMUM_INVOCATION_MS): () => number { + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) + throw new Error('timeoutMs must be a positive integer.'); + if (timeoutMs > MAXIMUM_INVOCATION_MS) + throw new Error('timeoutMs cannot exceed the production ten-minute ceiling.'); + const initialRemaining = invocationBudget(); + if (!Number.isSafeInteger(initialRemaining) || initialRemaining < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + const end = performance.now() + Math.min(initialRemaining, timeoutMs); + return () => { + const value = Math.min(invocationBudget(), Math.ceil(end - performance.now())); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + return value; + }; +} diff --git a/agents/container/changes.ts b/agents/container/changes.ts index df88f27e..b2640dbf 100644 --- a/agents/container/changes.ts +++ b/agents/container/changes.ts @@ -437,6 +437,83 @@ function treeDifferences(manifest: TaskChangeManifest): string[] { return found; } +function gitlinkMountDifferences(gitlinks: readonly string[]): string[] { + const found: string[] = [], q = JSON.stringify; + if (gitlinks.length > MAXIMUM_GITLINK_MOUNTS) + found.push(`the head has ${gitlinks.length} gitlinks, more than the ${MAXIMUM_GITLINK_MOUNTS} that can be mounted`); + const parents = gitlinkParents(gitlinks).length; + if (parents > MAXIMUM_GITLINK_MOUNTS) + found.push(`the head's gitlinks are below ${parents} directories, more than the ${MAXIMUM_GITLINK_MOUNTS} that can be pinned`); + const mountBytes = gitlinks.reduce((total, path) => total + Buffer.byteLength(path), 0) + + gitlinkParents(gitlinks).reduce((total, path) => total + 2 * Buffer.byteLength(path), 0); + if (mountBytes > MAXIMUM_GITLINK_MOUNT_BYTES) + found.push(`the gitlink mounts would name ${mountBytes} bytes of paths, more than the ${MAXIMUM_GITLINK_MOUNT_BYTES} a launch passes`); + for (const path of gitlinks) if (unmountable(path)) found.push(`gitlink ${q(path)} has a name Docker cannot mount`); + return found; +} + +export interface ConflictTreeOptions extends StorageScriptOptions { + readonly base: string; + /** The exact non-gitlink conflict paths whose dirty pre-agent state is allowed. */ + readonly paths: readonly string[]; + readonly metadataBaseline?: string; +} + +/** + * Pre-launch capability for a rebase-fix child. Unlike an ordinary item, the input is intentionally dirty, but every + * difference must be one of the exact conflict paths and no path may traverse a link or gitlink. + */ +export async function checkConflictTree(storage: TaskFilesystems, options: ConflictTreeOptions): Promise { + if (typeof options.base !== 'string' || !COMMIT_ID.test(options.base)) throw new Error('base must be a full commit ID.'); + if (!Array.isArray(options.paths) || options.paths.length === 0 || options.paths.length > MAXIMUM_DECLARED_LINKS) + throw new Error(`Conflict paths must contain between 1 and ${MAXIMUM_DECLARED_LINKS} entries.`); + const paths = [...new Set(options.paths)]; + if (paths.length !== options.paths.length) throw new Error('Conflict paths must not repeat.'); + for (const path of paths) assertDeclaredPath(path); + assertArguments(paths); + const baseline = resolveMetadataBaseline(storage, options.metadataBaseline); + let stdout: string; + try { + stdout = await runStorageScript(storage, { kind: 'inspect', operation: 'Conflict tree check', + consequence: 'the conflict tree cannot be checked', entrypoint: 'perl', maxOutputBytes: MAXIMUM_TREE_OUTPUT + OUTPUT_SLACK, + memory: INSPECTION_MEMORY, tmpBytes: INSPECTION_TMP, + args: ['-e', TREE_SCRIPT, 'prelaunch', baseline, options.base, ...paths] }, + { ...options, timeoutMs: options.timeoutMs ?? 120_000 }); + } catch (error) { + if (error instanceof DockerError && [6, 8, 9].includes(error.status ?? -1)) { + const reason = error.stderr.trim().split('\n').at(-1)!.slice(0, 512); + throw new TaskTreeRefused([`the conflict tree cannot be checked whole (${JSON.stringify(reason)})`]); + } + throw error; + } + const output = JSON.parse(stdout) as InspectOutput & { metadataOnly?: boolean; gitlinks?: unknown; declared?: unknown }; + const manifest = manifestOf(options.base, { baseline, links: [], recorded: {}, linkArgs: [] }, output); + const allowed = new Set(paths), differences: string[] = [], q = JSON.stringify; + if (manifest.metadataChanged) differences.push('the Git metadata differs from what the seeder recorded'); + if (manifest.agentCommits.length) differences.push('HEAD is not the recorded head'); + for (const change of manifest.changes) { + if (!allowed.has(change.path) || (change.oldPath !== undefined && !allowed.has(change.oldPath))) + differences.push(`${q(change.path)} differs outside the conflict set`); + if (change.underGit || [change.oldType, change.newType].some(type => type === 'gitlink' || type === 'directory' || type === 'other')) + differences.push(`${q(change.path)} has a type the conflict resolver cannot edit`); + } + for (const path of manifest.nestedGitlinkContent) differences.push(`gitlink ${q(path)} has content or cannot be read`); + if (manifest.linkTargetChanges.length) differences.push('a conflict path reaches through a symbolic link'); + const gitlinks = output.gitlinks, declared = output.declared as DeclaredFact[] | undefined; + if (!Array.isArray(gitlinks) || gitlinks.some(path => { try { assertDeclaredPath(path); return false; } catch { return true; } }) + || !Array.isArray(declared) || declared.length !== paths.length || declared.some((fact, index) => fact?.path !== paths[index])) + throw new Error('The conflict tree check returned an unexpected result.'); + for (const fact of declared) { + if (fact.blockedBy !== undefined) differences.push(`${q(fact.path)} lies beneath the ${fact.blockedByType} ${q(fact.blockedBy)}`); + if (!['absent', 'file', 'symlink'].includes(fact.type)) differences.push(`${q(fact.path)} is a ${fact.type}, not an editable conflict file`); + } + differences.push(...gitlinkMountDifferences(gitlinks as string[])); + if (differences.length) throw new TaskTreeRefused([...new Set(differences)]); + const check = Object.freeze({ base: options.base, gitlinks: Object.freeze([...gitlinks as string[]]) }); + treeChecks.set(check, { filesystems: storage, used: false }); + return check; +} + /** * Check task storage immediately before an execute or fix invocation (plan-format.md, "Clean invocation state" and * "Submodules in version 1"): read without following links, the work tree must be exactly the tree of `base` (no @@ -507,16 +584,7 @@ export async function checkTaskTree(storage: TaskFilesystems, options: TreeCheck if (type !== 'absent') differences.push(`${operation.kind} destination ${q(path)} is occupied by a ${type}`); } } - if (gitlinks.length > MAXIMUM_GITLINK_MOUNTS) - differences.push(`the head has ${gitlinks.length} gitlinks, more than the ${MAXIMUM_GITLINK_MOUNTS} that can be mounted`); - const parents = gitlinkParents(gitlinks as string[]).length; - if (parents > MAXIMUM_GITLINK_MOUNTS) - differences.push(`the head's gitlinks are below ${parents} directories, more than the ${MAXIMUM_GITLINK_MOUNTS} that can be pinned`); - const mountBytes = (gitlinks as string[]).reduce((total, path) => total + Buffer.byteLength(path), 0) - + gitlinkParents(gitlinks as string[]).reduce((total, path) => total + 2 * Buffer.byteLength(path), 0); - if (mountBytes > MAXIMUM_GITLINK_MOUNT_BYTES) - differences.push(`the gitlink mounts would name ${mountBytes} bytes of paths, more than the ${MAXIMUM_GITLINK_MOUNT_BYTES} a launch passes`); - for (const path of gitlinks as string[]) if (unmountable(path)) differences.push(`gitlink ${q(path)} has a name Docker cannot mount`); + differences.push(...gitlinkMountDifferences(gitlinks as string[])); if (differences.length) throw new TaskTreeRefused([...new Set(differences)]); const check = Object.freeze({ base: options.base, gitlinks: Object.freeze([...gitlinks as string[]]) }); treeChecks.set(check, { filesystems: storage, used: false }); diff --git a/agents/container/profile.ts b/agents/container/profile.ts index 177b7ef1..1beb1ba5 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -2,7 +2,7 @@ import { createHash, randomUUID } from 'node:crypto'; import { chmodSync, closeSync, constants, fstatSync, lstatSync, mkdtempSync, openSync, readSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { isAbsolute, join } from 'node:path'; import { assertCapturedInvocation, type InvocationInput, type Phase, type UnreleasedResource } from '../contract.ts'; import { assertBuiltAgentImage } from './image.ts'; import { assertTaskFilesystems, taskFilesystemOwner, type TaskFilesystems } from './storage.ts'; @@ -11,6 +11,7 @@ import { ownerLabelArgs, type ResourceOwner } from '../labels.ts'; import { assertVendorNetwork, removeVendorNetwork, vendorNetworkResources, type VendorNetwork } from '../network/network.ts'; import { assertAgentCommand, assertCommandSchema, assertPhasePolicy, type AgentCommand, type PhasePolicy } from '../policy.ts'; +import type { ProcessGroupLifecycle } from '../tracked-docker.ts'; export interface ContainerProfile { readonly name: string; readonly args: readonly string[]; @@ -48,8 +49,14 @@ export interface ProfileOptions { readonly treeCheck?: TaskTreeCheck; /** Remaining invocation budget for Docker-backed profile validation. */ readonly timeoutMs?: number; + /** Trusted monotonic budget carried from adapter admission. */ + readonly invocationBudget?: () => number; /** Cancels creation; whatever was staged is removed before the promise rejects. */ readonly signal?: AbortSignal; + /** Optional owner-only plain directory beneath which temporary profile snapshots are created. */ + readonly cleanupRoot?: string; + /** Durable ownership for Docker clients used during cleanup. */ + readonly processLifecycle?: ProcessGroupLifecycle; } export class ProfileCreationCleanupError extends AggregateError { @@ -82,7 +89,7 @@ interface FileIdentity { interface ProfileIdentity { readonly inputDirectory: string; readonly schema: FileIdentity; readonly auth?: FileIdentity; readonly cleanupDirectories: readonly string[]; readonly filesystems: TaskFilesystems; readonly clone: InvocationInput['clone']; readonly deadline: number; readonly network: VendorNetwork; - readonly policy: PhasePolicy; readonly invocation: InvocationInput } + readonly invocationBudget?: () => number; readonly policy: PhasePolicy; readonly invocation: InvocationInput } type InputIdentity = Pick; interface InputCapture extends InputIdentity { readonly content: Buffer } const identities = new WeakMap(); @@ -168,13 +175,13 @@ export function isContainerProfileAuthentic(profile: ContainerProfile): boolean /** Internal authenticity and host-file revalidation used at every launch boundary. */ export async function assertContainerProfile(profile: ContainerProfile, timeoutMs = 30_000, - signal?: AbortSignal): Promise { + signal?: AbortSignal, processLifecycle?: ProcessGroupLifecycle): Promise { const expected = identities.get(profile); if (!expected) throw new Error('Container profile was not created by the trusted profile builder.'); assertTaskFilesystems(expected.filesystems, expected.clone); // Every caller, including those using the default budget, is bounded by the invocation deadline. await assertVendorNetwork(expected.network, expected.invocation, profile.name, profileTimeout(profile, timeoutMs), - signal); + signal, processLifecycle); assertPhasePolicy(expected.policy, expected.invocation); const actual = captureInput(expected.inputDirectory); if (actual.inputDirectory !== expected.inputDirectory || !sameFile(actual.schema, expected.schema)) @@ -185,12 +192,12 @@ export async function assertContainerProfile(profile: ContainerProfile, timeoutM } } -/** Clamp a Docker budget to the captured invocation deadline, which no launch may outlive. */ +/** Clamp a Docker budget to the captured monotonic budget, or the wall deadline when no budget was carried. */ export function profileTimeout(profile: ContainerProfile, timeoutMs: number, now = Date.now()): number { const expected = identities.get(profile); if (!expected) throw new Error('Container profile was not created by the trusted profile builder.'); - const left = Math.floor(expected.deadline - now); - if (left < 1) throw new Error('Invocation deadline has passed.'); + const left = Math.floor(expected.invocationBudget?.() ?? expected.deadline - now); + if (!Number.isSafeInteger(left) || left < 1) throw new Error('Invocation deadline has passed.'); return Math.min(timeoutMs, left); } @@ -210,12 +217,13 @@ export function containerProfileResources(profile: ContainerProfile): readonly U } /** Remove runner-owned credential staging after this one-shot profile settles. */ -export async function disposeContainerProfile(profile: ContainerProfile, networkTimeoutMs = 30_000): Promise { +export async function disposeContainerProfile(profile: ContainerProfile, networkTimeoutMs = 30_000, + processLifecycle?: ProcessGroupLifecycle): Promise { const identity = identities.get(profile); if (!identity) return; const failures: unknown[] = []; try { removeOwnedDirectories(identity.cleanupDirectories); } catch (error) { failures.push(error); } - try { await removeVendorNetwork(identity.network, networkTimeoutMs); } catch (error) { failures.push(error); } + try { await removeVendorNetwork(identity.network, networkTimeoutMs, processLifecycle); } catch (error) { failures.push(error); } if (failures.length) throw new AggregateError(failures, 'Profile resource cleanup did not settle.'); identities.delete(profile); } @@ -242,15 +250,24 @@ export async function createContainerProfile(options: ProfileOptions): Promise { const failures: unknown[] = []; try { removeOwnedDirectories(cleanupDirectories); } catch (cleanupError) { failures.push(cleanupError); } - try { await removeVendorNetwork(options.network, budgetMs); } catch (cleanupError) { failures.push(cleanupError); } + try { await removeVendorNetwork(options.network, budgetMs, options.processLifecycle); } + catch (cleanupError) { failures.push(cleanupError); } if (failures.length) throw new AggregateError(failures, 'Profile resource cleanup did not settle.'); }; try { await cleanupProfileResources(); } diff --git a/agents/container/run.ts b/agents/container/run.ts index ab5301f9..9e0df5bf 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -1,7 +1,7 @@ import { realpathSync } from 'node:fs'; import { createOutcomeUnknown, DOCKER_ID } from '../client-outcome.ts'; import type { UnreleasedResource } from '../contract.ts'; -import { docker as runDockerCommand, pause, runDocker, type DockerOutcome } from '../docker.ts'; +import { docker as runDockerCommand, dockerEnvironment, pause, runDocker, type DockerOutcome } from '../docker.ts'; import { agentContainerOwner, assertContainerProfile, assertContainerProfileAuthenticity, disposeContainerProfile, GITLINK_MOUNT_BYTES, GITLINK_MOUNT_MODE, isContainerProfileAuthentic, profileTimeout, type ContainerProfile } from './profile.ts'; @@ -10,6 +10,7 @@ import { taskFilesystemOwner } from './storage.ts'; import { gitlinkParents } from './changes.ts'; import { hasOwnerLabels, ownerLabels } from '../labels.ts'; import { assertPhasePolicy } from '../policy.ts'; +import { runTrackedDocker, runTrackedProcess, type ProcessGroupLifecycle } from '../tracked-docker.ts'; export { prepareTaskFilesystems, removeTaskFilesystems, UnusableRepositoryError } from './storage.ts'; export type { TaskFilesystems, TaskStorageLimits } from './storage.ts'; @@ -21,8 +22,14 @@ const validateSecrets = (profile: ContainerProfile, secrets: Readonly>; - signal?: AbortSignal } = {}) => runDockerCommand(args, { timeoutMs: options.timeoutMs ?? 30_000, - secrets: options.secrets, signal: options.signal }); + signal?: AbortSignal; processLifecycle?: ProcessGroupLifecycle } = {}) => options.processLifecycle + ? runTrackedDocker(args, { timeoutMs: options.timeoutMs ?? 30_000, secrets: options.secrets, + signal: options.signal, lifecycle: options.processLifecycle }) + : runDockerCommand(args, { timeoutMs: options.timeoutMs ?? 30_000, + secrets: options.secrets, signal: options.signal }); +const dockerOutcome = (args: readonly string[], timeoutMs: number, processLifecycle?: ProcessGroupLifecycle) => + processLifecycle ? runTrackedProcess('docker', args, + { env: dockerEnvironment(), timeoutMs, lifecycle: processLifecycle }) : runDocker(args, { timeoutMs }); const validLimit = (value: number, name: string) => { if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be a positive integer.`); }; @@ -95,14 +102,15 @@ export function agentContainerResources(profile: ContainerProfile): readonly Unr labels: containerLabels.get(profile) ?? Object.freeze({ 'io.codeboost.invocation': profile.ownershipId }) })]); } // Cleanup is never cancelled: it runs to its own deadline so nothing is dropped. -const removeContainerOrThrow = async (profile: ContainerProfile, waitForSettle = false, timeoutMs = 30_000) => { +const removeContainerOrThrow = async (profile: ContainerProfile, waitForSettle = false, timeoutMs = 30_000, + processLifecycle?: ProcessGroupLifecycle) => { // Destructive cleanup acts only for the builder-registered profile; a copy's name and label are not a capability. if (!isContainerProfileAuthentic(profile)) throw new Error('Container profile was not created by the trusted profile builder.'); // A profile whose `docker create` never ran, or was refused by the daemon, owns no container: the name may belong // to another invocation, which must neither be inspected as ours nor block this profile's own cleanup. if (!createdContainers.has(profile)) { - await disposeContainerProfile(profile, timeoutMs); + await disposeContainerProfile(profile, timeoutMs, processLifecycle); return; } const settleUntil = unsettledCreates.get(profile) ?? 0; @@ -111,7 +119,7 @@ const removeContainerOrThrow = async (profile: ContainerProfile, waitForSettle = const capturedId = createdContainers.get(profile), target = capturedId ?? profile.name; let before: DockerOutcome; for (;;) { - before = await runDocker(['container', 'inspect', target], { timeoutMs: remaining() }); + before = await dockerOutcome(['container', 'inspect', target], remaining(), processLifecycle); if (before.status === 0) break; const missing = before.status !== null && !before.error && /No such (?:object|container)/i.test(`${before.stdout}\n${before.stderr}`); @@ -122,7 +130,7 @@ const removeContainerOrThrow = async (profile: ContainerProfile, waitForSettle = if (performance.now() >= settleUntil) { unsettledCreates.delete(profile); createdContainers.delete(profile); - await disposeContainerProfile(profile, remaining()); + await disposeContainerProfile(profile, remaining(), processLifecycle); return; } if (!waitForSettle) throw new Error('Agent container creation did not settle; staged credentials were retained.'); @@ -142,25 +150,26 @@ const removeContainerOrThrow = async (profile: ContainerProfile, waitForSettle = const id = inspected?.Id; if (!id || !DOCKER_ID.test(id) || (capturedId && id !== capturedId)) throw new Error('Failed to establish the agent container identity; staged credentials were retained.'); - const result = await runDocker(['rm', '--force', id], { timeoutMs: remaining() }); + const result = await dockerOutcome(['rm', '--force', id], remaining(), processLifecycle); if (result.status !== 0) { - const inspect = await runDocker(['container', 'inspect', id], { timeoutMs: remaining() }); + const inspect = await dockerOutcome(['container', 'inspect', id], remaining(), processLifecycle); const absent = inspect.status !== null && inspect.status !== 0 && !inspect.error && /No such (?:object|container)/i.test(`${inspect.stdout}\n${inspect.stderr}`); if (!absent) throw new Error('Failed to confirm removal of the agent container; staged credentials were retained.'); } unsettledCreates.delete(profile); createdContainers.delete(profile); - await disposeContainerProfile(profile, remaining()); + await disposeContainerProfile(profile, remaining(), processLifecycle); }; /** * Remove a validated invocation container, then its profile-owned staging and network resources. `timeoutMs` bounds * the whole removal: the container step and the network step share one deadline. */ -export async function disposeValidatedContainer(profile: ContainerProfile, timeoutMs = 30_000): Promise { +export async function disposeValidatedContainer(profile: ContainerProfile, timeoutMs = 30_000, + processLifecycle?: ProcessGroupLifecycle): Promise { assertContainerProfileAuthenticity(profile); - await removeContainerOrThrow(profile, false, timeoutMs); + await removeContainerOrThrow(profile, false, timeoutMs, processLifecycle); } type Inspect = { @@ -191,13 +200,15 @@ type Inspect = { /** Validate daemon-resolved configuration before starting an agent. */ export async function validateContainer(container: string, profile: ContainerProfile, timeoutMs = 30_000, - signal?: AbortSignal): Promise { + signal?: AbortSignal, processLifecycle?: ProcessGroupLifecycle): Promise { const remaining = createDeadline(timeoutMs); - await assertContainerProfile(profile, remaining(), signal); - const inspect = JSON.parse(await docker(['container', 'inspect', container], { timeoutMs: remaining(), signal }))[0] as + await assertContainerProfile(profile, remaining(), signal, processLifecycle); + const inspect = JSON.parse(await docker(['container', 'inspect', container], + { timeoutMs: remaining(), signal, processLifecycle }))[0] as Inspect | undefined; if (!inspect) throw new Error('Docker did not return the created container.'); - const image = JSON.parse(await docker(['image', 'inspect', profile.expectedImage], { timeoutMs: remaining(), signal }))[0] as + const image = JSON.parse(await docker(['image', 'inspect', profile.expectedImage], + { timeoutMs: remaining(), signal, processLifecycle }))[0] as { Id?: string; Config?: { User?: string; Env?: string[]; Entrypoint?: string[]; Labels?: Record } } | undefined; const imageId = image?.Id, labels = image?.Config?.Labels ?? {}; const host = inspect.HostConfig; @@ -306,7 +317,7 @@ export async function validateContainer(container: string, profile: ContainerPro throw new Error('Container task volumes do not match their captured identity.'); if (work.Source === metadata.Source) throw new Error('Worktree and Git metadata must use separate filesystems.'); const volumes = JSON.parse(await docker(['volume', 'inspect', work.Name!, metadata.Name!], - { timeoutMs: remaining(), signal })) as + { timeoutMs: remaining(), signal, processLifecycle })) as Array<{ Name: string; Driver: string; Labels: Record | null; Options: Record | null }>; // One owner for the whole check: the storage's runner, attempt and allocation label every volume and the keeper. const storageOwner = taskFilesystemOwner(profile.filesystems); @@ -324,7 +335,7 @@ export async function validateContainer(container: string, profile: ContainerPro throw new Error('Task volume does not match its bounded tmpfs allocation.'); } const keeper = JSON.parse(await docker(['container', 'inspect', profile.filesystems.keeper], - { timeoutMs: remaining(), signal }))[0] as + { timeoutMs: remaining(), signal, processLifecycle }))[0] as { State?: { Running?: boolean }; Config?: { Image?: string; User?: string; Labels?: Record }; HostConfig?: { ReadonlyRootfs?: boolean; Privileged?: boolean; NetworkMode?: string; CapDrop?: string[] | null; CapAdd?: string[] | null; SecurityOpt?: string[] | null; @@ -382,7 +393,7 @@ export async function validateContainer(container: string, profile: ContainerPro throw new Error('Credential profiles must not be combined or redirected.'); if (profile.vendor === 'claude' && (names.includes('CODEX_HOME') || !names.includes('CLAUDE_CODE_OAUTH_TOKEN'))) throw new Error('Credential profiles must not be combined.'); - await assertContainerProfile(profile, remaining(), signal); + await assertContainerProfile(profile, remaining(), signal, processLifecycle); remaining(); } @@ -391,7 +402,8 @@ export async function validateContainer(container: string, profile: ContainerPro * container (if the create may have landed) is removed before the promise rejects. */ export async function createValidatedContainer(profile: ContainerProfile, timeoutMs = 30_000, - secrets: Readonly> = {}, signal?: AbortSignal): Promise { + secrets: Readonly> = {}, signal?: AbortSignal, + processLifecycle?: ProcessGroupLifecycle): Promise { assertLaunchable(profile); const remaining = createDeadline(profileTimeout(profile, timeoutMs)); let createUnsettled = false; @@ -399,12 +411,14 @@ export async function createValidatedContainer(profile: ContainerProfile, timeou containerLabels.set(profile, Object.freeze({ 'io.codeboost.invocation': profile.ownershipId, ...ownerLabels(agentContainerOwner(assertPhasePolicy(profile.policy), profile.filesystems)) })); validateSecrets(profile, secrets); - await assertContainerProfile(profile, remaining(), signal); + await assertContainerProfile(profile, remaining(), signal, processLifecycle); signal?.throwIfAborted(); const createTimeout = remaining(); createUnsettled = true; try { - const id = await docker(profile.args, { timeoutMs: createTimeout, secrets, signal }); + const id = processLifecycle + ? await runTrackedDocker(profile.args, { timeoutMs: createTimeout, secrets, signal, lifecycle: processLifecycle }) + : await docker(profile.args, { timeoutMs: createTimeout, secrets, signal }); // An unexpected create output leaves the ID unknown, so cleanup falls back to a verified name lookup. createdContainers.set(profile, DOCKER_ID.test(id) ? id : undefined); } @@ -415,14 +429,14 @@ export async function createValidatedContainer(profile: ContainerProfile, timeou throw error; } createUnsettled = false; - await validateContainer(requireContainerId(profile), profile, remaining(), signal); - await assertContainerProfile(profile, remaining(), signal); + await validateContainer(requireContainerId(profile), profile, remaining(), signal, processLifecycle); + await assertContainerProfile(profile, remaining(), signal, processLifecycle); remaining(); signal?.throwIfAborted(); return profile.name; } catch (error) { if (createUnsettled) unsettledCreates.set(profile, performance.now() + CREATE_SETTLE_MS); - try { await removeContainerOrThrow(profile, createUnsettled); } + try { await removeContainerOrThrow(profile, createUnsettled, 30_000, processLifecycle); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Container creation failed and cleanup did not settle.'); } throw error; } @@ -433,7 +447,8 @@ export async function createValidatedContainer(profile: ContainerProfile, timeou * killed, and the container is still removed (cleanup is never cancelled) before the promise rejects. */ export async function startValidatedContainer(profile: ContainerProfile, timeoutMs = 60_000, - secrets: Readonly> = {}, signal?: AbortSignal): Promise { + secrets: Readonly> = {}, signal?: AbortSignal, + processLifecycle?: ProcessGroupLifecycle): Promise { assertLaunchable(profile); const remaining = createDeadline(profileTimeout(profile, timeoutMs)); let failure: unknown; @@ -441,14 +456,14 @@ export async function startValidatedContainer(profile: ContainerProfile, timeout validateSecrets(profile, secrets); // Validate and start the container this profile created, by ID: a same-named replacement must never run. const id = requireContainerId(profile); - await validateContainer(id, profile, remaining(), signal); - const output = await docker(['start', '--attach', id], { timeoutMs: remaining(), secrets, signal }); + await validateContainer(id, profile, remaining(), signal, processLifecycle); + const output = await docker(['start', '--attach', id], { timeoutMs: remaining(), secrets, signal, processLifecycle }); remaining(); return output; } catch (error) { failure = error; throw error; } finally { - try { await removeContainerOrThrow(profile); } + try { await removeContainerOrThrow(profile, false, 30_000, processLifecycle); } catch (cleanupError) { if (failure) throw new AggregateError([failure, cleanupError], 'Agent invocation failed and cleanup did not settle.'); throw cleanupError; @@ -458,15 +473,16 @@ export async function startValidatedContainer(profile: ContainerProfile, timeout /** Create, validate, start and remove the container. `signal` cancels at any step; cleanup still runs. */ export async function runContainer(profile: ContainerProfile, timeoutMs = 60_000, - secrets: Readonly> = {}, signal?: AbortSignal): Promise { + secrets: Readonly> = {}, signal?: AbortSignal, + processLifecycle?: ProcessGroupLifecycle): Promise { const remaining = createDeadline(profileTimeout(profile, timeoutMs)); - await createValidatedContainer(profile, remaining(), secrets, signal); + await createValidatedContainer(profile, remaining(), secrets, signal, processLifecycle); let startBudget: number; try { startBudget = remaining(); } catch (error) { - try { await removeContainerOrThrow(profile); } + try { await removeContainerOrThrow(profile, false, 30_000, processLifecycle); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Agent deadline and cleanup both failed.'); } throw error; } - return startValidatedContainer(profile, startBudget, secrets, signal); + return startValidatedContainer(profile, startBudget, secrets, signal, processLifecycle); } diff --git a/agents/container/storage.ts b/agents/container/storage.ts index f4e7aaea..194e755c 100644 --- a/agents/container/storage.ts +++ b/agents/container/storage.ts @@ -8,7 +8,8 @@ import { assertBuiltAgentImage } from './image.ts'; import { createOutcomeUnknown, DOCKER_ID } from '../client-outcome.ts'; import { DockerError, pause, runDocker, type DockerOutcome } from '../docker.ts'; import { MAXIMUM_TIMER_MS, runInProcessGroup, type ProcessGroup } from '../process-group.ts'; -import { TREE_SCRIPT } from './tree-script.ts'; +import { runTrackedProcess, type ProcessGroupLifecycle } from '../tracked-docker.ts'; +import { MAXIMUM_DECLARED_LINKS, MAXIMUM_NAME_BYTES, TREE_SCRIPT } from './tree-script.ts'; import { ALLOCATION_IN_USE, allocationListCommands, assertResourceOwner, claimAllocationId, hasOwnerLabels, ownerLabelArgs, releaseAllocationId, type ResourceOwner, UUID_V4 } from '../labels.ts'; @@ -128,6 +129,10 @@ export interface PreparationOptions { readonly signal?: AbortSignal; /** Called in the same turn as each Docker spawn with its process group, so the caller can record it durably. */ readonly onProcessGroup?: (group: ProcessGroup) => void; + /** Durable ownership around every individual Docker subprocess. */ + readonly processLifecycle?: ProcessGroupLifecycle; + /** Overall budget for allocation or cleanup. Callers with a larger lifecycle deadline reserve group settlement. */ + readonly timeoutMs?: number; } // The non-blocking driver: every Docker call runs in its own process group, and the promise settles only after it has // exited. An abort reaches the generator as an error at its next cancellable call or pause, so its own cleanup runs. @@ -144,9 +149,12 @@ async function runStepsAsync(steps: Steps, options: PreparationOptions): P await pause(step.sleepMs); next = steps.next(); } else { - next = steps.next(await runInProcessGroup('docker', step.args, { env: dockerEnvironment(), - timeoutMs: step.timeoutMs, signal: step.cancellable ? options.signal : undefined, - onProcessGroup: options.onProcessGroup, maxBuffer: step.maxBuffer, input: step.input })); + const processOptions = { env: dockerEnvironment(), timeoutMs: step.timeoutMs, + signal: step.cancellable ? options.signal : undefined, + onProcessGroup: options.onProcessGroup, maxBuffer: step.maxBuffer, input: step.input }; + next = steps.next(options.processLifecycle + ? await runTrackedProcess('docker', step.args, { ...processOptions, lifecycle: options.processLifecycle }) + : await runInProcessGroup('docker', step.args, processOptions)); } } return next.value; @@ -378,7 +386,7 @@ export function prepareTaskFilesystems(clone: TaskClone, limits: TaskStorageLimi * every group has exited. A create cut short by the abort counts as possibly created, as for a killed client. */ export async function prepareTaskFilesystemsAsync(clone: TaskClone, limits: TaskStorageLimits, imageId: string, - owner: ResourceOwner, options: PreparationOptions & { readonly timeoutMs?: number } = {}): Promise { + owner: ResourceOwner, options: PreparationOptions = {}): Promise { if (options.signal?.aborted) throw Object.assign(new Error('Task storage allocation was cancelled.'), { name: 'AbortError', code: 'ABORT_ERR' }); try { return await runStepsAsync(allocation(clone, limits, imageId, owner, options.timeoutMs ?? 60_000), options); } @@ -498,19 +506,22 @@ export function removeTaskFilesystems(filesystems: TaskFilesystems | RecoveredTa * `removeTaskFilesystems` without blocking the event loop: the same removal, each Docker call in its own process group. * It is never cancelled: it runs to its own deadline, so nothing is dropped, and settles only after every call exited. */ -export async function removeTaskFilesystemsAsync(filesystems: TaskFilesystems | RecoveredTaskStorage): Promise { +export async function removeTaskFilesystemsAsync(filesystems: TaskFilesystems | RecoveredTaskStorage, + options: PreparationOptions = {}): Promise { const recovered = recoveredStorage.get(filesystems as RecoveredTaskStorage); if (recovered) { const handle = filesystems as RecoveredTaskStorage; await runStepsAsync(cleanup(recovered.keeperId ? [recovered.keeperId] : [], - [handle.metadataVolume, handle.workVolume].filter((name): name is string => name !== undefined), recovered.owner), {}); + [handle.metadataVolume, handle.workVolume].filter((name): name is string => name !== undefined), recovered.owner, + new Set(), options.timeoutMs ?? 30_000), options); recoveredStorage.delete(handle); return; } const allocated = filesystems as TaskFilesystems; assertTaskFilesystems(allocated); const owner = taskFilesystemOwner(allocated); - await runStepsAsync(cleanup([allocated.keeper], [allocated.metadataVolume, allocated.workVolume], owner), {}); + await runStepsAsync(cleanup([allocated.keeper], [allocated.metadataVolume, allocated.workVolume], owner, + new Set(), options.timeoutMs ?? 30_000), options); allocations.delete(allocated); liveAllocations.delete(owner.allocationId); } @@ -523,6 +534,111 @@ export interface TaskDiff { /** Whether the diff was longer than `maxBytes` and was cut. */ readonly truncated: boolean; } + +export interface ExportedTaskPath { + readonly path: string; + readonly type: 'absent' | 'file' | 'symlink'; + readonly executable?: boolean; + readonly content?: Buffer; +} + +const EXPORT_PATHS_SCRIPT = String.raw` +const fs = require('node:fs'); +const path = require('node:path'); +const request = JSON.parse(fs.readFileSync(0, 'utf8')); +if (!request || !Array.isArray(request.paths) || !Number.isSafeInteger(request.maxBytes) || request.maxBytes < 1) process.exit(2); +let used = 0; +const entries = []; +for (const name of request.paths) { + if (typeof name !== 'string' || !name || path.posix.isAbsolute(name) || Buffer.byteLength(name) > ${MAXIMUM_NAME_BYTES} + || /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Cn}]/u.test(name) + || name.split('/').some(p => !p || p === '.' || p === '..') || name.split('/')[0] === '.git') process.exit(2); + used += Buffer.byteLength(name); + if (used > request.maxBytes) process.exit(4); + const parts = name.split('/'); + let cursor = '/work'; + let missingAncestor = false; + for (const part of parts.slice(0, -1)) { + cursor = path.join(cursor, part); + let stat; + try { stat = fs.lstatSync(cursor); } + catch (error) { + if (error && error.code === 'ENOENT') { missingAncestor = true; break; } + throw error; + } + if (!stat.isDirectory() || stat.isSymbolicLink()) process.exit(3); + } + if (missingAncestor) { entries.push({ path: name, type: 'absent' }); continue; } + const target = path.join('/work', ...parts); + let stat; + try { stat = fs.lstatSync(target); } + catch (error) { + if (error && error.code === 'ENOENT') { entries.push({ path: name, type: 'absent' }); continue; } + throw error; + } + let content; + if (stat.isSymbolicLink()) content = Buffer.from(fs.readlinkSync(target, { encoding: 'buffer' })); + else if (stat.isFile()) { + if (stat.size > request.maxBytes - used) process.exit(4); + if (!fs.constants.O_NOFOLLOW) process.exit(3); + const fd = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const opened = fs.fstatSync(fd); + if (!opened.isFile() || opened.dev !== stat.dev || opened.ino !== stat.ino) process.exit(3); + if (opened.size > request.maxBytes - used) process.exit(4); + content = fs.readFileSync(fd); + } finally { fs.closeSync(fd); } + } else process.exit(3); + used += content.length; + if (used > request.maxBytes) process.exit(4); + entries.push({ path: name, type: stat.isSymbolicLink() ? 'symlink' : 'file', + ...(stat.isFile() ? { executable: (stat.mode & 0o111) !== 0 } : {}), content: content.toString('base64') }); +} +process.stdout.write(JSON.stringify(entries)); +`; + +/** Export exact no-follow entries from task storage. The raw path plus content budget is enforced before output. */ +export async function exportTaskPaths(storage: TaskFilesystems | RecoveredTaskStorage, paths: readonly string[], + maxBytes: number, options: StorageScriptOptions): Promise { + if (!Array.isArray(paths) || paths.length > MAXIMUM_DECLARED_LINKS || paths.some(path => typeof path !== 'string' || !path + || path.startsWith('/') || /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Cn}]/u.test(path) || Buffer.byteLength(path) > MAXIMUM_NAME_BYTES + || path.split('/').some(part => !part || part === '.' || part === '..') || path.split('/')[0] === '.git')) + throw new Error('Conflict paths are invalid or exceed the entry limit.'); + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > 32 * 1024 * 1024) + throw new Error('Conflict path export needs a positive limit of at most 32 MiB.'); + let stdout: string; + try { + stdout = await runStorageScript(storage, { kind: 'export', operation: 'Conflict path export', + consequence: 'resolved conflict files cannot be exported', entrypoint: 'node', + args: ['-e', EXPORT_PATHS_SCRIPT], input: Buffer.from(JSON.stringify({ paths, maxBytes })), + maxOutputBytes: Math.ceil(maxBytes / 3) * 4 + + paths.reduce((n, path) => n + Buffer.byteLength(JSON.stringify(path)) + 96, 2) }, options); + } catch (error) { + if (error instanceof AggregateError || (error as Error | undefined)?.name === 'AbortError') throw error; + throw new Error('Conflict path export failed.', { cause: error }); + } + const decoded = JSON.parse(stdout) as { path?: unknown; type?: unknown; executable?: unknown; content?: unknown }[]; + if (!Array.isArray(decoded) || decoded.length !== paths.length) throw new Error('Conflict path export returned a malformed list.'); + let used = 0; + return Object.freeze(decoded.map((entry, index) => { + if (entry?.path !== paths[index] || !['absent', 'file', 'symlink'].includes(entry.type as string)) + throw new Error('Conflict path export returned an unexpected entry.'); + used += Buffer.byteLength(entry.path as string); + if (used > maxBytes) throw new Error('Conflict path export exceeded its byte limit.'); + if (entry.type === 'absent') { + if (entry.content !== undefined || entry.executable !== undefined) throw new Error('Conflict path export returned malformed absence.'); + return Object.freeze({ path: entry.path as string, type: 'absent' as const }); + } + if (typeof entry.content !== 'string' || (entry.type === 'file' ? typeof entry.executable !== 'boolean' : entry.executable !== undefined)) + throw new Error('Conflict path export returned malformed content.'); + const content = Buffer.from(entry.content, 'base64'); + if (content.toString('base64') !== entry.content) throw new Error('Conflict path export returned malformed base64.'); + used += content.length; + if (used > maxBytes) throw new Error('Conflict path export exceeded its byte limit.'); + return Object.freeze({ path: entry.path as string, type: entry.type as 'file' | 'symlink', + ...(entry.type === 'file' ? { executable: entry.executable as boolean } : {}), content }); + })); +} export interface ExportOptions extends PreparationOptions { /** The commit the storage was seeded from (the clone's head): a full commit ID. */ readonly base: string; @@ -748,10 +864,10 @@ export const EXPORT_SCRIPT = [ 'if [ "${statuses[1]}" -ne 0 ] || [ "${statuses[2]}" -ne 0 ] || [ "${statuses[3]}" -ne 0 ]; then echo "the export pipeline failed" >&2; exit 5; fi', ].join('\n'); -/** A container run over both volumes of task storage, read-only, as the storage user, with no network. */ +/** A container run over both volumes of task storage as the storage user with no network; only `import` may write work. */ interface StorageScript { /** The `io.codeboost.task-storage` kind the container carries; recovery removes a leftover one. */ - readonly kind: 'export' | 'inspect' | 'commit'; + readonly kind: 'export' | 'inspect' | 'commit' | 'import'; /** Names the operation in messages, for example "Task diff export". */ readonly operation: string; /** Ends "Task work volume is missing; ..." when a volume is gone. */ @@ -766,6 +882,8 @@ interface StorageScript { readonly tmpBytes?: '64m' | '512m'; /** Written to the script's stdin (`docker run -i`), which is then closed; without it stdin is not connected. */ readonly input?: Buffer; + /** Only the conflict-snapshot importer may write the work volume; metadata always remains read-only. */ + readonly writableWork?: boolean; } export interface StorageScriptOptions extends PreparationOptions { /** The immutable ID of the built agent image, whose tools run the script. */ @@ -793,7 +911,7 @@ function* storageScriptSteps(workVolume: string, metadataVolume: string, owner: '--read-only', '--user', '10001:10001', '--network=none', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--security-opt=seccomp=builtin', '--runtime=runc', '--pids-limit=64', `--memory=${script.memory ?? '256m'}`, '--cpus=.5', '--tmpfs', `/tmp:rw,nosuid,nodev,noexec,size=${script.tmpBytes ?? '64m'}`, - '--mount', `type=volume,source=${workVolume},target=/work,readonly`, + '--mount', `type=volume,source=${workVolume},target=/work${script.writableWork ? '' : ',readonly'}`, '--mount', `type=volume,source=${metadataVolume},target=/work/.git,readonly`, '--entrypoint', script.entrypoint, imageId, ...script.args]; const outcome = yield* run(args, remaining(), true, script.maxOutputBytes, script.input); @@ -816,14 +934,16 @@ function* storageScriptSteps(workVolume: string, metadataVolume: string, owner: } /** - * Run a script over task storage (the value `prepareTaskFilesystems` returned, or a recovery handle) in a read-only - * container with no network, and return its standard output. For D's own storage operations; not part of the contract. + * Run a script over task storage (the value `prepareTaskFilesystems` returned, or a recovery handle) in an isolated + * container with no network, and return its standard output. All callers mount both volumes read-only except the + * bounded conflict importer, which may write only the work volume. For D's own storage operations; not part of the contract. * On abort or at the deadline the client is stopped and the container, which outlives a killed client, is removed; the * promise settles only after both. An abort rejects with an `AbortError`, unless that container's cleanup did not * settle, which rejects with an `AggregateError`. */ export async function runStorageScript(storage: TaskFilesystems | RecoveredTaskStorage, script: StorageScript, options: StorageScriptOptions): Promise { + if (script.writableWork && script.kind !== 'import') throw new Error('Only a conflict import may write task storage.'); if (!/^sha256:[0-9a-f]{64}$/.test(options.imageId)) throw new Error(`${script.operation} requires the immutable built image ID.`); assertBuiltAgentImage(options.imageId); let owner: ResourceOwner, workVolume: string | undefined, metadataVolume: string | undefined; @@ -850,6 +970,106 @@ export async function runStorageScript(storage: TaskFilesystems | RecoveredTaskS } } +const IMPORT_PATHS_SCRIPT = String.raw` +const fs = require('node:fs'); +const path = require('node:path'); +const request = JSON.parse(fs.readFileSync(0, 'utf8')); +if (!request || !Array.isArray(request.entries) || !Number.isSafeInteger(request.maxBytes) || request.maxBytes < 1) process.exit(2); +let used = 0; +for (const entry of request.entries) { + const name = entry && entry.path; + if (typeof name !== 'string' || !name || path.posix.isAbsolute(name) || Buffer.byteLength(name) > ${MAXIMUM_NAME_BYTES} + || /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Cn}]/u.test(name) + || name.split('/').some(p => !p || p === '.' || p === '..') || name.split('/')[0] === '.git' + || !['absent', 'file', 'symlink'].includes(entry.type)) process.exit(2); + used += Buffer.byteLength(name); + let content; + if (entry.type === 'absent') { + if (entry.content !== undefined || entry.executable !== undefined) process.exit(2); + } else { + if (typeof entry.content !== 'string' || (entry.type === 'file' ? typeof entry.executable !== 'boolean' : entry.executable !== undefined)) process.exit(2); + content = Buffer.from(entry.content, 'base64'); + if (content.toString('base64') !== entry.content) process.exit(2); + used += content.length; + } + if (used > request.maxBytes) process.exit(4); +} +for (const entry of request.entries) { + const parts = entry.path.split('/'); + let cursor = '/work', missing = false; + for (const part of parts.slice(0, -1)) { + cursor = path.join(cursor, part); + let stat; + try { stat = fs.lstatSync(cursor); } + catch (error) { + if (!error || error.code !== 'ENOENT') throw error; + if (entry.type === 'absent') { missing = true; break; } + fs.mkdirSync(cursor, { mode: 0o755 }); + stat = fs.lstatSync(cursor); + } + if (!stat.isDirectory() || stat.isSymbolicLink()) process.exit(3); + } + if (missing) continue; + const target = path.join('/work', ...parts); + let current; + try { current = fs.lstatSync(target); } + catch (error) { if (!error || error.code !== 'ENOENT') throw error; } + if (current) { + if (!current.isFile() && !current.isSymbolicLink()) process.exit(3); + fs.unlinkSync(target); + } + if (entry.type === 'absent') continue; + const content = Buffer.from(entry.content, 'base64'); + if (entry.type === 'symlink') fs.symlinkSync(content, target); + else { + const fd = fs.openSync(target, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, + entry.executable ? 0o755 : 0o644); + try { fs.writeFileSync(fd, content); } finally { fs.closeSync(fd); } + fs.chmodSync(target, entry.executable ? 0o755 : 0o644); + } +} +`; + +/** Import a bounded, exact conflict snapshot into a clean allocation without exposing any host path to Docker. */ +export async function importTaskPaths(storage: TaskFilesystems, entries: readonly ExportedTaskPath[], maxBytes: number, + options: StorageScriptOptions): Promise { + if (!Array.isArray(entries) || entries.length === 0 || entries.length > MAXIMUM_DECLARED_LINKS) + throw new Error('Conflict path import needs a non-empty bounded entry list.'); + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > 32 * 1024 * 1024) + throw new Error('Conflict path import needs a positive limit of at most 32 MiB.'); + // Encoding follows user-controlled snapshot size and runs synchronously. Own one monotonic deadline before it, then + // admit Docker work only with the budget that remains after the payload is fully serialized. + const remaining = createDeadline(options.timeoutMs ?? 60_000); + let used = 0; + const encoded = entries.map(entry => { + if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.startsWith('/') + || /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Cn}]/u.test(entry.path) || Buffer.byteLength(entry.path) > MAXIMUM_NAME_BYTES + || entry.path.split('/').some((part: string) => !part || part === '.' || part === '..') || entry.path.split('/')[0] === '.git' + || !['absent', 'file', 'symlink'].includes(entry.type)) throw new Error('Conflict path import entry is invalid.'); + used += Buffer.byteLength(entry.path); + if (entry.type === 'absent') { + if (entry.content !== undefined || entry.executable !== undefined) throw new Error('Conflict path import absence is malformed.'); + return { path: entry.path, type: entry.type }; + } + if (!Buffer.isBuffer(entry.content) || (entry.type === 'file' ? typeof entry.executable !== 'boolean' : entry.executable !== undefined)) + throw new Error('Conflict path import content is malformed.'); + used += entry.content.length; + return { path: entry.path, type: entry.type, ...(entry.type === 'file' ? { executable: entry.executable } : {}), + content: entry.content.toString('base64') }; + }); + if (new Set(entries.map(entry => entry.path)).size !== entries.length) throw new Error('Conflict path import entries must not repeat.'); + if (used > maxBytes) throw new Error('Conflict path import exceeded its byte limit.'); + const input = Buffer.from(JSON.stringify({ entries: encoded, maxBytes })), timeoutMs = remaining(); + try { + await runStorageScript(storage, { kind: 'import', operation: 'Conflict path import', + consequence: 'the conflict snapshot cannot be imported', entrypoint: 'node', args: ['-e', IMPORT_PATHS_SCRIPT], + input, writableWork: true }, { ...options, timeoutMs }); + } catch (error) { + if (error instanceof AggregateError || (error as Error | undefined)?.name === 'AbortError') throw error; + throw new Error('Conflict path import failed.', { cause: error }); + } +} + /** * Export the diff of task storage against `base`, the commit it was seeded from, for a stopped attempt's partial * output (#51 item 6). It accepts the value `prepareTaskFilesystems` returned or a recovery handle, runs Git in a diff --git a/agents/network/network.ts b/agents/network/network.ts index 994265c5..25000339 100644 --- a/agents/network/network.ts +++ b/agents/network/network.ts @@ -4,7 +4,8 @@ import { createOutcomeUnknown, DOCKER_ID } from '../client-outcome.ts'; import { ALLOCATION_IN_USE, allocationListCommands, assertResourceOwner, claimAllocationId, hasOwnerLabels, ownerLabelArgs, ownerLabels, releaseAllocationId, type ResourceOwner } from '../labels.ts'; import { assertBuiltAgentImage } from '../container/image.ts'; -import { docker as runDockerCommand, pause, runDocker, type DockerOutcome } from '../docker.ts'; +import { docker as runDockerCommand, dockerEnvironment, pause, runDocker, type DockerOutcome } from '../docker.ts'; +import { runTrackedDocker, runTrackedProcess, type ProcessGroupLifecycle } from '../tracked-docker.ts'; export const VENDOR_HOSTS = Object.freeze({ claude: Object.freeze(['api.anthropic.com']), @@ -47,12 +48,19 @@ const networkResources = (name: string, proxyContainer: string, owner: ResourceO ]); }; // Fails closed: an unanswered list cannot prove the ID is unused. Throws when more than `expected` objects carry it. -// The three lists are independent, so they run in parallel. +// The three lists are independent, so ordinary callers run them in parallel. A durable owner permits only one +// subprocess at a time, so lifecycle callers serialize them and refresh the shared deadline before each command. const assertAllocationObjects = async (allocationId: string, expected: number, remaining: () => number, - signal?: AbortSignal) => { - const timeoutMs = remaining(); - const results = await Promise.all(allocationListCommands(allocationId) - .map(command => runDocker(command, { timeoutMs, signal }))); + signal?: AbortSignal, lifecycle?: ProcessGroupLifecycle) => { + const commands = allocationListCommands(allocationId); + const results: DockerOutcome[] = []; + if (lifecycle) { + for (const command of commands) results.push(await runTrackedProcess('docker', command, + { env: dockerEnvironment(), timeoutMs: remaining(), signal, lifecycle })); + } else { + results.push(...await Promise.all(commands.map(command => + runDocker(command, { timeoutMs: remaining(), signal })))); + } if (results.some(result => result.status !== 0)) throw new Error('Could not confirm that allocationId is unused.'); const found = results.reduce((count, result) => count + result.stdout.split('\n').filter(line => line.trim()).length, 0); if (found > expected) throw new Error(ALLOCATION_IN_USE); @@ -78,8 +86,12 @@ const deadline = (timeoutMs: number) => { return value; }; }; -const docker = (args: readonly string[], timeout: number, signal?: AbortSignal) => - runDockerCommand(args, { timeoutMs: timeout, signal }); +const docker = (args: readonly string[], timeout: number, signal?: AbortSignal, lifecycle?: ProcessGroupLifecycle) => + lifecycle ? runTrackedDocker(args, { timeoutMs: timeout, signal, lifecycle }) + : runDockerCommand(args, { timeoutMs: timeout, signal }); +const dockerOutcome = (args: readonly string[], timeout: number, lifecycle?: ProcessGroupLifecycle) => lifecycle + ? runTrackedProcess('docker', args, { env: dockerEnvironment(), timeoutMs: timeout, lifecycle }) + : runDocker(args, { timeoutMs: timeout }); const absent = (result: DockerOutcome) => result.status !== 0 && result.status !== null && !result.error && /(?:No such (?:object|container|network)|network .* not found)/i.test(`${result.stdout}\n${result.stderr}`); /** How long a network or proxy whose create client was killed may still materialize in the daemon. */ @@ -91,8 +103,8 @@ const CREATE_SETTLE_MS = 10_000; // even if its labels are wrong, as when validation refused it for that. A name proves nothing: an object found by // name is removed only if it carries the egress label and all three owner labels. const remove = async (object: 'container' | 'network', target: string, remaining: () => number, kind: string, - owner: ResourceOwner, settleBy = 0) => { - const inspect = (ref: string) => runDocker([object, 'inspect', ref], { timeoutMs: remaining() }); + owner: ResourceOwner, settleBy = 0, lifecycle?: ProcessGroupLifecycle) => { + const inspect = (ref: string) => dockerOutcome([object, 'inspect', ref], remaining(), lifecycle); let before: DockerOutcome; for (;;) { before = await inspect(target); @@ -110,14 +122,15 @@ const remove = async (object: 'container' | 'network', target: string, remaining const id = inspected?.Id; if (!id || !DOCKER_ID.test(id) || (DOCKER_ID.test(target) && id !== target)) throw new Error(`Failed to establish the identity of ${kind}.`); - const result = await runDocker(object === 'container' ? ['rm', '--force', id] : ['network', 'rm', id], - { timeoutMs: remaining() }); + const result = await dockerOutcome(object === 'container' ? ['rm', '--force', id] : ['network', 'rm', id], + remaining(), lifecycle); if (result.status === 0) return; if (!absent(await inspect(id))) throw new Error(`Failed to confirm removal of ${kind}.`); }; const validateVendorNetwork = async (network: VendorNetwork, invocation: InvocationInput | undefined, - agentName: string | undefined, remaining: () => number, signal?: AbortSignal): Promise => { + agentName: string | undefined, remaining: () => number, signal?: AbortSignal, + lifecycle?: ProcessGroupLifecycle): Promise => { const identity = identities.get(network); if (!identity) throw new Error('Vendor network was not created by the trusted network builder.'); if (invocation && (identity.invocation !== invocation || network.vendor !== invocation.vendor)) @@ -125,7 +138,7 @@ const validateVendorNetwork = async (network: VendorNetwork, invocation: Invocat assertBuiltAgentImage(identity.imageId); // Inspect by the captured IDs, so a removed-and-recreated network or proxy cannot stand in for the original. const inspectAllocated = async (args: readonly string[]) => { - try { return await docker(args, remaining(), signal); } + try { return await docker(args, remaining(), signal, lifecycle); } catch (cause) { throw new Error('Vendor network or proxy changed after allocation.', { cause }); } }; const inspect = JSON.parse(await inspectAllocated(['container', 'inspect', identity.proxyId]))[0] as @@ -140,7 +153,7 @@ const validateVendorNetwork = async (network: VendorNetwork, invocation: Invocat RestartPolicy?: { Name?: string; MaximumRetryCount?: number } | null }; NetworkSettings?: { Networks?: Record; Ports?: Record }; Mounts?: unknown[] } | undefined; - const image = JSON.parse(await docker(['image', 'inspect', identity.imageId], remaining(), signal))[0] as + const image = JSON.parse(await docker(['image', 'inspect', identity.imageId], remaining(), signal, lifecycle))[0] as { Config?: { Env?: string[] } } | undefined; const inspectedNetwork = JSON.parse(await inspectAllocated(['network', 'inspect', identity.networkId]))[0] as { Id?: string; Name?: string; Internal?: boolean; Driver?: string; Labels?: Record; IPAM?: { Config?: Array<{ Subnet?: string }> }; @@ -188,8 +201,8 @@ const validateVendorNetwork = async (network: VendorNetwork, invocation: Invocat // Async, so an invalid deadline rejects like every other failure instead of throwing synchronously. export async function assertVendorNetwork(network: VendorNetwork, invocation?: InvocationInput, agentName?: string, - timeoutMs = 30_000, signal?: AbortSignal): Promise { - return validateVendorNetwork(network, invocation, agentName, deadline(timeoutMs), signal); + timeoutMs = 30_000, signal?: AbortSignal, lifecycle?: ProcessGroupLifecycle): Promise { + return validateVendorNetwork(network, invocation, agentName, deadline(timeoutMs), signal, lifecycle); } /** @@ -197,7 +210,8 @@ export async function assertVendorNetwork(network: VendorNetwork, invocation?: I * and everything created so far is removed (cleanup itself is not cancelled) before the promise rejects. */ export async function createVendorNetwork(invocation: InvocationInput, imageId: string, allocationId: string, - timeoutMs = 60_000, signal?: AbortSignal): Promise { + timeoutMs = 60_000, signal?: AbortSignal, processLifecycle?: ProcessGroupLifecycle, + invocationBudget?: () => number): Promise { assertCapturedInvocation(invocation); assertBuiltAgentImage(imageId); // The network and proxy carry the invocation's runner and attempt and this caller-chosen allocation ID. @@ -207,7 +221,7 @@ export async function createVendorNetwork(invocation: InvocationInput, imageId: const vendor = invocation.vendor; // Setup runs inside the caller's budget minus a cleanup reserve, so failure cleanup cannot overrun timeoutMs. // No allocation may outlive the invocation it serves. - const invocationLeft = Math.floor(invocation.deadline - Date.now()); + const invocationLeft = Math.floor(invocationBudget?.() ?? invocation.deadline - Date.now()); if (invocationLeft < 1) throw new Error('Invocation deadline has passed.'); timeoutMs = Math.min(timeoutMs, invocationLeft); const overall = deadline(timeoutMs), cleanupReserve = Math.min(10_000, Math.floor(timeoutMs / 3)); @@ -229,7 +243,7 @@ export async function createVendorNetwork(invocation: InvocationInput, imageId: const create = async (object: string, args: readonly string[]) => { const timeout = remaining(); try { - const output = await docker(args, timeout, signal); + const output = await docker(args, timeout, signal, processLifecycle); created.add(object); return output; } catch (error) { @@ -259,19 +273,19 @@ export async function createVendorNetwork(invocation: InvocationInput, imageId: if (!mayExist('network')) networkGone = true; if (proxyPlanned && !proxyGone) try { await remove('container', proxyTarget, - budget, 'vendor proxy', owner, proxyId ? 0 : settleBy(proxyContainer)); + budget, 'vendor proxy', owner, proxyId ? 0 : settleBy(proxyContainer), processLifecycle); proxyGone = true; } catch (cleanupError) { failures.push(cleanupError); } if (networkPlanned && !networkGone) try { await remove('network', networkTarget, - budget, 'vendor network', owner, networkId ? 0 : settleBy(name)); + budget, 'vendor network', owner, networkId ? 0 : settleBy(name), processLifecycle); networkGone = true; } catch (cleanupError) { failures.push(cleanupError); } if (failures.length) throw new AggregateError(failures, 'Vendor network cleanup did not settle.'); }; claimAllocationId(allocationId); // Nothing created yet: a reused ID (still labelling objects from any earlier process) is refused here. - try { await assertAllocationObjects(allocationId, 0, remaining, signal); } + try { await assertAllocationObjects(allocationId, 0, remaining, signal, processLifecycle); } catch (error) { releaseAllocationId(allocationId); throw error; } try { networkPlanned = true; @@ -280,7 +294,7 @@ export async function createVendorNetwork(invocation: InvocationInput, imageId: '--label', `io.codeboost.egress=${allocationId}`, ...labels, name]), 'network'); // The check above and this create are not atomic across processes. Once our network exists, it must be the only // object carrying the ID: of two racing allocations, the later check sees both and backs out. - await assertAllocationObjects(allocationId, 1, remaining, signal); + await assertAllocationObjects(allocationId, 1, remaining, signal, processLifecycle); proxyPlanned = true; // Create and start separately: a refused create made nothing, while a failed start leaves a container we own by ID. proxyId = createdId(await create(proxyContainer, ['create', '--name', proxyContainer, '--read-only', '--user', '10001:10001', @@ -289,15 +303,15 @@ export async function createVendorNetwork(invocation: InvocationInput, imageId: '--label', `io.codeboost.egress=${allocationId}`, ...labels, '--env', `CODEBOOST_ALLOWED_HOSTS=${VENDOR_HOSTS[vendor].join(',')}`, '--entrypoint', 'node', imageId, '/usr/local/lib/codeboost-egress-proxy.mjs']), 'proxy'); - await docker(['start', proxyId], remaining(), signal); - await docker(['network', 'connect', 'bridge', proxyId], remaining(), signal); + await docker(['start', proxyId], remaining(), signal, processLifecycle); + await docker(['network', 'connect', 'bridge', proxyId], remaining(), signal, processLifecycle); await docker(['exec', proxyId, 'node', '-e', [ "const net=require('node:net');let attempts=0;", "const check=()=>{const socket=net.connect(3128,'127.0.0.1');", "socket.once('connect',()=>{socket.destroy();process.exit(0)});", "socket.once('error',()=>{socket.destroy();if(++attempts===50)process.exit(1);setTimeout(check,20)})};check();", - ].join('')], remaining(), signal); - const proxyInspect = JSON.parse(await docker(['container', 'inspect', proxyId], remaining(), signal))[0] as + ].join('')], remaining(), signal, processLifecycle); + const proxyInspect = JSON.parse(await docker(['container', 'inspect', proxyId], remaining(), signal, processLifecycle))[0] as { NetworkSettings?: { Networks?: Record } } | undefined; const proxyIp = proxyInspect?.NetworkSettings?.Networks?.[name]?.IPAddress; if (!proxyIp || !/^10\.254\.\d{1,3}\.\d{1,3}$/.test(proxyIp)) @@ -306,7 +320,7 @@ export async function createVendorNetwork(invocation: InvocationInput, imageId: identities.set(network, Object.freeze({ allocationId, owner, imageId, invocation, subnet, proxyIp, networkId, proxyId })); registered = network; - await validateVendorNetwork(network, invocation, undefined, remaining, signal); + await validateVendorNetwork(network, invocation, undefined, remaining, signal, processLifecycle); remaining(); signal?.throwIfAborted(); releaseAllocationId(allocationId); @@ -331,7 +345,8 @@ export async function createVendorNetwork(invocation: InvocationInput, imageId: } } -export async function removeVendorNetwork(network: VendorNetwork, timeoutMs = 30_000): Promise { +export async function removeVendorNetwork(network: VendorNetwork, timeoutMs = 30_000, + processLifecycle?: ProcessGroupLifecycle): Promise { const identity = identities.get(network); if (!identity) { if (removedNetworks.has(network)) return; @@ -344,12 +359,12 @@ export async function removeVendorNetwork(network: VendorNetwork, timeoutMs = 30 removedParts.set(network, removed); if (!removed.has('container')) try { await remove('container', identity.proxyId, - remaining, 'vendor proxy', identity.owner); + remaining, 'vendor proxy', identity.owner, 0, processLifecycle); removed.add('container'); } catch (error) { failures.push(error); } if (!removed.has('network')) try { await remove('network', identity.networkId, - remaining, 'vendor network', identity.owner); + remaining, 'vendor network', identity.owner, 0, processLifecycle); removed.add('network'); } catch (error) { failures.push(error); } if (failures.length) throw new AggregateError(failures, 'Vendor network cleanup did not settle.'); diff --git a/agents/process-group.ts b/agents/process-group.ts index dd7f712e..bd460e57 100644 --- a/agents/process-group.ts +++ b/agents/process-group.ts @@ -8,7 +8,7 @@ export interface ProcessGroup { readonly pgid: number; /** `Date.now()` right after the spawn, retained for lifecycle timing and diagnostics. */ readonly startedAt: number; - /** Linux boot ID plus kernel start ticks. Null means crash recovery must not signal this group automatically. */ + /** Linux boot ID plus kernel start ticks, retained for diagnostics and identity-bound process-control adapters. */ readonly identity: string | null; } @@ -43,6 +43,12 @@ export interface ProcessGroupOptions { * reading it all is not an error: what it did is in its status and output. */ readonly input?: Buffer; + /** + * Return a retained-owner error when descendants or inherited pipe holders outlive the bounded drain. This is safe + * only when the caller durably owns the reported group/resource and recovery will keep that ownership until release. + * Callers without durable ownership wait fail-closed instead of returning while unknown work is still live. + */ + readonly allowUnsettledReturn?: boolean; } const DEFAULT_GRACE_MS = 5_000; @@ -54,6 +60,8 @@ const DRAIN_LIMIT_MS = 10_000; // Once the group is gone, how long to wait for stdout and stderr to reach end of file. Only a process outside the group // (one that called setsid but kept the inherited pipe) can hold them open after that, and it must not hold this call. const STDIO_CLOSE_MS = 1_000; +/** Worst-case default SIGTERM grace plus post-exit group and pipe settlement. */ +export const DEFAULT_PROCESS_SETTLEMENT_MS = DEFAULT_GRACE_MS + DRAIN_LIMIT_MS + STDIO_CLOSE_MS; const pause = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); const notStarted = (message: string): DockerOutcome => ({ status: null, stdout: '', stderr: '', error: Object.assign(new Error(message), { code: NOT_STARTED }) }); @@ -65,25 +73,29 @@ const signalGroup = (pgid: number, signal: NodeJS.Signals | 0) => { try { process.kill(-pgid, signal); return true; } catch (error) { return (error as NodeJS.ErrnoException).code !== 'ESRCH'; } }; - -// The leader can exit while other members of its group still run (for example a child it forked). A group ID stays -// taken while any member exists, so until the group is empty the ID cannot name someone else's group. -const drainGroup = async (pgid: number) => { +// Once the leader exits, there is no portable identity-bound way in Node to signal its numeric process-group ID. Even +// a liveness/identity check followed by kill has a reuse race between the two syscalls. Poll only: a group that remains +// alive is returned to durable recovery instead of risking a signal to an unrelated process. +const drainGroup = async (group: ProcessGroup, bounded: boolean) => { + const { pgid } = group; const giveUpAt = performance.now() + DRAIN_LIMIT_MS; while (signalGroup(pgid, 0)) { - signalGroup(pgid, 'SIGKILL'); - if (performance.now() >= giveUpAt) return false; + if (bounded && performance.now() >= giveUpAt) return false; await pause(20); } return true; }; +/** After its leader exits, observe whether every member of a recorded process group has gone without signalling it. */ +export const drainProcessGroup = (group: ProcessGroup): Promise => drainGroup(group, true); + /** * Run one command as the leader of a new process group. The group is reported through `onProcessGroup` before this * returns control to the event loop. On abort or at the deadline the whole group gets SIGTERM, then SIGKILL after the * grace period, so a child that ignores SIGTERM is still bounded. The promise settles only after the leader has - * exited and every other member of its group is gone. Never rejects: failures are in the outcome, shaped like - * `runDocker`'s, so `status` is a number only when the command ran to completion. + * exited and every other member of its group is gone, unless a caller with durable recovery ownership explicitly + * allows an unsettled return. Never rejects: failures are in the outcome, shaped like `runDocker`'s, so `status` is a + * number only when the command ran to completion. */ export function runInProcessGroup(file: string, args: readonly string[], options: ProcessGroupOptions): Promise { @@ -123,14 +135,15 @@ export function runInProcessGroup(file: string, args: readonly string[], if (stopped || exited) return; stopped = reason; signalGroup(pgid, 'SIGTERM'); - graceTimer = setTimeout(() => signalGroup(pgid, 'SIGKILL'), graceMs); + graceTimer = setTimeout(() => { if (!exited) signalGroup(pgid, 'SIGKILL'); }, graceMs); }; // Armed at the spawn, before the caller records the group: time spent recording counts against the deadline. const deadline = setTimeout(() => stop('timeout'), options.timeoutMs); // If the caller cannot record the group, the child must not outlive this call: it is killed at once below, and // the call still settles only after the group has exited, with the caller's error. + const group = Object.freeze({ pgid, startedAt: Date.now(), identity: processIdentity(pgid) }); let unrecorded: unknown; - try { options.onProcessGroup?.(Object.freeze({ pgid, startedAt: Date.now(), identity: processIdentity(pgid) })); } + try { options.onProcessGroup?.(group); } catch (error) { unrecorded = error; } // Normally output past the limit stops the group and settles as ENOBUFS. A caller that classifies from durable state // may explicitly keep a bounded prefix and let the process finish instead. @@ -157,13 +170,14 @@ export function runInProcessGroup(file: string, args: readonly string[], if (finished) return; finished = true; exited = true; + clearTimeout(graceTimer); clearTimeout(deadline); options.signal?.removeEventListener('abort', onAbort); - void drainGroup(pgid).then(async drained => { - // The group is empty (or given up on): a later SIGKILL could reach a new group that reuses the ID. - clearTimeout(graceTimer); + void drainGroup(group, options.allowUnsettledReturn === true).then(async drained => { let stdioTimer: ReturnType | undefined; - await Promise.race([closed, new Promise(done => { stdioTimer = setTimeout(done, STDIO_CLOSE_MS); })]); + if (options.allowUnsettledReturn) { + await Promise.race([closed, new Promise(done => { stdioTimer = setTimeout(done, STDIO_CLOSE_MS); })]); + } else await closed; // Cleared so a finished call never keeps the process alive. clearTimeout(stdioTimer); // Timers run before I/O in each event-loop turn: after a long block the timer can win while the rest of the @@ -175,7 +189,8 @@ export function runInProcessGroup(file: string, args: readonly string[], const stdout = Buffer.concat(out).toString('utf8'), stderr = Buffer.concat(err).toString('utf8'); if (!drained) { resolve({ status: null, stdout, stderr, error: Object.assign( - new Error(`${file} left processes in its group that did not exit after SIGKILL.`), { code: 'EGROUPALIVE' }) }); + new Error(`${file} left processes in its group after its leader exited; ownership was retained without signalling a reusable group ID.`), + { code: 'EGROUPALIVE' }) }); return; } if (!pipesClosed) { diff --git a/agents/recovery.ts b/agents/recovery.ts index 61868f2b..99366cef 100644 --- a/agents/recovery.ts +++ b/agents/recovery.ts @@ -190,9 +190,10 @@ export async function recoverLeftovers(runnerOwner: string, timeoutMs = 120_000, for (const resource of owned.container) { const kind = kindOf(resource); if (kind === 'storage:keeper') keep(resource, 'keeper'); - // Seeders, export, inspection and commit containers are transient: a leftover one is removed, never kept with the + // Seeders, conflict imports, exports, inspections and commits are transient: a leftover one is removed, never kept with the // storage. - else if (kind === 'storage:seeder' || kind === 'storage:export' || kind === 'storage:inspect' || kind === 'storage:commit' + else if (kind === 'storage:seeder' || kind === 'storage:import' || kind === 'storage:export' + || kind === 'storage:inspect' || kind === 'storage:commit' || kind === 'agent' || kind === 'egress') remove.push(resource); else unknown(resource); diff --git a/agents/tracked-docker.ts b/agents/tracked-docker.ts new file mode 100644 index 00000000..d6b57545 --- /dev/null +++ b/agents/tracked-docker.ts @@ -0,0 +1,81 @@ +import { DockerError, dockerEnvironment, NOT_STARTED, type DockerOutcome } from './docker.ts'; +import { MAXIMUM_TIMER_MS, runInProcessGroup, type ProcessGroup, type ProcessGroupOptions } from './process-group.ts'; + +export type ProcessGroupOwner = ProcessGroup | 'spawning'; +export interface ProcessGroupLifecycle { + /** Persist the spawning state before the Docker client can exist. */ + readonly starting: () => void; + /** Replace spawning with the exact group synchronously after spawn. */ + readonly started: (group: ProcessGroup) => void; + /** Clear this exact owner only after the client and all group members settled. */ + readonly settled: (owner: ProcessGroupOwner) => void; + /** Preserve durable ownership when the group or its output did not settle. */ + readonly unsettled: (owner: ProcessGroupOwner, reason: 'group-alive' | 'stdio-held') => void; +} + +/** Run one subprocess under durable ownership; lifecycle write failures become an unsuccessful settled outcome. */ +export async function runTrackedProcess(file: string, args: readonly string[], + options: ProcessGroupOptions & { readonly lifecycle: ProcessGroupLifecycle }): Promise { + if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1 || options.timeoutMs > MAXIMUM_TIMER_MS) + return { status: null, stdout: '', stderr: '', error: Object.assign( + new Error(`Process deadline must be a positive integer of at most ${MAXIMUM_TIMER_MS} ms.`), { code: NOT_STARTED }) }; + if (options.signal?.aborted) + return { status: null, stdout: '', stderr: '', error: Object.assign( + new Error(`${file} was cancelled before it started.`), { code: NOT_STARTED }) }; + // Arm the monotonic deadline before the synchronous durable write. A timer cannot run while that hook blocks. + const deadline = performance.now() + options.timeoutMs; + let owner: ProcessGroupOwner = 'spawning'; + try { options.lifecycle.starting(); } + catch (error) { + return { status: null, stdout: '', stderr: '', error: Object.assign( + new Error(`${file} process ownership could not be recorded before spawn.`, { cause: error }), { code: NOT_STARTED }) }; + } + const remaining = Math.ceil(deadline - performance.now()); + if (remaining < 1) { + const timeout = Object.assign(new Error(`${file} ${args[0] ?? ''} ETIMEDOUT before spawn.`), { code: 'ETIMEDOUT' }); + try { options.lifecycle.settled('spawning'); } + catch (error) { + return { status: null, stdout: '', stderr: '', error: Object.assign(new Error( + `${file} process ownership settlement could not be recorded.`, { cause: new AggregateError([timeout, error]) }), + { code: 'ETIMEDOUT' }) }; + } + return { status: null, stdout: '', stderr: '', error: timeout }; + } + const outcome = await runInProcessGroup(file, args, { ...options, timeoutMs: remaining, allowUnsettledReturn: true, + onProcessGroup: group => { + options.lifecycle.started(group); + owner = group; + options.onProcessGroup?.(group); + } }); + const code = (outcome.error as { code?: unknown } | undefined)?.code; + try { + if (code === 'EGROUPALIVE') options.lifecycle.unsettled(owner, 'group-alive'); + else if (code === 'ESTDIOHELD') options.lifecycle.unsettled(owner, 'stdio-held'); + else options.lifecycle.settled(owner); + } catch (error) { + const original = outcome.error ?? (outcome.status !== null && outcome.status !== 0 + ? Object.assign(new Error(`${file} ${args[0] ?? ''} exited with status ${outcome.status}.`), { status: outcome.status }) + : undefined); + const settlement = new Error(`${file} process ownership settlement could not be recorded.`, + { cause: new AggregateError(original ? [original, error] : [error]) }); + if (typeof code === 'string') Object.assign(settlement, { code }); + return { status: null, stdout: outcome.stdout, stderr: outcome.stderr, + error: settlement }; + } + return outcome; +} + +/** Run a resource-creating Docker command under durable process-group ownership. */ +export async function runTrackedDocker(args: readonly string[], options: { + readonly timeoutMs: number; + readonly signal?: AbortSignal; + readonly secrets?: Readonly>; + readonly lifecycle: ProcessGroupLifecycle; +}): Promise { + const outcome = await runTrackedProcess('docker', args, { + env: dockerEnvironment(options.secrets), timeoutMs: options.timeoutMs, signal: options.signal, + lifecycle: options.lifecycle, + }); + if (outcome.status !== 0) throw new DockerError(args, outcome); + return outcome.stdout.trim(); +} diff --git a/docs/implementation/pre-merge-rebase.md b/docs/implementation/pre-merge-rebase.md index 506235a3..fb515fe4 100644 --- a/docs/implementation/pre-merge-rebase.md +++ b/docs/implementation/pre-merge-rebase.md @@ -1,24 +1,26 @@ # Pre-merge rebase and foreign-conflict boundary -**Issue and lane:** #22, lanes F3-F4. This is the trusted local rewrite, recovery, and foreign-conflict engine boundary. The production `rebase-fix` child-attempt adapter is not wired yet: the runner must not invoke this boundary until that adapter owns its container and storage durably. Pushing the rewritten head, running plan `cmd:` checks, waiting for GitHub checks, and handing a pair to the merge coordinator remain F5-F6 work in #22. +**Issue and lane:** #22, lanes F3-F4. This is the trusted local rewrite, recovery, and foreign-conflict engine boundary. The `rebase-fix` child adapter is assembled inside the production rebaser through lane D with durable resource ownership and bounded exact-path copy-in/copy-out. No production request starts that rebaser yet: F5-F6 must own admission plus the current base/head refresh before invoking it, then push the rewritten head, run plan `cmd:` checks, wait for GitHub checks, and hand the validated pair to the merge coordinator. Exposing the raw rebaser earlier would bypass those required guards. ## Ownership and lifecycle Before the first Git process, `Store.beginRebase` records one marker in `tasks.rebase_in_progress`: ```text -{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, resolvedConflicts, processGroup } +{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, resolvedConflicts, conflict, processGroup } ``` Admission checks the current plan revision, snapshot, review version, task state version, mergeable task status, cancellation state, active runner attempt, active merge, existing rebase, both ends of the reviewed snapshot, and the complete ordered old history captured by review. Recording the marker advances the task state version. While it exists, another rebase, the legacy rewrite path, merge admission, runner admission, reassignment, and ordinary status changes fail closed. Cancellation remains pending until the owned rebase has settled and exact-attempt cleanup clears the marker; that cleanup then closes the task. -`processGroup` is null only while no Git call owns work. Immediately before spawn it becomes `spawning`; the synchronous spawn callback replaces that with `{ pgid, startedAt, identity }`. On Linux, `identity` combines the host boot ID with the leader's `/proc` start ticks. After that exact group is confirmed settled, a compare-and-swap clears it. A group that cannot be drained remains recorded, and a descendant that escapes the group leaves `unsettled`; restart recovery fails closed on either unknown state. A duplicate spawn or late settlement therefore cannot replace or erase another live process owner. +`processGroup` is null only while no Git call owns work. Immediately before spawn it becomes `spawning`; the synchronous spawn callback replaces that with `{ pgid, startedAt, identity }`. On Linux, `identity` combines the host boot ID with the leader's `/proc` start ticks. While the owned leader is live, timeout or cancellation may signal its group. The instant the leader exits, delayed signals are disabled and cleanup only observes whether the group disappears: a numeric PGID may be reused, and a separate identity check followed by `kill` is not atomic. After settlement is confirmed, a compare-and-swap clears the owner. A group that remains alive or a descendant that escapes the group leaves durable ownership for fail-closed restart recovery. A duplicate spawn or late settlement therefore cannot replace or erase another live process owner. `GitRebaser` works only in `//rebases/`, beside the runner-owned bare repository. It accepts at most 500 commits and requires the old history to be complete, linear, descended from the recorded base, and byte-for-byte equal to the ordered history in the durable marker. Git runs with user/system configuration, hooks, replacement objects, lazy fetches, maintenance, and submodule recursion disabled. Before and after replay, a bounded verifier reads the index as NUL-delimited bytes, checks path spellings against raw directory entries, and compares file modes, symlink bytes, and path-filtered file hashes before requiring the index tree to equal the expected commit tree. It never enters a gitlink or trusts Git's stat cache. One monotonic deadline covers history inspection, worktree creation, replay, validation, ref retention, process-group teardown, and cleanup, with explicit budget reserved for teardown and cleanup. Rebase diagnostics retain only a bounded prefix without stopping Git; conflict classification reads the unmerged index instead of diagnostic text. Other abnormal process outcomes retain their timeout, cancellation, output-limit, or unsettled-group reason. If cleanup also fails, both failures are retained with the original as the cause. The rewrite uses one commit for every old commit, including empty or already-applied commits, so ledger mappings stay one-to-one. A successful head is retained at `refs/codeboost/rebases/`; its mutable worktree is removed before the call returns. After replay settles and before its first result-ref write, the rebaser supplies the Store's preparation hook only the ordered rewritten history. The Store zips that history by position with the independently captured `oldHistory`, then durably binds the resulting complete mapping and exact intended ref value as `prepared`. The atomic create outcome advances it to `ready`, `uncertain`, or `refused`; an already-current no-op uses `prepared` then `ready` without a ref write. `Store.finishRebase` accepts only a `ready`, byte-for-byte mapping from the same attempt while the captured plan, snapshot, review version, task version, old base, old head, and target base are unchanged. A non-empty mapping must end at both the captured old head and the committed rewritten head; an empty mapping is valid only when the captured history is empty and the result equals the new base. The snapshot, rewrite map, and owned/foreign ledger provenance commit atomically, and the marker clears in that transaction. A stale result changes nothing and leaves its marker for exact-attempt cleanup. -For each conflict, classification uses only the supplied trusted ledger. A missing entry and an explicit `{ owner: null, origin: "foreign" }` entry take the foreign branch; trailers are never read. An owned entry is refused until the separate owned-commit resolver exists. A gitlink conflict is also refused for manual resolution before the file resolver runs; the boundary never enters a nested repository. The injected foreign resolver receives the source commit and exact non-gitlink conflicting path set as a read-only string array. Raw Git path streams use the same explicit 32 MiB and 262,144-entry bounds as checkout verification, travel between processes as base64, and undergo fatal UTF-8 decoding so malformed names fail closed without rejecting a valid literal replacement character. Before and after the resolver runs, one killable helper batches the index scan and hashes every non-conflict changed path against the same pinned `HEAD`; separate bounded helpers hash every index-only gitlink entry and the byte-exact rebase control directory without following links. These helpers run inside the operation deadline instead of blocking the server event loop or spawning once per path. After the resolver settles, the rebaser proves both `HEAD` and `REBASE_HEAD` are unchanged, rejects every tracked or untracked edit outside the conflict set, refuses any change to the remaining rebase operation, stages only literal NUL-delimited path data on stdin, then rechecks the gitlink index so an allowed file cannot become an embedded repository. It also refuses any remaining unmerged entry. A rejection or invocation failure aborts and removes the partial rewrite. The production adapter must expose the rebase worktree only through lane D's bounded task storage; this engine callback is trusted orchestration, not agent code, and is not permission to mount the host worktree or its shared Git metadata directly. +For each conflict, classification uses only the supplied trusted ledger. A missing entry and an explicit `{ owner: null, origin: "foreign" }` entry take the foreign branch; trailers are never read. An owned entry is refused until the separate owned-commit resolver exists. A gitlink conflict is also refused for manual resolution before the file resolver runs; the boundary never enters a nested repository. The injected foreign resolver receives the source commit and exact non-gitlink conflicting path set as a read-only string array. Raw Git path streams use the same explicit 32 MiB and 262,144-entry bounds as checkout verification, travel between processes as base64, and undergo fatal UTF-8 decoding so malformed names fail closed without rejecting a valid literal replacement character. Before and after the resolver runs, one killable helper batches the index scan and hashes every non-conflict changed path against the same pinned `HEAD`; separate bounded helpers hash every index-only gitlink entry and the byte-exact rebase control directory without following links. These helpers run inside the operation deadline instead of blocking the server event loop or spawning once per path. After the resolver settles, the rebaser proves both `HEAD` and `REBASE_HEAD` are unchanged, rejects every tracked or untracked edit outside the conflict set, refuses any change to the remaining rebase operation, stages only literal NUL-delimited path data on stdin, then rechecks the gitlink index so an allowed file cannot become an embedded repository. It also refuses any remaining unmerged entry. A rejection or invocation failure aborts and removes the partial rewrite. + +Before any clone or Docker operation, the production adapter records a unique child attempt plus distinct storage and network allocation IDs in `conflict`. It clones the pinned pre-conflict `HEAD`, snapshots only the exact conflict leaves without following ancestor links, and allocates ordinary bounded task storage from that still-clean clone. Only after seeding succeeds does a no-network stdin-backed importer write the bounded snapshot into the work volume; metadata stays read-only. The importer owns a monotonic deadline before base64/JSON serialization and rechecks it before any Docker probe. The path set is capped at 32 KiB before ownership is claimed, keeping the provider prompt below the operating system's single-argument limit. Every individual Git or Docker client—validation, execution, and cleanup included—moves the durable marker through `spawning`, its exact process group, and settled or retained ownership; a retained client prevents every later subprocess from starting. Each child subprocess budget leaves 17 seconds for the default 5-second stop grace, 10-second group drain, 1-second pipe drain, and durable settlement, and every asynchronous child stage shares the parent's rebase-work boundary. The resolver carries that same monotonic budget through vendor-network creation, profile validation, and supervisor launch; those stages do not re-read the transported wall deadline. If a stage cannot settle by that boundary, the resolver returns with durable ownership retained instead of consuming the parent's 30-second cleanup reserve. A specialized pre-launch tree check allows the expected dirty conflict paths but rejects repository metadata, commits, outside paths, unsafe types, links, populated gitlinks, and unmountable gitlink layouts. The ordinary `fix` profile then runs with no command capability. Its host-side input snapshot is created beneath the canonical spelling of the durably known conflict staging directory, so retaining the parent workspace also retains every reported directory resource for restart cleanup. On success, the adapter audits the complete manifest, exports only the exact approved leaves with a 32 MiB raw-content budget, and copies them back without exposing the host worktree or shared Git metadata to the agent. Storage and staging cleanup complete before the marker clears; an unreleased resource keeps the marker and parent workspace durable. Startup recovery removes agent/network leftovers through lane D, aborts the parent worktree (including profile snapshots), removes an exactly matching child storage handle, and only then clears child and parent ownership. The Store records each resolved source SHA before the result ref write. Preparation refuses duplicates, sources outside the captured history, owned sources, and sources that did not actually rewrite. On publication, the mapped commit retains `owner: null`, `origin: foreign`, and `sourceSha`, plus `conflictResolved: true`. Linking therefore keeps its lines in Unplanned while the review UI adds the text label “conflict resolved by agent.” The commit author is preserved by Git's replay and no Plan-Item trailer is added. @@ -26,10 +28,10 @@ All other conflicts are a review outcome (`RebaseConflict`). Cancellation and fa ## Restart recovery -Startup validates every stored marker before touching its resource. If its recorded process group is still alive, recovery re-reads the kernel identity immediately before signalling and requires the boot ID and `/proc` start ticks to match; a legacy or current marker with a null identity fails closed instead of signalling a numeric ID. Signal grace and post-kill settlement use monotonic bounded deadlines. If the group has disappeared, recovery independently proves that no process still has the rebase workspace open before clearing that group owner; Linux performs that potentially blocking `/proc` walk in an owned, killable subprocess whose settlement is inside the overall deadline. The production recovery adapter accepts only the configured plan key, then removes the matching worktree. It deletes a result ref only with Git's exact-old-value guard using the durably recorded `resultHead`; absence is already-clean success. A `prepared` marker preserves a different pre-existing value, while a `ready` marker fails closed if its ref changed. A legacy, pre-result, or reused-attempt marker without an intended value preserves any existing ref. Only after cleanup succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, process or cleanup failure, workspace user, or changed marker blocks startup and retains the durable ownership evidence. +Startup first validates every stored marker and child identity. A recovered `spawning` marker is ambiguous because the runner may have crashed before the synchronous spawn callback recorded the child. A recovered exact process group is also ambiguous even after its leader or numeric group disappears: the runner may have crashed before an escaped output-pipe holder was durably marked `unsettled`, and that process need not use the workspace. Recovery requests identity-bound termination for an exact live group when its process controller can provide it; the host controller instead fails closed on a live reusable numeric group. Once no live exact group blocks the transition, recovery changes either ambiguous owner to `unsettled` and keeps admission closed. The operator must stop any unknown process and run `--release-rebase-process ` under the runner lock. Release revalidates the complete runner/owner/rebases chain as owner-only plain directories before it inspects the attempt leaf, and it refuses any visible workspace user; preparation release applies the same check to the attempts chain. Lane D then removes this runner's labelled agent containers, proxies, and networks and returns authenticated storage handles. The production recovery adapter accepts only the configured plan key, then removes the matching worktree. It deletes a result ref only with Git's exact-old-value guard using the durably recorded `resultHead`; absence is already-clean success. A `prepared` marker preserves a different pre-existing value, while a `ready` marker fails closed if its ref changed. A legacy, pre-result, or reused-attempt marker without an intended value preserves any existing ref. An exact child storage identity is removed after the parent workspace and before either marker clears; a partial match fails closed. A live run whose child cleanup is uncertain retains the parent workspace instead of racing those resources. Only after cleanup succeeds does `Store.abortRebase` compare-and-swap the same attempt ID to clear the marker. A missing adapter, malformed marker, wrong plan, recovered or unknown process, cleanup failure, unsafe ancestor, workspace user, or changed marker blocks startup and retains the durable ownership evidence. This ordering is intentionally local-only. The later push integration must extend the durable lifecycle before its first external write and reconcile an ambiguous push outcome; it must not reuse this local completion point as proof that GitHub moved. ## Regression evidence -`test/runner-rebase.test.ts` covers clean one-to-one replay, the already-current no-op, process-group ownership, the whole-operation deadline, foreign conflict resolution, clean sibling files, gitlink-pointer auditing and manual gitlink-conflict routing, embedded-repository refusal, literal replacement-character paths, invalid ledger classification, outside-file, resolver-commit and rebase-control refusal, owned-conflict refusal, cancellation before and during Git, exact cleanup, retained refs, and a read-only source checkout. `test/runner-lifecycle-store.test.ts` covers admission, durable process ownership, compare-and-swap races, resolved-source validation, and owned/foreign provenance. `test/runner-recovery.test.ts` covers marker validation, process termination and restart ordering plus fail-closed absence of a recovery implementation. `test/history.test.ts` and `test/browser/review.spec.ts` cover Unplanned attribution and the visible conflict-resolution label. `test/runner-production.test.ts` covers exact-plan recovery routing. +`test/runner-rebase.test.ts` covers clean one-to-one replay, the already-current no-op, process-group ownership, the whole-operation deadline, foreign conflict resolution, clean sibling files, gitlink-pointer auditing and manual gitlink-conflict routing, embedded-repository refusal, literal replacement-character paths, invalid ledger classification, outside-file, resolver-commit and rebase-control refusal, owned-conflict refusal, cancellation before and during Git, exact cleanup, retained refs, and a read-only source checkout. `test/runner-rebase-conflict.test.ts` covers claim-before-prepare ordering, distinct allocation identities, bounded no-follow host snapshots, nested modify/delete copy-in, copy-back audit, child settlement deadlines, cancellation settlement, cleanup ordering, and retained ownership for unreleased resources. `test/runner-lifecycle-store.test.ts` covers admission, durable process and child ownership, compare-and-swap races, resolved-source validation, and owned/foreign provenance. `test/runner-recovery.test.ts` covers marker validation, live-group fail-closed recovery, exact child-storage recovery, partial-match refusal, and restart ordering. `test/agent-process-group.test.ts` covers delayed-signal suppression after exit and retained ownership for an orphaned group. The real-container suite covers dirty-conflict admission and bounded exact-path export. `test/history.test.ts` and `test/browser/review.spec.ts` cover Unplanned attribution and the visible conflict-resolution label. `test/runner-production.test.ts` covers exact-plan recovery routing and adapter construction. diff --git a/docs/implementation/runner-lifecycle.md b/docs/implementation/runner-lifecycle.md index 5a12e7b5..01d5f7b9 100644 --- a/docs/implementation/runner-lifecycle.md +++ b/docs/implementation/runner-lifecycle.md @@ -267,7 +267,7 @@ codeboost never writes the user's repository. The commits it makes live in a **r | Holder | Owner | Rule | |---|---|---| | Partial output of a stopped writable attempt | F, using a D export | Done in D by #51 item 6: `exportTaskDiff(storage, { base, imageId, maxBytes, signal, onProcessGroup, timeoutMs, metadataBaseline })` (`agents/container/storage.ts`) takes the storage value or a recovery handle and `base`, the full ID of the commit it was seeded from, and returns `{ diff, truncated }`: at most `maxBytes` (1 MiB at most) of raw diff bytes. It checks both volumes' labels, runs Git in a read-only container with no network that writes nothing to either volume, covers edits and new untracked files (agents cannot commit or stage: the metadata is read-only to them, and the export first checks it against the seeder's baseline, refusing if it changed; a recovery handle needs the `metadataBaseline` F recorded), and cuts the output inside the container, so the Docker work stops at the limit. A changed file over 8 MiB, an untracked nested repository, or an untracked path Git will not add (a part named `.git`, or a symlink named `.gitmodules`) is named in a `codeboost:` notice line instead of diffed, so one large file cannot exhaust the export's memory or deadline; untracked symlinks are diffed as links. What Git skips without a word is named in a notice line too: an entry named `.git` (and, if it holds a repository, the new files beside it), a fifo or socket, an ignored untracked path, and anything in a submodule directory. A changed file whose `ident` or `working-tree-encoding` attribute changes what the diff shows is named too. Anything it cannot read fails the export, and a Git failure's reason includes Git's last two error lines. On abort or at its deadline it stops the client and removes the export container before it settles (an `AbortError`, or an `AggregateError` if that removal did not settle); recovery removes a leftover one. The task filesystems are Docker volumes behind a keeper container. F allocates them with `prepareTaskFilesystems` and removes them with `removeTaskFilesystems`; D's supervisor does not remove them at settlement. But F cannot read a volume from the host. So the order is: `settled` → a **D-provided bounded export** (a new D operation, for example `exportTaskDiff(filesystems, maxBytes)`, which runs `git diff` against the commit the storage was seeded from in a read-only container and returns at most 1 MiB) → F saves it → the terminal write stores `diagnostic_ref` → `removeTaskFilesystems` → the slot is freed. **The export has an overall deadline** (60 seconds, passed to D as an `AbortSignal`), because `maxBytes` bounds only the returned data, not the Docker work. On timeout or any export failure, F records "Partial output could not be exported: " as the diagnostic and continues with the terminal write and removal; shutdown and recovery never wait on an export past its deadline. F saves the export to a runner-owned diagnostics directory, with a total byte cap. **Retention never deletes a file an attempt row still references.** When the cap is exceeded, retention first deletes unreferenced files, oldest first. If that is not enough, it takes the oldest referenced file and, in one transaction, sets that row's `diagnostic_ref` to null and appends "(partial output removed by retention)" to its diagnostic. Only after that commit does it delete the file. No row is ever left pointing at a missing file. The attempt row references it as `diagnostic_ref`. If the capture fails, the row records that it failed. The task filesystem is never reused. | -| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, processGroup }`. F3 sets it by compare-and-swap from null with the complete ordered reviewed history, tracks each Git process group until settlement, records the complete ordered mapping plus the intended result-ref value as `prepared` before the first ref write, and records the atomic outcome as `ready`, `uncertain`, or `refused`. Startup recovery aborts every recorded rebase through the runner before it hands the task to I3. Exact owned deletion treats absence as success; `prepared` blocks on a same-valued ref because intent is not ownership, `uncertain` deletes only the exact intended value, `refused` preserves the winner, and `ready` fails closed if its ref changed. A legacy or pre-result marker without an ownership value preserves the ref. | +| Interrupted rebase | F3 | F3 records `rebase in progress` durably before starting a rebase and clears it after. It is stored as `tasks.rebase_in_progress`: null, or JSON `{ attemptId, oldBase, oldHead, onto, oldHistory, startedAt, resultState, resultHead, resultMappings, resolvedConflicts, conflict, processGroup }`. `resolvedConflicts` lists the source commits whose foreign conflicts F4 resolved. `conflict` is null or the exact active child ownership record `{ attemptId, allocationId, networkAllocationId, source }`; recovery must settle and clear that child before it clears the parent. F3 sets the parent marker by compare-and-swap from null with the complete ordered reviewed history, tracks each Git process group until settlement, records the complete ordered mapping plus the intended result-ref value as `prepared` before the first ref write, and records the atomic outcome as `ready`, `uncertain`, or `refused`. An `unsettled` process marker means an escaped output-pipe holder may remain unidentified after restart. Recovery converts both a crash-before-callback `spawning` marker and every recovered exact group to `unsettled` because a crash may precede the durable identity or settlement transition; it first requests identity-bound termination for an exact live group when available and otherwise keeps that group fail-closed. Startup stays closed until an operator stops any unknown process and runs `--release-rebase-process `. Manual release first revalidates the full owner-only, no-symlink ancestor chain and then refuses any visible workspace user. Startup recovery then aborts every recorded rebase through the runner before it hands the task to I3. Exact owned deletion treats absence as success; `prepared` blocks on a same-valued ref because intent is not ownership, `uncertain` deletes only the exact intended value, `refused` preserves the winner, and `ready` fails closed if its ref changed. A legacy or pre-result marker without an ownership value preserves the ref. | | Workspace rebuild | I3 | I3 never rebuilds before both rows above are resolved for the task. | **Process shutdown is a hard stop.** When the process stops, the running task does not finish. Its attempt ends `cancelled` with the reason "Stopped by shutdown". Restart recovery (lane I3) puts the task back in the queue. This is different from "Stop the queue" (lane I), which lets the running task finish. diff --git a/git/clone.ts b/git/clone.ts index 6ff37997..c1b23814 100644 --- a/git/clone.ts +++ b/git/clone.ts @@ -4,6 +4,7 @@ import { lstatSync, mkdtempSync, opendirSync, realpathSync, rmSync } from 'node: import { isAbsolute, join, relative, resolve } from 'node:path'; import type { TaskClone } from '../agents/contract.ts'; import { runInProcessGroup, type ProcessGroup } from '../agents/process-group.ts'; +import { runTrackedProcess, type ProcessGroupLifecycle } from '../agents/tracked-docker.ts'; interface DirectoryIdentity { readonly dev: number; readonly ino: number } interface CloneIdentity { readonly directory: string; readonly root: DirectoryIdentity; readonly metadata: DirectoryIdentity } @@ -122,11 +123,17 @@ function* cloneSteps(options: CloneOptions): Generator + ['EGROUPALIVE', 'ESTDIOHELD'].includes(String((error as NodeJS.ErrnoException | undefined)?.code)); + /** * Prepare an independent committed snapshot. This is trusted staging, not the * writable execution filesystem: D2 must reserve bounded storage and separate @@ -160,6 +167,8 @@ export interface AsyncCloneOptions extends CloneOptions { readonly signal?: AbortSignal; /** Called in the same turn as each Git spawn with its process group, so the caller can record it durably. */ readonly onProcessGroup?: (group: ProcessGroup) => void; + /** Durable ownership around every individual Git subprocess. */ + readonly processLifecycle?: ProcessGroupLifecycle; } /** @@ -189,8 +198,11 @@ async function runCloneSteps(options: AsyncCloneOptions): Promise { next = signal?.aborted ? steps.throw(aborted()) : steps.next(); continue; } - const outcome = await runInProcessGroup('git', [...GIT_OPTIONS, ...step.args], { cwd: step.cwd, env, - timeoutMs: step.timeoutMs, signal, onProcessGroup: options.onProcessGroup, maxBuffer: 1024 * 1024 }); + const processOptions = { cwd: step.cwd, env, timeoutMs: step.timeoutMs, signal, + onProcessGroup: options.onProcessGroup, maxBuffer: 1024 * 1024 }; + const outcome = options.processLifecycle + ? await runTrackedProcess('git', [...GIT_OPTIONS, ...step.args], { ...processOptions, lifecycle: options.processLifecycle }) + : await runInProcessGroup('git', [...GIT_OPTIONS, ...step.args], processOptions); const cancelled = (outcome.error as NodeJS.ErrnoException | undefined)?.code === 'ABORT_ERR'; next = steps.next(outcome.status === 0 ? outcome : { ...outcome, error: cancelled ? aborted() : outcome.error ?? new Error(`git ${step.args[0] ?? ''} failed (exit ${outcome.status}): ${outcome.stderr.trim().slice(0, 512)}`) }); diff --git a/runner/production.ts b/runner/production.ts index 7066078a..090d6a36 100644 --- a/runner/production.ts +++ b/runner/production.ts @@ -20,6 +20,7 @@ import { PullRequestPublisher } from './publish.ts'; import type { ReviewService } from './review.ts'; import { openRunnerRepository, ownerOnlyDirectory, type RunnerRepository } from './runner-repository.ts'; import { GitRebaser } from './rebase.ts'; +import { createForeignConflictResolver } from './rebase-conflict.ts'; import { createTaskWorkspace, workspaceFilesystems } from './workspace.ts'; /** @@ -207,6 +208,9 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd review.runnerRepository = repository.path; return repository; }); + // F3-F4 assemble the trusted local rewrite/conflict engine here so startup can recover its durable markers. No + // production action starts it yet: F5-F6 must first own base/head refresh, admission, checks, push and merge handoff. + // Exposing the raw rebaser before that coordinator exists would let a caller bypass those required guards. const getRebaser = () => rebaserPromise ??= getRepository().then(repository => new GitRebaser({ repository, runnerRoot: config.root, runnerOwner, committer: config.committer, onProcessStarting: attemptId => service.store.setRebaseProcessGroup(rebasePlanKey, attemptId, null, 'spawning'), @@ -216,7 +220,10 @@ export async function setUpRunner(o: { service: ReviewService; capability: Shutd onResultPrepared: (attemptId, head, history, conflicts) => service.store.prepareRebaseResult(rebasePlanKey, attemptId, head, history, conflicts), onResultState: (attemptId, state) => state === 'ready' ? service.store.completeRebaseResult(rebasePlanKey, attemptId) - : service.store.setRebaseResultState(rebasePlanKey, attemptId, state) })); + : service.store.setRebaseResultState(rebasePlanKey, attemptId, state), + resolveForeignConflict: createForeignConflictResolver({ store: service.store, identity, planKey: rebasePlanKey, + repository, runnerOwner, image, token, + limits: { ...EXECUTE_STORAGE, ...config.limits } }) })); const recovery = await recoverStartup({ store: service.store, runnerOwner, runnerRoot: config.root, diagnosticsDir, diagnosticsCapBytes: config.diagnosticsCapBytes, deps: o.recovery ? o.recovery(image) : dRecoveryDeps(image, diff --git a/runner/rebase-conflict.ts b/runner/rebase-conflict.ts new file mode 100644 index 00000000..5c440591 --- /dev/null +++ b/runner/rebase-conflict.ts @@ -0,0 +1,375 @@ +import { randomUUID } from 'node:crypto'; +import { chmodSync, constants, lstatSync, mkdirSync, openSync, closeSync, fstatSync, readFileSync, readlinkSync, + realpathSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { startClaudeInvocation } from '../agents/adapters/claude.ts'; +import type { AgentAdapterOptions, AgentAdapterRequest } from '../agents/adapters/types.ts'; +import { captureInvocation, type InvocationHandle, type TaskClone } from '../agents/contract.ts'; +import { checkConflictTree, inspectTaskChanges, MAXIMUM_DECLARED_LINKS, MAXIMUM_NAME_BYTES, snapshotDeclaredLinks, + type TaskChangeManifest, type TaskTreeCheck } from '../agents/container/changes.ts'; +import { exportTaskPaths, importTaskPaths, prepareTaskFilesystemsAsync, removeTaskFilesystemsAsync, + type ExportedTaskPath, type PreparationOptions, type TaskFilesystems, type TaskStorageLimits } from '../agents/container/storage.ts'; +import { DEFAULT_PROCESS_SETTLEMENT_MS, type ProcessGroup } from '../agents/process-group.ts'; +import { runTrackedProcess, type ProcessGroupLifecycle } from '../agents/tracked-docker.ts'; +import type { PlanIdentity } from '../core/identity.ts'; +import { createTaskCloneAsync } from '../git/clone.ts'; +import { sameContext } from './lifecycle.ts'; +import { RebaseResourcesUnsettled, type RebaseConflictInput } from './rebase.ts'; +import type { RunnerRepository } from './runner-repository.ts'; +import type { Store } from './store.ts'; + +export const MAX_CONFLICT_SNAPSHOT_BYTES = 32 * 1024 * 1024; +export const MAX_CONFLICT_PATH_BYTES = 32 * 1024; +// Reserve the tracked process's complete default stop/drain/pipe budget, plus one second for durable settlement writes. +export const CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS = DEFAULT_PROCESS_SETTLEMENT_MS + 1_000; +const SCHEMA = '{"$schema":"https://json-schema.org/draft/2020-12/schema","title":"codeboost conflict resolution summary","type":"string"}\n'; + +interface ResolverDeps { + clone(options: Parameters[0]): Promise; + allocate: typeof prepareTaskFilesystemsAsync; + snapshotLinks: typeof snapshotDeclaredLinks; + checkTree: typeof checkConflictTree; + start(request: AgentAdapterRequest, token: string, options?: AgentAdapterOptions): InvocationHandle; + inspect: typeof inspectTaskChanges; + exportPaths: typeof exportTaskPaths; + importPaths: typeof importTaskPaths; + remove(filesystems: TaskFilesystems, options?: PreparationOptions): Promise; + removeStaging(path: string, timeoutMs: number, lifecycle: ProcessGroupLifecycle): Promise; +} +const REMOVE_STAGING_SCRIPT = "const fs=require('node:fs');fs.rmSync(fs.readFileSync(0,'utf8'),{recursive:true,force:true})"; +const defaults: ResolverDeps = { + clone: createTaskCloneAsync, allocate: prepareTaskFilesystemsAsync, snapshotLinks: snapshotDeclaredLinks, + checkTree: checkConflictTree, start: startClaudeInvocation, inspect: inspectTaskChanges, + exportPaths: exportTaskPaths, importPaths: importTaskPaths, remove: removeTaskFilesystemsAsync, + removeStaging: async (path, timeoutMs, lifecycle) => { + const outcome = await runTrackedProcess(process.execPath, ['-e', REMOVE_STAGING_SCRIPT], { + cwd: dirname(path), env: {}, input: Buffer.from(path), timeoutMs, lifecycle, + }); + if (outcome.status !== 0) throw new Error('Conflict staging cleanup did not settle.', + { cause: outcome.error ?? new Error(outcome.stderr.trim() || `node exited with status ${outcome.status}.`) }); + }, +}; + +export interface ForeignConflictResolverOptions { + readonly store: Store; + readonly identity: PlanIdentity; + readonly planKey: string; + readonly repository: RunnerRepository; + readonly runnerOwner: string; + /** Lazy so startup recovery can construct the rebaser without building an image merely to abort it. */ + readonly image: () => string; + readonly token: string; + readonly limits: TaskStorageLimits; + readonly deps?: Partial; +} + +const assertRelativePath = (path: string): string[] => { + if (!path || path.startsWith('/') || /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Cn}]/u.test(path) + || Buffer.byteLength(path) > MAXIMUM_NAME_BYTES || path.split('/')[0] === '.git' + || path.split('/').some(part => !part || part === '.' || part === '..')) + throw new Error(`Conflict path ${JSON.stringify(path)} is invalid.`); + return path.split('/'); +}; + +function assertConflictPathSet(paths: readonly string[]): void { + if (!Array.isArray(paths) || paths.length === 0 || paths.length > MAXIMUM_DECLARED_LINKS) + throw new Error(`Conflict paths must contain between 1 and ${MAXIMUM_DECLARED_LINKS} entries.`); + let pathBytes = 0; + for (const path of paths) { + assertRelativePath(path); + pathBytes += Buffer.byteLength(path); + if (pathBytes > MAX_CONFLICT_PATH_BYTES) + throw new Error(`Conflict paths exceed the ${MAX_CONFLICT_PATH_BYTES}-byte command-input limit.`); + } + if (new Set(paths).size !== paths.length) throw new Error('Conflict paths must not repeat.'); +} + +function assertAncestors(root: string, parts: readonly string[]): void { + let current = root; + const rootStat = lstatSync(root); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('Conflict snapshot root is not a plain directory.'); + for (const part of parts.slice(0, -1)) { + current = join(current, part); + const stat = lstatSync(current, { throwIfNoEntry: false }); + if (!stat?.isDirectory() || stat.isSymbolicLink()) + throw new Error(`Conflict path ancestor ${JSON.stringify(part)} is not a plain directory.`); + } +} + +function prepareDestinationAncestors(root: string, parts: readonly string[], createMissing: boolean): boolean { + let current = root; + const rootStat = lstatSync(root); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) + throw new Error('Conflict snapshot root is not a plain directory.'); + for (const part of parts.slice(0, -1)) { + current = join(current, part); + let stat = lstatSync(current, { throwIfNoEntry: false }); + if (!stat) { + if (!createMissing) return false; + mkdirSync(current, { mode: 0o755 }); + stat = lstatSync(current); + } + if (!stat.isDirectory() || stat.isSymbolicLink()) + throw new Error(`Conflict path ancestor ${JSON.stringify(part)} is not a plain directory.`); + } + return true; +} + +function readEntry(root: string, path: string, remaining: number): ExportedTaskPath { + const parts = assertRelativePath(path); + assertAncestors(root, parts); + const target = join(root, ...parts), stat = lstatSync(target, { throwIfNoEntry: false }); + if (!stat) return Object.freeze({ path, type: 'absent' }); + let content: Buffer; + if (stat.isSymbolicLink()) content = Buffer.from(readlinkSync(target, { encoding: 'buffer' })); + else if (stat.isFile()) { + if (stat.size > remaining) throw new Error('Conflict snapshot exceeds its byte limit.'); + const fd = openSync(target, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const opened = fstatSync(fd); + if (!opened.isFile() || opened.dev !== stat.dev || opened.ino !== stat.ino) + throw new Error('Conflict snapshot entry changed while it was read.'); + content = readFileSync(fd); + } finally { closeSync(fd); } + } else throw new Error(`Conflict path ${JSON.stringify(path)} is not a file, symlink or absence.`); + if (content.length > remaining) throw new Error('Conflict snapshot exceeds its byte limit.'); + return Object.freeze({ path, type: stat.isSymbolicLink() ? 'symlink' : 'file', + ...(stat.isFile() ? { executable: (stat.mode & 0o111) !== 0 } : {}), content }); +} + +function writeEntry(root: string, entry: ExportedTaskPath): void { + const parts = assertRelativePath(entry.path); + if (!prepareDestinationAncestors(root, parts, entry.type !== 'absent')) return; + const target = join(root, ...parts), current = lstatSync(target, { throwIfNoEntry: false }); + if (current) { + if (!current.isFile() && !current.isSymbolicLink()) throw new Error(`Conflict path ${JSON.stringify(entry.path)} became a non-file.`); + unlinkSync(target); + } + if (entry.type === 'absent') return; + if (!entry.content) throw new Error('Conflict snapshot entry has no content.'); + if (entry.type === 'symlink') symlinkSync(entry.content, target); + else { + writeFileSync(target, entry.content, { flag: 'wx', mode: entry.executable ? 0o755 : 0o644 }); + chmodSync(target, entry.executable ? 0o755 : 0o644); + } +} + +/** Read only exact leaf entries; no ancestor symlink is followed and the complete snapshot is bounded. */ +function snapshotConflictPaths(source: string, paths: readonly string[], maxBytes = MAX_CONFLICT_SNAPSHOT_BYTES): readonly ExportedTaskPath[] { + assertConflictPathSet(paths); + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > MAX_CONFLICT_SNAPSHOT_BYTES) + throw new Error('Invalid conflict snapshot limit.'); + const entries: ExportedTaskPath[] = []; + let used = 0; + for (const path of paths) { + const pathBytes = Buffer.byteLength(path); + if (used + pathBytes > maxBytes) throw new Error('Conflict snapshot exceeds its byte limit.'); + const entry = readEntry(source, path, maxBytes - used - pathBytes); + used += pathBytes + (entry.content?.length ?? 0); entries.push(entry); + } + return entries; +} + +/** Copy only exact leaf entries; no ancestor symlink is followed and the complete snapshot is bounded before writes. */ +export function copyConflictPaths(source: string, destination: string, paths: readonly string[], + maxBytes = MAX_CONFLICT_SNAPSHOT_BYTES): void { + for (const entry of snapshotConflictPaths(source, paths, maxBytes)) writeEntry(destination, entry); +} + +function assertResolvedManifest(manifest: TaskChangeManifest, files: readonly string[]): void { + if (manifest.metadataChanged || manifest.agentCommits.length || manifest.nestedGitlinkContent.length) + throw new Error('The conflict resolver changed protected repository state.'); + const allowed = new Set(files); + for (const change of manifest.changes) { + if (!allowed.has(change.path) || (change.oldPath !== undefined && !allowed.has(change.oldPath)) || change.underGit + || [change.oldType, change.newType].some(type => type === 'gitlink' || type === 'directory' || type === 'other')) + throw new Error(`The conflict resolver changed an unapproved path or type: ${JSON.stringify(change.path)}.`); + } + for (const change of manifest.linkTargetChanges) + if (!allowed.has(change.link) || change.change !== 'retargeted') + throw new Error(`The conflict resolver wrote through a symbolic link: ${JSON.stringify(change.link)}.`); +} + +const stopReason = (signal: AbortSignal) => + (signal.reason as { code?: unknown } | undefined)?.code === 'ETIMEDOUT' ? 'timeout' as const : 'cancelled' as const; + +export function createForeignConflictResolver(options: ForeignConflictResolverOptions): (input: RebaseConflictInput) => Promise { + const deps = { ...defaults, ...options.deps }; + return async input => { + assertConflictPathSet(input.files); + if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(input.baseHead)) throw new Error('Conflict base must be a full commit ID.'); + const wallRemaining = input.deadline - Date.now(); + if (!Number.isSafeInteger(input.deadline) || !Number.isSafeInteger(wallRemaining) || wallRemaining < 1) + throw new Error('Conflict deadline has passed.'); + // Convert the caller's wall-clock transport value once. Every later stage shares this monotonic boundary even if + // the system clock moves; wall time is reconstructed only for the child invocation contract. + const deadline = performance.now() + wallRemaining; + // A pre-cancelled request owns nothing: check immediately before the first durable child claim. + input.signal?.throwIfAborted(); + const childAttemptId = randomUUID(), allocationId = randomUUID(), networkAllocationId = randomUUID(); + options.store.beginRebaseConflict(options.planKey, input.attemptId, + { attemptId: childAttemptId, allocationId, networkAllocationId, source: input.commit }); + let staging = join(input.repository, `.codeboost-conflict-${childAttemptId}`); + let filesystems: TaskFilesystems | undefined, primary: unknown, handle: InvocationHandle | undefined, retainOwnership = false; + let imageId = ''; + let processGroup: ProcessGroup | 'spawning' | 'unsettled' | null = null; + const processLifecycle: ProcessGroupLifecycle = { + starting: () => { + if (retainOwnership || processGroup !== null) + throw new Error('A prior conflict subprocess has not settled; no later subprocess may start.'); + options.store.setRebaseProcessGroup(options.planKey, input.attemptId, processGroup, 'spawning'); + processGroup = 'spawning'; + }, + started: group => { + options.store.setRebaseProcessGroup(options.planKey, input.attemptId, processGroup, group); + processGroup = group; + }, + settled: owner => { + options.store.setRebaseProcessGroup(options.planKey, input.attemptId, owner, null); + processGroup = null; + }, + unsettled: (owner, reason) => { + retainOwnership = true; + if (reason === 'stdio-held' && owner !== 'spawning') { + options.store.setRebaseProcessGroup(options.planKey, input.attemptId, owner, 'unsettled'); + processGroup = 'unsettled'; + } + }, + }; + const deadlineError = () => Object.assign(new Error('Conflict child settlement exceeded the rebase work deadline.'), + { code: 'ETIMEDOUT' }); + const operationBudget = () => { + const remaining = Math.floor(deadline - performance.now() - CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS); + if (remaining < 1) throw deadlineError(); + return remaining; + }; + const bounded = async (operation: Promise): Promise => { + const remaining = Math.floor(deadline - performance.now()); + if (remaining < 1) { + retainOwnership = true; + void operation.catch(() => { /* durable recovery owns any late result */ }); + throw new RebaseResourcesUnsettled('Conflict child settlement exceeded the rebase work deadline.'); + } + let timer: ReturnType | undefined; + const expired = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + retainOwnership = true; + handle?.cancel('timeout'); + reject(new RebaseResourcesUnsettled('Conflict child settlement exceeded the rebase work deadline.')); + }, remaining); + }); + try { return await Promise.race([operation, expired]); } + finally { + if (timer) clearTimeout(timer); + if (retainOwnership) void operation.catch(() => { /* durable recovery owns any late result */ }); + } + }; + try { + imageId = options.image(); + input.signal?.throwIfAborted(); + mkdirSync(staging, { mode: 0o700 }); + // Profile creation accepts only a canonical cleanup root. Preserve the UUID leaf while resolving any configured + // root aliases (for example macOS /var -> /private/var) before handing the path across subsystem boundaries. + staging = realpathSync(staging); + const clone = await bounded(deps.clone({ source: options.repository.path, parent: staging, + taskId: options.planKey, head: input.baseHead, timeoutMs: operationBudget(), signal: input.signal, processLifecycle })); + const conflictSnapshot = snapshotConflictPaths(input.repository, input.files); + try { + filesystems = await bounded(deps.allocate(clone, options.limits, imageId, + { runnerOwner: options.runnerOwner, attemptId: childAttemptId, allocationId }, + { signal: input.signal, processLifecycle, timeoutMs: operationBudget() })); + } catch (error) { + if (error instanceof AggregateError) retainOwnership = true; + throw error; + } + await bounded(deps.importPaths(filesystems, conflictSnapshot, MAX_CONFLICT_SNAPSHOT_BYTES, + { imageId, signal: input.signal, processLifecycle, timeoutMs: operationBudget() })); + const links = await bounded(deps.snapshotLinks(filesystems!, input.files, + { imageId, signal: input.signal, processLifecycle, timeoutMs: operationBudget() })); + const treeCheck: TaskTreeCheck = await bounded(deps.checkTree(filesystems!, + { base: input.baseHead, paths: input.files, imageId, signal: input.signal, processLifecycle, + timeoutMs: operationBudget() })); + const inputDirectory = join(staging, 'input'); + mkdirSync(inputDirectory, { mode: 0o755 }); + writeFileSync(join(inputDirectory, 'schema.json'), SCHEMA, { mode: 0o444, flag: 'wx' }); + chmodSync(join(inputDirectory, 'schema.json'), 0o444); + const context = options.store.currentContext(options.identity); + const invocation = captureInvocation({ clone, phase: 'fix', vendor: 'claude', approvedArgv: [], + deadline: Date.now() + operationBudget(), + attemptId: childAttemptId, runnerOwner: options.runnerOwner, context }); + const prompt = `Resolve the in-progress rebase conflict in exactly these paths: ${JSON.stringify(input.files)}. ` + + 'Edit only those paths. Do not create commits or change repository metadata. Preserve the intent of both sides and leave each path in its final resolved form.'; + handle = deps.start({ invocation, filesystems, inputDirectory, imageId, prompt, + networkAllocationId, treeCheck, cleanupRoot: staging, processLifecycle }, options.token, + { invocationBudget: operationBudget }); + const cancel = () => handle!.cancel(stopReason(input.signal!)); + if (input.signal?.aborted) cancel(); else input.signal?.addEventListener('abort', cancel, { once: true }); + const result = await bounded(handle.settled).finally(() => input.signal?.removeEventListener('abort', cancel)); + if (processGroup !== null) { + retainOwnership = true; + throw new Error('The conflict resolver could not confirm settlement of its Docker client.'); + } + if (result.unreleased !== undefined) { + retainOwnership = true; + throw new Error('The conflict resolver could not confirm removal of its container resources.'); + } + if (result.attemptId !== childAttemptId || !sameContext(result.context, context)) + throw new Error('The conflict resolver returned a result for another invocation.'); + if (result.exitCode !== 0 || result.stopReason) throw new Error(`The conflict resolver failed${result.stderr ? `: ${JSON.stringify(result.stderr.slice(0, 2048))}` : '.'}`); + input.signal?.throwIfAborted(); + const manifest = await bounded(deps.inspect(filesystems!, { base: input.baseHead, linkSnapshot: links, + imageId, signal: input.signal, processLifecycle, timeoutMs: operationBudget() })); + assertResolvedManifest(manifest, input.files); + const entries = await bounded(deps.exportPaths(filesystems!, input.files, MAX_CONFLICT_SNAPSHOT_BYTES, + { imageId, signal: input.signal, processLifecycle, timeoutMs: operationBudget() })); + input.signal?.throwIfAborted(); + if (entries.length !== input.files.length || entries.some((entry, index) => entry.path !== input.files[index])) + throw new Error('The conflict export did not return the exact approved path set.'); + let exportedBytes = 0; + for (const entry of entries) { + if (!['absent', 'file', 'symlink'].includes(entry.type) + || (entry.type === 'absent' ? entry.content !== undefined || entry.executable !== undefined + : !Buffer.isBuffer(entry.content) || (entry.type === 'file' ? typeof entry.executable !== 'boolean' : entry.executable !== undefined))) + throw new Error('The conflict export returned a malformed entry.'); + if (entry.path === '.gitmodules' && entry.type === 'symlink') + throw new Error('The conflict resolver cannot produce a symlink named .gitmodules.'); + exportedBytes += Buffer.byteLength(entry.path) + (entry.content?.length ?? 0); + if (exportedBytes > MAX_CONFLICT_SNAPSHOT_BYTES) throw new Error('The conflict export exceeded its byte limit.'); + } + for (const entry of entries) { + if (performance.now() >= deadline) throw deadlineError(); + writeEntry(input.repository, entry); + } + if (performance.now() >= deadline) throw deadlineError(); + } catch (error) { + // An AggregateError from any Docker-backed stage means its own cleanup did not settle. Keep the durable child + // claim and all remaining resources for startup recovery instead of racing that still-owned work here. + if (error instanceof AggregateError) retainOwnership = true; + primary = error; + } + const cleanup: unknown[] = []; + if (handle && input.signal?.aborted) handle.cancel(stopReason(input.signal)); + if (filesystems && !retainOwnership) try { + await bounded(deps.remove(filesystems!, { processLifecycle, timeoutMs: operationBudget() })); + } catch (error) { cleanup.push(error); retainOwnership = true; } + if (!retainOwnership) try { + await bounded(deps.removeStaging(staging, operationBudget(), processLifecycle)); + } catch (error) { cleanup.push(error); retainOwnership = true; } + if (!cleanup.length && !retainOwnership) { + try { options.store.clearRebaseConflict(options.planKey, input.attemptId, childAttemptId); } + catch (error) { cleanup.push(error); } + } + if (retainOwnership) { + const cause = primary !== undefined && cleanup.length + ? new AggregateError([primary, ...cleanup], 'Conflict resolution and cleanup failed.', { cause: primary }) + : primary ?? cleanup[0]; + throw new RebaseResourcesUnsettled( + `Conflict resolution retained resources for startup recovery${primary instanceof Error ? `: ${primary.message}` : '.'}`, + cause === undefined ? undefined : { cause }); + } + if (primary && cleanup.length) throw new AggregateError([primary, ...cleanup], 'Conflict resolution and cleanup failed.', { cause: primary }); + if (primary) throw primary; + if (cleanup.length > 1) throw new AggregateError(cleanup, 'Conflict resolver cleanup failed.'); + if (cleanup.length) throw cleanup[0]; + }; +} diff --git a/runner/rebase.ts b/runner/rebase.ts index 8f21a6ff..16642e4b 100644 --- a/runner/rebase.ts +++ b/runner/rebase.ts @@ -20,9 +20,13 @@ export interface RebaseMapping { oldSha: string; newSha: string } export interface RebaseConflictInput { readonly attemptId: string; readonly commit: string; + /** The pinned HEAD whose tree the resolver snapshot is based on. */ + readonly baseHead: string; readonly files: readonly string[]; /** The isolated runner-owned worktree. A production resolver must expose it only through lane D's bounded storage. */ readonly repository: string; + /** Wall-clock form of this rebase operation's already-bounded monotonic work deadline. */ + readonly deadline: number; readonly signal?: AbortSignal; } export interface RebaseResult { @@ -68,6 +72,8 @@ const configuredGit = (): { executable: string; environment: NodeJS.ProcessEnv } /** A conflict is a review outcome, not an infrastructure failure. No rewritten ref is retained. */ export class RebaseConflict extends Error {} +/** The resolver retained child resources for startup recovery, so live cleanup must not remove their parent workspace. */ +export class RebaseResourcesUnsettled extends Error {} /** The durable ref that keeps a completed rewrite reachable in the runner-owned bare repository. */ export function rebaseRef(attemptId: string): string { @@ -174,7 +180,8 @@ export class GitRebaser { const outsideBefore = await this.#outsideConflictState(path, allowed, priorHead, scope); const operationBefore = await this.#rebaseState(path, scope); this.#assertResultCurrent(scope); - await this.#resolveConflict({ attemptId, commit, files, repository: path }, scope); + await this.#resolveConflict({ attemptId, commit, baseHead: priorHead, files, repository: path, + deadline: Date.now() + Math.max(0, Math.floor(scope.workDeadline - performance.now())) }, scope); this.#assertResultCurrent(scope); if (await this.#git(path, ['rev-parse', '--verify', 'REBASE_HEAD^{commit}'], scope) !== commit) throw new RebaseConflict('The conflict resolver changed the rebase operation.'); @@ -227,7 +234,7 @@ export class GitRebaser { result = { oldHead, base: onto, head, mappings, resolvedConflicts }; } catch (error) { primary = error; } const cleanupFailures: unknown[] = []; - if (added || lstatSync(path, { throwIfNoEntry: false })) { + if (!(primary instanceof RebaseResourcesUnsettled) && (added || lstatSync(path, { throwIfNoEntry: false }))) { try { await this.#remove(path, scope); } catch (error) { cleanupFailures.push(error); } } // Cleanup is deliberately uninterruptible, but an abort that arrived while it ran still cancels the operation. @@ -423,6 +430,8 @@ export class GitRebaser { // Await settlement after abort: the resolver owns its container/storage until its promise ends. try { await resolveConflict(Object.freeze({ ...input, signal: controller.signal })); } catch (error) { + // Retained ownership outranks cancellation/deadline reporting: live cleanup must preserve its parent workspace. + if (error instanceof RebaseResourcesUnsettled) throw error; abortExpired(); controller.signal.throwIfAborted(); throw error; @@ -498,6 +507,7 @@ export class GitRebaser { const outcome = await runInProcessGroup(file, args, { cwd, env, timeoutMs, graceMs: 1_000, signal: cleanup ? undefined : scope.signal, input, onProcessGroup: value => { this.#options.onProcessGroup(scope.attemptId, value); group = value; }, maxBuffer, discardExcessOutput, + allowUnsettledReturn: true, }); const code = (outcome.error as NodeJS.ErrnoException | undefined)?.code; let callbackFailure: unknown; diff --git a/runner/recovery.ts b/runner/recovery.ts index f8036b99..eb85ec21 100644 --- a/runner/recovery.ts +++ b/runner/recovery.ts @@ -9,7 +9,7 @@ import type { RebaseMarker, Store } from './store.ts'; import { WRITABLE_KINDS, isUuidV4 } from './lifecycle.ts'; import { partialOutput, saveDiagnostic } from './diagnostics.ts'; import { ownerOnlyDirectory } from './runner-repository.ts'; -import { processIdentity, runInProcessGroup } from '../agents/process-group.ts'; +import { runInProcessGroup } from '../agents/process-group.ts'; import type { DockerOutcome } from '../agents/docker.ts'; /** @@ -98,38 +98,20 @@ export function acquireRunnerLock(databasePath: string, options: { lockRoot?: st } export interface ProcessControl { - /** The numeric group still exists. Exact ownership is revalidated only at the signal boundary. */ + /** The numeric group still exists. */ isAlive(pgid: number): boolean; - /** Revalidate its kernel identity, SIGTERM, SIGKILL after grace, and resolve only once the group has exited. */ + /** Stop the group only through an implementation with identity-bound signalling; otherwise fail closed. */ terminate(pgid: number, identity: string | null, graceMs: number): Promise; } const groupAlive = (pgid: number) => { try { process.kill(-pgid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code !== 'ESRCH'; } }; -const ownsGroupAtSignal = (pgid: number, identity: string | null): boolean => { - if (!groupAlive(pgid)) return false; - const current = processIdentity(pgid); - if (identity === null || current === null) - throw new Error('Could not prove the recorded process group still owns its ID; refusing to signal it.'); - if (current !== identity) throw new Error('The recorded process group identity changed before recovery could signal it.'); - return true; -}; export const hostProcesses: ProcessControl = { isAlive(pgid) { return groupAlive(pgid); }, - async terminate(pgid, identity, graceMs) { - if (!ownsGroupAtSignal(pgid, identity)) return; - try { process.kill(-pgid, 'SIGTERM'); } - catch (error) { if ((error as NodeJS.ErrnoException).code === 'ESRCH') return; throw error; } - const until = performance.now() + graceMs; - while (groupAlive(pgid) && performance.now() < until) await new Promise(resolve => setTimeout(resolve, 50)); - if (ownsGroupAtSignal(pgid, identity)) { - try { process.kill(-pgid, 'SIGKILL'); } - catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; } - } - const settlement = performance.now() + graceMs; - while (groupAlive(pgid) && performance.now() < settlement) await new Promise(resolve => setTimeout(resolve, 50)); - if (groupAlive(pgid)) throw new Error('The recorded process group did not exit after SIGKILL; retaining its ownership.'); + async terminate(pgid) { + if (!groupAlive(pgid)) return; + throw new Error('A recovered process group is still alive; the host cannot signal its reusable numeric ID safely, so ownership is retained.'); }, }; @@ -193,6 +175,45 @@ function validResolvedConflicts(history: unknown, mappings: unknown, conflicts: return new Set(conflicts).size === conflicts.length && conflicts.every(value => typeof value === 'string' && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(value) && history.includes(value) && sources.get(value) !== value); } +interface RecoveryRebase { + planKey: string; + marker: RebaseMarker; +} +function recoveryRebase(planKey: string, value: unknown): RecoveryRebase | null { + const marker = value as Partial | null; + const processGroup = marker?.processGroup, processGroupIdentity = processGroup && typeof processGroup === 'object' + ? processGroup.identity ?? null : null, oldHistory = marker?.oldHistory ?? null, + resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null, + resultState = marker?.resultState ?? 'none', rawResolvedConflicts = marker && Object.hasOwn(marker, 'resolvedConflicts') + ? marker.resolvedConflicts : [], rawConflict = marker && Object.hasOwn(marker, 'conflict') ? marker.conflict : null; + const conflict = rawConflict === null ? null : rawConflict as { + attemptId?: unknown; allocationId?: unknown; networkAllocationId?: unknown; source?: unknown }; + if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldBase) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldHead) || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.onto) || + !Number.isSafeInteger(marker.startedAt) || marker.startedAt! < 0 || + !validOldHistory(marker, oldHistory) || + (oldHistory === null && (resultState !== 'none' || resultHead !== null || resultMappings !== null)) || + (resultHead !== null && (typeof resultHead !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(resultHead))) || + !['none', 'prepared', 'uncertain', 'refused', 'ready'].includes(resultState) || + (resultState === 'none' ? resultHead !== null || resultMappings !== null : resultMappings === null) || + !validRebaseResult(marker, oldHistory, resultHead, resultMappings) || + !validResolvedConflicts(oldHistory, resultMappings ?? [], rawResolvedConflicts) || + (resultState === 'none' && Array.isArray(rawResolvedConflicts) && rawResolvedConflicts.length !== 0) || + (processGroup !== null && (typeof processGroup === 'string' ? !['spawning', 'unsettled'].includes(processGroup) : + (!processGroup || !Number.isSafeInteger(processGroup.pgid) || processGroup.pgid <= 1 || + !Number.isSafeInteger(processGroup.startedAt) || processGroup.startedAt < 0 || + (processGroupIdentity !== null && !/^linux:[0-9a-f-]{36}:\d+$/.test(processGroupIdentity))))) || + (conflict !== null && (!isUuidV4(conflict.attemptId) || !isUuidV4(conflict.allocationId) || + !isUuidV4(conflict.networkAllocationId) || conflict.allocationId === conflict.networkAllocationId || conflict.attemptId === marker.attemptId || + typeof conflict.source !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(conflict.source) || + !Array.isArray(oldHistory) || !oldHistory.includes(conflict.source) || (rawResolvedConflicts as unknown[]).includes(conflict.source) || resultState !== 'none'))) + return null; + return { planKey, marker: { ...marker as RebaseMarker, oldHistory: oldHistory as string[] | null, + resultState, resultHead: resultHead as string | null, + resultMappings: resultMappings as RebaseMarker['resultMappings'], resolvedConflicts: rawResolvedConflicts as string[], + conflict: conflict === null ? null : conflict as RebaseMarker['conflict'] } }; +} const EXPORT_LIMIT = 1024 * 1024; /** @@ -202,6 +223,40 @@ const EXPORT_LIMIT = 1024 * 1024; export async function recoverStartup(o: RecoveryOptions): Promise { const now = o.now ?? Date.now, processes = o.deps.processes ?? hostProcesses; if (!/^[0-9a-f]{32}$/.test(o.runnerOwner)) throw new Error('Invalid runner owner token.'); + // Preflight every rebase marker before recovery terminates a process or changes durable ownership. In particular, + // one unidentified escaped process blocks the whole pass: settling earlier work would make a refused startup partial. + const rawRebases = o.store.rebasesInProgress(); + const rebases = rawRebases.map(value => recoveryRebase(value.planKey, value.marker)); + if (rebases.some(value => value === null)) + throw new RecoveryBlocked('An interrupted rebase has an invalid recovery marker', rawRebases.map(value => value.planKey)); + const activeRebases = rebases as RecoveryRebase[], childIdentities = new Set(), childAllocations = new Set(); + for (const { planKey, marker } of activeRebases) if (marker.conflict) { + if (childIdentities.has(marker.conflict.attemptId) || childAllocations.has(marker.conflict.allocationId) + || childAllocations.has(marker.conflict.networkAllocationId)) + throw new RecoveryBlocked('Interrupted rebases claim the same conflict child resources', [planKey]); + childIdentities.add(marker.conflict.attemptId); childAllocations.add(marker.conflict.allocationId); + childAllocations.add(marker.conflict.networkAllocationId); + } + const unsettled = activeRebases.find(({ marker }) => marker.processGroup === 'unsettled'); + if (unsettled) + throw new RecoveryBlocked(`An interrupted rebase has an unidentified escaped process; stop it, then run --release-rebase-process ${unsettled.marker.attemptId}`, + [unsettled.marker.attemptId]); + // `spawning` means a crash may have happened before the synchronous spawn callback identified the child. A persisted + // exact group proves only that the leader was recorded, not that every descendant settled: the runner may have crashed + // after the leader/group disappeared but before the lifecycle callback durably recorded `unsettled`. Stop a still-live + // exact group while its identity is trustworthy, then require the same explicit release in every ambiguous case. + // Workspace open-file scans cannot prove that an escaped pipe holder no longer exists. + const recoveredProcess = activeRebases.find(({ marker }) => marker.processGroup !== null); + if (recoveredProcess) { + const { planKey, marker } = recoveredProcess; + if (marker.processGroup === null) throw new Error('Recovered process ownership disappeared during startup recovery.'); + const processGroup = marker.processGroup; + if (typeof processGroup !== 'string' && processes.isAlive(processGroup.pgid)) + await processes.terminate(processGroup.pgid, processGroup.identity, o.graceMs ?? 5_000); + o.store.setRebaseProcessGroup(planKey, marker.attemptId, processGroup, 'unsettled'); + throw new RecoveryBlocked(`An interrupted rebase may have an unidentified escaped process; stop it, then run --release-rebase-process ${marker.attemptId}`, + [marker.attemptId]); + } const interrupted = o.store.interruptedAttempts(); // Step 7's input is taken before finalization: an interrupted attempt that started preparation but never saved its group. const unowned = interrupted.filter(a => a.preparationStartedAt !== null && a.preparationPgid === null).map(a => a.id); @@ -212,9 +267,23 @@ export async function recoverStartup(o: RecoveryOptions): Promise isUuidV4(s.attemptId) && isUuidV4(s.allocationId) && o.store.attemptAllocation(s.attemptId) === s.allocationId); - const unmatchedStorage = recovered.storage.filter(s => !matched.includes(s)).map(s => s.attemptId); + const rebaseStorage = new Map(); + for (const storage of recovered.storage) { + const owner = activeRebases.find(value => value.marker.conflict?.attemptId === storage.attemptId + && value.marker.conflict.allocationId === storage.allocationId); + if (!owner) { + const partial = activeRebases.find(value => value.marker.conflict?.attemptId === storage.attemptId + || value.marker.conflict?.allocationId === storage.allocationId); + if (partial) throw new RecoveryBlocked('Recovered storage does not match its conflict child ownership marker', [partial.planKey]); + continue; + } + if (rebaseStorage.has(owner.planKey)) + throw new RecoveryBlocked('An interrupted rebase has more than one recovered handle for its conflict child', [owner.planKey]); + rebaseStorage.set(owner.planKey, storage); + } + const unmatchedStorage = recovered.storage.filter(s => !matched.includes(s) && ![...rebaseStorage.values()].includes(s)).map(s => s.attemptId); // 3. Export phase: stopped writable attempts only, outside any transaction, fixed names, bounded deadline. const exports: Record = {}; ownerOnlyDirectory(o.diagnosticsDir); @@ -243,45 +312,15 @@ export async function recoverStartup(o: RecoveryOptions): Promise | null; - const processGroup = marker?.processGroup, processGroupIdentity = processGroup && typeof processGroup === 'object' - ? processGroup.identity ?? null : null, oldHistory = marker?.oldHistory ?? null, - resultHead = marker?.resultHead ?? null, resultMappings = marker?.resultMappings ?? null, - resultState = marker?.resultState ?? 'none', rawResolvedConflicts = marker && Object.hasOwn(marker, 'resolvedConflicts') - ? marker.resolvedConflicts : []; - if (!marker || !isUuidV4(marker.attemptId) || typeof marker.oldBase !== 'string' || typeof marker.oldHead !== 'string' || typeof marker.onto !== 'string' || - !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldBase) || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.oldHead) || - !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(marker.onto) || - !Number.isSafeInteger(marker.startedAt) || marker.startedAt! < 0 || - !validOldHistory(marker, oldHistory) || - (oldHistory === null && (resultState !== 'none' || resultHead !== null || resultMappings !== null)) || - (resultHead !== null && (typeof resultHead !== 'string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(resultHead))) || - !['none', 'prepared', 'uncertain', 'refused', 'ready'].includes(resultState) || - (resultState === 'none' ? resultHead !== null || resultMappings !== null : resultMappings === null) || - !validRebaseResult(marker, oldHistory, resultHead, resultMappings) || - !validResolvedConflicts(oldHistory, resultMappings ?? [], rawResolvedConflicts) || - (resultState === 'none' && Array.isArray(rawResolvedConflicts) && rawResolvedConflicts.length !== 0) || - (processGroup !== null && (!processGroup || typeof processGroup === 'string' || !Number.isSafeInteger(processGroup.pgid) || processGroup.pgid <= 1 || - !Number.isSafeInteger(processGroup.startedAt) || processGroup.startedAt < 0 || - (processGroupIdentity !== null && !/^linux:[0-9a-f-]{36}:\d+$/.test(processGroupIdentity))))) - throw new RecoveryBlocked('An interrupted rebase has an invalid recovery marker', [rebase.planKey]); - const resolvedConflicts = rawResolvedConflicts as string[]; - if (processGroup) { - if (processes.isAlive(processGroup.pgid)) - await processes.terminate(processGroup.pgid, processGroupIdentity, o.graceMs ?? 5_000); - // A descendant can escape the recorded process group before it is terminated. Whether the group was initially - // alive or dead, prove that no process still uses the workspace before releasing its durable owner. - const workspace = join(o.runnerRoot, o.runnerOwner, 'rebases', marker.attemptId); - const stat = lstatSync(workspace, { throwIfNoEntry: false }); - if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) - throw new RecoveryBlocked('An interrupted rebase workspace is not a plain directory', [workspace]); - const users = stat ? await (o.openFiles ? o.openFiles(workspace) : hostOpenFilesBounded(workspace)) : []; - if (users.length) throw new RecoveryBlocked('A process still uses an interrupted rebase workspace', users); - o.store.setRebaseProcessGroup(rebase.planKey, marker.attemptId, processGroup, null); + const marker = rebase.marker; + await o.deps.abortRebase(rebase.planKey, marker); + if (marker.conflict) { + const storage = rebaseStorage.get(rebase.planKey); + if (storage) await o.deps.removeTaskFilesystems(storage.handle); + o.store.clearRebaseConflict(rebase.planKey, marker.attemptId, marker.conflict.attemptId); } - await o.deps.abortRebase(rebase.planKey, { ...marker, oldHistory, resultState, resultHead, resultMappings, resolvedConflicts, processGroup: null }); if (!o.store.abortRebase(rebase.planKey, marker.attemptId)) throw new RecoveryBlocked('An interrupted rebase changed while recovery aborted it', [rebase.planKey]); } @@ -316,7 +355,8 @@ async function stopped(exporting: Promise, graceMs: number, attemptId: /** --release-preparation: remove an attempt directory only when no process has a file open or a working directory in it. */ export function releasePreparation(o: { store: Store; runnerRoot: string; runnerOwner: string; attemptId: string; openFiles?: (dir: string) => string[] }): void { if (!isUuidV4(o.attemptId) || o.store.attemptOwner(o.attemptId) === null) throw new Error('Unknown attempt.'); - const dir = join(o.runnerRoot, o.runnerOwner, 'attempts', o.attemptId), st = lstatSync(dir, { throwIfNoEntry: false }); + const attempts = ownerOnlyDirectory(o.runnerRoot, o.runnerOwner, 'attempts'); + const dir = join(attempts, o.attemptId), st = lstatSync(dir, { throwIfNoEntry: false }); if (st && (!st.isDirectory() || st.isSymbolicLink())) throw new Error('The attempt path is not a plain directory.'); if (st) { const users = (o.openFiles ?? hostOpenFiles)(dir); @@ -325,6 +365,27 @@ export function releasePreparation(o: { store: Store; runnerRoot: string; runner } if (!o.store.clearPreparationMarker(o.attemptId)) throw new Error('The attempt is not waiting for release.'); } + +/** --release-rebase-process: clear an unidentified escaped-process marker only after explicit operator confirmation. */ +export function releaseRebaseProcess(o: { store: Store; runnerRoot: string; runnerOwner: string; attemptId: string; + openFiles?: (dir: string) => string[] }): void { + if (!isUuidV4(o.attemptId)) throw new Error('Unknown rebase attempt.'); + const matches = o.store.rebasesInProgress().filter(value => { + const marker = value.marker as { attemptId?: unknown; processGroup?: unknown } | null; + return marker?.attemptId === o.attemptId; + }); + if (matches.length !== 1) throw new Error('Unknown rebase attempt.'); + const marker = matches[0]!.marker as { processGroup?: unknown }; + if (marker.processGroup !== 'unsettled') throw new Error('The rebase attempt is not waiting for escaped-process release.'); + const rebases = ownerOnlyDirectory(o.runnerRoot, o.runnerOwner, 'rebases'); + const dir = join(rebases, o.attemptId), st = lstatSync(dir, { throwIfNoEntry: false }); + if (st && (!st.isDirectory() || st.isSymbolicLink())) throw new Error('The rebase path is not a plain directory.'); + if (st) { + const users = (o.openFiles ?? hostOpenFiles)(dir); + if (users.length) throw new RecoveryBlocked('A process is still using the rebase directory', users); + } + o.store.setRebaseProcessGroup(matches[0]!.planKey, o.attemptId, 'unsettled', null); +} /** Processes with a file open or a working directory under dir. Throws if the check cannot run (fail closed). */ export function hostOpenFiles(dir: string): string[] { // /proc links and lsof report resolved paths, so compare against the resolved directory. diff --git a/runner/store.ts b/runner/store.ts index 0bed520e..bd847585 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -95,6 +95,8 @@ export interface RebaseMarker { resultMappings: { oldSha: string; newSha: string }[] | null; /** Source commits whose foreign conflicts were resolved by the sandboxed F4 agent. */ resolvedConflicts: string[]; + /** The one sandboxed child whose Docker/storage resources the parent rebase currently owns. */ + conflict: { attemptId: string; allocationId: string; networkAllocationId: string; source: string } | null; processGroup: { pgid: number; startedAt: number; identity: string | null } | 'spawning' | 'unsettled' | null; } export interface AttemptRecord { @@ -759,7 +761,7 @@ export class Store { assertUuidV4(attemptId, 'Rebase attempt ID'); if (!Number.isSafeInteger(startedAt) || startedAt < 0) throw new Error('Invalid rebase start time.'); const marker: RebaseMarker = { attemptId, oldBase: input.oldBase, oldHead: input.oldHead, onto: input.onto, oldHistory, startedAt, - resultState: 'none', resultHead: null, resultMappings: null, resolvedConflicts: [], processGroup: null }; + resultState: 'none', resultHead: null, resultMappings: null, resolvedConflicts: [], conflict: null, processGroup: null }; const key = identityKey(identity); return this.#transaction(() => { this.#expect(key, expected); @@ -779,6 +781,57 @@ export class Store { return marker; }); } + /** Claim the exact child identity before its first clone, storage, network or container write. */ + beginRebaseConflict(planKey: string, attemptId: string, + input: { attemptId: string; allocationId: string; networkAllocationId: string; source: string }): void { + assertUuidV4(attemptId, 'Rebase attempt ID'); + assertUuidV4(input.attemptId, 'Conflict attempt ID'); + assertUuidV4(input.allocationId, 'Conflict allocation ID'); + assertUuidV4(input.networkAllocationId, 'Conflict network allocation ID'); + sha(input.source); + if (input.attemptId === attemptId) throw new Error('A conflict child needs its own attempt identity.'); + if (input.allocationId === input.networkAllocationId) + throw new Error('Conflict storage and network need distinct allocation identities.'); + this.#transaction(() => { + if (this.#get('SELECT 1 FROM attempts WHERE id=?', input.attemptId)) + throw new GuardRefusal('An existing runner attempt owns this conflict child identity.'); + if (this.#get('SELECT 1 FROM attempts WHERE allocation_id IN (?,?)', input.allocationId, input.networkAllocationId)) + throw new GuardRefusal('An existing runner attempt owns this conflict child allocation.'); + // A child identity or allocation is never allowed to name resources of another active rebase. + for (const row of this.#db.prepare('SELECT plan_key, rebase_in_progress FROM tasks WHERE rebase_in_progress IS NOT NULL').all()) { + const active = decode>(row.rebase_in_progress); + if (row.plan_key !== planKey && active.conflict && + (active.conflict.attemptId === input.attemptId || + [active.conflict.allocationId, active.conflict.networkAllocationId].includes(input.allocationId) || + [active.conflict.allocationId, active.conflict.networkAllocationId].includes(input.networkAllocationId))) + throw new GuardRefusal('Another rebase already owns this conflict child identity.'); + } + const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); + if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); + if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if (marker.conflict) throw new GuardRefusal('A conflict child is already active for this rebase.'); + if (marker.resultState !== 'none' || marker.oldHistory === null || !marker.oldHistory.includes(input.source) + || marker.resolvedConflicts.includes(input.source)) + throw new GuardRefusal('This conflict does not belong to the active rebase.'); + if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', + encode({ ...marker, conflict: { ...input } }), planKey, task.rebase_in_progress as string).changes !== 1) + throw new GuardRefusal('This rebase attempt no longer owns the task.'); + }); + } + /** Release a child only after its invocation, storage and staging files have all settled. */ + clearRebaseConflict(planKey: string, attemptId: string, conflictAttemptId: string): void { + assertUuidV4(attemptId, 'Rebase attempt ID'); assertUuidV4(conflictAttemptId, 'Conflict attempt ID'); + this.#transaction(() => { + const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); + if (marker?.attemptId !== attemptId || marker.conflict?.attemptId !== conflictAttemptId) + throw new GuardRefusal('This conflict child no longer owns the rebase.'); + if (marker.processGroup !== null) + throw new GuardRefusal('The conflict child process has not settled.'); + if (this.#run('UPDATE tasks SET rebase_in_progress=? WHERE plan_key=? AND rebase_in_progress=?', + encode({ ...marker, conflict: null }), planKey, task.rebase_in_progress as string).changes !== 1) + throw new GuardRefusal('This conflict child no longer owns the rebase.'); + }); + } /** Record or clear the exact Git process group currently owned by a rebase. This does not advance task context. */ setRebaseProcessGroup(planKey: string, attemptId: string, expected: RebaseMarker['processGroup'], group: RebaseMarker['processGroup']): void { @@ -809,6 +862,7 @@ export class Store { const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if (marker.conflict) throw new GuardRefusal('The conflict child has not settled.'); if (marker.oldHistory === null || rewrittenHistory.length !== marker.oldHistory.length) throw new GuardRefusal('The rebase result does not cover the complete captured history.'); if (resolvedConflicts.some(value => !marker.oldHistory!.includes(value))) @@ -849,6 +903,7 @@ export class Store { const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if (marker.conflict) throw new GuardRefusal('The conflict child has not settled.'); if (marker.resultState === state) return; if (marker.resultState !== 'prepared' || marker.resultMappings === null) throw new GuardRefusal('The rebase result was not prepared before its ref write.'); @@ -875,6 +930,7 @@ export class Store { if (task.state_version !== expectedTaskStateVersion + 1) throw new GuardRefusal('The task changed during the rebase. Discard its result.'); if (marker?.attemptId !== attemptId) throw new GuardRefusal('This rebase attempt no longer owns the task.'); if (marker.processGroup !== null) throw new GuardRefusal('The rebase Git process has not settled.'); + if (marker.conflict) throw new GuardRefusal('The conflict child has not settled.'); if (marker.resultState !== 'ready' || marker.resultMappings === null || (marker.onto === marker.oldBase ? marker.resultHead !== null : marker.resultHead !== head)) throw new GuardRefusal('The rebase result does not own its retained ref.'); @@ -899,6 +955,7 @@ export class Store { const task = this.#task(planKey), marker = task.rebase_in_progress === null ? null : decode(task.rebase_in_progress); if (marker?.attemptId !== attemptId) return false; if (marker.processGroup !== null) return false; + if (marker.conflict) return false; const encoded = task.rebase_in_progress as string; if (this.#run('UPDATE tasks SET rebase_in_progress=NULL WHERE plan_key=? AND rebase_in_progress=?', planKey, encoded).changes !== 1) return false; if (task.cancel_requested !== null && !this.#closed(task.status as TaskStatus)) this.#closeTask(planKey, 'cancelled', task.cancel_requested as string); diff --git a/test/agent-adapter-cleanup.test.ts b/test/agent-adapter-cleanup.test.ts index 59a004fd..42bd09d9 100644 --- a/test/agent-adapter-cleanup.test.ts +++ b/test/agent-adapter-cleanup.test.ts @@ -6,11 +6,19 @@ const TEST_RUNNER_OWNER = '0123456789abcdef0123456789abcdef'; // When profile creation and its cleanup both fail, the adapter keeps retrying that one cleanup inside the bounded // window. It must not add a second network removal with a fresh deadline on every retry (#51 item 1). const state = vi.hoisted(() => ({ budgets: [] as { at: number; budget: number }[], networkRemovals: 0, - profileFails: true, profileDisposals: 0 })); + profileFails: true, profileDisposals: 0, advanceWall: false, wallOffset: 0, + networkBudget: undefined as number | undefined, profileBudget: undefined as number | undefined, + networkHangs: false })); vi.mock('../agents/network/network.ts', async importOriginal => ({ ...await importOriginal(), - createVendorNetwork: () => ({ name: 'codeboost-egress-claude-x', proxyContainer: 'codeboost-proxy-claude-x', - proxyUrl: 'http://10.254.0.2:3128', vendor: 'claude' }), + createVendorNetwork: async (...args: unknown[]) => { + if (state.advanceWall) state.wallOffset = 60_000; + state.networkBudget = (args[6] as (() => number) | undefined)?.(); + if (state.networkHangs) await new Promise((_resolve, reject) => + (args[4] as AbortSignal).addEventListener('abort', () => reject(new Error('network setup cancelled')), { once: true })); + return { name: 'codeboost-egress-claude-x', proxyContainer: 'codeboost-proxy-claude-x', + proxyUrl: 'http://10.254.0.2:3128', vendor: 'claude' }; + }, removeVendorNetwork: () => { state.networkRemovals += 1; }, })); // The placeholder image is not built here; image trust is not what this test is about. @@ -20,7 +28,8 @@ vi.mock('../agents/container/image.ts', async importOriginal => ({ vi.mock('../agents/container/profile.ts', async importOriginal => { const actual = await importOriginal(); return { ...actual, disposeContainerProfile: async () => { state.profileDisposals += 1; }, - createContainerProfile: () => { + createContainerProfile: (options: { invocationBudget?: () => number }) => { + state.profileBudget = options.invocationBudget?.(); // Otherwise a profile the supervisor refuses at handoff (not issued by the real builder). if (!state.profileFails) return { name: 'codeboost-agent-x', network: {}, policy: {} }; throw new actual.ProfileCreationCleanupError(new Error('schema input is not readable'), @@ -38,6 +47,8 @@ describe('adapter profile-creation cleanup', () => { beforeEach(() => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }); state.budgets = []; state.networkRemovals = 0; state.profileFails = true; state.profileDisposals = 0; + state.advanceWall = false; state.wallOffset = 0; state.networkBudget = undefined; state.profileBudget = undefined; + state.networkHangs = false; }); afterEach(() => { vi.useRealTimers(); }); @@ -77,6 +88,46 @@ describe('adapter profile-creation cleanup', () => { expect(box.result?.stderr).toContain('trusted profile builder'); expect(state.profileDisposals).toBe(1); }); + + it('carries a supplied monotonic budget through real adapter setup after wall time moves forward', async () => { + state.profileFails = false; state.advanceWall = true; + const wall = Date.now(), clock = vi.spyOn(Date, 'now').mockImplementation(() => wall + state.wallOffset); + try { + const invocation = captureInvocation({ runnerOwner: TEST_RUNNER_OWNER, + clone: { id: 'clone', taskId: 'task', directory: '/tmp/task', head: 'a'.repeat(40) }, + phase: 'review', vendor: 'claude', approvedArgv: [], deadline: wall + 5_000, + attemptId: 'adapter-monotonic-handoff', + context: { snapshotId: 's', planId: 'p', planRevision: 1, assignmentId: 'a', referencedCodeHash: 'c', stateVersion: 1 }, + }); + const end = performance.now() + 5_000, invocationBudget = () => Math.ceil(end - performance.now()); + const handle = startClaudeInvocation({ invocation, filesystems: {} as never, inputDirectory: '/unused', + imageId: `sha256:${'a'.repeat(64)}`, prompt: 'unused', networkAllocationId: randomUUID() }, 'token', + { invocationBudget }); + const result = await handle.settled; + expect(result.stopReason).toBe('capture-failure'); // The intentionally fake profile reaches the real handoff. + expect(state.networkBudget).toBeGreaterThan(0); + expect(state.profileBudget).toBeGreaterThan(0); + expect(state.profileDisposals).toBe(1); + } finally { clock.mockRestore(); } + }); + + it('keeps a shorter configured timeout over a longer carried budget during adapter setup', async () => { + state.networkHangs = true; + const invocation = captureInvocation({ runnerOwner: TEST_RUNNER_OWNER, + clone: { id: 'clone', taskId: 'task', directory: '/tmp/task', head: 'a'.repeat(40) }, + phase: 'review', vendor: 'claude', approvedArgv: [], deadline: Date.now() + 60_000, + attemptId: 'adapter-configured-cap', + context: { snapshotId: 's', planId: 'p', planRevision: 1, assignmentId: 'a', referencedCodeHash: 'c', stateVersion: 1 }, + }); + const handle = startClaudeInvocation({ invocation, filesystems: {} as never, inputDirectory: '/unused', + imageId: `sha256:${'a'.repeat(64)}`, prompt: 'unused', networkAllocationId: randomUUID() }, 'token', + { timeoutMs: 1_000, invocationBudget: () => 60_000 }); + const box: { result?: InvocationResult } = {}; + void handle.settled.then(result => { box.result = result; }); + await vi.advanceTimersByTimeAsync(1_001); + expect(box.result?.stopReason).toBe('timeout'); + expect(state.profileBudget).toBeUndefined(); + }); }); describe('adapter start input', () => { diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts index 8bdb47c0..b3d5696a 100644 --- a/test/agent-adapter.test.ts +++ b/test/agent-adapter.test.ts @@ -7,7 +7,7 @@ import { describe, expect, it, vi } from 'vitest'; import { parseClaudeOutput, readPlanningSchema, startClaudeInvocation } from '../agents/adapters/claude.ts'; import { startCodexInvocation } from '../agents/adapters/codex.ts'; import { isInvocationActive, OUTPUT_LIMITS, retainSetupCleanup } from '../agents/adapters/supervisor.ts'; -import { createAdapterInvocationBudget, createInvocationBudget } from '../agents/adapters/types.ts'; +import { capAdapterInvocationBudget, createAdapterInvocationBudget, createInvocationBudget } from '../agents/adapters/types.ts'; import { captureInvocation } from '../agents/contract.ts'; const TEST_RUNNER_OWNER = '0123456789abcdef0123456789abcdef'; @@ -181,4 +181,12 @@ describe('production agent adapters', () => { expect(remaining()).toBeLessThanOrEqual(250); expect(() => createAdapterInvocationBudget(invocation, 10 * 60_000 + 1)).toThrow('ten-minute ceiling'); }); + + it('caps a carried monotonic budget without extending either deadline', () => { + const carriedEnd = performance.now() + 60_000; + const remaining = capAdapterInvocationBudget(() => Math.ceil(carriedEnd - performance.now()), 250); + expect(remaining()).toBeGreaterThan(0); + expect(remaining()).toBeLessThanOrEqual(250); + expect(() => capAdapterInvocationBudget(() => 60_000, 10 * 60_000 + 1)).toThrow('ten-minute ceiling'); + }); }); diff --git a/test/agent-clone-async.test.ts b/test/agent-clone-async.test.ts index 9f326343..09cbed5b 100644 --- a/test/agent-clone-async.test.ts +++ b/test/agent-clone-async.test.ts @@ -6,7 +6,8 @@ import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; // Every Git call passes through; a test can act after one call settles and before the next is requested. -const calls = vi.hoisted(() => ({ made: [] as string[][], afterCall: undefined as undefined | ((args: readonly string[]) => void) })); +const calls = vi.hoisted(() => ({ made: [] as string[][], afterCall: undefined as undefined | ((args: readonly string[]) => void), + trackedFailure: undefined as undefined | 'EGROUPALIVE' | 'ESTDIOHELD' })); vi.mock('../agents/process-group.ts', async importOriginal => { const actual = await importOriginal(); return { ...actual, runInProcessGroup: async (...call: Parameters) => { @@ -16,7 +17,16 @@ vi.mock('../agents/process-group.ts', async importOriginal => { return outcome; } }; }); -const { assertTaskClone, createTaskCloneAsync } = await import('../git/clone.ts'); +vi.mock('../agents/tracked-docker.ts', async importOriginal => { + const actual = await importOriginal(); + return { ...actual, runTrackedProcess: async (...call: Parameters) => { + const outcome = await actual.runTrackedProcess(...call); + if (calls.trackedFailure && call[0] === 'git' && call[1].includes('clone') && outcome.status === 0) + return { ...outcome, status: null, error: Object.assign(new Error('injected unsettled clone'), { code: calls.trackedFailure }) }; + return outcome; + } }; +}); +const { assertTaskClone, cloneFailureRetainsDirectory, createTaskCloneAsync } = await import('../git/clone.ts'); import type { ProcessGroup } from '../agents/process-group.ts'; // The asynchronous clone (#51 item 5): every Git call runs in its own process group, and an abort settles only after @@ -48,11 +58,28 @@ const withStubbornClone = async (root: string, started: string, run: () => Pr try { return await run(); } finally { process.env.PATH = path; } }; afterEach(() => { - calls.afterCall = undefined; calls.made = []; + calls.afterCall = undefined; calls.trackedFailure = undefined; calls.made = []; for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); describe('asynchronous task clone', () => { + it('retains a partial clone only for outcomes that prove process or pipe ownership is unsettled', () => { + expect(cloneFailureRetainsDirectory(Object.assign(new Error('alive'), { code: 'EGROUPALIVE' }))).toBe(true); + expect(cloneFailureRetainsDirectory(Object.assign(new Error('pipe'), { code: 'ESTDIOHELD' }))).toBe(true); + expect(cloneFailureRetainsDirectory(Object.assign(new Error('cancelled'), { code: 'ABORT_ERR' }))).toBe(false); + }); + + it.each(['EGROUPALIVE', 'ESTDIOHELD'] as const)( + 'retains the actual partial clone after a tracked %s outcome', async code => { + const input = fixture(); calls.trackedFailure = code; + const lifecycle = { starting: () => {}, started: () => {}, settled: () => {}, unsettled: () => {} }; + await expect(createTaskCloneAsync({ ...input, processLifecycle: lifecycle })).rejects.toMatchObject({ code }); + const retained = readdirSync(input.parent); + expect(retained).toHaveLength(1); + expect(retained[0]).toMatch(/^codeboost-task-/); + expect(existsSync(join(input.parent, retained[0]!))).toBe(true); + }); + it('clones like the synchronous helper and reports each Git process group, all gone once it settles', async () => { const input = fixture(), groups: ProcessGroup[] = []; const clone = await createTaskCloneAsync({ ...input, onProcessGroup: group => { groups.push(group); } }); diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index d3708c86..6a948dcb 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -1,22 +1,23 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { fixtureGit } from './fixtures/git.ts'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; -import { chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; +import { chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, + symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { captureInvocation, type InvocationInput, type Phase } from '../agents/contract.ts'; import { AGENT_IMAGE, assertBuiltAgentImage, buildAgentImage } from '../agents/container/image.ts'; import { assertContainerProfile, createContainerProfile, disposeContainerProfile, - isContainerProfileAuthentic } from '../agents/container/profile.ts'; + containerProfileResources, isContainerProfileAuthentic } from '../agents/container/profile.ts'; import { createValidatedContainer, disposeValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems, runContainer, startValidatedContainer, hasExactOptions, validateContainer } from '../agents/container/run.ts'; import { createTaskClone } from '../git/clone.ts'; import { hasOwnerLabels } from '../agents/labels.ts'; -import { exportTaskDiff, isRecoveredTaskStorage, prepareTaskFilesystemsAsync, taskFilesystemOwner, EXPORT_SCRIPT, +import { exportTaskDiff, exportTaskPaths, importTaskPaths, isRecoveredTaskStorage, prepareTaskFilesystemsAsync, taskFilesystemOwner, EXPORT_SCRIPT, UnusableRepositoryError } from '../agents/container/storage.ts'; -import { checkTaskTree, commitTaskChanges, inspectTaskChanges, manifestDigest, MAXIMUM_CHANGES, MAXIMUM_NAME_BYTES, snapshotDeclaredLinks, +import { checkConflictTree, checkTaskTree, commitTaskChanges, inspectTaskChanges, manifestDigest, MAXIMUM_CHANGES, MAXIMUM_NAME_BYTES, snapshotDeclaredLinks, TASK_COMMIT_REF, TaskCommitRefused, TaskTreeRefused, type TaskChangeManifest, type TaskTreeCheck } from '../agents/container/changes.ts'; import { TREE_SCRIPT } from '../agents/container/tree-script.ts'; import { recoverLeftovers } from '../agents/recovery.ts'; @@ -24,6 +25,20 @@ import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupE type VendorNetwork } from '../agents/network/network.ts'; import { createClaudeCommand, createIsolationProbeCommand, createPhasePolicy, assertPhasePolicy, type AgentCommand, type IsolationProbe } from '../agents/policy.ts'; +const stagingFault = vi.hoisted(() => ({ chmodPathPrefix: '', realpathPathPrefix: '' })); +vi.mock('node:fs', async importOriginal => { + const actual = await importOriginal(); + return { ...actual, chmodSync: (path: Parameters[0], mode: number) => { + if (typeof path === 'string' && stagingFault.chmodPathPrefix && path.startsWith(stagingFault.chmodPathPrefix) + && path.endsWith('/schema.json')) + throw Object.assign(new Error('staging filesystem is full'), { code: 'ENOSPC' }); + return actual.chmodSync(path, mode); + }, realpathSync: ((path: Parameters[0]) => { + if (typeof path === 'string' && stagingFault.realpathPathPrefix && path.startsWith(stagingFault.realpathPathPrefix)) + throw Object.assign(new Error('staging canonicalization failed'), { code: 'EIO' }); + return actual.realpathSync(path); + }) as typeof actual.realpathSync }; +}); const TEST_RUNNER_OWNER = '0123456789abcdef0123456789abcdef'; const testOwner = (attemptId = 'fixture') => ({ runnerOwner: TEST_RUNNER_OWNER, attemptId, allocationId: randomUUID() }); @@ -82,7 +97,7 @@ const governed = async (captured: InvocationInput, probe: IsolationProbe = 'noop async function profile(data: ReturnType, phase: Phase, command: IsolationProbe | ((policy: ReturnType) => AgentCommand), options: { vendor?: 'codex' | 'claude'; authProbe?: boolean; codexAuthFile?: string; claudeToken?: string; deadlineMs?: number; - treeCheck?: TaskTreeCheck; + treeCheck?: TaskTreeCheck; cleanupRoot?: string; invocationBudget?: () => number; } = {}) { const vendor = options.vendor ?? 'codex'; const captured = invocation(data.clone, phase, vendor, options.deadlineMs); @@ -94,6 +109,8 @@ async function profile(data: ReturnType, phase: Phase, ? options.treeCheck ?? await checkTaskTree(data.filesystems, { base: data.clone.head, operations: [], imageId }) : undefined; const base = await createContainerProfile({ invocation: captured, policy, network, filesystems: data.filesystems, inputDirectory: data.input, command: trustedCommand, imageId, treeCheck, + cleanupRoot: options.cleanupRoot, + invocationBudget: options.invocationBudget, codexAuthFile: vendor === 'codex' ? (options.codexAuthFile ?? data.fakeAuth) : undefined, claudeToken: vendor === 'claude' ? options.claudeToken : undefined }); profiles.push(base); @@ -118,6 +135,37 @@ afterAll(async () => { }, 120_000); describe('real Docker agent isolation', () => { + it.each([ + ['input setup', 'codeboost-input-', 'chmodPathPrefix', 'staging filesystem is full'], + ['auth canonicalization', 'codeboost-auth-', 'realpathPathPrefix', 'staging canonicalization failed'], + ] as const)('removes a newly created profile directory when %s fails', async (_case, prefix, fault, message) => { + const data = fixture(), requestedCleanupRoot = join(data.root, 'profile-construction-failure'); + mkdirSync(requestedCleanupRoot, { mode: 0o700 }); + const cleanupRoot = realpathSync(requestedCleanupRoot); + stagingFault[fault] = `${cleanupRoot}/${prefix}`; + try { + await expect(profile(data, 'planning', 'noop', { cleanupRoot })).rejects.toThrow(message); + expect(readdirSync(cleanupRoot)).toEqual([]); + } finally { stagingFault[fault] = ''; } + }, 60_000); + + it('places temporary profile snapshots beneath an owner-only durable cleanup root', async () => { + const data = fixture(), requestedCleanupRoot = join(data.root, 'profile-staging'); + mkdirSync(requestedCleanupRoot, { mode: 0o700 }); + const cleanupRoot = realpathSync(requestedCleanupRoot); + const valid = await profile(data, 'planning', 'noop', { cleanupRoot }); + const directories = containerProfileResources(valid).filter(resource => resource.kind === 'directory'); + expect(directories).toHaveLength(2); + expect(directories.every(resource => resource.name.startsWith(`${cleanupRoot}/codeboost-`))).toBe(true); + expect(valid.args).toContainEqual(expect.stringMatching( + new RegExp(`^type=bind,source=${cleanupRoot.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/codeboost-input-`))); + + const requestedUnsafeRoot = join(data.root, 'unsafe-profile-staging'); + mkdirSync(requestedUnsafeRoot, { mode: 0o755 }); + const unsafeRoot = realpathSync(requestedUnsafeRoot); + await expect(profile(data, 'planning', 'noop', { cleanupRoot: unsafeRoot })).rejects.toThrow('owner-only directory'); + }, 60_000); + it.each(['planning', 'questions', 'review', 'execute', 'fix'] as const)( '%s applies its enforced worktree access profile', async phase => { const data = fixture(); @@ -1702,6 +1750,49 @@ describe('real Docker agent isolation', () => { return (error as TaskTreeRefused).differences; }; + it('accepts only the exact dirty conflict set and exports exact no-follow leaf entries', async () => { + const data = fixture(); + asAgent(data.filesystems, 'printf resolved > file.txt'); + expect(await checkConflictTree(data.filesystems, { base: data.clone.head, paths: ['file.txt'], imageId })) + .toEqual({ base: data.clone.head, gitlinks: [] }); + asAgent(data.filesystems, 'ln -s file.txt link; rm -f absent; mkdir -p nested; printf gone > nested/deleted; rm -rf nested'); + const exported = await exportTaskPaths(data.filesystems, ['file.txt', 'link', 'absent', 'nested/deleted'], 1024, { imageId }); + expect(exported.map(entry => ({ path: entry.path, type: entry.type, content: entry.content?.toString() }))).toEqual([ + { path: 'file.txt', type: 'file', content: 'resolved' }, + { path: 'link', type: 'symlink', content: 'file.txt' }, + { path: 'absent', type: 'absent', content: undefined }, + { path: 'nested/deleted', type: 'absent', content: undefined }, + ]); + await expect(exportTaskPaths(data.filesystems, ['file.txt'], 2, { imageId })).rejects.toThrow(/Conflict path export/); + asAgent(data.filesystems, 'printf outside > outside.txt'); + expect(await refusal(checkConflictTree(data.filesystems, { base: data.clone.head, paths: ['file.txt'], imageId }))) + .toContain('"outside.txt" differs outside the conflict set'); + }, 180_000); + + it('imports a bounded conflict snapshot only after clean storage allocation', async () => { + const data = fixture(); + await importTaskPaths(data.filesystems, [ + { path: 'file.txt', type: 'file', executable: false, content: Buffer.from('conflicted\n') }, + { path: 'link', type: 'symlink', content: Buffer.from('file.txt') }, + { path: 'nested/added.txt', type: 'file', executable: true, content: Buffer.from('added\n') }, + { path: 'missing/path.txt', type: 'absent' }, + ], 1024, { imageId }); + expect(await checkConflictTree(data.filesystems, + { base: data.clone.head, paths: ['file.txt', 'link', 'nested/added.txt', 'missing/path.txt'], imageId })) + .toEqual({ base: data.clone.head, gitlinks: [] }); + const exported = await exportTaskPaths(data.filesystems, + ['file.txt', 'link', 'nested/added.txt', 'missing/path.txt'], 1024, { imageId }); + expect(exported).toEqual([ + { path: 'file.txt', type: 'file', executable: false, content: Buffer.from('conflicted\n') }, + { path: 'link', type: 'symlink', content: Buffer.from('file.txt') }, + { path: 'nested/added.txt', type: 'file', executable: true, content: Buffer.from('added\n') }, + { path: 'missing/path.txt', type: 'absent' }, + ]); + await expect(importTaskPaths(data.filesystems, + [{ path: '.git/config', type: 'file', executable: false, content: Buffer.from('unsafe') }], 1024, { imageId })) + .rejects.toThrow(/invalid/); + }, 180_000); + it('mounts every gitlink empty and read-only, so an agent\'s write beneath one is refused', async () => { const data = withGitlinks(['sm', 'deps/inner', 'x/y/z']); const check = await checkTaskTree(data.filesystems, { base: data.clone.head, imageId, @@ -2317,6 +2408,16 @@ describe('real Docker agent isolation', () => { await expect(runContainer(late, 60_000)).rejects.toThrow('deadline has passed'); }, 60_000); + it('carries a monotonic invocation budget through real container launch after wall time moves forward', async () => { + const data = fixture(), end = performance.now() + 60_000; + const valid = await profile(data, 'planning', 'noop', { + deadlineMs: 5 * 60_000, invocationBudget: () => Math.ceil(end - performance.now()), + }); + const wall = Date.now(), clock = vi.spyOn(Date, 'now').mockReturnValue(wall + 10 * 60_000); + try { expect(await runContainer(valid, 60_000)).toBe(''); } + finally { clock.mockRestore(); } + }, 60_000); + it('refuses to build a profile once the invocation deadline has passed', async () => { const data = fixture(), trusted = await governed(invocation(data.clone, 'planning', 'codex', 5_000)); const wait = Math.max(0, trusted.invocation.deadline - Date.now() + 500); diff --git a/test/agent-process-group.test.ts b/test/agent-process-group.test.ts index 729317f8..1a4edfb1 100644 --- a/test/agent-process-group.test.ts +++ b/test/agent-process-group.test.ts @@ -1,10 +1,14 @@ import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { EventEmitter } from 'node:events'; +import type { ChildProcess } from 'node:child_process'; +import { describe, expect, it, vi } from 'vitest'; import { runInProcessGroup, type ProcessGroup } from '../agents/process-group.ts'; import { NOT_STARTED } from '../agents/docker.ts'; import { createOutcomeUnknown } from '../agents/client-outcome.ts'; +import { runTrackedProcess, type ProcessGroupOwner } from '../agents/tracked-docker.ts'; +import { createAttachExitGuard, signalAttachedChild, startOwnedAttachClient, watchAttachedChild } from '../agents/adapters/supervisor.ts'; // Preparation subprocesses run in their own process group, and settle only once the whole group has exited (#51 item 5). const env = { PATH: process.env.PATH }; @@ -13,6 +17,118 @@ const groupAlive = (pgid: number) => { }; describe('runInProcessGroup', () => { + it('does not let a delayed attach kill signal a numeric PGID after leader exit', async () => { + vi.useFakeTimers(); + const guard = createAttachExitGuard(), childKill = vi.fn(), groupKill = vi.spyOn(process, 'kill').mockReturnValue(true); + const child = Object.assign(new EventEmitter(), { pid: 999_999, kill: childKill }) as unknown as ChildProcess; + try { + setTimeout(() => signalAttachedChild(guard, child, 'SIGKILL'), 1_000); + guard.markExited(); + await vi.advanceTimersByTimeAsync(1_000); + expect(groupKill).not.toHaveBeenCalled(); + expect(childKill).not.toHaveBeenCalled(); + } finally { groupKill.mockRestore(); vi.useRealTimers(); } + }); + + it('honours synchronous cancellation from attach lifecycle hooks before and after spawn', () => { + let stopped = false, spawned = 0, attached = 0, terminated = 0; + const child = Object.assign(new EventEmitter(), { pid: 999_999, kill: () => true }) as unknown as ChildProcess; + const lifecycle = { + starting: () => {}, + started: () => { stopped = true; }, + settled: () => {}, + unsettled: () => {}, + }; + startOwnedAttachClient({ lifecycle, stopped: () => stopped, + spawn: () => { spawned++; return child; }, attach: (_child, owner, group) => { + attached++; expect(owner).toEqual(group); expect(group?.pgid).toBe(999_999); + }, terminate: () => { terminated++; }, recordFailure: error => { throw error; } }); + expect({ spawned, attached, terminated }).toEqual({ spawned: 1, attached: 1, terminated: 1 }); + + stopped = true; + expect(() => startOwnedAttachClient({ lifecycle: { ...lifecycle, starting: () => {} }, stopped: () => stopped, + spawn: () => { spawned++; return child; }, attach: () => {}, terminate: () => {}, + recordFailure: error => { throw error; } })).toThrow(/before the Docker attach client started/); + expect(spawned).toBe(1); + }); + + it('finishes a spawned attach client on error even when no exit event follows', () => { + const child = new EventEmitter() as unknown as ChildProcess; + let finishes = 0, failures = 0; + const events: string[] = []; + watchAttachedChild(child, (code, signal) => { finishes++; events.push('finish'); expect(code).toBeNull(); expect(signal).toBeNull(); }, + error => { failures++; events.push('failure'); expect(error.message).toBe('kill failed'); }); + child.emit('error', new Error('kill failed')); + expect({ finishes, failures }).toEqual({ finishes: 1, failures: 1 }); + expect(events).toEqual(['finish', 'failure']); + }); + + it('never signals an attach PGID after exit while inherited pipes remain open', async () => { + const child = new EventEmitter() as unknown as ChildProcess; + const guard = createAttachExitGuard(), pipeClosed = Promise.withResolvers(); + let signals = 0; + watchAttachedChild(child, () => { + guard.markExited(); + void pipeClosed.promise.then(() => undefined); + }, error => { throw error; }); + child.emit('exit', 0, null); + // The group is now empty but an escaped descendant still owns a pipe. A deadline firing here must not use the + // reusable numeric PGID; finish may continue waiting for bounded pipe closure without enabling termination. + if (guard.canSignal()) signals++; + expect(signals).toBe(0); + pipeClosed.resolve(); + await pipeClosed.promise; + expect(guard.canSignal()).toBe(false); + }); + + it('composes durable lifecycle ownership with the caller process-group observer', async () => { + let owner: ProcessGroupOwner | null = null; + const events: string[] = []; + const outcome = await runTrackedProcess('sh', ['-c', 'exit 0'], { env, timeoutMs: 5_000, + lifecycle: { + starting: () => { expect(owner).toBeNull(); owner = 'spawning'; events.push('starting'); }, + started: group => { expect(owner).toBe('spawning'); owner = group; events.push('started'); }, + settled: expected => { expect(owner).toEqual(expected); owner = null; events.push('settled'); }, + unsettled: () => { throw new Error('unexpected unsettled process'); }, + }, + onProcessGroup: group => { expect(owner).toEqual(group); events.push('observer'); }, + }); + expect(outcome.status).toBe(0); + expect(events).toEqual(['starting', 'started', 'observer', 'settled']); + expect(owner).toBeNull(); + }); + + it('does not spawn after the durable starting write exhausts the original deadline', async () => { + let spawned = false, settled: ProcessGroupOwner | undefined; + const outcome = await runTrackedProcess('sh', ['-c', 'exit 0'], { env, timeoutMs: 10, + lifecycle: { + starting: () => { const until = performance.now() + 20; while (performance.now() < until); }, + started: () => { throw new Error('a process must not start after its deadline'); }, + settled: owner => { settled = owner; }, + unsettled: () => { throw new Error('an unstarted process cannot remain unsettled'); }, + }, + onProcessGroup: () => { spawned = true; }, + }); + expect((outcome.error as NodeJS.ErrnoException).code).toBe('ETIMEDOUT'); + expect(spawned).toBe(false); + expect(settled).toBe('spawning'); + }); + + it('preserves a pre-spawn timeout when recording its settlement also fails', async () => { + const outcome = await runTrackedProcess('sh', ['-c', 'exit 0'], { env, timeoutMs: 10, + lifecycle: { + starting: () => { const until = performance.now() + 20; while (performance.now() < until); }, + started: () => { throw new Error('must not start'); }, + settled: () => { throw new Error('settlement write failed'); }, + unsettled: () => { throw new Error('must not become unsettled'); }, + }, + }); + expect(outcome).toMatchObject({ status: null, error: { code: 'ETIMEDOUT' } }); + expect(((outcome.error as Error).cause as AggregateError).errors).toMatchObject([ + { message: expect.stringContaining('ETIMEDOUT'), code: 'ETIMEDOUT' }, { message: 'settlement write failed' }, + ]); + }); + it('reports the group in the same turn as the spawn, and returns a normal exit', async () => { let reported: ProcessGroup | undefined; const pending = runInProcessGroup('sh', ['-c', 'echo out; echo err >&2; exit 3'], @@ -50,14 +166,32 @@ describe('runInProcessGroup', () => { expect(groupAlive(group!.pgid)).toBe(false); }); - it('kills what the leader left in its group before settling, even after a normal exit', async () => { - let group: ProcessGroup | undefined; + it('retains ownership without signalling a numeric group after its leader exits', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orphaned-group-')), pidFile = join(dir, 'pid'); const began = performance.now(); - const outcome = await runInProcessGroup('sh', ['-c', 'sleep 60 >/dev/null 2>&1 & echo started'], - { env, timeoutMs: 30_000, onProcessGroup: reported => { group = reported; } }); - expect(outcome).toMatchObject({ status: 0, stdout: 'started\n' }); - expect(groupAlive(group!.pgid)).toBe(false); - expect(performance.now() - began).toBeLessThan(10_000); + const script = `const {spawn}=require('node:child_process'),fs=require('node:fs');` + + `const c=spawn('sleep',['60'],{stdio:'ignore'});fs.writeFileSync(${JSON.stringify(pidFile)},String(c.pid));c.unref();`; + try { + const outcome = await runInProcessGroup(process.execPath, ['-e', script], { + env, timeoutMs: 30_000, allowUnsettledReturn: true, + }); + expect(outcome.status).toBeNull(); + expect((outcome.error as NodeJS.ErrnoException).code).toBe('EGROUPALIVE'); + expect(performance.now() - began).toBeGreaterThanOrEqual(10_000); + expect(() => process.kill(Number(readFileSync(pidFile, 'utf8')), 0)).not.toThrow(); + } finally { + try { process.kill(Number(readFileSync(pidFile, 'utf8')), 'SIGKILL'); } catch {} + rmSync(dir, { recursive: true, force: true }); + } + }, 20_000); + + it('does not release an untracked caller while a descendant remains in the group', async () => { + const script = `const {spawn}=require('node:child_process');spawn('sleep',['0.2'],{stdio:'ignore'}).unref();`; + const began = performance.now(); + const outcome = await runInProcessGroup(process.execPath, ['-e', script], { env, timeoutMs: 5_000 }); + expect(outcome).toMatchObject({ status: 0 }); + expect(outcome.error).toBeUndefined(); + expect(performance.now() - began).toBeGreaterThanOrEqual(150); }); it('bounds a group that ignores SIGTERM: SIGKILL after the grace period, then settles as a timeout', async () => { @@ -113,7 +247,9 @@ describe('runInProcessGroup', () => { + `String(c.pid)); c.unref(); console.log('leader done');`; try { const began = performance.now(); - const outcome = await runInProcessGroup(process.execPath, ['-e', escape], { env, timeoutMs: 30_000 }); + const outcome = await runInProcessGroup(process.execPath, ['-e', escape], { + env, timeoutMs: 30_000, allowUnsettledReturn: true, + }); // The output may be incomplete, so it is not reported as a success; what was read is kept. expect(outcome).toMatchObject({ status: null, stdout: 'leader done\n' }); expect((outcome.error as NodeJS.ErrnoException).code).toBe('ESTDIOHELD'); @@ -124,6 +260,32 @@ describe('runInProcessGroup', () => { } }, 30_000); + it('does not release an untracked caller while a process outside the group holds its output pipe', async () => { + const dir = mkdtempSync(join(tmpdir(), 'escaped-untracked-')), pidFile = join(dir, 'pid'); + const escape = `const c = require('node:child_process').spawn('sleep', ['60'], { detached: true, ` + + `stdio: ['ignore', 'inherit', 'inherit'] }); require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, ` + + `String(c.pid)); c.unref(); console.log('leader done');`; + let settled = false; + try { + const pending = runInProcessGroup(process.execPath, ['-e', escape], { env, timeoutMs: 30_000 }); + void pending.then(() => { settled = true; }); + const until = performance.now() + 5_000; + while (!readFileSync(pidFile, { encoding: 'utf8', flag: 'a+' }).trim()) { + if (performance.now() >= until) throw new Error('escaped pipe holder did not start'); + await new Promise(resolve => setTimeout(resolve, 10)); + } + await new Promise(resolve => setTimeout(resolve, 1_100)); + expect(settled).toBe(false); + process.kill(Number(readFileSync(pidFile, 'utf8')), 'SIGKILL'); + const outcome = await pending; + expect(outcome).toMatchObject({ status: 0, stdout: 'leader done\n' }); + expect(outcome.error).toBeUndefined(); + } finally { + try { process.kill(Number(readFileSync(pidFile, 'utf8')), 'SIGKILL'); } catch { /* already gone */ } + rmSync(dir, { recursive: true, force: true }); + } + }, 10_000); + it('counts time spent recording the group against the deadline', async () => { const began = performance.now(); const outcome = await runInProcessGroup('sleep', ['60'], { env, timeoutMs: 1_000, graceMs: 100, diff --git a/test/agent-recovery.test.ts b/test/agent-recovery.test.ts index 087bf9dd..84ed0ead 100644 --- a/test/agent-recovery.test.ts +++ b/test/agent-recovery.test.ts @@ -161,7 +161,7 @@ describe('recoverLeftovers', () => { expect(names()).toContain(agent!.name); }); - it.each(['export', 'inspect', 'commit'])('removes a leftover %s container with the other transient containers, keeping the storage', async kind => { + it.each(['import', 'export', 'inspect', 'commit'])('removes a leftover %s container with the other transient containers, keeping the storage', async kind => { const mine = attempt(A, 'attempt-a'), storageLabels = mine.objects[0]!.labels; const transient: FakeObject = { kind: 'container', id: id(), name: `codeboost-${kind}-${randomUUID()}`, labels: { ...storageLabels, 'io.codeboost.task-storage': kind } }; diff --git a/test/agent-settlement-supervisor.test.ts b/test/agent-settlement-supervisor.test.ts index c138b15d..8bb88474 100644 --- a/test/agent-settlement-supervisor.test.ts +++ b/test/agent-settlement-supervisor.test.ts @@ -158,6 +158,38 @@ describe('startProfileInvocation bounded cleanup', () => { expect(state.spawned).toBe(0); }); + it('uses a carried monotonic budget after the transported wall deadline moves forward', async () => { + state.disposeOk = true; + const profile = fakeProfile('carried-monotonic-budget'), clock = vi.spyOn(Date, 'now').mockReturnValue(Date.now() + 20 * 60_000); + try { + const end = performance.now() + 5_000; + const handle = startProfileInvocation(profile, { timeoutMs: 5_000, + invocationBudget: () => Math.ceil(end - performance.now()) }); + const box = watch(handle.settled); + await vi.advanceTimersByTimeAsync(0); + expect(state.spawned).toBe(1); + expect(box.result?.stopReason).toBeUndefined(); + expect(box.result?.exitCode).toBe(0); + } finally { clock.mockRestore(); } + }); + + it('does not spawn an attach client after its durable starting hook exhausts the deadline', async () => { + state.disposeOk = true; + const settledOwners: unknown[] = []; + const handle = startProfileInvocation(fakeProfile('attach-start-deadline'), { timeoutMs: 5_000, + processLifecycle: { + starting: () => { vi.advanceTimersByTime(5_001); }, + started: () => { throw new Error('an attach client must not start after its deadline'); }, + settled: owner => { settledOwners.push(owner); }, + unsettled: () => { throw new Error('an unstarted attach client cannot remain unsettled'); }, + } }); + const box = watch(handle.settled); + await vi.advanceTimersByTimeAsync(0); + expect(box.result?.stopReason).toBe('timeout'); + expect(state.spawned).toBe(0); + expect(settledOwners).toContain('spawning'); + }); + it('forwards a cancel made during adapter setup to the supervisor it hands off to', async () => { state.disposeOk = true; const profile = fakeProfile('handoff-cancel'); diff --git a/test/agent-storage-async.test.ts b/test/agent-storage-async.test.ts index 380accfb..720bc6f1 100644 --- a/test/agent-storage-async.test.ts +++ b/test/agent-storage-async.test.ts @@ -31,7 +31,7 @@ vi.mock('node:child_process', async importOriginal => { vi.mock('../agents/container/image.ts', async importOriginal => ({ ...await importOriginal(), assertBuiltAgentImage: () => {}, })); -const { adoptRecoveredTaskStorage, exportTaskDiff, hasLiveTaskStorage, prepareTaskFilesystemsAsync, +const { adoptRecoveredTaskStorage, exportTaskDiff, hasLiveTaskStorage, importTaskPaths, prepareTaskFilesystemsAsync, removeTaskFilesystems, removeTaskFilesystemsAsync } = await import('../agents/container/storage.ts'); const { createTaskClone } = await import('../git/clone.ts'); const { commitTaskChanges, inspectTaskChanges, manifestDigest, MAXIMUM_BUNDLE_BYTES, MAXIMUM_DECLARED_LINKS, @@ -263,6 +263,32 @@ describe('asynchronous task storage allocation', () => { }); }); +describe('conflict path import', () => { + it('starts no Docker work when snapshot encoding consumes the remaining monotonic budget', async () => { + const filesystems = await prepareTaskFilesystemsAsync(clone(), LIMITS, IMAGE, owner()); + calls.made = []; + let now = 0; + const clock = vi.spyOn(performance, 'now').mockImplementation(() => now); + const content = Buffer.from('conflicted\n'); + Object.defineProperty(content, 'toString', { value: function(this: Buffer, encoding?: BufferEncoding) { + now = 2; + return Buffer.prototype.toString.call(this, encoding); + } }); + const starting = vi.fn(); + try { + await expect(importTaskPaths(filesystems, + [{ path: 'file.txt', type: 'file', executable: false, content }], 1024, { imageId: IMAGE, timeoutMs: 1, + processLifecycle: { starting, started: vi.fn(), settled: vi.fn(), unsettled: vi.fn() } })) + .rejects.toThrow(/overall deadline/); + expect(starting).not.toHaveBeenCalled(); + expect(calls.made).toEqual([]); + } finally { + clock.mockRestore(); + removeTaskFilesystems(filesystems); + } + }); +}); + describe('task diff export', () => { const BASE = 'b'.repeat(40); const exports = () => stored().filter(file => file.startsWith('codeboost-export-')); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts index 212ac152..1244c342 100644 --- a/test/agent-supervisor.test.ts +++ b/test/agent-supervisor.test.ts @@ -17,6 +17,7 @@ import { createValidatedContainer, disposeValidatedContainer, prepareTaskFilesys startValidatedContainer } from '../agents/container/run.ts'; import { createVendorNetwork } from '../agents/network/network.ts'; import { createIsolationProbeCommand, createPhasePolicy, type IsolationProbe } from '../agents/policy.ts'; +import type { ProcessGroupLifecycle, ProcessGroupOwner } from '../agents/tracked-docker.ts'; import { createTaskClone } from '../git/clone.ts'; const TEST_RUNNER_OWNER = '0123456789abcdef0123456789abcdef'; const testOwner = (attemptId = 'fixture') => ({ runnerOwner: TEST_RUNNER_OWNER, attemptId, allocationId: randomUUID() }); @@ -100,6 +101,26 @@ describe('container invocation supervisor', () => { expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); }, 60_000); + it('owns every validation, create, attach and cleanup Docker client without overlap', async () => { + const current = await profile(fixture(), 'finite-output', 'tracked-docker-clients'); + let owner: ProcessGroupOwner | null = null; + const events: string[] = []; + const lifecycle: ProcessGroupLifecycle = { + starting: () => { expect(owner).toBeNull(); owner = 'spawning'; events.push('starting'); }, + started: group => { expect(owner).toBe('spawning'); owner = group; events.push(`started:${group.pgid}`); }, + settled: expected => { expect(owner).toEqual(expected); owner = null; events.push('settled'); }, + unsettled: () => { throw new Error('Docker client unexpectedly remained unsettled.'); }, + }; + const result = await startProfileInvocation(current, { processLifecycle: lifecycle }).settled; + expect(result).toMatchObject({ exitCode: 0 }); + expect(result.stopReason).toBeUndefined(); + expect(owner).toBeNull(); + const starts = events.filter(event => event === 'starting').length; + expect(starts).toBeGreaterThan(10); + expect(events.filter(event => event.startsWith('started:'))).toHaveLength(starts); + expect(events.filter(event => event === 'settled')).toHaveLength(starts); + }, 60_000); + it.each([ ['infinite-stdout', { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 96 * 1024 }], ['infinite-stderr', { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 96 * 1024 }], diff --git a/test/cli.test.ts b/test/cli.test.ts index 78ad8561..aaefcd9e 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -86,13 +86,15 @@ it('refuses a runner without github.baseBranch before the lock and before the to expect([missing.status, missing.stderr.trim()]).toEqual([1, RUNNER_NEEDS_GITHUB]); } finally { held.release(); } }); -it('refuses a bad port and an unknown preparation with a message, never a stack', () => { +it('refuses a bad port and unknown manual-release targets with a message, never a stack', () => { const { run } = review(); const port = run('--port', 'x'); expect([port.status, port.stderr.trim()]).toEqual([1, 'Invalid port.']); for (let n = 0; n < 2; n++) { const release = run('--release-preparation', randomUUID()); expect([release.status, release.stderr.trim()]).toEqual([1, 'Unknown attempt.']); + const rebase = run('--release-rebase-process', randomUUID()); + expect([rebase.status, rebase.stderr.trim()]).toEqual([1, 'Unknown rebase attempt.']); } }); @@ -128,6 +130,10 @@ it('reports that --release-preparation needs --config, instead of printing help const result = spawnSync(process.execPath, [cli, '--release-preparation', randomUUID()], { encoding: 'utf8', timeout: 20_000 }); expect([result.status, result.stderr.trim()]).toEqual([1, '--release-preparation needs --config.']); }); +it('reports that --release-rebase-process needs --config, instead of printing help and exiting 0', () => { + const result = spawnSync(process.execPath, [cli, '--release-rebase-process', randomUUID()], { encoding: 'utf8', timeout: 20_000 }); + expect([result.status, result.stderr.trim()]).toEqual([1, '--release-rebase-process needs --config.']); +}); it('ignores a runner block in a demo configuration opened with --config, and serves the demo', async () => { const root = mkdtempSync(join(tmpdir(), 'codeboost-cli-')); roots.push(root); const demo = createDemo(join(root, 'demo')); diff --git a/test/runner-lifecycle-store.test.ts b/test/runner-lifecycle-store.test.ts index f39c4d9f..4fa406ff 100644 --- a/test/runner-lifecycle-store.test.ts +++ b/test/runner-lifecycle-store.test.ts @@ -545,7 +545,7 @@ describe('pre-merge rebase ownership', () => { { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)], startedAt: 123 }); expect(store.getTask(identity).rebaseInProgress).toEqual({ attemptId: marker.attemptId, oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)], startedAt: 123, - resultState: 'none', resultHead: null, resultMappings: null, resolvedConflicts: [], processGroup: null }); + resultState: 'none', resultHead: null, resultMappings: null, resolvedConflicts: [], conflict: null, processGroup: null }); const group = { pgid: 4242, startedAt: 456, identity: 'linux:00000000-0000-0000-0000-000000000000:1' }; expect(() => store.setRebaseProcessGroup(store.getTask(identity).planKey, marker.attemptId, null, { pgid: 1, startedAt: 456, identity: null })).toThrow(/Invalid rebase process group/); @@ -569,6 +569,34 @@ describe('pre-merge rebase ownership', () => { expect(() => store.setAssignment(identity, store.getTask(identity).stateVersion, 'other', 'other-head')).toThrow(/rebase is in progress/); }); + it('keeps a unique conflict child durable until its resources settle', () => { + const { store } = fixture(); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const marker = store.beginRebase(identity, reviewed(store), store.getTask(identity).stateVersion, + { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] }); + const planKey = store.getTask(identity).planKey, child = randomUUID(), allocation = randomUUID(), networkAllocation = randomUUID(); + expect(() => store.beginRebaseConflict(planKey, marker.attemptId, + { attemptId: marker.attemptId, allocationId: allocation, networkAllocationId: networkAllocation, source: oid(2) })).toThrow(/own attempt identity/); + expect(() => store.beginRebaseConflict(planKey, marker.attemptId, + { attemptId: child, allocationId: allocation, networkAllocationId: allocation, source: oid(2) })).toThrow(/distinct allocation/); + store.beginRebaseConflict(planKey, marker.attemptId, + { attemptId: child, allocationId: allocation, networkAllocationId: networkAllocation, source: oid(2) }); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { + attemptId: child, allocationId: allocation, networkAllocationId: networkAllocation, source: oid(2) } }); + expect(() => store.beginRebaseConflict(planKey, marker.attemptId, + { attemptId: randomUUID(), allocationId: randomUUID(), networkAllocationId: randomUUID(), source: oid(2) })).toThrow(/already active/); + expect(() => store.prepareRebaseResult(planKey, marker.attemptId, oid(4), [oid(4)], [oid(2)])).toThrow(/child has not settled/); + expect(store.abortRebase(planKey, marker.attemptId)).toBe(false); + expect(() => store.clearRebaseConflict(planKey, marker.attemptId, randomUUID())).toThrow(/no longer owns/); + store.setRebaseProcessGroup(planKey, marker.attemptId, null, 'spawning'); + expect(() => store.clearRebaseConflict(planKey, marker.attemptId, child)).toThrow(/process has not settled/); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { attemptId: child }, processGroup: 'spawning' }); + store.setRebaseProcessGroup(planKey, marker.attemptId, 'spawning', null); + store.clearRebaseConflict(planKey, marker.attemptId, child); + expect(store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null }); + expect(store.abortRebase(planKey, marker.attemptId)).toBe(true); + }); + it('publishes only a current attempt and preserves owned and foreign ledger provenance', () => { const { store } = fixture(); store.recordHistory(identity, { revision: 1, snapshotId: store.getSnapshot(identity).id }, oid(1), oid(4), [ diff --git a/test/runner-rebase-conflict.test.ts b/test/runner-rebase-conflict.test.ts new file mode 100644 index 00000000..da334a03 --- /dev/null +++ b/test/runner-rebase-conflict.test.ts @@ -0,0 +1,455 @@ +import { randomUUID } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, readlinkSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { InvocationHandle, InvocationResult, TaskClone } from '../agents/contract.ts'; +import type { AgentAdapterRequest } from '../agents/adapters/types.ts'; +import type { TaskChangeManifest, TaskTreeCheck } from '../agents/container/changes.ts'; +import type { ExportedTaskPath, TaskFilesystems } from '../agents/container/storage.ts'; +import type { ProcessGroupLifecycle } from '../agents/tracked-docker.ts'; +import { DEFAULT_PROCESS_SETTLEMENT_MS } from '../agents/process-group.ts'; +import type { Plan, PlanContext } from '../core/plan.ts'; +import type { AsyncCloneOptions } from '../git/clone.ts'; +import { CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS, copyConflictPaths, createForeignConflictResolver, + MAX_CONFLICT_PATH_BYTES } from '../runner/rebase-conflict.ts'; +import { RebaseResourcesUnsettled, type RebaseConflictInput } from '../runner/rebase.ts'; +import type { RunnerRepository } from '../runner/runner-repository.ts'; +import { Store } from '../runner/store.ts'; + +const identity = { repositoryId: 'repo', taskId: 'task', planId: 'plan' }; +const context: PlanContext = { identity, issue: 22, baseEntries: [{ path: 'conflict.txt', kind: 'file' }], pathKey: p => p, allowedCommands: [] }; +const plan: Plan = { schema_version: 1, revision: 1, issue: 22, summary: 'Resolve', questions: [], items: [{ id: 'P1', title: 'Resolve', intent: 'Resolve', + files: [{ path: 'conflict.txt', kind: 'edit', renamed_from: null, change: 'Resolve' }], acceptance: [{ type: 'check', text: 'Works' }], depends_on: [] }] }; +const oid = (n: number) => n.toString(16).padStart(40, '0'); +const roots: string[] = [], stores: Store[] = []; +afterEach(() => { vi.restoreAllMocks(); for (const store of stores.splice(0)) store.close(); for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'codeboost-conflict-resolver-')); roots.push(root); + const store = new Store(join(root, 'state.sqlite')); stores.push(store); + store.createPlan(JSON.stringify(plan), 'json', context, oid(1), oid(2)); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(identity), expected = { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(identity) }; + const marker = store.beginRebase(identity, expected, store.getTask(identity).stateVersion, + { oldBase: oid(1), oldHead: oid(2), onto: oid(3), oldHistory: [oid(2)] }); + const repository = join(root, 'rebase'); mkdirSync(repository); writeFileSync(join(repository, 'conflict.txt'), 'conflicted\n'); + return { root, store, marker, planKey: store.getTask(identity).planKey, repository }; +} + +const manifest = (changes: TaskChangeManifest['changes'] = [{ path: 'conflict.txt', kind: 'modify', oldType: 'file', newType: 'file', underGit: false, ignored: false }]) => ({ + base: oid(2), changes, agentCommits: [], metadataChanged: false, linkTargetChanges: [], nestedGitlinkContent: [], digest: 'd'.repeat(64), +}) satisfies TaskChangeManifest; +type ProcessOptions = { processLifecycle?: ProcessGroupLifecycle; timeoutMs?: number }; + +function deps(f: ReturnType, over: Record = {}) { + const events: string[] = []; + let request: AgentAdapterRequest | undefined; + let storageAllocationId: string | undefined; + let imported: readonly ExportedTaskPath[] | undefined; + const storage = { keeper: 'keeper' } as TaskFilesystems; + const d = { + clone: async (options: AsyncCloneOptions) => { + events.push('clone'); + for (const group of [{ pgid: 4242, startedAt: 1, identity: null }, { pgid: 4343, startedAt: 2, identity: null }]) { + options.processLifecycle?.starting(); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { source: oid(2) }, processGroup: 'spawning' }); + options.processLifecycle?.started(group); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: group }); + options.processLifecycle?.settled(group); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: null }); + } + const directory = join(options.parent, 'clone'); mkdirSync(directory); mkdirSync(join(directory, '.git')); writeFileSync(join(directory, 'conflict.txt'), 'base\n'); + return { id: randomUUID(), taskId: f.planKey, directory, head: oid(2) } as TaskClone; + }, + allocate: async (_clone: TaskClone, _limits: unknown, _image: string, owner: { allocationId: string }, options: ProcessOptions) => { + events.push(`allocate:${readFileSync((_clone as TaskClone).directory + '/conflict.txt', 'utf8').trim()}`); + storageAllocationId = owner.allocationId; + const group = { pgid: 5252, startedAt: 2, identity: null }; + options.processLifecycle?.starting(); options.processLifecycle?.started(group); options.processLifecycle?.settled(group); + return storage; + }, + importPaths: async (_storage: TaskFilesystems, entries: readonly ExportedTaskPath[]) => { + imported = entries; + events.push(`import:${entries[0]?.content?.toString().trim() ?? entries[0]?.type}`); + }, + snapshotLinks: async () => ({ links: [], recorded: {}, linkArgs: [] }) as never, + checkTree: async (_filesystems: TaskFilesystems, options: ProcessOptions) => { + const group = { pgid: 6262, startedAt: 3, identity: null }; + options.processLifecycle?.starting(); options.processLifecycle?.started(group); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: { pgid: 6262 } }); + options.processLifecycle?.settled(group); + return { base: oid(2), gitlinks: [] } as TaskTreeCheck; + }, + start: (value: typeof request) => { + request = value; events.push('start'); + return { attemptId: value!.invocation.attemptId, cancel: vi.fn(), settled: Promise.resolve({ + attemptId: value!.invocation.attemptId, context: value!.invocation.context, exitCode: 0, signal: null, stdout: '', stderr: '', + } satisfies InvocationResult) } satisfies InvocationHandle; + }, + inspect: async () => manifest(), + exportPaths: async () => [{ path: 'conflict.txt', type: 'file', executable: false, content: Buffer.from('resolved\n') }] as readonly ExportedTaskPath[], + remove: async () => { events.push('remove'); expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: expect.any(Object) }); }, + removeStaging: async (path: string) => { rmSync(path, { recursive: true }); }, + ...over, + }; + return { d, events, storage, get request() { return request; }, get storageAllocationId() { return storageAllocationId; }, + get imported() { return imported; } }; +} + +const input = (f: ReturnType, signal?: AbortSignal): RebaseConflictInput => ({ attemptId: f.marker.attemptId, + commit: oid(2), baseHead: oid(2), files: ['conflict.txt'], repository: f.repository, deadline: Date.now() + 60_000, signal }); + +describe('production foreign conflict resolver', () => { + it('reserves the complete default tracked-process settlement budget plus its durable-write margin', () => { + expect(CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS).toBe(DEFAULT_PROCESS_SETTLEMENT_MS + 1_000); + expect(DEFAULT_PROCESS_SETTLEMENT_MS).toBe(16_000); + }); + + it.each([-60_000, 60_000])('keeps every stage and child invocation on one monotonic deadline when wall time moves by %i ms', async wallStep => { + vi.useFakeTimers(); + try { + const f = fixture(), x = deps(f), budgets: number[] = []; + const clone = x.d.clone, allocate = x.d.allocate; + x.d.clone = async (options: AsyncCloneOptions) => { + budgets.push(options.timeoutMs!); + const result = await clone(options); + vi.setSystemTime(Date.now() + wallStep); + return result; + }; + x.d.allocate = async (...args: Parameters) => { + budgets.push(args[4].timeoutMs!); + return allocate(...args); + }; + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', + limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await resolve({ ...input(f), deadline: Date.now() + CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS + 5_000 }); + expect(budgets).toHaveLength(2); + expect(budgets[1]).toBeLessThanOrEqual(budgets[0]!); + expect(x.request!.invocation.deadline - Date.now()).toBeGreaterThan(0); + expect(x.request!.invocation.deadline - Date.now()).toBeLessThanOrEqual(budgets[0]!); + } finally { vi.useRealTimers(); } + }); + + it('carries the resolver monotonic budget into adapter setup across a later wall-clock step', async () => { + vi.useFakeTimers(); + try { + const f = fixture(), x = deps(f), start = x.d.start; + x.d.start = ((value: AgentAdapterRequest, _token: string, + options?: { invocationBudget?: () => number }) => { + vi.setSystemTime(Date.now() + 60_000); + expect(options?.invocationBudget?.()).toBeGreaterThan(0); + return start(value); + }) as typeof x.d.start; + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', + limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + + await resolve({ ...input(f), deadline: Date.now() + CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS + 5_000 }); + } finally { vi.useRealTimers(); } + }); + + it('does not claim a conflict child for an already-aborted request', async () => { + const f = fixture(), x = deps(f), reason = new Error('cancelled before conflict admission'); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: vi.fn(() => 'sha256:' + 'b'.repeat(64)), token: 'secret', + limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f, AbortSignal.abort(reason)))).rejects.toBe(reason); + expect(x.events).toEqual([]); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null, processGroup: null }); + }); + + it('bounds the complete host snapshot before changing the destination', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-conflict-copy-')); roots.push(root); + const source = join(root, 'source'), destination = join(root, 'destination'); + mkdirSync(source); mkdirSync(destination); + writeFileSync(join(source, 'a'), '12345'); writeFileSync(join(source, 'b'), '67890'); + writeFileSync(join(destination, 'a'), 'old-a'); writeFileSync(join(destination, 'b'), 'old-b'); + expect(() => copyConflictPaths(source, destination, ['a', 'b'], 8)).toThrow(/byte limit/); + expect(readFileSync(join(destination, 'a'), 'utf8')).toBe('old-a'); + expect(readFileSync(join(destination, 'b'), 'utf8')).toBe('old-b'); + }); + + it('never follows a conflict path through a host symlink ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-conflict-link-')); roots.push(root); + const source = join(root, 'source'), destination = join(root, 'destination'), outside = join(root, 'outside'); + mkdirSync(source); mkdirSync(destination); mkdirSync(outside); writeFileSync(join(outside, 'file'), 'outside'); + symlinkSync(outside, join(source, 'link')); mkdirSync(join(destination, 'link')); + expect(() => copyConflictPaths(source, destination, ['link/file'])).toThrow(/ancestor/); + }); + + it('recreates missing plain destination ancestors for a nested modify/delete conflict', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-conflict-nested-')); roots.push(root); + const source = join(root, 'source'), destination = join(root, 'destination'); + mkdirSync(source); mkdirSync(destination); mkdirSync(join(source, 'dir')); + writeFileSync(join(source, 'dir', 'file'), 'modified side\n'); + copyConflictPaths(source, destination, ['dir/file']); + expect(readFileSync(join(destination, 'dir', 'file'), 'utf8')).toBe('modified side\n'); + }); + + it('does not recreate missing ancestors when copying an absent nested conflict path', () => { + const root = mkdtempSync(join(tmpdir(), 'codeboost-conflict-delete-')); roots.push(root); + const source = join(root, 'source'), destination = join(root, 'destination'); + mkdirSync(source); mkdirSync(destination); mkdirSync(join(source, 'dir')); + copyConflictPaths(source, destination, ['dir/missing']); + expect(readdirSync(destination)).toEqual([]); + }); + + it('rejects an invalid conflict set before claiming ownership or preparing resources', async () => { + const f = fixture(), image = vi.fn(() => 'sha256:' + 'b'.repeat(64)), x = deps(f); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), image, + token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve({ ...input(f), files: ['.git/config'] })).rejects.toThrow(/invalid/); + expect(image).not.toHaveBeenCalled(); expect(x.events).toEqual([]); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null }); + }); + + it('rejects a conflict path set that cannot fit safely in one provider argument before claiming ownership', async () => { + const f = fixture(), image = vi.fn(() => 'sha256:' + 'b'.repeat(64)), x = deps(f); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), image, + token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + const files = Array.from({ length: Math.ceil(MAX_CONFLICT_PATH_BYTES / 1000) + 1 }, (_, index) => + `${String(index).padStart(3, '0')}-${'x'.repeat(995)}`); + await expect(resolve({ ...input(f), files })).rejects.toThrow(/command-input limit/); + expect(image).not.toHaveBeenCalled(); expect(x.events).toEqual([]); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null }); + }); + + it('allocates from the clean clone before importing the conflict snapshot, then copies back only the audited path', async () => { + const f = fixture(), x = deps(f); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await resolve(input(f)); + expect(x.events).toEqual(['clone', 'allocate:base', 'import:conflicted', 'start', 'remove']); + expect(x.imported).toEqual([{ path: 'conflict.txt', type: 'file', executable: false, content: Buffer.from('conflicted\n') }]); + expect(x.request).toMatchObject({ invocation: { phase: 'fix', approvedArgv: [], runnerOwner: 'a'.repeat(32) }, + prompt: expect.stringContaining('["conflict.txt"]'), cleanupRoot: expect.stringContaining('.codeboost-conflict-') }); + expect(x.request!.networkAllocationId).not.toBe(x.storageAllocationId); + expect(readFileSync(join(f.repository, 'conflict.txt'), 'utf8')).toBe('resolved\n'); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null, processGroup: null }); + }); + + it('canonicalizes a cleanup root whose configured spelling traverses a symlink', async () => { + const f = fixture(), aliasParent = join(f.root, 'root-alias'), alias = join(aliasParent, 'rebase'); + symlinkSync(f.root, aliasParent); + const x = deps(f); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, + metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await resolve({ ...input(f), repository: alias }); + const canonicalRepository = realpathSync(f.repository); + expect(x.request!.cleanupRoot).toMatch(new RegExp(`^${canonicalRepository.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/\\.codeboost-conflict-`)); + expect(x.request!.cleanupRoot).not.toContain('root-alias'); + }); + + it('reserves process settlement time and retains ownership when the adapter cannot settle by the work deadline', async () => { + vi.useFakeTimers(); + try { + const f = fixture(), cancel = vi.fn(), pending = Promise.withResolvers(); + let request: AgentAdapterRequest | undefined; + const x = deps(f, { start: (value: AgentAdapterRequest) => { + request = value; + return { attemptId: value.invocation.attemptId, cancel, settled: pending.promise } satisfies InvocationHandle; + } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, + metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + const settlementDeadline = Date.now() + CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS + 1_000; + const running = resolve({ ...input(f), deadline: settlementDeadline }); + const rejected = expect(running).rejects.toThrow(/retained resources/); + for (let turn = 0; turn < 20 && !request; turn++) await Promise.resolve(); + expect(request?.invocation.deadline).toBe(settlementDeadline - CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS); + await vi.advanceTimersByTimeAsync(CONFLICT_PROCESS_SETTLEMENT_RESERVE_MS + 1_000); + await rejected; + expect(cancel).toHaveBeenCalledWith('timeout'); + expect(x.events).not.toContain('remove'); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { source: oid(2) } }); + pending.resolve({ attemptId: request!.invocation.attemptId, context: request!.invocation.context, + exitCode: null, signal: null, stopReason: 'timeout', stdout: '', stderr: '' }); + await Promise.resolve(); + } finally { vi.useRealTimers(); } + }); + + it('rejects an outside change without copying it back and releases a settled child', async () => { + const f = fixture(), x = deps(f, { inspect: async () => manifest([{ path: 'outside.txt', kind: 'add', newType: 'file', underGit: false, ignored: false }]) }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f))).rejects.toThrow(/unapproved path/); + expect(readFileSync(join(f.repository, 'conflict.txt'), 'utf8')).toBe('conflicted\n'); + expect(x.events).toContain('remove'); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null }); + }); + + it('retains the marker and staging tree when final storage cleanup does not settle', async () => { + const f = fixture(), x = deps(f, { remove: async () => { throw new Error('storage cleanup failed'); } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, + metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f))).rejects.toThrow(/retained resources/); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { source: oid(2) } }); + expect(readdirSync(f.repository).some(name => name.startsWith('.codeboost-conflict-'))).toBe(true); + }); + + it('preserves both the primary failure and cleanup failure when retaining ownership', async () => { + const f = fixture(), primary = new Error('manifest inspection failed'), cleanup = new Error('storage cleanup failed'); + const x = deps(f, { inspect: async () => { throw primary; }, remove: async () => { throw cleanup; } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, + metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + const failure = await resolve(input(f)).then(() => undefined, error => error as Error); + expect(failure).toBeInstanceOf(RebaseResourcesUnsettled); + expect(failure?.cause).toBeInstanceOf(AggregateError); + expect((failure?.cause as AggregateError).errors).toEqual([primary, cleanup]); + expect((failure?.cause as AggregateError).cause).toBe(primary); + }); + + it.each([[[]], [[{ kind: 'container' as const, name: 'left' }]]])( + 'retains durable ownership when the invocation reports an unreleased inventory %#', async unreleased => { + const f = fixture(), remove = vi.fn(async () => undefined); + const x = deps(f, { remove, start: (value: AgentAdapterRequest) => ({ attemptId: value.invocation.attemptId, cancel: vi.fn(), + settled: Promise.resolve({ attemptId: value.invocation.attemptId, context: value.invocation.context, exitCode: null, signal: null, + stopReason: 'capture-failure', stdout: '', stderr: '', unreleased }) }) }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f))).rejects.toThrow(/could not confirm removal/); + expect(remove).not.toHaveBeenCalled(); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { source: oid(2) } }); + }); + + it('retains the allocation identity when failed allocation cleanup does not settle', async () => { + const f = fixture(), x = deps(f, { allocate: async (_clone: TaskClone, _limits: unknown, _image: string, + _owner: unknown, options: ProcessOptions) => { + const group = { pgid: 7373, startedAt: 4, identity: null }; + options.processLifecycle?.starting(); options.processLifecycle?.started(group); options.processLifecycle?.settled(group); + throw new AggregateError([new Error('volume cleanup failed')], 'allocation cleanup did not settle'); + } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f))).rejects.toThrow(/startup recovery/); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { source: oid(2) }, processGroup: null }); + }); + + it('retains storage when conflict import cleanup does not settle', async () => { + const f = fixture(), remove = vi.fn(async () => undefined), x = deps(f, { + remove, + importPaths: async () => { throw new AggregateError([new Error('import container remains')], 'import cleanup did not settle'); }, + }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, + metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f))).rejects.toThrow(/startup recovery/); + expect(remove).not.toHaveBeenCalled(); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: { source: oid(2) } }); + }); + + it('retains an exact process group and staging tree when a helper remains alive', async () => { + const f = fixture(), x = deps(f, { clone: async (options: AsyncCloneOptions) => { + const group = { pgid: 8383, startedAt: 5, identity: null }; + options.processLifecycle?.starting(); options.processLifecycle?.started(group); + options.processLifecycle?.unsettled(group, 'group-alive'); + throw Object.assign(new Error('group remains alive'), { code: 'EGROUPALIVE' }); + } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f))).rejects.toThrow(/startup recovery/); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ + conflict: { source: oid(2) }, processGroup: { pgid: 8383 } }); + expect(readdirSync(f.repository).some(name => name.startsWith('.codeboost-conflict-'))).toBe(true); + }); + + it('durably owns an adapter Docker client until that exact process group settles', async () => { + const f = fixture(), child = Promise.withResolvers(); + let request: AgentAdapterRequest | undefined; + const group = { pgid: 8484, startedAt: 6, identity: 'linux:00000000-0000-0000-0000-000000000000:2' }; + const x = deps(f, { start: (value: AgentAdapterRequest) => { + request = value; + value.processLifecycle!.starting(); + value.processLifecycle!.started(group); + return { attemptId: value.invocation.attemptId, cancel: vi.fn(), settled: child.promise }; + } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, + metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + const running = resolve(input(f)); + await vi.waitFor(() => expect(request).toBeDefined()); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: group, + conflict: { attemptId: request!.invocation.attemptId } }); + request!.processLifecycle!.settled(group); + child.resolve({ attemptId: request!.invocation.attemptId, context: request!.invocation.context, + exitCode: 0, signal: null, stdout: '', stderr: '' }); + await running; + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: null, conflict: null }); + }); + + it('does not start cleanup after an adapter Docker client remains unsettled', async () => { + const f = fixture(), remove = vi.fn(async () => undefined); + const group = { pgid: 8585, startedAt: 7, identity: 'linux:00000000-0000-0000-0000-000000000000:4' }; + const x = deps(f, { remove, start: (value: AgentAdapterRequest) => { + value.processLifecycle!.starting(); value.processLifecycle!.started(group); + value.processLifecycle!.unsettled(group, 'group-alive'); + return { attemptId: value.invocation.attemptId, cancel: vi.fn(), settled: Promise.resolve({ + attemptId: value.invocation.attemptId, context: value.invocation.context, exitCode: null, signal: null, + stopReason: 'capture-failure', stdout: '', stderr: '', + }) }; + } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, + metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve(input(f))).rejects.toThrow(/startup recovery/); + expect(remove).not.toHaveBeenCalled(); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ processGroup: group, + conflict: { source: oid(2) } }); + }); + + it('refuses a symlink-valued .gitmodules result before host copyback', async () => { + const f = fixture(); symlinkSync('original', join(f.repository, '.gitmodules')); + const x = deps(f, { + inspect: async () => manifest([{ path: '.gitmodules', kind: 'modify', oldType: 'symlink', newType: 'symlink', underGit: false, ignored: false }]), + exportPaths: async () => [{ path: '.gitmodules', type: 'symlink', content: Buffer.from('replacement') }], + }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + await expect(resolve({ ...input(f), files: ['.gitmodules'] })).rejects.toThrow(/symlink named .gitmodules/); + expect(readlinkSync(join(f.repository, '.gitmodules'))).toBe('original'); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null }); + }); + + it('cancels and awaits the child before releasing storage and durable ownership', async () => { + const f = fixture(), controller = new AbortController(), settled = Promise.withResolvers(); + const cancel = vi.fn((reason: InvocationResult['stopReason']) => settled.resolve({ attemptId: request!.invocation.attemptId, + context: request!.invocation.context, exitCode: null, signal: 'SIGTERM', stopReason: reason, stdout: '', stderr: '' })); + let request: AgentAdapterRequest | undefined; + const x = deps(f, { start: (value: AgentAdapterRequest) => { + request = value; return { attemptId: value.invocation.attemptId, cancel, settled: settled.promise }; + } }); + const resolve = createForeignConflictResolver({ store: f.store, identity, planKey: f.planKey, + repository: { path: join(f.root, 'bare.git') } as RunnerRepository, runnerOwner: 'a'.repeat(32), + image: () => 'sha256:' + 'b'.repeat(64), token: 'secret', limits: { workBytes: 1, workInodes: 1, metadataBytes: 1, metadataInodes: 1 }, deps: x.d as never }); + const running = resolve(input(f, controller.signal)); + await vi.waitFor(() => expect(request).toBeDefined()); + controller.abort(Object.assign(new Error('cancel'), { code: 'ABORT_ERR' })); + await expect(running).rejects.toThrow(/failed/); + expect(cancel).toHaveBeenCalledWith('cancelled'); + expect(x.events.at(-1)).toBe('remove'); + expect(f.store.getTask(identity).rebaseInProgress).toMatchObject({ conflict: null }); + }); +}); diff --git a/test/runner-rebase.test.ts b/test/runner-rebase.test.ts index 551f9cb3..d711c3a3 100644 --- a/test/runner-rebase.test.ts +++ b/test/runner-rebase.test.ts @@ -5,7 +5,7 @@ import { dirname, join } from 'node:path'; import { randomUUID } from 'node:crypto'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { fixtureGit as git } from './fixtures/git.ts'; -import { GitRebaser, RebaseConflict, rebaseRef, type GitRebaserOptions } from '../runner/rebase.ts'; +import { GitRebaser, RebaseConflict, RebaseResourcesUnsettled, rebaseRef, type GitRebaserOptions } from '../runner/rebase.ts'; import { ensureCommit, openRunnerRepository } from '../runner/runner-repository.ts'; import { splitBoundedIndexRecords, verifyCheckout } from '../runner/verify-checkout.ts'; import { readBoundedRebaseStateNames } from '../runner/hash-rebase-state.ts'; @@ -650,6 +650,16 @@ describe('trusted pre-merge rebase', () => { expect(expireNextRead).toBe(false); }); + it('retains the parent workspace when conflict child resources need startup recovery', async () => { + const s = await setup('foreign'), attemptId = randomUUID(); + const runner = createRebaser(s, { resolveForeignConflict: async () => { + throw new RebaseResourcesUnsettled('child resources remain'); + } }); + await expect(runner.run({ attemptId, oldBase: s.base, oldHead: s.foreign, oldHistory: s.history, onto: s.onto, + ledger: [{ sha: s.foreign, owner: null, origin: 'foreign' }] })).rejects.toThrow(/child resources remain/); + expect(existsSync(join(s.runnerRoot, OWNER, 'rebases', attemptId))).toBe(true); + }); + it('never sends an owned conflict through the foreign resolver', async () => { const s = await setup(true), attemptId = randomUUID(), resolveForeignConflict = vi.fn(async () => {}); const runner = createRebaser(s, { resolveForeignConflict }); diff --git a/test/runner-recovery.test.ts b/test/runner-recovery.test.ts index 374a76e8..239f601c 100644 --- a/test/runner-recovery.test.ts +++ b/test/runner-recovery.test.ts @@ -9,7 +9,8 @@ import { DatabaseSync } from 'node:sqlite'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { Store } from '../runner/store.ts'; import { processIdentity } from '../agents/process-group.ts'; -import { LockHeld, RecoveryBlocked, acquireRunnerLock, hostOpenFiles, hostOpenFilesBounded, hostProcesses, recoverStartup, releasePreparation, type RecoveryDeps, type RunnerLock } from '../runner/recovery.ts'; +import { LockHeld, RecoveryBlocked, acquireRunnerLock, hostOpenFiles, hostOpenFilesBounded, hostProcesses, recoverStartup, + releasePreparation, releaseRebaseProcess, type RecoveryDeps, type RunnerLock } from '../runner/recovery.ts'; import type { PlanIdentity } from '../core/identity.ts'; import type { Plan, PlanContext } from '../core/plan.ts'; @@ -243,7 +244,7 @@ describe('startup recovery sequence', () => { expect(store.getAttempt(id(1), attempt.id).state).toBe('running'); } }); - it('aborts each exact interrupted rebase before clearing its durable ownership marker', async () => { + it('requires explicit release before aborting an exact interrupted rebase', async () => { const { d: root, store } = fixture(); store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); @@ -258,11 +259,103 @@ describe('startup recovery sequence', () => { const { d, calls } = deps({ abortRebase: async (planKey, value) => { calls.push('abort-rebase'); seen.push({ planKey, value, stillOwned: store.getTask(id(1)).rebaseInProgress }); } }); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: d })) + .rejects.toThrow(/--release-rebase-process/); + expect(seen).toEqual([]); + expect(calls).toEqual(['terminate:5151']); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); + releaseRebaseProcess({ store, runnerRoot: join(root, 'r'), runnerOwner: token, attemptId: marker.attemptId, + openFiles: () => [] }); await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: d }); expect(seen).toEqual([{ planKey: store.getTask(id(1)).planKey, value: cleared, stillOwned: cleared }]); - expect(calls).toEqual(['recover', 'terminate:5151', 'abort-rebase']); + expect(calls).toEqual(['terminate:5151', 'recover', 'abort-rebase']); expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); }); + it('keeps an unsettled pipe-holder fail-closed until the operator explicitly releases the exact rebase', async () => { + const { d: root, store, admit } = fixture(2); + const interrupted = admit(id(2), { kind: 'review' }); + const preparationIdentity = 'linux:00000000-0000-0000-0000-000000000000:42'; + store.markPreparationStarting(id(2), interrupted.id, Date.now()); + store.recordPreparationGroup(id(2), interrupted.id, 4242, Date.now(), preparationIdentity); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), planKey = store.getTask(id(1)).planKey; + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); + const child = randomUUID(), allocationId = randomUUID(), networkAllocationId = randomUUID(); + store.beginRebaseConflict(planKey, marker.attemptId, { attemptId: child, allocationId, networkAllocationId, source: snapshot.head }); + store.setRebaseProcessGroup(planKey, marker.attemptId, null, 'spawning'); + store.setRebaseProcessGroup(planKey, marker.attemptId, 'spawning', 'unsettled'); + mkdirSync(join(root, 'r', token, 'rebases', marker.attemptId), { recursive: true }); + const calls: string[] = [], isAlive = vi.fn(() => true), terminate = vi.fn(async () => undefined); + const openFiles = vi.fn(() => [] as string[]), abortRebase = vi.fn(async () => { calls.push('abort-rebase'); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: null, conflict: { attemptId: child } }); }); + const removeTaskFilesystems = vi.fn(async () => { calls.push('remove-child'); }); + const options = { store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), openFiles, + deps: deps({ abortRebase, removeTaskFilesystems, processes: { isAlive, terminate }, recoverLeftovers: async () => { + calls.push('recover'); return { storage: [{ attemptId: child, allocationId, handle: 'child' }], unowned: [] }; + } }).d }; + await expect(recoverStartup(options)).rejects.toThrow(/--release-rebase-process/); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled', conflict: { attemptId: child } }); + expect(store.interruptedAttempts().find(attempt => attempt.id === interrupted.id)) + .toMatchObject({ preparationPgid: 4242, preparationIdentity }); + expect(calls).toEqual([]); expect(isAlive).not.toHaveBeenCalled(); expect(terminate).not.toHaveBeenCalled(); + expect(openFiles).not.toHaveBeenCalled(); + expect(() => releaseRebaseProcess({ store, runnerRoot: join(root, 'r'), runnerOwner: token, + attemptId: marker.attemptId, openFiles: () => ['escaped-process'] })).toThrow(/still using/); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); + releaseRebaseProcess({ store, runnerRoot: join(root, 'r'), runnerOwner: token, attemptId: marker.attemptId, + openFiles: () => [] }); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: null }); + await recoverStartup(options); + expect(calls).toEqual(['recover', 'abort-rebase', 'remove-child']); + expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); + }); + it('removes an interrupted conflict child before clearing its parent rebase', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), planKey = store.getTask(id(1)).planKey; + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); + const child = randomUUID(), allocationId = randomUUID(), networkAllocationId = randomUUID(); + store.beginRebaseConflict(planKey, marker.attemptId, { attemptId: child, allocationId, networkAllocationId, source: snapshot.head }); + const dockerIdentity = 'linux:00000000-0000-0000-0000-000000000000:3'; + store.setRebaseProcessGroup(planKey, marker.attemptId, null, 'spawning'); + store.setRebaseProcessGroup(planKey, marker.attemptId, 'spawning', { pgid: 6161, startedAt: 789, identity: dockerIdentity }); + const calls: string[] = [], abortRebase = vi.fn(async () => { calls.push('abort-rebase'); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ conflict: { attemptId: child } }); }); + const removeTaskFilesystems = vi.fn(async () => { calls.push('remove-child'); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ conflict: { attemptId: child } }); }); + const options = { store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase, removeTaskFilesystems, recoverLeftovers: async () => { + calls.push('recover'); return { storage: [{ attemptId: child, allocationId, handle: 'child' }], unowned: [] }; + }, processes: { isAlive: () => true, terminate: async pgid => { calls.push(`terminate:${pgid}`); } } }).d }; + await expect(recoverStartup(options)).rejects.toThrow(/--release-rebase-process/); + expect(calls).toEqual(['terminate:6161']); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled', conflict: { attemptId: child } }); + releaseRebaseProcess({ store, runnerRoot: join(root, 'r'), runnerOwner: token, attemptId: marker.attemptId, + openFiles: () => [] }); + const report = await recoverStartup(options); + expect(calls).toEqual(['terminate:6161', 'recover', 'abort-rebase', 'remove-child']); + expect(removeTaskFilesystems).toHaveBeenCalledWith('child'); + expect(report.unmatchedStorage).toEqual([]); + expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); + }); + it('retains a conflict marker when recovered storage only partially matches its identity', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), planKey = store.getTask(id(1)).planKey; + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); + const child = randomUUID(), allocationId = randomUUID(), networkAllocationId = randomUUID(); + store.beginRebaseConflict(planKey, marker.attemptId, { attemptId: child, allocationId, networkAllocationId, source: snapshot.head }); + const abortRebase = vi.fn(async () => undefined), removeTaskFilesystems = vi.fn(async () => undefined); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase, removeTaskFilesystems, recoverLeftovers: async () => ({ + storage: [{ attemptId: child, allocationId: randomUUID(), handle: 'wrong' }], unowned: [], + }) }).d })).rejects.toThrow(/does not match/); + expect(abortRebase).not.toHaveBeenCalled(); expect(removeTaskFilesystems).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ conflict: { attemptId: child, allocationId } }); + }); it('passes only the exact durably owned result ref to rebase recovery', async () => { const { d: root, store } = fixture(); store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); @@ -289,7 +382,7 @@ describe('startup recovery sequence', () => { await recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase }).d }); expect(abortRebase).toHaveBeenCalledWith(planKey, { ...marker, oldHistory: null, resultState: 'none', - resultHead: null, resultMappings: null, resolvedConflicts: [] }); + resultHead: null, resultMappings: null, resolvedConflicts: [], conflict: null }); expect(store.getTask(id(1)).rebaseInProgress).toBeNull(); }); it('retains a recovery marker whose resolved-conflict provenance is outside its captured history', async () => { @@ -337,7 +430,7 @@ describe('startup recovery sequence', () => { expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); } }); - it('retains a dead-group marker while another process still uses its rebase workspace', async () => { + it('retains a dead recovered group when explicit release finds a workspace user', async () => { const { d: root, store } = fixture(); store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); @@ -350,11 +443,51 @@ describe('startup recovery sequence', () => { const abortRebase = vi.fn(async () => undefined); await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase, processes: { isAlive: () => false, terminate: async () => undefined } }).d, - openFiles: () => ['escaped-child'] })).rejects.toThrow(/still uses/); + openFiles: () => { throw new Error('startup must not use workspace scans to prove process settlement'); } })).rejects.toThrow(/--release-rebase-process/); expect(abortRebase).not.toHaveBeenCalled(); - expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: group }); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); + expect(() => releaseRebaseProcess({ store, runnerRoot: join(root, 'r'), runnerOwner: token, + attemptId: marker.attemptId, openFiles: () => ['escaped-child'] })).toThrow(/still using/); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); }); - it('retains a terminated-group marker while an escaped descendant still uses its rebase workspace', async () => { + it('requires explicit release when a recovered exact group disappeared before settlement was recorded', async () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), planKey = store.getTask(id(1)).planKey; + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); + store.setRebaseProcessGroup(planKey, marker.attemptId, null, 'spawning'); + const group = { pgid: 5151, startedAt: 456, identity: 'linux:00000000-0000-0000-0000-000000000000:1' }; + store.setRebaseProcessGroup(planKey, marker.attemptId, 'spawning', group); + const abortRebase = vi.fn(async () => undefined), recoverLeftovers = vi.fn(async () => ({ storage: [], unowned: [] })); + + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ abortRebase, recoverLeftovers, processes: { isAlive: () => false, terminate: async () => undefined } }).d, + openFiles: () => [] })).rejects.toThrow(/--release-rebase-process/); + + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ attemptId: marker.attemptId, processGroup: 'unsettled' }); + expect(abortRebase).not.toHaveBeenCalled(); + expect(recoverLeftovers).not.toHaveBeenCalled(); + }); + it('refuses explicit release through a replaced rebase-directory ancestor', () => { + const { d: root, store } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), planKey = store.getTask(id(1)).planKey; + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); + store.setRebaseProcessGroup(planKey, marker.attemptId, null, 'spawning'); + store.setRebaseProcessGroup(planKey, marker.attemptId, 'spawning', 'unsettled'); + const runnerRoot = join(root, 'r'), owner = join(runnerRoot, token), redirected = join(root, 'redirected-rebases'); + mkdirSync(owner, { recursive: true }); mkdirSync(join(redirected, marker.attemptId), { recursive: true }); + symlinkSync(redirected, join(owner, 'rebases')); + const openFiles = vi.fn(() => [] as string[]); + + expect(() => releaseRebaseProcess({ store, runnerRoot, runnerOwner: token, + attemptId: marker.attemptId, openFiles })).toThrow(/directory owned by you|plain directory/); + expect(openFiles).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); + }); + it('retains a terminated recovered group when explicit release finds an escaped workspace user', async () => { const { d: root, store } = fixture(); store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); @@ -367,10 +500,13 @@ describe('startup recovery sequence', () => { const terminate = vi.fn(async () => undefined), abortRebase = vi.fn(async () => undefined); await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), deps: deps({ abortRebase, processes: { isAlive: () => true, terminate } }).d, - openFiles: () => ['escaped-child'] })).rejects.toThrow(/still uses/); + openFiles: () => { throw new Error('startup must not use workspace scans to prove process settlement'); } })).rejects.toThrow(/--release-rebase-process/); expect(terminate).toHaveBeenCalledWith(group.pgid, group.identity, 5_000); expect(abortRebase).not.toHaveBeenCalled(); - expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: group }); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); + expect(() => releaseRebaseProcess({ store, runnerRoot: join(root, 'r'), runnerOwner: token, + attemptId: marker.attemptId, openFiles: () => ['escaped-child'] })).toThrow(/still using/); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); }); it('fails closed and retains an interrupted rebase when no recovery implementation is present', async () => { const { d: root, store } = fixture(); @@ -382,18 +518,33 @@ describe('startup recovery sequence', () => { .rejects.toThrow(/needs F3 to abort/); expect(store.getTask(id(1)).rebaseInProgress).toEqual(marker); }); - it('fails closed without cleanup when a crash left the spawned process identity unknown', async () => { + it('moves a crash-before-spawn owner into the explicit-release path without cleanup', async () => { const { d: root, store } = fixture(); store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); const snapshot = store.getSnapshot(id(1)); const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); store.setRebaseProcessGroup(store.getTask(id(1)).planKey, marker.attemptId, null, 'spawning'); - const abortRebase = vi.fn(async () => undefined); + const abortRebase = vi.fn(async () => undefined), recoverLeftovers = vi.fn(async () => ({ storage: [], unowned: [] })); await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), - deps: deps({ abortRebase }).d })).rejects.toThrow(/invalid recovery marker/); + deps: deps({ abortRebase, recoverLeftovers }).d })).rejects.toThrow(/--release-rebase-process/); expect(abortRebase).not.toHaveBeenCalled(); - expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'spawning' }); + expect(recoverLeftovers).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toMatchObject({ processGroup: 'unsettled' }); + }); + it('rejects an unknown string process owner before recovery actions', async () => { + const { d: root, store, raw } = fixture(); + store.transitionTask(id(1), store.getTask(id(1)).stateVersion, 'approved but merge blocked'); + const snapshot = store.getSnapshot(id(1)), planKey = store.getTask(id(1)).planKey; + const marker = store.beginRebase(id(1), { revision: 1, snapshotId: snapshot.id, reviewVersion: store.reviewVersion(id(1)) }, + store.getTask(id(1)).stateVersion, { oldBase: snapshot.base, oldHead: snapshot.head, onto: oid(3), oldHistory: [snapshot.head] }); + const malformed = { ...marker, processGroup: 'unknown-owner' }; + raw(`UPDATE tasks SET rebase_in_progress='${JSON.stringify(malformed)}' WHERE plan_key='${planKey}'`); + const recoverLeftovers = vi.fn(async () => ({ storage: [], unowned: [] })); + await expect(recoverStartup({ store, runnerOwner: token, runnerRoot: join(root, 'r'), diagnosticsDir: join(root, 'd'), + deps: deps({ recoverLeftovers }).d })).rejects.toThrow(/invalid recovery marker/); + expect(recoverLeftovers).not.toHaveBeenCalled(); + expect(store.getTask(id(1)).rebaseInProgress).toEqual(malformed); }); it('rejects process-group ID 1 without sending any recovery signal', async () => { const { d: root, store, raw } = fixture(); @@ -529,6 +680,19 @@ describe('startup recovery sequence', () => { expect(existsSync(dirPath)).toBe(false); await expect(run()).resolves.toMatchObject({ finalized: [] }); }); + it('refuses preparation release through a replaced attempts-directory ancestor', () => { + const { d: root, store, admit } = fixture(); + const attempt = admit(id(1)); store.markPreparationStarting(id(1), attempt.id, Date.now()); + const runnerRoot = join(root, 'r'), owner = join(runnerRoot, token), redirected = join(root, 'redirected-attempts'); + mkdirSync(owner, { recursive: true }); mkdirSync(join(redirected, attempt.id), { recursive: true }); + symlinkSync(redirected, join(owner, 'attempts')); + const openFiles = vi.fn(() => [] as string[]); + + expect(() => releasePreparation({ store, runnerRoot, runnerOwner: token, + attemptId: attempt.id, openFiles })).toThrow(/directory owned by you|plain directory/); + expect(openFiles).not.toHaveBeenCalled(); + expect(store.interruptedAttempts().find(value => value.id === attempt.id)?.preparationStartedAt).not.toBeNull(); + }); }); describe('host process checks', () => { @@ -537,14 +701,14 @@ describe('host process checks', () => { expect(hostProcesses.isAlive(4242)).toBe(true); }); - it('uses a kernel identity and never runs a PATH-selected ps with runner secrets', async () => { + it('never runs a PATH-selected ps or signals a recovered numeric group', async () => { const root = dir(), marker = join(root, 'ps-ran'), ps = join(root, 'ps'), path = process.env.PATH; writeFileSync(ps, `#!/bin/sh\nprintf '%s' "$CODEBOOST_TEST_SECRET" > '${marker}'\n`, { mode: 0o755 }); process.env.PATH = `${root}:${path ?? ''}`; process.env.CODEBOOST_TEST_SECRET = 'must-not-leak'; try { vi.spyOn(process, 'kill').mockImplementation(() => { throw Object.assign(new Error('denied'), { code: 'EPERM' }); }); expect(hostProcesses.isAlive(4242)).toBe(true); - await expect(hostProcesses.terminate(4242, null, 10)).rejects.toThrow(/prove the recorded process group/); + await expect(hostProcesses.terminate(4242, null, 10)).rejects.toThrow(/cannot signal its reusable numeric ID/); expect(existsSync(marker)).toBe(false); } finally { if (path === undefined) delete process.env.PATH; else process.env.PATH = path; @@ -552,16 +716,14 @@ describe('host process checks', () => { } }); - it('revalidates the exact kernel identity before signalling a process group', async () => { + it('fails closed for a live recovered group even when its recorded identity is exact', async () => { const child = spawn('/bin/sh', ['-c', 'exec sleep 30'], { detached: true, stdio: 'ignore' }); children.push(child); await once(child, 'spawn'); const pgid = child.pid!; try { expect(hostProcesses.isAlive(pgid)).toBe(true); - const identity = processIdentity(pgid), wrong = identity === null ? 'linux:00000000-0000-0000-0000-000000000000:1' - : `${identity}0`; - await expect(hostProcesses.terminate(pgid, wrong, 10)).rejects.toThrow(/identity|recorded process group/); + await expect(hostProcesses.terminate(pgid, processIdentity(pgid), 10)).rejects.toThrow(/cannot signal its reusable numeric ID/); expect(() => process.kill(pgid, 0)).not.toThrow(); } finally { try { process.kill(-pgid, 'SIGKILL'); } catch {} @@ -570,24 +732,6 @@ describe('host process checks', () => { } }); - it.skipIf(process.platform !== 'linux')('bounds settlement after SIGKILL and retains ownership on failure', async () => { - const child = spawn('/bin/sh', ['-c', 'exec sleep 30'], { detached: true, stdio: 'ignore' }); - children.push(child); await once(child, 'spawn'); - const pgid = child.pid!, identity = processIdentity(pgid)!; - vi.useFakeTimers(); - const signal = vi.spyOn(process, 'kill').mockImplementation(() => true); - const wallClock = vi.spyOn(Date, 'now').mockImplementation(() => 1_000_000 - performance.now()); - try { - const pending = hostProcesses.terminate(pgid, identity, 100); - const rejected = expect(pending).rejects.toThrow(/did not exit after SIGKILL/); - await vi.advanceTimersByTimeAsync(500); - await rejected; - } finally { - wallClock.mockRestore(); signal.mockRestore(); vi.useRealTimers(); - try { process.kill(-pgid, 'SIGKILL'); } catch {} - await once(child, 'exit'); children.splice(children.indexOf(child), 1); - } - }); it('finds a process working in the attempt directory when the path goes through a symlink', async () => { const root = dir(), real = join(root, 'real'), attempt = join(real, 'attempt'); mkdirSync(attempt, { recursive: true }); symlinkSync(real, join(root, 'link')); diff --git a/test/tracked-process-unsettled.test.ts b/test/tracked-process-unsettled.test.ts new file mode 100644 index 00000000..3d083387 --- /dev/null +++ b/test/tracked-process-unsettled.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ outcome: { status: null as number | null, stdout: 'partial', stderr: '', + error: Object.assign(new Error('descendant remains'), { code: 'EGROUPALIVE' }) as Error | undefined } })); +vi.mock('../agents/process-group.ts', async importOriginal => { + const actual = await importOriginal(); + return { ...actual, runInProcessGroup: async () => state.outcome }; +}); + +const { runTrackedProcess } = await import('../agents/tracked-docker.ts'); + +describe('tracked subprocess unsettled ownership', () => { + it('preserves the unsettled outcome code when the durable ownership callback also fails', async () => { + const result = await runTrackedProcess('git', ['clone'], { env: {}, timeoutMs: 1_000, + lifecycle: { + starting: () => {}, started: () => {}, settled: () => {}, + unsettled: () => { throw new Error('durable marker write failed'); }, + } }); + expect(result).toMatchObject({ status: null, stdout: 'partial', error: { code: 'EGROUPALIVE' } }); + const causes = ((result.error as Error).cause as AggregateError).errors; + expect(causes).toMatchObject([{ message: 'descendant remains', code: 'EGROUPALIVE' }, + { message: 'durable marker write failed' }]); + }); + + it('preserves a timeout or cancellation reason when the settlement callback also fails', async () => { + for (const [code, name] of [['ETIMEDOUT', 'Error'], ['ABORT_ERR', 'AbortError']] as const) { + state.outcome = { status: null, stdout: 'partial', stderr: '', + error: Object.assign(new Error(code === 'ETIMEDOUT' ? 'original timeout' : 'original cancellation'), { code, name }) }; + const result = await runTrackedProcess('git', ['clone'], { env: {}, timeoutMs: 1_000, + lifecycle: { starting: () => {}, started: () => {}, unsettled: () => {}, + settled: () => { throw new Error('settlement write failed'); } } }); + expect(result).toMatchObject({ status: null, error: { code } }); + expect(((result.error as Error).cause as AggregateError).errors).toMatchObject([ + { message: code === 'ETIMEDOUT' ? 'original timeout' : 'original cancellation', code }, + { message: 'settlement write failed' }, + ]); + } + }); + + it('preserves a nonzero exit and composes it with settlement failure', async () => { + state.outcome = { status: 7, stdout: '', stderr: 'failed', error: undefined }; + const result = await runTrackedProcess('git', ['clone'], { env: {}, timeoutMs: 1_000, + lifecycle: { starting: () => {}, started: () => {}, unsettled: () => {}, + settled: () => { throw new Error('settlement write failed'); } } }); + expect(result).toMatchObject({ status: null, stderr: 'failed' }); + expect(((result.error as Error).cause as AggregateError).errors).toMatchObject([ + { message: 'git clone exited with status 7.', status: 7 }, { message: 'settlement write failed' }, + ]); + }); +}); diff --git a/web/cli.ts b/web/cli.ts index 237382bc..6bd670c7 100644 --- a/web/cli.ts +++ b/web/cli.ts @@ -5,21 +5,25 @@ import { startServer, type RunnerSetup } from './server.ts'; import { productionPlanning } from './planning.ts'; import { createDemo } from '../scripts/demo.ts'; import { Store, requireSupportedNode } from '../runner/store.ts'; -import { acquireRunnerLock, releasePreparation } from '../runner/recovery.ts'; +import { acquireRunnerLock, releasePreparation, releaseRebaseProcess } from '../runner/recovery.ts'; import { RUNNER_NEEDS_GITHUB, baseBranch, parseRunnerConfig, recoveryWarnings, setUpRunner } from '../runner/production.ts'; requireSupportedNode(); /** A refusal the person acts on (bad input, a held lock, a blocked recovery): its message and exit 1, not a stack. */ function refuse(error: unknown): never { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } const checked = (fn: () => T): T => { try { return fn(); } catch (error) { return refuse(error); } }; -const { values } = parseArgs({ options: { demo: { type:'boolean' }, directory:{type:'string'}, config:{type:'string'}, port:{type:'string'}, help:{type:'boolean'}, 'release-preparation':{type:'string'} } }); -if (values.help || (!values.demo && !values.config && values['release-preparation'] === undefined)) { - console.log('codeboost local review\n\nDemo: npm run demo\nExisting store: npm start -- --config /absolute/path/review.json\nOptions: --port 4318 --directory /path/to/demo\n\nThe configuration binds a trusted repository, database, plan identity, and known path identity. Configure the question agent in Settings; Ask runs it in a Docker container (Claude Code needs CLAUDE_CODE_OAUTH_TOKEN; Codex cannot answer questions yet). A github block enables the guarded merge gate and plan suggestions (Claude Code in a Docker container, like Ask); demos never merge or plan. A runner block (root, committer, optional diagnosticsDir and limits) turns on the runner: it needs Docker, the github block with a baseBranch for its pull requests, and CLAUDE_CODE_OAUTH_TOKEN, and runs startup recovery before the server opens.\n\n--release-preparation : after startup recovery reports a preparation whose process was never recorded, and you have stopped that process, remove its attempt directory (refused while any process still uses it).'); -} else if (values['release-preparation'] !== undefined) { - const attemptId = values['release-preparation']; +const { values } = parseArgs({ options: { demo: { type:'boolean' }, directory:{type:'string'}, config:{type:'string'}, port:{type:'string'}, help:{type:'boolean'}, + 'release-preparation':{type:'string'}, 'release-rebase-process':{type:'string'} } }); +if (values['release-preparation'] !== undefined && values['release-rebase-process'] !== undefined) + refuse(new Error('Choose only one release operation.')); +const release = values['release-preparation'] !== undefined ? 'preparation' : values['release-rebase-process'] !== undefined ? 'rebase' : undefined; +if (values.help || (!values.demo && !values.config && release === undefined)) { + console.log('codeboost local review\n\nDemo: npm run demo\nExisting store: npm start -- --config /absolute/path/review.json\nOptions: --port 4318 --directory /path/to/demo\n\nThe configuration binds a trusted repository, database, plan identity, and known path identity. Configure the question agent in Settings; Ask runs it in a Docker container (Claude Code needs CLAUDE_CODE_OAUTH_TOKEN; Codex cannot answer questions yet). A github block enables the guarded merge gate and plan suggestions (Claude Code in a Docker container, like Ask); demos never merge or plan. A runner block (root, committer, optional diagnosticsDir and limits) turns on the runner: it needs Docker, the github block with a baseBranch for its pull requests, and CLAUDE_CODE_OAUTH_TOKEN, and runs startup recovery before the server opens.\n\n--release-preparation : after startup recovery reports a preparation whose process was never recorded, and you have stopped that process, remove its attempt directory (refused while any process still uses it).\n--release-rebase-process : after startup recovery reports an unidentified escaped rebase process, stop that process and explicitly release its retained marker (refused while any process still uses its rebase directory).'); +} else if (release !== undefined) { + const attemptId = release === 'preparation' ? values['release-preparation']! : values['release-rebase-process']!; checked(() => { - if (!values.config) throw new Error('--release-preparation needs --config.'); + if (!values.config) throw new Error(`--release-${release === 'preparation' ? 'preparation' : 'rebase-process'} needs --config.`); const config = JSON.parse(readFileSync(resolve(values.config), 'utf8')); - if (config.runner === undefined) throw new Error('This review has no runner block, so it has no preparations to release.'); + if (config.runner === undefined) throw new Error(`This review has no runner block, so it has no ${release === 'preparation' ? 'preparations' : 'rebases'} to release.`); const runnerConfig = parseRunnerConfig(config.runner); // Under the runner lock, so no runner of this database is starting or running while the directory is checked. const lock = acquireRunnerLock(config.database); @@ -27,7 +31,8 @@ if (values.help || (!values.demo && !values.config && values['release-preparatio const store = new Store(config.database); try { lock.verify(); - releasePreparation({ store, runnerRoot: runnerConfig.root, runnerOwner: store.runnerOwnerToken(lock.file), attemptId }); + const input = { store, runnerRoot: runnerConfig.root, runnerOwner: store.runnerOwnerToken(lock.file), attemptId }; + if (release === 'preparation') releasePreparation(input); else releaseRebaseProcess(input); console.log('Released. Start codeboost again.'); } finally { store.close(); } } finally { lock.release(); }