diff --git a/.github/workflows/agent-isolation.yml b/.github/workflows/agent-isolation.yml index 395e87e..5cd1af0 100644 --- a/.github/workflows/agent-isolation.yml +++ b/.github/workflows/agent-isolation.yml @@ -29,4 +29,4 @@ jobs: - run: npm ci --ignore-scripts - run: npm run typecheck # The Docker suites share one image tag and daemon, so run test files one at a time. - - run: npx vitest run --no-file-parallelism test/agent-contract.test.ts test/agent-clone.test.ts test/agent-container.test.ts test/agent-network.test.ts test/agent-policy.test.ts test/agent-proxy.test.ts + - run: npx vitest run --no-file-parallelism test/agent-contract.test.ts test/agent-clone.test.ts test/agent-container.test.ts test/agent-network.test.ts test/agent-policy.test.ts test/agent-proxy.test.ts test/agent-adapter.test.ts test/agent-supervisor.test.ts test/agent-output.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 330b0dc..fc5a962 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,6 @@ jobs: - run: npm run typecheck # The Docker agent suites run one file at a time in the Agent isolation workflow; running them here # would put them in parallel against the same image tag and daemon. - - run: npm test -- --exclude test/agent-container.test.ts --exclude test/agent-network.test.ts + - run: npm test -- --exclude test/agent-container.test.ts --exclude test/agent-network.test.ts --exclude test/agent-adapter.test.ts --exclude test/agent-supervisor.test.ts - run: npx playwright install --with-deps chromium - run: npm run test:browser diff --git a/agents/adapters/claude.ts b/agents/adapters/claude.ts new file mode 100644 index 0000000..902d3dd --- /dev/null +++ b/agents/adapters/claude.ts @@ -0,0 +1,54 @@ +import type { InvocationHandle } from '../contract.ts'; +import { createContainerProfile, ProfileCreationCleanupError } from '../container/profile.ts'; +import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupError, + type VendorNetwork } from '../network/network.ts'; +import { createClaudeCommand, createPhasePolicy } from '../policy.ts'; +import { retainNetworkCleanup, retainSetupCleanup, startProfileInvocation } from './supervisor.ts'; +import { createAdapterInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; + +export function parseClaudeOutput(raw: Buffer): { text: string; providerFailed: boolean } { + const envelope = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(raw)) as + { result?: unknown; is_error?: unknown }; + if (typeof envelope.result !== 'string' || typeof envelope.is_error !== 'boolean') + throw new Error('Claude returned a malformed output envelope.'); + return Object.freeze({ text: envelope.result, providerFailed: envelope.is_error }); +} + +export function startClaudeInvocation(request: AgentAdapterRequest, + oauthToken: string, options: AgentAdapterOptions = {}): InvocationHandle { + if (!oauthToken || oauthToken.includes('\0')) throw new Error('Claude OAuth token is malformed.'); + const policy = createPhasePolicy(request.invocation); + const remaining = createAdapterInvocationBudget(request.invocation, options.timeoutMs); + let network: VendorNetwork; + try { network = createVendorNetwork(request.invocation, request.imageId, Math.min(60_000, remaining())); } + catch (error) { + if (error instanceof VendorNetworkCreationCleanupError) + return retainSetupCleanup(request.invocation, error.retryCleanup, error.startupError, error, + 'network creation cleanup'); + throw error; + } + try { + const profile = createContainerProfile({ ...request, policy, network, + command: createClaudeCommand(policy, request.prompt), claudeToken: oauthToken, + timeoutMs: Math.min(60_000, remaining()) }); + return startProfileInvocation(profile, { ...options, secrets: { CLAUDE_CODE_OAUTH_TOKEN: oauthToken }, + invocationBudget: remaining, + decode: (_profile, raw) => parseClaudeOutput(raw) }); + } catch (error) { + if (error instanceof ProfileCreationCleanupError) { + const retryCleanup = (networkTimeoutMs = 30_000) => { + const failures: unknown[] = []; + try { error.retryCleanup(); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(network, networkTimeoutMs); } catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Adapter setup cleanup did not settle.'); + }; + try { retryCleanup(Math.min(30_000, remaining())); } + catch (cleanupError) { return retainSetupCleanup(request.invocation, () => retryCleanup(), + error.startupError, cleanupError, 'profile and network cleanup'); } + throw error.startupError; + } + try { removeVendorNetwork(network, Math.min(30_000, remaining())); } + catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } + throw error; + } +} diff --git a/agents/adapters/codex.ts b/agents/adapters/codex.ts new file mode 100644 index 0000000..158f8fe --- /dev/null +++ b/agents/adapters/codex.ts @@ -0,0 +1,57 @@ +import type { InvocationHandle } from '../contract.ts'; +import { createContainerProfile, ProfileCreationCleanupError } from '../container/profile.ts'; +import { createVendorNetwork, removeVendorNetwork, VendorNetworkCreationCleanupError, + type VendorNetwork } from '../network/network.ts'; +import { createCodexCommand, createPhasePolicy } from '../policy.ts'; +import { readBoundedContainerFile, retainNetworkCleanup, retainSetupCleanup, + startProfileInvocation } from './supervisor.ts'; +import { createAdapterInvocationBudget, type AgentAdapterOptions, type AgentAdapterRequest } from './types.ts'; + +export const CODEX_OUTPUT_FILE = '/run/codeboost-output/final.txt'; + +export async function readCodexOutput(container: string, maximumBytes: number, timeoutMs = 30_000, + signal?: AbortSignal) { + const output = await readBoundedContainerFile(container, CODEX_OUTPUT_FILE, maximumBytes, timeoutMs, signal); + const text = new TextDecoder('utf-8', { fatal: true }).decode(output); + return Object.freeze({ text, additionalBytes: output.length }); +} + +export function startCodexInvocation(request: AgentAdapterRequest, + authFile: string, options: AgentAdapterOptions = {}): InvocationHandle { + if (!authFile || authFile.includes('\0')) throw new Error('Codex auth path is malformed.'); + const policy = createPhasePolicy(request.invocation); + const remaining = createAdapterInvocationBudget(request.invocation, options.timeoutMs); + let network: VendorNetwork; + try { network = createVendorNetwork(request.invocation, request.imageId, Math.min(60_000, remaining())); } + catch (error) { + if (error instanceof VendorNetworkCreationCleanupError) + return retainSetupCleanup(request.invocation, error.retryCleanup, error.startupError, error, + 'network creation cleanup'); + throw error; + } + try { + const profile = createContainerProfile({ ...request, policy, network, + command: createCodexCommand(policy, request.prompt), codexAuthFile: authFile, deferredOutput: true, + timeoutMs: Math.min(60_000, remaining()) }); + return startProfileInvocation(profile, { ...options, + invocationBudget: remaining, + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal) }); + } catch (error) { + if (error instanceof ProfileCreationCleanupError) { + const retryCleanup = (networkTimeoutMs = 30_000) => { + const failures: unknown[] = []; + try { error.retryCleanup(); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(network, networkTimeoutMs); } catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Adapter setup cleanup did not settle.'); + }; + try { retryCleanup(Math.min(30_000, remaining())); } + catch (cleanupError) { return retainSetupCleanup(request.invocation, () => retryCleanup(), + error.startupError, cleanupError, 'profile and network cleanup'); } + throw error.startupError; + } + try { removeVendorNetwork(network, Math.min(30_000, remaining())); } + catch (cleanupError) { return retainNetworkCleanup(request.invocation, network, error, cleanupError); } + throw error; + } +} diff --git a/agents/adapters/supervisor.ts b/agents/adapters/supervisor.ts new file mode 100644 index 0000000..502f38f --- /dev/null +++ b/agents/adapters/supervisor.ts @@ -0,0 +1,610 @@ +import { execFile, spawn, type ChildProcess } from 'node:child_process'; +import type { InvocationHandle, InvocationInput, InvocationResult, StopReason } from '../contract.ts'; +import { assertPhasePolicy } from '../policy.ts'; +import { createValidatedContainer, disposeValidatedContainer, validateContainer } from '../container/run.ts'; +import { assertContainerProfileAuthenticity, disposeContainerProfile, isContainerProfileAuthentic, + type ContainerProfile } from '../container/profile.ts'; +import { removeVendorNetwork, type VendorNetwork } from '../network/network.ts'; + +export const OUTPUT_LIMITS = Object.freeze({ + stdoutBytes: 16 * 1024 * 1024, + stderrBytes: 4 * 1024 * 1024, + combinedBytes: 20 * 1024 * 1024, +}); +const DEFAULT_TIMEOUT_MS = 10 * 60_000; +const CAPTURE_ABORT_GRACE_MS = 1_000; +const DIAGNOSTIC_BYTES = 1024; +const active = new Map(); +const activeProfiles = new WeakSet(); +const cleanupRecoveries = new Set(); +const hasCleanupRecovery = (attemptId: string) => + Array.from(cleanupRecoveries).some(handle => handle.attemptId === attemptId); +const ownsAttempt = (attemptId: string) => active.has(attemptId) || hasCleanupRecovery(attemptId); + +export interface CaptureLimits { + readonly stdoutBytes: number; + readonly stderrBytes: number; + readonly combinedBytes: number; +} +export interface DecodedOutput { + readonly text: string; + /** Bytes captured outside process stdout, such as Codex's final-output file. */ + readonly additionalBytes?: number; + readonly providerFailed?: boolean; +} +export interface SupervisorOptions { + readonly secrets?: Readonly>; + readonly timeoutMs?: number; + readonly limits?: Partial; + /** Trusted monotonic budget carried from synchronous adapter setup. */ + readonly invocationBudget?: () => number; + readonly decode?: (profile: ContainerProfile, rawStdout: Buffer, maximumBytes: number, + timeoutMs: number, signal: AbortSignal) => DecodedOutput | Promise; +} +export class OutputLimitError extends Error {} +export class CaptureDeadlineError extends Error {} + +const dockerEnvironment = () => ({ PATH: process.env.PATH, DOCKER_HOST: process.env.DOCKER_HOST }); +const positiveInteger = (value: number, name: string) => { + if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be a positive integer.`); +}; +const captureLimits = (override: Partial | undefined): CaptureLimits => { + const limits = Object.freeze({ ...OUTPUT_LIMITS, ...override }); + positiveInteger(limits.stdoutBytes, 'stdoutBytes'); + positiveInteger(limits.stderrBytes, 'stderrBytes'); + positiveInteger(limits.combinedBytes, 'combinedBytes'); + if (limits.stdoutBytes > OUTPUT_LIMITS.stdoutBytes || limits.stderrBytes > OUTPUT_LIMITS.stderrBytes + || limits.combinedBytes > OUTPUT_LIMITS.combinedBytes) + throw new Error('Capture limits cannot exceed the production hard limits.'); + return limits; +}; +const diagnosticFor = (reason: StopReason, detail?: string) => Buffer.from( + `[codeboost: ${reason}${detail ? `: ${detail.replace(/[\r\n]+/g, ' ').slice(0, 512)}` : ''}]\n`); +/** + * Retains captured output in fixed-size blocks, so memory scales with retained bytes rather than with the number + * of write events a container emits. + */ +export class ByteCollector { + static readonly BLOCK_BYTES = 64 * 1024; + private readonly blocks: Buffer[] = []; + private used = 0; + push(data: Buffer): void { + for (let offset = 0; offset < data.length;) { + let block = this.blocks.at(-1); + if (!block || this.used === block.length) { + block = Buffer.allocUnsafe(ByteCollector.BLOCK_BYTES); + this.blocks.push(block); + this.used = 0; + } + const count = Math.min(data.length - offset, block.length - this.used); + data.copy(block, this.used, offset, offset + count); + this.used += count; + offset += count; + } + } + toBuffer(): Buffer { + if (!this.blocks.length) return Buffer.alloc(0); + return Buffer.concat([...this.blocks.slice(0, -1), this.blocks.at(-1)!.subarray(0, this.used)]); + } + get blockCount(): number { return this.blocks.length; } +} + +// `cleanup` is what recovery retries: container-level disposal by default, or profile-only disposal for a +// rejection that happened before this profile created any container. +const retainCleanupOwnership = (profile: ContainerProfile, detail: string, register = true, + cleanup: (profile: ContainerProfile) => void = disposeValidatedContainer): InvocationHandle => { + const invocation = assertPhasePolicy(profile.policy); + let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; + let cancelReason: StopReason | undefined; + let handle!: InvocationHandle; + let timer: ReturnType | undefined; + const settled = new Promise(resolve => { resolveSettled = resolve; }); + const schedule = () => { + if (timer) return; + timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); + }; + const retry = () => { + if (cleaning || complete) return; + cleaning = true; + try { + cleanup(profile); + if (timer) clearTimeout(timer); + timer = undefined; + complete = true; + if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); + activeProfiles.delete(profile); + cleanupRecoveries.delete(handle); + resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, + exitCode: null, signal: null, stopReason: cancelReason ?? 'capture-failure', stdout: '', + stderr: diagnosticFor(cancelReason ?? 'capture-failure', detail).toString('utf8') })); + } catch { + cleaning = false; + schedule(); + return; + } + cleaning = false; + }; + handle = Object.freeze({ attemptId: invocation.attemptId, settled, + cancel: (reason: StopReason) => { + cancelReason ??= reason; + if (timer) clearTimeout(timer); + timer = undefined; + retry(); + } }); + activeProfiles.add(profile); + if (register) active.set(invocation.attemptId, handle); + else cleanupRecoveries.add(handle); + schedule(); + return handle; +}; + +/** Retain attempt ownership while retrying a network allocated before profile construction failed. */ +export function retainNetworkCleanup(invocation: InvocationInput, network: VendorNetwork, + startupError: unknown, cleanupError: unknown): InvocationHandle { + return retainSetupCleanup(invocation, () => removeVendorNetwork(network), startupError, cleanupError, + 'network cleanup'); +} + +/** Retain attempt ownership while retrying resources allocated during synchronous adapter setup. */ +export function retainSetupCleanup(invocation: InvocationInput, retryCleanup: () => void, + startupError: unknown, cleanupError: unknown, kind = 'setup cleanup'): InvocationHandle { + const register = !ownsAttempt(invocation.attemptId); + let resolveSettled!: (result: InvocationResult) => void, cleaning = false, complete = false; + let cancelReason: StopReason | undefined; + let handle!: InvocationHandle; + let timer: ReturnType | undefined; + const settled = new Promise(resolve => { resolveSettled = resolve; }); + const detail = `Adapter startup failed and ${kind} remains unsettled: ${String(startupError)}; ${String(cleanupError)}`; + const retry = () => { + if (cleaning || complete) return; + cleaning = true; + try { + retryCleanup(); + if (timer) clearTimeout(timer); + timer = undefined; + complete = true; + if (active.get(invocation.attemptId) === handle) active.delete(invocation.attemptId); + cleanupRecoveries.delete(handle); + resolveSettled(Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, + exitCode: null, signal: null, stopReason: cancelReason ?? 'capture-failure', stdout: '', + stderr: diagnosticFor(cancelReason ?? 'capture-failure', detail).toString('utf8') })); + } catch { + cleaning = false; + if (!timer) { + timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); + } + return; + } + cleaning = false; + }; + handle = Object.freeze({ attemptId: invocation.attemptId, settled, + cancel: (reason: StopReason) => { + cancelReason ??= reason; + if (timer) clearTimeout(timer); + timer = undefined; + retry(); + } }); + if (register) active.set(invocation.attemptId, handle); + else cleanupRecoveries.add(handle); + timer = setTimeout(() => { timer = undefined; retry(); }, 1_000); + return handle; +} +const withDiagnostic = (stderr: Buffer, stdoutBytes: number, reason: StopReason, limits: CaptureLimits, + detail?: string) => { + const diagnostic = diagnosticFor(reason, detail).subarray(0, DIAGNOSTIC_BYTES); + const maximum = Math.max(0, Math.min(limits.stderrBytes, limits.combinedBytes - stdoutBytes)); + if (maximum <= diagnostic.length) return diagnostic.subarray(0, maximum); + return Buffer.concat([stderr.subarray(0, maximum - diagnostic.length), diagnostic]); +}; +/** Read a running container's tmpfs file with a pinned no-follow bounded reader. */ +export function readBoundedContainerFile(container: string, source: string, maximumBytes: number, + timeoutMs = 30_000, signal?: AbortSignal): Promise { + positiveInteger(maximumBytes, 'maximumBytes'); + positiveInteger(timeoutMs, 'timeoutMs'); + if (maximumBytes > OUTPUT_LIMITS.stdoutBytes) + throw new Error('Adapter output read cannot exceed the production stdout limit.'); + if (!/^\/run\/codeboost-output\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(source)) + throw new Error('Adapter output must come from the bounded output directory.'); + const reader = [ + "const fs=require('node:fs'),path=process.argv[1],maximum=Number(process.argv[2]),directory='/run/codeboost-output';", + 'let dirfd,fd;try{dirfd=fs.openSync(directory,fs.constants.O_RDONLY|fs.constants.O_DIRECTORY|fs.constants.O_NOFOLLOW);', + "fd=fs.openSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),", + 'fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK);', + "const before=fs.fstatSync(fd,{bigint:true});if(before.size>BigInt(maximum))throw new Error('OUTPUT_LIMIT');", + "if(!before.isFile()||before.nlink!==1n)throw new Error('UNSAFE_FILE');", + 'const output=Buffer.allocUnsafe(maximum+1);let length=0,count=0;', + 'do{count=fs.readSync(fd,output,length,output.length-length,null);length+=count}', + "while(count>0&&lengthmaximum)throw new Error('OUTPUT_LIMIT');", + 'const after=fs.fstatSync(fd,{bigint:true});if(before.dev!==after.dev||before.ino!==after.ino||before.size!==after.size', + '||before.mtimeMs!==after.mtimeMs||before.ctimeMs!==after.ctimeMs||after.nlink!==1n', + "||!after.isFile())throw new Error('CHANGED_FILE');", + "const named=fs.lstatSync('/proc/self/fd/'+dirfd+'/'+path.slice(directory.length+1),{bigint:true,throwIfNoEntry:false});", + "if(!named||!named.isFile()||named.dev!==after.dev||named.ino!==after.ino||named.nlink!==1n)throw new Error('REPLACED_FILE');", + "process.stdout.write(output.subarray(0,length))}catch(error){const codes={OUTPUT_LIMIT:42,UNSAFE_FILE:43,CHANGED_FILE:44,REPLACED_FILE:45};process.exitCode=codes[error.message]||46}", + 'finally{if(fd!==undefined)fs.closeSync(fd);if(dirfd!==undefined)fs.closeSync(dirfd)}', + ].join(''); + return new Promise((resolve, reject) => { + execFile('docker', ['exec', container, 'node', '-e', reader, source, String(maximumBytes)], { + env: dockerEnvironment(), timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: maximumBytes + 1, + encoding: 'buffer', signal, + }, (error, stdout, stderr) => { + if (!error) { resolve(stdout); return; } + if (error.code === 42) { + reject(new OutputLimitError('Adapter output exceeds its capture limit.')); return; + } + if ('killed' in error && error.killed) { + reject(new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')); return; + } + const reason = error.code === 43 ? 'unsafe type or link count' : error.code === 44 ? 'changed while reading' + : error.code === 45 ? 'pathname identity changed' : 'reader failure'; + reject(new Error(`Adapter output is not a stable bounded unlinked regular file (${reason}).`)); + }); + }); +} + +export function isInvocationActive(attemptId: string): boolean { + return ownsAttempt(attemptId); +} + +export function startProfileInvocation(profile: ContainerProfile, options: SupervisorOptions = {}): InvocationHandle { + assertContainerProfileAuthenticity(profile); + const invocation = assertPhasePolicy(profile.policy); + // Rejections before container creation own no container, so they release (and retry) only the profile's own + // staging and network; a name held by another invocation or a failing inspect cannot block that. + const rejectWithCleanup = (error: unknown, register = true): InvocationHandle => { + try { disposeContainerProfile(profile); } + catch (cleanupError) { + return retainCleanupOwnership(profile, + `Invocation was rejected and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`, register, + disposeContainerProfile); + } + throw error; + }; + if (ownsAttempt(invocation.attemptId)) { + if (activeProfiles.has(profile)) + throw new Error('This container profile already owns the active invocation.'); + // This profile never created a container; the name belongs to the active invocation, so only + // release this profile's own staging and network. + return rejectWithCleanup(new Error('An invocation with this attempt ID is still active.'), false); + } + let limits: CaptureLimits, configuredTimeout: number, carriedBudget: number; + try { + limits = captureLimits(options.limits); + configuredTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + positiveInteger(configuredTimeout, 'timeoutMs'); + if (configuredTimeout > DEFAULT_TIMEOUT_MS) + throw new Error('timeoutMs cannot exceed the production ten-minute ceiling.'); + carriedBudget = options.invocationBudget?.() ?? DEFAULT_TIMEOUT_MS; + positiveInteger(carriedBudget, 'invocationBudget'); + if (carriedBudget > DEFAULT_TIMEOUT_MS) + throw new Error('invocationBudget cannot exceed the production ten-minute ceiling.'); + } catch (error) { + return rejectWithCleanup(error); + } + const wallRemaining = invocation.deadline - Date.now(); + if (!Number.isSafeInteger(wallRemaining) || wallRemaining < 1) { + return rejectWithCleanup(new Error('Invocation deadline has already expired.')); + } + const duration = Math.min(wallRemaining, configuredTimeout, carriedBudget), deadline = performance.now() + duration; + const remaining = () => { + const value = Math.ceil(deadline - performance.now()); + if (value < 1) throw new Error('Invocation deadline has already expired.'); + return value; + }; + try { + createValidatedContainer(profile, remaining(), options.secrets ?? {}); + } catch (error) { + if (!isContainerProfileAuthentic(profile)) throw error; + try { disposeValidatedContainer(profile); } + catch (cleanupError) { + const detail = `Container validation failed and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`; + return retainCleanupOwnership(profile, detail); + } + throw error; + } + + const stdoutChunks = new ByteCollector(), stderrChunks = new ByteCollector(); + let stdoutBytes = 0, stderrBytes = 0, combinedBytes = 0; + let stopReason: StopReason | undefined, failureDetail: string | undefined; + let closed = false, terminating = false, settlementComplete = false; + let decodedOutput: DecodedOutput | undefined, decodePromise: Promise | undefined; + let decodeAbort: AbortController | undefined; + let protocolToken: string | undefined, protocolStarted = false, protocolReady = false; + let protocolBuffer = Buffer.alloc(0); + try { validateContainer(profile.name, profile, remaining()); } + catch (error) { + try { disposeValidatedContainer(profile); } + catch (cleanupError) { + const detail = `Final container validation failed and cleanup remains unsettled: ${String(error)}; ${String(cleanupError)}`; + return retainCleanupOwnership(profile, detail); + } + throw error; + } + const child = spawn('docker', ['start', '--attach', profile.name], { + env: dockerEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], + }); + const timers = new Set>(); + const controls = new Set>(); + const runControl = (args: readonly string[], timeoutMs = 5_000) => { + const operation = new Promise(resolve => { + const control = spawn('docker', [...args], { env: dockerEnvironment(), stdio: 'ignore' }); + const timer = setTimeout(() => control.kill('SIGKILL'), timeoutMs); + timer.unref(); + let completed = false; + const done = (success: boolean) => { + if (completed) return; + completed = true; clearTimeout(timer); resolve(success); + }; + control.once('close', code => done(code === 0)); + control.once('error', () => done(false)); + }); + controls.add(operation); + void operation.finally(() => controls.delete(operation)); + return operation; + }; + const acknowledgeDeferredOutput = (token: string) => { + const script = [ + "const fs=require('node:fs'),token=process.argv[1],directory='/run/codeboost-control';", + 'let dirfd,fd;try{dirfd=fs.openSync(directory,fs.constants.O_RDONLY|fs.constants.O_DIRECTORY|fs.constants.O_NOFOLLOW);', + "fd=fs.openSync('/proc/self/fd/'+dirfd+'/collected-'+token,", + 'fs.constants.O_WRONLY|fs.constants.O_CREAT|fs.constants.O_EXCL|fs.constants.O_NOFOLLOW,0o444);', + "fs.writeFileSync(fd,token);fs.fsyncSync(fd);const stat=fs.fstatSync(fd,{bigint:true});", + "if(!stat.isFile()||stat.nlink!==1n)throw new Error('UNSAFE_ACK')}finally{if(fd!==undefined)fs.closeSync(fd);", + 'if(dirfd!==undefined)fs.closeSync(dirfd)}', + ].join(''); + return runControl(['exec', '--user', '0', profile.name, 'node', '-e', script, token]); + }; + const later = (callback: () => void, delay: number) => { + const timer = setTimeout(() => { timers.delete(timer); callback(); }, delay); + timer.unref(); timers.add(timer); return timer; + }; + const terminate = () => { + if (terminating || closed) return; + terminating = true; + child.stdout?.resume(); child.stderr?.resume(); + void runControl(['stop', '--signal=TERM', '--time=1', profile.name]); + later(() => { if (!closed) void runControl(['kill', '--signal=KILL', profile.name]); }, 1_500); + later(() => { + if (!closed) { + void runControl(['rm', '--force', profile.name]); + child.kill('SIGKILL'); + } + }, 4_000); + }; + const stop = (reason: StopReason) => { + if (settlementComplete || stopReason) return; + stopReason = reason; + decodeAbort?.abort(); + if (!closed) terminate(); + }; + const capture = (stream: 'stdout' | 'stderr', value: Buffer | string, final = false) => { + if (!final && (stopReason || closed)) return; + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); + const streamBytes = stream === 'stdout' ? stdoutBytes : stderrBytes; + const streamLimit = stream === 'stdout' ? limits.stdoutBytes : limits.stderrBytes; + const available = Math.max(0, Math.min(streamLimit - streamBytes, limits.combinedBytes - combinedBytes)); + if (available > 0) { + const retained = chunk.subarray(0, available); + (stream === 'stdout' ? stdoutChunks : stderrChunks).push(retained); + if (stream === 'stdout') stdoutBytes += retained.length; + else stderrBytes += retained.length; + combinedBytes += retained.length; + } + if (chunk.length > available) { + if (final) stopReason ??= 'output-limit'; + else stop('output-limit'); + } + }; + const consumeProtocol = (length: number) => { + const available = Math.max(0, Math.min(limits.stderrBytes - stderrBytes, + limits.combinedBytes - combinedBytes)); + const consumed = Math.min(length, available); + stderrBytes += consumed; + combinedBytes += consumed; + if (length > available) stop('output-limit'); + }; + const decodeOutput = () => { + if (decodePromise) return decodePromise; + if (!options.decode || decodedOutput || stopReason) return Promise.resolve(); + decodePromise = (async () => { + try { + const budget = Math.ceil(deadline - performance.now()); + if (budget < 1) throw new CaptureDeadlineError('Invocation deadline expired before output capture.'); + const controller = new AbortController(); + decodeAbort = controller; + const aborted = new Promise((_resolve, reject) => { + controller.signal.addEventListener('abort', () => reject(stopReason === 'timeout' + ? new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.') + : new Error('Adapter output capture was cancelled.')), { once: true }); + }); + const raw = stdoutChunks.toBuffer(); + const operation = Promise.resolve(options.decode!(profile, raw, + Math.max(1, Math.min(limits.stdoutBytes - stdoutBytes, limits.combinedBytes - combinedBytes)), + budget, controller.signal)); + let decodeTimer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + decodeTimer = setTimeout(() => { + stop('timeout'); + reject(new CaptureDeadlineError('Adapter output capture exceeded the invocation deadline.')); + }, budget); + }); + let decoded: DecodedOutput; + try { decoded = await Promise.race([operation, timeout, aborted]); } + catch (error) { + controller.abort(); + let graceTimer: ReturnType | undefined; + const grace = new Promise(resolve => { + graceTimer = setTimeout(resolve, CAPTURE_ABORT_GRACE_MS); + }); + await Promise.race([operation.then(() => undefined, () => undefined), grace]); + if (graceTimer) clearTimeout(graceTimer); + void operation.catch(() => { /* prevent a detached noncooperative decoder from becoming unhandled */ }); + throw error; + } finally { + if (decodeTimer) clearTimeout(decodeTimer); + if (decodeAbort === controller) decodeAbort = undefined; + } + if (stopReason) return; + if (performance.now() >= deadline) + throw new CaptureDeadlineError('Invocation deadline expired while decoding output.'); + const additional = decoded.additionalBytes ?? 0; + const textBytes = Buffer.byteLength(decoded.text); + if (!Number.isSafeInteger(additional) || additional < 0) + throw new Error('Adapter returned an invalid byte count.'); + if ((additional > 0 && additional < textBytes) || textBytes > limits.stdoutBytes + || textBytes + stderrBytes > limits.combinedBytes) + throw new OutputLimitError('Decoded adapter output exceeds its capture limit.'); + if (stdoutBytes + additional > limits.stdoutBytes || combinedBytes + additional > limits.combinedBytes) + throw new OutputLimitError('Adapter output exceeds its capture limit.'); + if (performance.now() >= deadline) + throw new CaptureDeadlineError('Invocation deadline expired while validating decoded output.'); + decodedOutput = decoded; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const reason: StopReason = stopReason ?? (performance.now() >= deadline ? 'timeout' + : error instanceof OutputLimitError || /exceeds its capture limit/i.test(message) + ? 'output-limit' : error instanceof CaptureDeadlineError ? 'timeout' : 'capture-failure'); + failureDetail ??= message; + if (closed) stopReason ??= reason; + else stop(reason); + } + })(); + return decodePromise; + }; + const protocolLine = (line: Buffer) => { + const text = line.toString('utf8').trim(); + const started = /^\x1eCODEBOOST_START:([0-9a-f-]{36})\x1e$/.exec(text); + if (started) { + consumeProtocol(line.length); + if (stopReason) return true; + if (protocolStarted) { + failureDetail ??= 'Deferred output emitted a duplicate START frame.'; + stop('capture-failure'); + return true; + } + if (protocolToken && protocolToken !== started[1]) return false; + protocolStarted = true; + protocolToken = started[1]; + return true; + } + const ready = /^\x1eCODEBOOST_READY:([0-9a-f-]{36}):([0-9]+)\x1e$/.exec(text); + if (!ready || ready[1] !== protocolToken) return false; + consumeProtocol(line.length); + if (stopReason) return true; + if (protocolReady) { + failureDetail ??= 'Deferred output emitted a duplicate READY frame.'; + stop('capture-failure'); + return true; + } + protocolReady = true; + const readyToken = ready[1]!; + void decodeOutput().then(() => { + if (decodedOutput && !stopReason) { + void acknowledgeDeferredOutput(readyToken) + .then(success => { + if (!success) { + failureDetail ??= 'Deferred output acknowledgement failed.'; + stop('capture-failure'); + } + }); + } + }); + return true; + }; + const captureStderr = (value: Buffer | string) => { + if (stopReason || closed) return; + if (!profile.deferredOutput) { capture('stderr', value); return; } + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); + protocolBuffer = Buffer.concat([protocolBuffer, chunk]); + let newline: number; + while ((newline = protocolBuffer.indexOf(0x0a)) >= 0) { + const line = protocolBuffer.subarray(0, newline + 1); + protocolBuffer = protocolBuffer.subarray(newline + 1); + if (!protocolLine(line)) capture('stderr', line); + } + if (protocolBuffer.length > 1024) { + const flush = protocolBuffer.subarray(0, protocolBuffer.length - 128); + protocolBuffer = protocolBuffer.subarray(protocolBuffer.length - 128); + capture('stderr', flush); + } + }; + child.stdout?.on('data', value => capture('stdout', value)); + child.stderr?.on('data', captureStderr); + child.stdout?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); + child.stderr?.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); + child.once('error', error => { failureDetail ??= error.message; stop('capture-failure'); }); + later(() => stop('timeout'), Math.max(1, Math.ceil(deadline - performance.now()))); + + let resolveSettled!: (result: InvocationResult) => void; + let wakeCleanup: (() => void) | undefined; + const settled = new Promise(resolve => { resolveSettled = resolve; }); + const handle: InvocationHandle = Object.freeze({ + attemptId: invocation.attemptId, + settled, + cancel: (reason: StopReason) => { stop(reason); wakeCleanup?.(); }, + }); + active.set(invocation.attemptId, handle); + activeProfiles.add(profile); + + child.once('close', async (code, signal) => { + for (const timer of timers) clearTimeout(timer); + timers.clear(); + closed = true; + if (!stopReason && performance.now() >= deadline) stop('timeout'); + if (protocolBuffer.length) { + if (!protocolLine(protocolBuffer)) capture('stderr', protocolBuffer, true); + protocolBuffer = Buffer.alloc(0); + } + let finalStdout = stdoutChunks.toBuffer(), finalStderr = stderrChunks.toBuffer(); + let exitCode = code, finalSignal = signal; + if (!stopReason && options.decode && !profile.deferredOutput) await decodeOutput(); + if (decodePromise) await decodePromise; + if (!stopReason && profile.deferredOutput && !decodedOutput) { + stopReason = 'capture-failure'; failureDetail ??= 'Deferred output protocol did not complete.'; + } + if (decodedOutput && !stopReason) { + finalStdout = Buffer.from(decodedOutput.text); + if (decodedOutput.providerFailed) exitCode = exitCode === 0 ? 1 : exitCode; + } + await Promise.all([...controls]); + while (true) { + try { + disposeValidatedContainer(profile); + wakeCleanup = undefined; + break; + } catch (error) { + stopReason ??= 'capture-failure'; + failureDetail ??= error instanceof Error ? error.message : String(error); + await new Promise(resolve => { + let finished = false; + const wake = () => { if (finished) return; finished = true; clearTimeout(timer); resolve(); }; + const timer = setTimeout(wake, 1_000); + wakeCleanup = wake; + }); + } + } + // Publish only strictly valid UTF-8: replacement characters would grow the result past the byte ceilings. + // Only output cut at a capture limit may end in an incomplete character, which the streaming decode then drops; + // otherwise the decode flushes, so a trailing lone lead byte fails closed. + const truncated = stopReason === 'output-limit'; + const strictText = (value: Buffer) => { + try { return new TextDecoder('utf-8', { fatal: true }).decode(value, truncated ? { stream: true } : undefined); } + catch { return undefined; } + }; + const stdoutText = strictText(finalStdout), stderrText = strictText(finalStderr); + if (stdoutText === undefined || stderrText === undefined) { + stopReason ??= 'capture-failure'; + failureDetail ??= 'Captured output is not valid UTF-8.'; + } + finalStdout = Buffer.from(stdoutText ?? ''); finalStderr = Buffer.from(stderrText ?? ''); + if (stopReason) finalStderr = withDiagnostic(finalStderr, finalStdout.length, stopReason, limits, failureDetail); + const result = Object.freeze({ attemptId: invocation.attemptId, context: invocation.context, + exitCode, signal: finalSignal, ...(stopReason ? { stopReason } : {}), + stdout: finalStdout.toString('utf8'), stderr: finalStderr.toString('utf8') }); + settlementComplete = true; + activeProfiles.delete(profile); + active.delete(invocation.attemptId); + resolveSettled(result); + }); + return handle; +} diff --git a/agents/adapters/types.ts b/agents/adapters/types.ts new file mode 100644 index 0000000..01291b9 --- /dev/null +++ b/agents/adapters/types.ts @@ -0,0 +1,42 @@ +import type { InvocationInput } from '../contract.ts'; +import type { TaskFilesystems } from '../container/storage.ts'; +import type { CaptureLimits } from './supervisor.ts'; + +export interface AgentAdapterRequest { + readonly invocation: InvocationInput; + readonly filesystems: TaskFilesystems; + readonly inputDirectory: string; + readonly imageId: string; + readonly prompt: string; +} +export interface AgentAdapterOptions { + readonly timeoutMs?: number; + readonly limits?: Partial; +} +const MAXIMUM_INVOCATION_MS = 10 * 60_000; + +/** Convert an absolute wall-clock deadline once, then enforce it with a monotonic clock. */ +export function createInvocationBudget(invocation: InvocationInput, maximumMs: number): () => number { + if (!Number.isSafeInteger(maximumMs) || maximumMs < 1) + throw new Error('Invocation setup budget must be a positive integer.'); + const wallRemaining = invocation.deadline - Date.now(); + if (!Number.isSafeInteger(wallRemaining) || wallRemaining < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + const duration = Math.min(maximumMs, wallRemaining); + const end = performance.now() + duration; + return () => { + const value = Math.ceil(end - performance.now()); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error('Invocation deadline expired during adapter setup.'); + return value; + }; +} + +export function createAdapterInvocationBudget(invocation: InvocationInput, + timeoutMs = MAXIMUM_INVOCATION_MS): () => number { + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) + throw new Error('timeoutMs must be a positive integer.'); + if (timeoutMs > MAXIMUM_INVOCATION_MS) + throw new Error('timeoutMs cannot exceed the production ten-minute ceiling.'); + return createInvocationBudget(invocation, timeoutMs); +} diff --git a/agents/container/probe.sh b/agents/container/probe.sh index 70f9f24..06a4290 100644 --- a/agents/container/probe.sh +++ b/agents/container/probe.sh @@ -70,6 +70,13 @@ case "$CODEBOOST_VENDOR" in require_option "$CODEX_HOME" nodev require_option "$CODEX_HOME/auth.json" ro require_ceiling "$CODEX_HOME" 4194304 256 + [ "$(findmnt --noheadings --output FSTYPE --target /run/codeboost-output)" = 'tmpfs' ] \ + || fail 'Codex output must use tmpfs' + require_option /run/codeboost-output rw + require_option /run/codeboost-output nosuid + require_option /run/codeboost-output nodev + require_option /run/codeboost-output noexec + require_ceiling /run/codeboost-output 20971520 64 ;; claude) [ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] || fail 'Claude credential is missing' @@ -81,4 +88,25 @@ esac [ "$(codex --version)" = 'codex-cli 0.153.4' ] || fail 'unexpected Codex version' [ "$(claude --version | awk '{print $1}')" = '2.1.281' ] || fail 'unexpected Claude version' +if [ "${CODEBOOST_DEFERRED_OUTPUT:-}" = '1' ]; then + [ "$(findmnt --noheadings --output FSTYPE --target /run/codeboost-control)" = 'tmpfs' ] \ + || fail 'deferred control must use tmpfs' + require_option /run/codeboost-control rw + require_option /run/codeboost-control nosuid + require_option /run/codeboost-control nodev + require_option /run/codeboost-control noexec + require_ceiling /run/codeboost-control 65536 16 + [ ! -w /run/codeboost-control ] || fail 'agent must not write deferred control markers' + token="$(cat /proc/sys/kernel/random/uuid)" + printf '\036CODEBOOST_START:%s\036\n' "$token" >&2 + set +e + "$@" + status="$?" + set -e + printf '\n\036CODEBOOST_READY:%s:%s\036\n' "$token" "$status" >&2 + acknowledgement="/run/codeboost-control/collected-$token" + while [ ! -f "$acknowledgement" ] || [ -L "$acknowledgement" ] \ + || [ "$(cat "$acknowledgement" 2>/dev/null || true)" != "$token" ]; do sleep 0.05; done + exit "$status" +fi exec "$@" diff --git a/agents/container/profile.ts b/agents/container/profile.ts index 02c5f26..cb8754d 100644 --- a/agents/container/profile.ts +++ b/agents/container/profile.ts @@ -21,6 +21,7 @@ export interface ContainerProfile { readonly ownershipId: string; readonly network: VendorNetwork; readonly policy: PhasePolicy; + readonly deferredOutput: boolean; } export interface ProfileOptions { readonly invocation: InvocationInput; @@ -32,6 +33,20 @@ export interface ProfileOptions { readonly claudeToken?: string; readonly network: VendorNetwork; readonly policy: PhasePolicy; + readonly deferredOutput?: boolean; + /** Remaining invocation budget for Docker-backed profile validation. */ + readonly timeoutMs?: number; +} + +export class ProfileCreationCleanupError extends AggregateError { + readonly startupError: unknown; + readonly retryCleanup: () => void; + + constructor(startupError: unknown, cleanupError: unknown, retryCleanup: () => void) { + super([startupError, cleanupError], 'Profile creation and cleanup both failed.'); + this.startupError = startupError; + this.retryCleanup = retryCleanup; + } } interface FileIdentity { @@ -112,6 +127,15 @@ const captureInput = (directory: string): InputCapture => { return Object.freeze({ inputDirectory: canonical, schema, content: captured.content }); }; +/** Prove that a profile object is the exact capability issued by this module. */ +export function assertContainerProfileAuthenticity(profile: ContainerProfile): void { + if (!identities.has(profile)) throw new Error('Container profile was not created by the trusted profile builder.'); +} + +export function isContainerProfileAuthentic(profile: ContainerProfile): boolean { + return identities.has(profile); +} + /** Internal authenticity and host-file revalidation used at every launch boundary. */ export function assertContainerProfile(profile: ContainerProfile, timeoutMs = 30_000): void { const expected = identities.get(profile); @@ -129,10 +153,6 @@ export function assertContainerProfile(profile: ContainerProfile, timeoutMs = 30 } } -export function isContainerProfileAuthentic(profile: ContainerProfile): boolean { - return identities.has(profile); -} - /** Clamp a Docker budget to the captured invocation deadline, which no launch may outlive. */ export function profileTimeout(profile: ContainerProfile, timeoutMs: number, now = Date.now()): number { const expected = identities.get(profile); @@ -174,7 +194,7 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil assertTaskFilesystems(filesystems, invocation.clone); const invocationLeft = Math.floor(invocation.deadline - Date.now()); if (invocationLeft < 1) throw new Error('Invocation deadline has passed.'); - assertVendorNetwork(options.network, invocation, undefined, Math.min(30_000, invocationLeft)); + assertVendorNetwork(options.network, invocation, undefined, Math.min(options.timeoutMs ?? 30_000, invocationLeft)); if (claimedNetworks.has(options.network)) throw new Error('Vendor network already belongs to another container profile.'); // Own the network from here on, so any later failure removes it rather than leaking it. claimedNetworks.add(options.network); @@ -229,8 +249,14 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil '--mount', mount({ type: 'volume', source: filesystems.workVolume, target: '/work', readonly: readOnlyWork }), '--mount', mount({ type: 'volume', source: filesystems.metadataVolume, target: '/work/.git', readonly: true }), '--mount', mount({ type: 'bind', source: inputIdentity.inputDirectory, target: '/run/codeboost-input', readonly: true })]; + if (options.deferredOutput) { + if (invocation.vendor !== 'codex') throw new Error('Deferred output is available only for Codex.'); + args.push('--env', 'CODEBOOST_DEFERRED_OUTPUT=1', + '--tmpfs', '/run/codeboost-control:rw,nosuid,nodev,noexec,size=65536,nr_inodes=16,uid=0,gid=0,mode=0711'); + } if (invocation.vendor === 'codex') { args.push('--env', 'CODEX_HOME=/run/codeboost-auth/codex', + '--tmpfs', '/run/codeboost-output:rw,nosuid,nodev,noexec,size=20971520,nr_inodes=64,uid=10001,gid=10001,mode=0700', '--tmpfs', '/run/codeboost-auth/codex:rw,nosuid,nodev,size=4194304,nr_inodes=256,uid=10001,gid=10001,mode=0700', '--mount', mount({ type: 'bind', source: codexAuthFile!, target: '/run/codeboost-auth/codex/auth.json', readonly: true })); } else args.push('--env', 'CLAUDE_CODE_OAUTH_TOKEN'); @@ -239,17 +265,22 @@ export function createContainerProfile(options: ProfileOptions): ContainerProfil const profile = Object.freeze({ name, args: Object.freeze(args), expectedImage: options.imageId, phase: invocation.phase, vendor: invocation.vendor, filesystems: capturedFilesystems, inputDirectory: inputIdentity.inputDirectory, codexAuthFile, - command: Object.freeze([...command]), ownershipId, network: options.network, policy: options.policy }); + command: Object.freeze([...command]), ownershipId, network: options.network, policy: options.policy, + deferredOutput: options.deferredOutput === true }); identities.set(profile, Object.freeze({ inputDirectory: inputIdentity.inputDirectory, schema: inputIdentity.schema, auth: authIdentity, cleanupDirectories: Object.freeze([...cleanupDirectories]), filesystems, clone: invocation.clone, deadline: invocation.deadline, network: options.network, policy: options.policy, invocation })); return profile; } catch (error) { - const failures: unknown[] = []; - try { removeOwnedDirectories(cleanupDirectories); } catch (cleanupError) { failures.push(cleanupError); } - try { removeVendorNetwork(options.network); } catch (cleanupError) { failures.push(cleanupError); } - if (failures.length) throw new AggregateError([error, ...failures], 'Profile creation and cleanup both failed.'); + const cleanupProfileResources = () => { + const failures: unknown[] = []; + try { removeOwnedDirectories(cleanupDirectories); } catch (cleanupError) { failures.push(cleanupError); } + try { removeVendorNetwork(options.network); } catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Profile resource cleanup did not settle.'); + }; + try { cleanupProfileResources(); } + catch (cleanupError) { throw new ProfileCreationCleanupError(error, cleanupError, cleanupProfileResources); } throw error; } } diff --git a/agents/container/run.ts b/agents/container/run.ts index e9d7b7f..85f7d44 100644 --- a/agents/container/run.ts +++ b/agents/container/run.ts @@ -1,6 +1,7 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { realpathSync } from 'node:fs'; -import { assertContainerProfile, disposeContainerProfile, isContainerProfileAuthentic, profileTimeout, +import { assertContainerProfile, assertContainerProfileAuthenticity, disposeContainerProfile, + isContainerProfileAuthentic, profileTimeout, type ContainerProfile } from './profile.ts'; import { BASE_IMAGE, CLAUDE_VERSION, CODEX_VERSION } from './image.ts'; import { taskFilesystemAllocationId } from './storage.ts'; @@ -48,12 +49,14 @@ const canonicalDockerBindSource = (source: string) => { /** How long a killed `docker create` may still materialize its container in the daemon. */ const CREATE_SETTLE_MS = 10_000; const sleep = (ms: number) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = false) => { +// When a killed `docker create` for a profile stops counting as possibly in flight (performance.now() timestamp). +const unsettledCreates = new WeakMap(); +const removeContainerOrThrow = (profile: ContainerProfile, waitForSettle = false) => { // Destructive cleanup acts only for the builder-registered profile; a copy's name and label are not a capability. if (!isContainerProfileAuthentic(profile)) throw new Error('Container profile was not created by the trusted profile builder.'); - const remaining = createDeadline(30_000 + (createUnsettled ? CREATE_SETTLE_MS : 0)); - const settleBy = performance.now() + (createUnsettled ? CREATE_SETTLE_MS : 0); + const settleUntil = unsettledCreates.get(profile) ?? 0; + const remaining = createDeadline(30_000 + (waitForSettle ? Math.max(0, Math.ceil(settleUntil - performance.now())) : 0)); let before: ReturnType; for (;;) { before = spawnSync('docker', ['container', 'inspect', profile.name], { @@ -62,13 +65,15 @@ const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = fal if (before.status === 0) break; const missing = !before.error && /No such (?:object|container)/i.test(`${before.stdout ?? ''}\n${before.stderr ?? ''}`); if (!missing) throw new Error('Failed to establish ownership of the agent container; staged credentials were retained.'); - if (!createUnsettled) { + // A killed create may still land in the daemon; absence only counts once its settle window has passed, on every + // path. Only the create path waits here; later cleanup (such as a supervisor recovery) reports "not settled" + // inside the window and retries later. + if (performance.now() >= settleUntil) { + unsettledCreates.delete(profile); disposeContainerProfile(profile); return; } - // A killed create may still land in the daemon; absence is not proof until the settle window passes. - if (performance.now() >= settleBy) - throw new Error('Agent container creation did not settle; staged credentials were retained.'); + if (!waitForSettle) throw new Error('Agent container creation did not settle; staged credentials were retained.'); sleep(250); } const inspected = JSON.parse(String(before.stdout || '[]'))[0] as { Config?: { Labels?: Record } } | undefined; @@ -85,9 +90,16 @@ const removeContainerOrThrow = (profile: ContainerProfile, createUnsettled = fal && /No such (?:object|container)/i.test(`${inspect.stdout ?? ''}\n${inspect.stderr ?? ''}`); if (!absent) throw new Error('Failed to confirm removal of the agent container; staged credentials were retained.'); } + unsettledCreates.delete(profile); disposeContainerProfile(profile); }; +/** Remove a validated invocation container, then its profile-owned staging and network resources. */ +export function disposeValidatedContainer(profile: ContainerProfile): void { + assertContainerProfileAuthenticity(profile); + removeContainerOrThrow(profile); +} + type Inspect = { Image: string; Config: { Image: string; User: string; Env: string[]; Entrypoint: string[] | null; Cmd: string[] | null; @@ -166,7 +178,11 @@ export function validateContainer(container: string, profile: ContainerProfile, ['/tmp', ['rw', 'nosuid', 'nodev', 'size=33554432', 'nr_inodes=4096', 'mode=1777']], ['/home/codeboost', ['rw', 'nosuid', 'nodev', 'size=1048576', 'nr_inodes=128', 'uid=10001', 'gid=10001', 'mode=0700']], ...(profile.vendor === 'codex' ? [['/run/codeboost-auth/codex', - ['rw', 'nosuid', 'nodev', 'size=4194304', 'nr_inodes=256', 'uid=10001', 'gid=10001', 'mode=0700']] as const] : []), + ['rw', 'nosuid', 'nodev', 'size=4194304', 'nr_inodes=256', 'uid=10001', 'gid=10001', 'mode=0700']] as const, + ['/run/codeboost-output', + ['rw', 'nosuid', 'nodev', 'noexec', 'size=20971520', 'nr_inodes=64', 'uid=10001', 'gid=10001', 'mode=0700']] as const] : []), + ...(profile.deferredOutput ? [['/run/codeboost-control', + ['rw', 'nosuid', 'nodev', 'noexec', 'size=65536', 'nr_inodes=16', 'uid=0', 'gid=0', 'mode=0711']] as const] : []), ]); if (Object.keys(tmpfs).length !== expectedTmpfs.size) throw new Error('Container tmpfs mount set changed.'); for (const [path, expected] of expectedTmpfs) { @@ -239,6 +255,7 @@ export function validateContainer(container: string, profile: ContainerProfile, const allowedEnvironment = new Set(['PATH', 'NODE_VERSION', 'YARN_VERSION', 'HOME', 'CODEBOOST_PHASE', 'CODEBOOST_VENDOR', 'CODEBOOST_WORK_BYTES', 'CODEBOOST_WORK_INODES', 'CODEBOOST_METADATA_BYTES', 'CODEBOOST_METADATA_INODES', 'npm_config_cache', 'XDG_CACHE_HOME', 'HTTPS_PROXY', 'HTTP_PROXY', 'NO_PROXY', + ...(profile.deferredOutput ? ['CODEBOOST_DEFERRED_OUTPUT'] : []), ...(profile.vendor === 'codex' ? ['CODEX_HOME'] : ['CLAUDE_CODE_OAUTH_TOKEN'])]); if (new Set(names).size !== names.length || names.some(name => !allowedEnvironment.has(name))) throw new Error('Container includes an unexpected environment variable.'); @@ -255,6 +272,8 @@ export function validateContainer(container: string, profile: ContainerProfile, || environment.get('HTTP_PROXY') !== profile.network.proxyUrl || environment.get('NO_PROXY') !== 'localhost,127.0.0.1') throw new Error('Container isolation environment changed.'); + if (profile.deferredOutput && environment.get('CODEBOOST_DEFERRED_OUTPUT') !== '1') + throw new Error('Container deferred-output protocol changed.'); if (profile.vendor === 'codex' && (names.includes('CLAUDE_CODE_OAUTH_TOKEN') || environment.get('CODEX_HOME') !== '/run/codeboost-auth/codex')) throw new Error('Credential profiles must not be combined or redirected.'); @@ -285,6 +304,7 @@ export function createValidatedContainer(profile: ContainerProfile, timeoutMs = remaining(); return profile.name; } catch (error) { + if (createUnsettled) unsettledCreates.set(profile, performance.now() + CREATE_SETTLE_MS); try { removeContainerOrThrow(profile, createUnsettled); } catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Container creation failed and cleanup did not settle.'); } throw error; diff --git a/agents/network/network.ts b/agents/network/network.ts index 56c451a..e62fb08 100644 --- a/agents/network/network.ts +++ b/agents/network/network.ts @@ -18,6 +18,16 @@ interface NetworkIdentity { readonly allocationId: string; readonly imageId: str readonly subnet: string; readonly proxyIp: string; /** Daemon object IDs captured at creation; a same-named replacement has a different ID. */ readonly networkId: string; readonly proxyId: string } +export class VendorNetworkCreationCleanupError extends AggregateError { + readonly startupError: unknown; + readonly retryCleanup: () => void; + + constructor(startupError: unknown, cleanupError: unknown, retryCleanup: () => void) { + super([startupError, cleanupError], 'Vendor network creation and cleanup failed.'); + this.startupError = startupError; + this.retryCleanup = retryCleanup; + } +} const identities = new WeakMap(); const removedNetworks = new WeakSet(); const environment = () => ({ PATH: process.env.PATH, DOCKER_HOST: process.env.DOCKER_HOST }); @@ -168,6 +178,24 @@ export function createVendorNetwork(invocation: InvocationInput, imageId: string throw error; } }; + // The first cleanup shares the caller's overall deadline; a later retry gets its own budget. Killed + // creates get a settle window, bounded by whatever that budget has left. + const cleanupPlannedResources = (budget: () => number = deadline(30_000)) => { + let budgetLeft = 0; + try { budgetLeft = budget(); } catch { /* the budget is spent */ } + const settleBy = (object: string) => unsettled.has(object) + ? performance.now() + Math.min(CREATE_SETTLE_MS, budgetLeft) : 0; + const failures: unknown[] = []; + // Target the created IDs; names only for a create whose ID never came back, which alone gets a settle window. + const proxyTarget = proxyId ?? proxyContainer, networkTarget = networkId ?? name; + if (proxyPlanned) try { remove(['rm', '--force', proxyTarget], ['container', 'inspect', proxyTarget], + budget, 'vendor proxy', allocationId, proxyId ? 0 : settleBy(proxyContainer)); } + catch (cleanupError) { failures.push(cleanupError); } + if (networkPlanned) try { remove(['network', 'rm', networkTarget], ['network', 'inspect', networkTarget], + budget, 'vendor network', allocationId, networkId ? 0 : settleBy(name)); } + catch (cleanupError) { failures.push(cleanupError); } + if (failures.length) throw new AggregateError(failures, 'Vendor network cleanup did not settle.'); + }; try { networkPlanned = true; networkId = createdId(create(name, ['network', 'create', '--internal', '--driver', 'bridge', '--subnet', subnet, @@ -196,26 +224,15 @@ export function createVendorNetwork(invocation: InvocationInput, imageId: string remaining(); return network; } catch (error) { - const failures: unknown[] = []; - // Killed creates get a settle window, but only inside the cleanup reserve of the caller's budget. - let reserveLeft = 0; - try { reserveLeft = overall(); } catch { /* the overall budget is spent */ } - const settleBy = (object: string) => unsettled.has(object) - ? performance.now() + Math.min(CREATE_SETTLE_MS, reserveLeft) : 0; - const cleanupBudget = overall; - const proxyTarget = proxyId ?? proxyContainer, networkTarget = networkId ?? name; - if (proxyPlanned) try { remove(['rm', '--force', proxyTarget], ['container', 'inspect', proxyTarget], - cleanupBudget, 'vendor proxy', allocationId, proxyId ? 0 : settleBy(proxyContainer)); } - catch (cleanupError) { failures.push(cleanupError); } - if (networkPlanned) try { remove(['network', 'rm', networkTarget], ['network', 'inspect', networkTarget], - cleanupBudget, 'vendor network', allocationId, networkId ? 0 : settleBy(name)); } - catch (cleanupError) { failures.push(cleanupError); } - if (failures.length) throw new AggregateError([error, ...failures], 'Vendor network creation and cleanup failed.'); + try { cleanupPlannedResources(overall); } + catch (cleanupError) { + throw new VendorNetworkCreationCleanupError(error, cleanupError, () => cleanupPlannedResources()); + } throw error; } } -export function removeVendorNetwork(network: VendorNetwork): void { +export function removeVendorNetwork(network: VendorNetwork, timeoutMs = 30_000): void { const identity = identities.get(network); if (!identity) { if (removedNetworks.has(network)) return; @@ -223,7 +240,7 @@ export function removeVendorNetwork(network: VendorNetwork): void { } assertBuiltAgentImage(identity.imageId); const allocationId = identity.allocationId; - const remaining = deadline(30_000), failures: unknown[] = []; + const remaining = deadline(timeoutMs), failures: unknown[] = []; // Remove by the captured IDs; a same-named replacement is not ours to delete and keeps the network busy. try { remove(['rm', '--force', identity.proxyId], ['container', 'inspect', identity.proxyId], remaining, 'vendor proxy', allocationId); } catch (error) { failures.push(error); } diff --git a/agents/policy.ts b/agents/policy.ts index 17c6349..0d1ebaf 100644 --- a/agents/policy.ts +++ b/agents/policy.ts @@ -84,12 +84,16 @@ export function createCodexCommand(policy: PhasePolicy, prompt: string): AgentCo if (!prompt || prompt.includes('\0')) throw new Error('Codex prompt must be nonempty and contain no NUL.'); const sandbox = policy.worktree === 'read-write' ? 'workspace-write' : 'read-only'; // `--` ends option parsing, so a prompt beginning with `-` stays prompt data. - return command(policy, [...codexBaseArguments(policy), 'exec', '--sandbox', sandbox, '--skip-git-repo-check', '--', - prompt]); + return command(policy, [...codexBaseArguments(policy), 'exec', '--sandbox', sandbox, '--skip-git-repo-check', + '--output-last-message', '/run/codeboost-output/final.txt', '--', prompt]); } export type IsolationProbe = 'noop' | 'phase-worktree' | 'read-only-isolation' | 'persist-write' - | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker'; + | 'persist-read' | 'capacity' | 'metadata' | 'must-not-run' | 'input-marker' | 'finite-output' + | 'infinite-stdout' | 'infinite-stderr' | 'infinite-mixed' | 'ignore-term' | 'symlink-output' + | 'oversized-output' | 'fifo-output' | 'invalid-utf8-output' | 'invalid-utf8-stderr' | 'truncated-utf8-stderr' + | 'replace-output-directory' + | 'nonzero-output' | 'duplicate-protocol' | 'newline-free-deferred-output'; /** Fixed startup probes validate the sandbox itself without granting an agent a process tool. */ export function createIsolationProbeCommand(policy: PhasePolicy, probe: IsolationProbe): AgentCommand { @@ -112,6 +116,21 @@ export function createIsolationProbeCommand(policy: PhasePolicy, probe: Isolatio 'must-not-run': 'touch /tmp/command-ran', 'input-marker': 'set -eu; grep -q codeboost-schema-marker /run/codeboost-input/schema.json; ' + 'test ! -e /run/codeboost-input/extra.json', + 'finite-output': 'printf stdout-marker; printf stderr-marker >&2', + 'invalid-utf8-stderr': "printf 'bad-\\377\\377-stderr' >&2", + 'truncated-utf8-stderr': "printf 'cut-\\342' >&2", + 'infinite-stdout': "while :; do head -c 4096 /dev/zero | tr '\\0' x; done", + 'infinite-stderr': "while :; do head -c 4096 /dev/zero | tr '\\0' x >&2; done", + 'infinite-mixed': "while :; do head -c 4096 /dev/zero | tr '\\0' x; head -c 4096 /dev/zero | tr '\\0' y >&2; done", + 'ignore-term': "trap '' TERM; while :; do sleep 1; done", + 'symlink-output': 'ln -s /etc/passwd /run/codeboost-output/final.txt', + 'oversized-output': 'head -c 131072 /dev/zero > /run/codeboost-output/final.txt', + 'fifo-output': 'mkfifo /run/codeboost-output/final.txt', + 'invalid-utf8-output': "printf '\\377' > /run/codeboost-output/final.txt", + 'replace-output-directory': 'rm -rf /run/codeboost-output; ln -s /etc /run/codeboost-output', + 'nonzero-output': 'printf encoded-output; exit 7', + 'duplicate-protocol': "printf '\\036CODEBOOST_START:00000000-0000-0000-0000-000000000000\\036\\n' >&2", + 'newline-free-deferred-output': "printf captured > /run/codeboost-output/final.txt; printf trailing-diagnostic >&2", }; return command(policy, probe === 'noop' ? ['true'] : ['sh', '-c', scripts[probe]]); } diff --git a/test/agent-adapter.test.ts b/test/agent-adapter.test.ts new file mode 100644 index 0000000..0d727c4 --- /dev/null +++ b/test/agent-adapter.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it, vi } from 'vitest'; +import { parseClaudeOutput, startClaudeInvocation } from '../agents/adapters/claude.ts'; +import { CODEX_OUTPUT_FILE, startCodexInvocation } from '../agents/adapters/codex.ts'; +import { isInvocationActive, OUTPUT_LIMITS, retainSetupCleanup } from '../agents/adapters/supervisor.ts'; +import { createAdapterInvocationBudget, createInvocationBudget } from '../agents/adapters/types.ts'; +import { captureInvocation } from '../agents/contract.ts'; +import { createCodexCommand, createPhasePolicy } from '../agents/policy.ts'; + +describe('production agent adapters', () => { + const capturedInvocation = (attemptId: string, deadline: number, vendor: 'codex' | 'claude' = 'codex') => + captureInvocation({ + clone: { id: 'clone', taskId: 'task', directory: '/tmp/task', head: 'a'.repeat(40) }, + phase: 'planning', vendor, approvedArgv: [], deadline, attemptId, + context: { snapshotId: 's', planId: 'p', planRevision: 1, assignmentId: 'a', + referencedCodeHash: 'c', stateVersion: 1 }, + }, deadline - 1); + + it('parses recorded Claude success and failure envelopes', () => { + expect(parseClaudeOutput(Buffer.from('{"result":"planned","is_error":false}'))) + .toEqual({ text: 'planned', providerFailed: false }); + expect(parseClaudeOutput(Buffer.from('{"result":"login required","is_error":true}'))) + .toEqual({ text: 'login required', providerFailed: true }); + expect(() => parseClaudeOutput(Buffer.from('{"result":3,"is_error":false}'))).toThrow('malformed'); + expect(() => parseClaudeOutput(Buffer.from('not json'))).toThrow(); + expect(() => parseClaudeOutput(Buffer.from([0xff]))).toThrow(); + }); + + it('routes Codex final output to the bounded scratch directory', () => { + const invocation = captureInvocation({ + clone: { id: 'clone', taskId: 'task', directory: '/tmp/task', head: 'a'.repeat(40) }, + phase: 'planning', vendor: 'codex', approvedArgv: [], deadline: 2_000, attemptId: 'adapter-command', + context: { snapshotId: 's', planId: 'p', planRevision: 1, assignmentId: 'a', referencedCodeHash: 'c', stateVersion: 1 }, + }, 1_000); + const argv = createCodexCommand(createPhasePolicy(invocation), 'Plan this.').argv; + expect(argv.slice(argv.indexOf('--output-last-message'), argv.indexOf('--output-last-message') + 2)) + .toEqual(['--output-last-message', CODEX_OUTPUT_FILE]); + }); + + it('publishes immutable production output ceilings', () => { + expect(OUTPUT_LIMITS).toEqual({ stdoutBytes: 16 * 1024 * 1024, stderrBytes: 4 * 1024 * 1024, + combinedBytes: 20 * 1024 * 1024 }); + expect(Object.isFrozen(OUTPUT_LIMITS)).toBe(true); + }); + + it.each(['codex', 'claude'] as const)('rejects expired %s setup before allocating a network', vendor => { + const invocation = capturedInvocation(`expired-${vendor}`, Date.now() - 1, vendor); + const request = { invocation, filesystems: {} as never, inputDirectory: '/unused', + imageId: `sha256:${'a'.repeat(64)}`, prompt: 'unused' }; + const start = () => vendor === 'codex' + ? startCodexInvocation(request, '/unused/auth.json') + : startClaudeInvocation(request, 'token'); + expect(start).toThrow('deadline expired during adapter setup'); + expect(isInvocationActive(invocation.attemptId)).toBe(false); + }); + + it('retains every colliding setup cleanup owner until all retries succeed', async () => { + const invocation = capturedInvocation('setup-recovery', Date.now() + 60_000); + let releaseFirst = false, releaseSecond = false; + const first = retainSetupCleanup(invocation, () => { + if (!releaseFirst) throw new Error('first still busy'); + }, new Error('startup failed'), new Error('cleanup failed')); + const second = retainSetupCleanup(invocation, () => { + if (!releaseSecond) throw new Error('second still busy'); + }, new Error('duplicate startup failed'), new Error('duplicate cleanup failed')); + expect(isInvocationActive(invocation.attemptId)).toBe(true); + releaseFirst = true; + first.cancel('cancelled'); + await first.settled; + expect(isInvocationActive(invocation.attemptId)).toBe(true); + releaseSecond = true; + second.cancel('cancelled'); + const result = await second.settled; + expect(result.stopReason).toBe('cancelled'); + expect(result.stderr).toContain('[codeboost: cancelled:'); + expect(result.stderr).toContain('setup cleanup remains unsettled'); + expect(isInvocationActive(invocation.attemptId)).toBe(false); + }); + + it('does not extend an invocation budget when the wall clock moves backward', () => { + const wall = Date.now(); + const invocation = capturedInvocation('monotonic-budget', wall + 5_000); + const clock = vi.spyOn(Date, 'now').mockReturnValueOnce(wall).mockReturnValue(wall - 60_000); + try { + const remaining = createInvocationBudget(invocation, 1_000); + expect(remaining()).toBeGreaterThan(0); + expect(remaining()).toBeLessThanOrEqual(1_000); + } finally { clock.mockRestore(); } + }); + + it('applies the configured timeout to the original adapter setup budget', () => { + const invocation = capturedInvocation('configured-budget', Date.now() + 60_000); + const remaining = createAdapterInvocationBudget(invocation, 250); + expect(remaining()).toBeGreaterThan(0); + expect(remaining()).toBeLessThanOrEqual(250); + expect(() => createAdapterInvocationBudget(invocation, 10 * 60_000 + 1)).toThrow('ten-minute ceiling'); + }); +}); diff --git a/test/agent-container.test.ts b/test/agent-container.test.ts index dd2295f..0320c88 100644 --- a/test/agent-container.test.ts +++ b/test/agent-container.test.ts @@ -6,9 +6,10 @@ import { join } from 'node:path'; import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; import { captureInvocation, type InvocationInput, type Phase } from '../agents/contract.ts'; import { AGENT_IMAGE, assertBuiltAgentImage, buildAgentImage } from '../agents/container/image.ts'; -import { assertContainerProfile, createContainerProfile, disposeContainerProfile } from '../agents/container/profile.ts'; -import { createValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems, runContainer, startValidatedContainer, - hasExactOptions, validateContainer } from '../agents/container/run.ts'; +import { assertContainerProfile, createContainerProfile, disposeContainerProfile, + isContainerProfileAuthentic } from '../agents/container/profile.ts'; +import { createValidatedContainer, disposeValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems, runContainer, + startValidatedContainer, hasExactOptions, validateContainer } from '../agents/container/run.ts'; import { createTaskClone } from '../git/clone.ts'; import { createVendorNetwork, removeVendorNetwork, type VendorNetwork } from '../agents/network/network.ts'; import { createClaudeCommand, createCodexCommand, createIsolationProbeCommand, createPhasePolicy, @@ -272,12 +273,13 @@ describe('real Docker agent isolation', () => { docker(...first.args); containers.add(first.name); expect(() => createValidatedContainer(duplicate)).toThrow('Container creation failed and cleanup did not settle.'); expect(existsSync(duplicate.codexAuthFile!)).toBe(true); + expect(isContainerProfileAuthentic(duplicate)).toBe(true); const state = JSON.parse(docker('container', 'inspect', first.name))[0] as { State: { Status: string } }; expect(state.State.Status).toBe('created'); docker('rm', '--force', first.name); containers.delete(first.name); }, 60_000); - it('retains credentials when a killed create cannot be proven absent', () => { + it('releases a killed create once its settle window passes with no container', () => { const data = fixture(), unsettled = profile(data, 'planning', 'noop'); const shim = join(data.root, 'docker-shim'); mkdirSync(shim); const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); @@ -286,8 +288,35 @@ describe('real Docker agent isolation', () => { `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); const path = process.env.PATH; process.env.PATH = `${shim}:${path}`; - try { expect(() => createValidatedContainer(unsettled, 1_000)).toThrow('cleanup did not settle'); } + const started = performance.now(); + // The create path waits out the settle window, then treats absence as settled and releases the profile. + try { expect(() => createValidatedContainer(unsettled, 3_000)).toThrow('ETIMEDOUT'); } finally { process.env.PATH = path; } + expect(performance.now() - started).toBeGreaterThanOrEqual(10_000); + expect(isContainerProfileAuthentic(unsettled)).toBe(false); + expect(existsSync(unsettled.codexAuthFile!)).toBe(false); + }, 60_000); + + it('keeps a killed create unsettled for later cleanup until its settle window passes', () => { + const data = fixture(), unsettled = profile(data, 'planning', 'noop'); + const shim = join(data.root, 'docker-shim'); mkdirSync(shim); + const created = join(shim, 'created'), failed = join(shim, 'failed'); + const realDocker = execFileSync('sh', ['-c', 'command -v docker'], { encoding: 'utf8' }).trim(); + // The create client hangs until killed, and the first inspect after it fails for an unrelated reason, so the + // create path gives up early, inside the settle window. + writeFileSync(join(shim, 'docker'), ['#!/bin/sh', + `if [ "$1" = create ]; then touch '${created}'; exec sleep 30; fi`, + `if [ "$1" = container ] && [ "$2" = inspect ] && [ -e '${created}' ] && [ ! -e '${failed}' ]; then`, + ` touch '${failed}'; echo 'daemon unavailable' >&2; exit 1`, 'fi', + `exec '${realDocker}' "$@"`].join('\n'), { mode: 0o755 }); + const path = process.env.PATH; + process.env.PATH = `${shim}:${path}`; + try { + expect(() => createValidatedContainer(unsettled, 3_000)).toThrow('cleanup did not settle'); + // A follow-up cleanup inside the window must not treat absence as proof and release the profile. + expect(() => disposeValidatedContainer(unsettled)).toThrow('did not settle'); + } finally { process.env.PATH = path; } + expect(isContainerProfileAuthentic(unsettled)).toBe(true); expect(existsSync(unsettled.codexAuthFile!)).toBe(true); }, 60_000); diff --git a/test/agent-output.test.ts b/test/agent-output.test.ts new file mode 100644 index 0000000..51b4c90 --- /dev/null +++ b/test/agent-output.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; +import { ByteCollector } from '../agents/adapters/supervisor.ts'; + +describe('captured output storage', () => { + it('keeps storage proportional to bytes, not to the number of writes', () => { + const collector = new ByteCollector(); + for (let i = 0; i < 100_000; i++) collector.push(Buffer.from([i % 256])); + expect(collector.blockCount).toBeLessThanOrEqual(2); + const output = collector.toBuffer(); + expect(output.length).toBe(100_000); + expect(output.every((byte, index) => byte === index % 256)).toBe(true); + }); + + it('splits a large write across blocks without losing or reordering bytes', () => { + const collector = new ByteCollector(), large = Buffer.alloc(3 * ByteCollector.BLOCK_BYTES + 17, 7); + collector.push(Buffer.from('head')); + collector.push(large); + expect(collector.blockCount).toBe(4); + expect(collector.toBuffer()).toEqual(Buffer.concat([Buffer.from('head'), large])); + expect(new ByteCollector().toBuffer().length).toBe(0); + }); +}); diff --git a/test/agent-supervisor.test.ts b/test/agent-supervisor.test.ts new file mode 100644 index 0000000..7204045 --- /dev/null +++ b/test/agent-supervisor.test.ts @@ -0,0 +1,398 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { chmodSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { startClaudeInvocation } from '../agents/adapters/claude.ts'; +import { readCodexOutput, startCodexInvocation } from '../agents/adapters/codex.ts'; +import { isInvocationActive, readBoundedContainerFile, retainSetupCleanup, + startProfileInvocation } from '../agents/adapters/supervisor.ts'; +import { captureInvocation, type InvocationInput } from '../agents/contract.ts'; +import { buildAgentImage } from '../agents/container/image.ts'; +import { createContainerProfile, disposeContainerProfile, isContainerProfileAuthentic, + type ContainerProfile } from '../agents/container/profile.ts'; +import { disposeValidatedContainer, prepareTaskFilesystems, removeTaskFilesystems } from '../agents/container/run.ts'; +import { createVendorNetwork } from '../agents/network/network.ts'; +import { createIsolationProbeCommand, createPhasePolicy, type IsolationProbe } from '../agents/policy.ts'; +import { createTaskClone } from '../git/clone.ts'; + +const roots: string[] = [], profiles: ContainerProfile[] = []; +const allocations: ReturnType[] = []; +let imageId = ''; +const git = (cwd: string, ...args: string[]) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', ...args], + { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'agent-supervisor-')); roots.push(root); + const source = join(root, 'source'), staging = join(root, 'staging'), input = join(root, 'input'); + mkdirSync(source); mkdirSync(staging); mkdirSync(input); + git(source, 'init'); git(source, 'config', 'user.name', 'Test'); git(source, 'config', 'user.email', 'test@example.com'); + writeFileSync(join(source, 'file.txt'), 'trusted\n'); git(source, 'add', '.'); git(source, 'commit', '-m', 'baseline'); + writeFileSync(join(input, 'schema.json'), '{}\n'); chmodSync(join(input, 'schema.json'), 0o444); chmodSync(input, 0o555); + const clone = createTaskClone({ source, parent: staging, taskId: 'supervisor', head: git(source, 'rev-parse', 'HEAD') }); + const filesystems = prepareTaskFilesystems(clone, { + workBytes: 16 * 1024 * 1024, workInodes: 512, metadataBytes: 16 * 1024 * 1024, metadataInodes: 512, + }, imageId); allocations.push(filesystems); + const auth = join(root, 'auth.json'); writeFileSync(auth, '{}', { mode: 0o600 }); + return { root, input, clone, filesystems, auth }; +} +function invocation(data: ReturnType, attemptId: string, deadlineMs = 2 * 60_000, + vendor: 'codex' | 'claude' = 'codex'): InvocationInput { + return captureInvocation({ clone: data.clone, phase: 'planning', vendor, approvedArgv: [], + deadline: Date.now() + deadlineMs, attemptId, + context: { snapshotId: 'snapshot', planId: 'plan', planRevision: 1, assignmentId: 'assignment', + referencedCodeHash: 'code', stateVersion: 1 } }); +} +function profile(data: ReturnType, probe: IsolationProbe, + attempt: string | InvocationInput = `attempt-${Math.random()}`, deadlineMs = 2 * 60_000, deferredOutput = false) { + // An attempt can be captured once, so a duplicate-attempt profile reuses the captured invocation. + const captured = typeof attempt === 'string' ? invocation(data, attempt, deadlineMs) : attempt; + const policy = createPhasePolicy(captured); + const network = createVendorNetwork(captured, imageId); + const value = createContainerProfile({ invocation: captured, policy, network, filesystems: data.filesystems, + inputDirectory: data.input, command: createIsolationProbeCommand(policy, probe), imageId, codexAuthFile: data.auth, + deferredOutput }); + profiles.push(value); return value; +} + +beforeAll(() => { imageId = buildAgentImage(); }, 10 * 60_000); +afterAll(() => { + for (const profile of profiles) disposeContainerProfile(profile); + for (const allocation of allocations.reverse()) removeTaskFilesystems(allocation); + for (const root of roots.reverse()) { + chmodSync(join(root, 'input'), 0o700); + rmSync(root, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); + } +}, 3 * 60_000); + +describe('container invocation supervisor', () => { + it('preserves the first cancellation reason while retained setup cleanup settles', async () => { + const data = fixture(), captured = invocation(data, 'cancel-setup-cleanup'); + let attempts = 0; + const handle = retainSetupCleanup(captured, () => { + attempts += 1; + if (attempts === 1) return; + throw new Error('unexpected repeated cleanup'); + }, new Error('startup failed'), new Error('cleanup failed')); + handle.cancel('shutdown'); + handle.cancel('cancelled'); + const result = await handle.settled; + expect(result.stopReason).toBe('shutdown'); + expect(result.stderr).toContain('[codeboost: shutdown:'); + expect(attempts).toBe(1); + expect(isInvocationActive('cancel-setup-cleanup')).toBe(false); + }); + + it('captures finite output and releases ownership only after cleanup', async () => { + const current = profile(fixture(), 'finite-output', 'finite'); + const handle = startProfileInvocation(current); + expect(isInvocationActive('finite')).toBe(true); + const result = await handle.settled; + expect(result).toMatchObject({ attemptId: 'finite', exitCode: 0, signal: null, + stdout: 'stdout-marker', stderr: 'stderr-marker' }); + expect(result.stopReason).toBeUndefined(); + expect(isInvocationActive('finite')).toBe(false); + expect(spawnSync('docker', ['container', 'inspect', current.name]).status).not.toBe(0); + expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); + }, 60_000); + + it.each([ + ['infinite-stdout', { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 96 * 1024 }], + ['infinite-stderr', { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 96 * 1024 }], + ['infinite-mixed', { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 48 * 1024 }], + ] as const)('terminates %s at bounded output limits', async (probe, limits) => { + const attemptId = `limit-${probe}`, handle = startProfileInvocation(profile(fixture(), probe, attemptId), { limits }); + const result = await handle.settled; + expect(result.stopReason).toBe('output-limit'); + expect(Buffer.byteLength(result.stdout)).toBeLessThanOrEqual(limits.stdoutBytes); + expect(Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(limits.stderrBytes); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(limits.combinedBytes); + expect(isInvocationActive(attemptId)).toBe(false); + }, 60_000); + + it('stops buffering deferred newline-free stderr after the limit is reached', async () => { + const attemptId = 'deferred-stderr-limit'; + const handle = startProfileInvocation(profile(fixture(), 'infinite-stderr', attemptId, 2 * 60_000, true), { + limits: { stdoutBytes: 64 * 1024, stderrBytes: 32 * 1024, combinedBytes: 64 * 1024 }, + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal), + }); + const result = await handle.settled; + expect(result.stopReason).toBe('output-limit'); + expect(Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(32 * 1024); + expect(isInvocationActive(attemptId)).toBe(false); + }, 60_000); + + it('preserves the first cancellation reason until an ignored SIGTERM fully settles', async () => { + const current = profile(fixture(), 'ignore-term', 'cancelled'); + const handle = startProfileInvocation(current, { timeoutMs: 30_000 }); + let settled = false; void handle.settled.then(() => { settled = true; }); + handle.cancel('cancelled'); handle.cancel('shutdown'); + await Promise.resolve(); + expect(settled).toBe(false); + expect(isInvocationActive('cancelled')).toBe(true); + const result = await handle.settled; + expect(result.stopReason).toBe('cancelled'); + expect(result.stderr).toContain('[codeboost: cancelled]'); + expect(isInvocationActive('cancelled')).toBe(false); + }, 60_000); + + it('enforces a finite wall deadline and force-settles the container', async () => { + const started = Date.now(); + const handle = startProfileInvocation(profile(fixture(), 'ignore-term', 'timeout', 8_000), { timeoutMs: 30_000 }); + const result = await handle.settled; + expect(result.stopReason).toBe('timeout'); + expect(Date.now() - started).toBeLessThan(20_000); + expect(isInvocationActive('timeout')).toBe(false); + }, 60_000); + + it('records timeout when close delivery resumes after the monotonic deadline', async () => { + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'late-close-delivery', 30_000), + { timeoutMs: 3_000 }); + const end = performance.now() + 3_500; + while (performance.now() < end) { /* delay both close and timer delivery */ } + const result = await handle.settled; + expect(result.stopReason).toBe('timeout'); + }, 15_000); + + it('fails capture instead of publishing replacement characters for invalid UTF-8 stderr', async () => { + const result = await startProfileInvocation(profile(fixture(), 'invalid-utf8-stderr'), { timeoutMs: 30_000 }).settled; + expect(result.stopReason).toBe('capture-failure'); + expect(result.stderr).not.toContain('\uFFFD'); + expect(result.stderr).not.toContain('bad-'); + }, 60_000); + + it('fails capture when output ends in an incomplete character without reaching a limit', async () => { + const result = await startProfileInvocation(profile(fixture(), 'truncated-utf8-stderr'), { timeoutMs: 30_000 }).settled; + expect(result.stopReason).toBe('capture-failure'); + expect(result.stderr).not.toContain('cut-'); + }, 60_000); + + it('releases only the profile when rejecting before creation, even if its name is held elsewhere', () => { + const current = profile(fixture(), 'noop'); + // A foreign container occupies the deterministic name, so container-level cleanup could never settle. + execFileSync('docker', ['create', '--name', current.name, '--label', 'io.codeboost.invocation=someone-else', + '--entrypoint', 'true', imageId], { stdio: 'ignore' }); + try { + expect(() => startProfileInvocation(current, { timeoutMs: 10 * 60_000 + 1 })).toThrow('ceiling'); + expect(isContainerProfileAuthentic(current)).toBe(false); + expect(spawnSync('docker', ['container', 'inspect', current.name], { stdio: 'ignore' }).status).toBe(0); + } finally { spawnSync('docker', ['rm', '--force', current.name], { stdio: 'ignore' }); } + }, 60_000); + + it('blocks a duplicate attempt while the original container remains active', async () => { + const data = fixture(), duplicate = invocation(data, 'duplicate'); + const first = startProfileInvocation(profile(data, 'ignore-term', duplicate), { timeoutMs: 30_000 }); + expect(() => startProfileInvocation(profile(data, 'finite-output', duplicate))).toThrow('still active'); + expect(isInvocationActive('duplicate')).toBe(true); + first.cancel('shutdown'); + expect((await first.settled).stopReason).toBe('shutdown'); + }, 60_000); + + it('rejects reuse of the same active profile without disposing its container', async () => { + const current = profile(fixture(), 'ignore-term', 'same-profile-duplicate'); + const first = startProfileInvocation(current, { timeoutMs: 30_000 }); + expect(() => startProfileInvocation(current)).toThrow('already owns the active invocation'); + expect(isInvocationActive('same-profile-duplicate')).toBe(true); + expect(spawnSync('docker', ['container', 'inspect', current.name]).status).toBe(0); + first.cancel('shutdown'); + expect((await first.settled).stopReason).toBe('shutdown'); + }, 60_000); + + it('rejects a cloned profile without disposing the authentic active container', async () => { + const current = profile(fixture(), 'ignore-term', 'cloned-profile'); + const first = startProfileInvocation(current, { timeoutMs: 30_000 }); + const clone = Object.freeze({ ...current }); + expect(() => disposeValidatedContainer(clone)).toThrow('not created by the trusted profile builder'); + expect(() => startProfileInvocation(clone)).toThrow('not created by the trusted profile builder'); + expect(isInvocationActive('cloned-profile')).toBe(true); + expect(spawnSync('docker', ['container', 'inspect', current.name]).status).toBe(0); + first.cancel('shutdown'); + expect((await first.settled).stopReason).toBe('shutdown'); + }, 60_000); + + it('records decoder failure without publishing a successful result', async () => { + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'capture-failure'), { + decode: () => { throw new Error('simulated capture failure'); }, + }); + const result = await handle.settled; + expect(result.stopReason).toBe('capture-failure'); + expect(result.stderr).toContain('[codeboost: capture-failure'); + expect(isInvocationActive('capture-failure')).toBe(false); + }, 60_000); + + it('preserves cancellation while post-close decoding is still unsettled', async () => { + let begin!: () => void, release!: () => void; + const started = new Promise(resolve => { begin = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'cancel-during-decode'), { + decode: async () => { begin(); await gate; return { text: 'must-not-publish' }; }, + }); + await started; + handle.cancel('cancelled'); + release(); + const result = await handle.settled; + expect(result.stopReason).toBe('cancelled'); + expect(result.stdout).not.toContain('must-not-publish'); + }, 60_000); + + it('keeps post-close decoding inside the invocation deadline', async () => { + const started = Date.now(); + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-timeout', 30_000), { + timeoutMs: 3_000, + decode: (_current, _raw, _maximum, _timeout, signal) => new Promise((_resolve, reject) => + signal.addEventListener('abort', () => reject(new Error('decoder aborted')), { once: true })), + }).settled; + expect(result.stopReason).toBe('timeout'); + expect(Date.now() - started).toBeLessThan(15_000); + expect(isInvocationActive('decode-timeout')).toBe(false); + }, 30_000); + + it('does not wedge when an injected decoder ignores abort', async () => { + const started = Date.now(); + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-ignores-abort', 30_000), { + timeoutMs: 3_000, + decode: () => new Promise(() => {}), + }).settled; + expect(result.stopReason).toBe('timeout'); + expect(Date.now() - started).toBeLessThan(15_000); + expect(isInvocationActive('decode-ignores-abort')).toBe(false); + }, 30_000); + + it('settles cancellation promptly when an injected decoder ignores abort', async () => { + let begin!: () => void; + const started = new Promise(resolve => { begin = resolve; }); + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'cancel-ignored-decode', 30_000), { + timeoutMs: 30_000, + decode: () => { begin(); return new Promise(() => {}); }, + }); + await started; + const cancelledAt = performance.now(); + handle.cancel('cancelled'); + const result = await handle.settled; + expect(result.stopReason).toBe('cancelled'); + expect(performance.now() - cancelledAt).toBeLessThan(5_000); + expect(isInvocationActive('cancel-ignored-decode')).toBe(false); + }, 15_000); + + it('does not publish a synchronous decode that finishes after the monotonic deadline', async () => { + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-over-deadline', 30_000), { + timeoutMs: 3_000, + decode: (_current, _raw, _maximum, timeoutMs) => { + const end = performance.now() + timeoutMs + 50; + while (performance.now() < end) { /* deliberately block the timer queue */ } + return { text: 'must-not-publish' }; + }, + }).settled; + expect(result.stopReason).toBe('timeout'); + expect(result.stdout).not.toContain('must-not-publish'); + }, 15_000); + + it('classifies a decoder failure after the monotonic deadline as timeout', async () => { + const result = await startProfileInvocation(profile(fixture(), 'finite-output', 'decode-fails-late', 30_000), { + timeoutMs: 3_000, + decode: (_current, _raw, _maximum, timeoutMs) => { + const end = performance.now() + timeoutMs + 50; + while (performance.now() < end) { /* deliberately block the timer queue */ } + throw new Error('late decoder failure'); + }, + }).settled; + expect(result.stopReason).toBe('timeout'); + }, 15_000); + + it('validates and decodes provider output even when the process exits nonzero', async () => { + const result = await startProfileInvocation(profile(fixture(), 'nonzero-output', 'nonzero-decode'), { + decode: (_current, raw) => ({ text: `decoded:${raw.toString('utf8')}` }), + }).settled; + expect(result).toMatchObject({ exitCode: 7, stdout: 'decoded:encoded-output' }); + expect(result.stopReason).toBeUndefined(); + }, 60_000); + + it('bounds decoded text independently of adapter byte accounting', async () => { + const handle = startProfileInvocation(profile(fixture(), 'finite-output', 'decoded-limit'), { + limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, + decode: () => ({ text: 'x'.repeat(64 * 1024 + 1), additionalBytes: 0 }), + }); + expect((await handle.settled).stopReason).toBe('output-limit'); + }, 60_000); + + it('rejects traversal before starting an output read', () => { + expect(() => readBoundedContainerFile('unused', + '/run/codeboost-output/../../run/codeboost-auth/codex/auth.json', 1024)).toThrow('bounded output directory'); + expect(() => readBoundedContainerFile('unused', '/run/codeboost-output/final.txt', + 16 * 1024 * 1024 + 1)).toThrow('production stdout limit'); + }); + + it.each([ + ['symlink-output', 'capture-failure'], + ['oversized-output', 'output-limit'], + ['fifo-output', 'capture-failure'], + ['invalid-utf8-output', 'capture-failure'], + ['replace-output-directory', 'capture-failure'], + ['duplicate-protocol', 'capture-failure'], + ] as const)('rejects unsafe Codex output from %s', async (probe, reason) => { + const handle = startProfileInvocation(profile(fixture(), probe, `file-${probe}`, 2 * 60_000, true), { + limits: { stdoutBytes: 64 * 1024, stderrBytes: 64 * 1024, combinedBytes: 128 * 1024 }, + decode: (current, _raw, maximum, timeoutMs) => readCodexOutput(current.name, maximum, timeoutMs), + }); + const result = await handle.settled; + expect(result.stopReason, result.stderr).toBe(reason); + }, 60_000); + + it('rejects limits above the production ceilings and cleans the unused profile', () => { + const current = profile(fixture(), 'finite-output', 'invalid-limit'); + expect(() => startProfileInvocation(current, { limits: { stdoutBytes: 16 * 1024 * 1024 + 1 } })) + .toThrow('production hard limits'); + expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); + }, 60_000); + + it('rejects timeouts above the production ceiling and cleans the unused profile', () => { + const current = profile(fixture(), 'finite-output', 'invalid-timeout'); + expect(() => startProfileInvocation(current, { timeoutMs: 10 * 60_000 + 1 })) + .toThrow('ten-minute ceiling'); + expect(spawnSync('docker', ['network', 'inspect', current.network.name]).status).not.toBe(0); + }, 60_000); + + it('fails closed when deferred output is never produced', async () => { + const handle = startProfileInvocation(profile(fixture(), 'nonzero-output', 'missing-deferred', 2 * 60_000, true), { + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal), + }); + expect((await handle.settled).stopReason).toBe('capture-failure'); + }, 60_000); + + it('delimits READY after finite newline-free stderr', async () => { + const result = await startProfileInvocation( + profile(fixture(), 'newline-free-deferred-output', 'newline-free-ready', 2 * 60_000, true), { + decode: (current, _raw, maximum, timeoutMs, signal) => + readCodexOutput(current.name, maximum, timeoutMs, signal), + }).settled; + expect(result.stopReason, result.stderr).toBeUndefined(); + expect(result.stdout).toBe('captured'); + expect(result.stderr).toContain('trailing-diagnostic'); + }, 60_000); + + if (process.env.CODEBOOST_RUN_AUTH_PROBES === '1') { + it('runs the production Codex adapter and collects its bounded output file', async () => { + const data = fixture(), authFile = process.env.CODEBOOST_CODEX_AUTH_FILE; + if (!authFile) throw new Error('CODEBOOST_CODEX_AUTH_FILE is required.'); + const result = await startCodexInvocation({ invocation: invocation(data, 'live-codex', 6 * 60_000), + filesystems: data.filesystems, inputDirectory: data.input, imageId, + prompt: 'Reply only with this exact marker: codeboost-adapter-marker' }, authFile).settled; + expect(result.stopReason, result.stderr).toBeUndefined(); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('codeboost-adapter-marker'); + }, 8 * 60_000); + + it('runs the production Claude adapter and parses its bounded envelope', async () => { + const data = fixture(), token = process.env.CLAUDE_CODE_OAUTH_TOKEN; + if (!token) throw new Error('CLAUDE_CODE_OAUTH_TOKEN is required.'); + const result = await startClaudeInvocation({ invocation: invocation(data, 'live-claude', 6 * 60_000, 'claude'), + filesystems: data.filesystems, inputDirectory: data.input, imageId, + prompt: 'Reply only with this exact marker: codeboost-adapter-marker' }, token).settled; + expect(result.stopReason).toBeUndefined(); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('codeboost-adapter-marker'); + }, 8 * 60_000); + } +});