diff --git a/runner/coordinator.ts b/runner/coordinator.ts new file mode 100644 index 0000000..ee46c7b --- /dev/null +++ b/runner/coordinator.ts @@ -0,0 +1,231 @@ +import { identityKey, type PlanIdentity } from '../core/identity.ts'; +import { captureInvocation, type InvocationHandle, type InvocationInput, type InvocationResult, type StopReason, type TaskClone } from '../agents/contract.ts'; +import type { AttemptRecord, Store } from './store.ts'; +import { ATTEMPT_PHASES, GuardRefusal, WRITABLE_KINDS, bounded, sameContext, type AttemptKind, type Classification, type FirstReason } from './lifecycle.ts'; + +/** What F's host-side preparation hands to D's start call. */ +export interface PreparedAttempt { + readonly clone: TaskClone; + readonly vendor: 'claude' | 'codex'; + readonly approvedArgv: readonly (readonly string[])[]; +} +export interface RunnerDeps { + /** + * Host-side preparation (clone, prompt). On abort it must stop and await every subprocess it started, then reject. + * It never leaves work running after it settles. + */ + prepare(attempt: AttemptRecord, signal: AbortSignal): Promise; + /** Remove host-side preparation files only. Task storage waits for the terminal write. */ + cleanupPreparation(attempt: AttemptRecord): Promise; + /** D's start call: returns a handle at once, or throws with nothing left running. */ + start(input: InvocationInput, prepared: PreparedAttempt): InvocationHandle; + /** Validate a clean result; throw with an actionable reason if it is invalid. Returns the value to persist. */ + validate(attempt: AttemptRecord, result: InvocationResult): unknown; + now?(): number; +} +export interface SlotLimits { readonly writable: number; readonly readOnly: number } +export interface StartRequest { + expectedStateVersion: number; kind: AttemptKind; item?: string | null; deadline: number; budgetMs?: number; retryOf?: string; + expectedContext: AttemptRecord['context']; +} +export interface RunnerStatus { + active: boolean; + stopRequested: { attemptId: string; reason: FirstReason; saved: boolean } | null; + unresolved: { attemptId: string; reason: 'result-not-saved' | 'start-not-saved' } | null; +} +type Group = 'writable' | 'readOnly'; +interface Job { + identity: PlanIdentity; key: string; group: Group; attemptId: string; attempt?: AttemptRecord; + firstReason: FirstReason | null; reasonSaved: boolean; preparationTimedOut: boolean; + controller: AbortController; handle?: InvocationHandle; timers: ReturnType[]; done?: Promise; +} +interface Marker { group: Group; attemptId: string; reason: 'result-not-saved' | 'start-not-saved' } + +const D_REASON: Record = { cancelled: 'cancelled', stale: 'cancelled', shutdown: 'shutdown', 'time-limit': 'timeout' }; +/** setTimeout accepts at most 2^31-1 ms; longer waits are re-armed. */ +const MAX_TIMER = 2_147_483_647; + +/** + * One runner coordinator per process and Store. Owns in-memory jobs, slots and unresolved markers. + * See docs/implementation/runner-lifecycle.md ("Slots and concurrency", "Launch", "Rules for the running state"). + */ +export class RunnerCoordinator { + #store: Store; #deps: RunnerDeps; #limits: SlotLimits; + #jobs = new Map(); #markers = new Map(); + #closing = false; + constructor(store: Store, deps: RunnerDeps, limits: SlotLimits = { writable: 1, readOnly: 1 }) { + if (![limits.writable, limits.readOnly].every(n => Number.isSafeInteger(n) && n >= 1)) throw new Error('Slot limits must be positive integers.'); + this.#store = store; this.#deps = deps; this.#limits = limits; + } + get closing(): boolean { return this.#closing; } + #now(): number { return this.#deps.now?.() ?? Date.now(); } + #used(group: Group): number { + let used = 0; + for (const job of this.#jobs.values()) if (job.group === group) used++; + for (const marker of this.#markers.values()) if (marker.group === group) used++; + return used; + } + /** + * Admission. In-memory checks and the slot reservation happen in one synchronous turn before the Store transaction; + * a refused transaction releases the reservation in the same turn. + */ + start(identity: PlanIdentity, request: StartRequest): AttemptRecord { + if (this.#closing) throw new GuardRefusal('The runner is shutting down.'); + const key = identityKey(identity); + if (this.#jobs.has(key)) throw new GuardRefusal('An attempt is already active for this task.'); + const marker = this.#markers.get(key); + if (marker) throw new GuardRefusal('Needs restart: the last result could not be saved.'); + if (!(request.kind in ATTEMPT_PHASES)) throw new GuardRefusal('Unknown attempt kind.'); + const group: Group = WRITABLE_KINDS.includes(request.kind) ? 'writable' : 'readOnly'; + if (this.#used(group) >= this.#limits[group]) throw new GuardRefusal('No free runner slot. Try again when the current attempt finishes.'); + const job: Job = { identity: { ...identity }, key, group, attemptId: '', firstReason: null, reasonSaved: true, preparationTimedOut: false, controller: new AbortController(), timers: [] }; + this.#jobs.set(key, job); + let attempt: AttemptRecord; + try { attempt = this.#store.admitAttempt(identity, { ...request, now: this.#now() }); } + catch (error) { this.#jobs.delete(key); throw error; } + job.attemptId = attempt.id; job.attempt = attempt; + this.#arm(job, attempt); + job.done = this.#run(job, attempt).catch(error => this.#unexpected(job, error)); + return attempt; + } + /** Retry is a new attempt bound to the latest failed or cancelled one. */ + retry(identity: PlanIdentity, attemptId: string, request: Omit): AttemptRecord { + return this.start(identity, { ...request, retryOf: attemptId }); + } + /** User stop or detected staleness. The first reason wins; nothing is freed until settlement. */ + stop(identity: PlanIdentity, attemptId: string, reason: 'cancelled' | 'stale'): boolean { + const job = this.#jobs.get(identityKey(identity)); + if (!job || job.attemptId !== attemptId) return false; + return this.#requestStop(job, reason); + } + /** Cancel task: the Store records the reason and the pending close; the coordinator stops the running work. */ + cancelTask(identity: PlanIdentity, expectedStateVersion: number, actionId: string): 'closed' | 'stopping' { + const outcome = this.#store.cancelTask(identity, expectedStateVersion, actionId); + const job = this.#jobs.get(identityKey(identity)); + if (outcome === 'stopping' && job) this.#requestStop(job, 'cancelled'); + return outcome; + } + isActive(identity: PlanIdentity): boolean { return this.#jobs.has(identityKey(identity)); } + status(identity: PlanIdentity): RunnerStatus { + const key = identityKey(identity), job = this.#jobs.get(key), marker = this.#markers.get(key); + return { + active: !!job, + stopRequested: job?.firstReason ? { attemptId: job.attemptId, reason: job.firstReason, saved: job.reasonSaved } : null, + unresolved: marker ? { attemptId: marker.attemptId, reason: marker.reason } : null, + }; + } + /** Resolves when the task's current job has settled (or immediately if there is none). */ + async settled(identity: PlanIdentity): Promise { await this.#jobs.get(identityKey(identity))?.done; } + /** + * Shutdown step 4: reject admission, record `shutdown` only where no reason is set, stop everything and await settlement. + * No timer abandons a job (decision 4); the Store stays open for the caller to close afterwards. + */ + async close(): Promise { + this.#closing = true; + const jobs = [...this.#jobs.values()]; + for (const job of jobs) this.#requestStop(job, 'shutdown'); + await Promise.all(jobs.map(job => job.done)); + } + + #requestStop(job: Job, reason: FirstReason): boolean { + if (job.firstReason) { job.handle?.cancel(D_REASON[job.firstReason]); return false; } + job.firstReason = reason; + try { + if (job.attemptId && !this.#store.recordFirstReason(job.identity, job.attemptId, reason)) { + // Another writer (for example cancel task) recorded a reason first; adopt the durable one. + const durable = this.#store.getAttempt(job.identity, job.attemptId).firstReason; + if (durable) job.firstReason = durable; + } + job.reasonSaved = true; + } catch { job.reasonSaved = false; } + job.controller.abort(new Error(`Stopped: ${job.firstReason}`)); + job.handle?.cancel(D_REASON[job.firstReason]); + return true; + } + /** Task budget and, before launch, the attempt deadline. D enforces the deadline once it runs. */ + #arm(job: Job, attempt: AttemptRecord): void { + const at = (when: number, fire: () => void) => { + const wait = () => { + const remaining = when - this.#now(); + if (remaining <= 0) return fire(); + job.timers.push(setTimeout(wait, Math.min(remaining, MAX_TIMER))); + }; + wait(); + }; + const budget = this.#store.getTask(job.identity).budgetDeadline; + if (budget !== null) at(budget, () => this.#requestStop(job, 'time-limit')); + at(attempt.deadline, () => { if (!job.handle && !job.firstReason) { job.preparationTimedOut = true; job.controller.abort(new Error('Timed out while preparing.')); } }); + } + async #run(job: Job, attempt: AttemptRecord): Promise { + try { + let prepared: PreparedAttempt; + try { prepared = await this.#deps.prepare(attempt, job.controller.signal); } + catch (error) { return await this.#endBeforeLaunch(job, attempt, this.#preparationDetail(job, error)); } + if (job.firstReason || job.preparationTimedOut) return await this.#endBeforeLaunch(job, attempt, this.#preparationDetail(job)); + // Launch check: one synchronous turn, no await between the checks and D's start call. + const now = this.#now(), row = this.#store.getAttempt(job.identity, attempt.id), task = this.#store.getTask(job.identity); + if (row.firstReason && !job.firstReason) job.firstReason = row.firstReason; + if (row.state !== 'pending' || job.firstReason) return await this.#endBeforeLaunch(job, attempt, {}); + if (task.budgetDeadline !== null && now >= task.budgetDeadline) { this.#requestStop(job, 'time-limit'); return await this.#endBeforeLaunch(job, attempt, {}); } + if (now >= attempt.deadline) return await this.#endBeforeLaunch(job, attempt, { stopReason: 'timeout', detail: 'Timed out while preparing.' }); + if (!sameContext(row.context, this.#store.currentContext(job.identity))) return await this.#endBeforeLaunch(job, attempt, {}); + let handle: InvocationHandle; + try { + const input = captureInvocation({ clone: prepared.clone, phase: ATTEMPT_PHASES[attempt.kind], vendor: prepared.vendor, + approvedArgv: prepared.approvedArgv, deadline: attempt.deadline, attemptId: attempt.id, context: attempt.context }, now); + handle = this.#deps.start(input, prepared); + } catch (error) { return await this.#endBeforeLaunch(job, attempt, { detail: `Launch failed: ${message(error)}` }); } + job.handle = handle; + let running: boolean | undefined; + try { running = this.#store.markRunning(job.identity, attempt.id); } catch { running = undefined; } + if (running === undefined) { + // A storage error, not a stop: keep ownership until D settles, then hold the slot under a marker. + handle.cancel('capture-failure'); + await handle.settled.catch(() => undefined); + this.#markers.set(job.key, { group: job.group, attemptId: attempt.id, reason: 'start-not-saved' }); + return; + } + if (!running) { + // Refused because a first reason is now recorded: a normal stop. + const durable = this.#store.getAttempt(job.identity, attempt.id).firstReason; + if (durable && !job.firstReason) job.firstReason = durable; + handle.cancel(D_REASON[job.firstReason ?? 'cancelled']); + } else if (job.firstReason) handle.cancel(D_REASON[job.firstReason]); + const result = await handle.settled; + let valid = false, value: unknown, detail = result.stderr ? bounded(result.stderr) : undefined; + if (!job.firstReason && result.exitCode === 0 && !result.stopReason) { + try { value = this.#deps.validate(attempt, result); valid = true; } + catch (error) { detail = `Invalid output: ${message(error)}`; } + } + this.#settle(job, { stopReason: result.stopReason, exitCode: result.exitCode, signal: result.signal, valid, result: value, detail }); + } finally { + for (const timer of job.timers) clearTimeout(timer); + if (this.#jobs.get(job.key) === job) this.#jobs.delete(job.key); + } + } + #preparationDetail(job: Job, error?: unknown): { stopReason?: StopReason; detail?: string } { + if (job.preparationTimedOut && !job.firstReason) return { stopReason: 'timeout', detail: 'Timed out while preparing.' }; + return error === undefined || job.firstReason ? {} : { detail: `Preparation failed: ${message(error)}` }; + } + /** Ending without a handle: host-side cleanup, then the terminal write from the first reason. */ + async #endBeforeLaunch(job: Job, attempt: AttemptRecord, s: { stopReason?: StopReason; detail?: string }): Promise { + await this.#deps.cleanupPreparation(attempt).catch(() => undefined); + this.#settle(job, { stopReason: s.stopReason, exitCode: null, signal: null, valid: false, detail: s.detail }); + } + #settle(job: Job, s: { stopReason?: StopReason; exitCode: number | null; signal: string | null; valid: boolean; result?: unknown; detail?: string }): Classification | undefined { + try { + return this.#store.settleAttempt(job.identity, job.attemptId, { ...s, firstReason: job.firstReason }); + } catch { + // The row's outcome is unknown: hold the slot until startup recovery reconciles it. + this.#markers.set(job.key, { group: job.group, attemptId: job.attemptId, reason: 'result-not-saved' }); + return undefined; + } + } + #unexpected(job: Job, error: unknown): void { + // Fail closed: an unexpected error keeps the slot held until restart. + this.#markers.set(job.key, { group: job.group, attemptId: job.attemptId, reason: 'result-not-saved' }); + if (this.#jobs.get(job.key) === job) this.#jobs.delete(job.key); + console.error(`Runner job ${job.attemptId} failed unexpectedly: ${message(error)}`); + } +} +const message = (error: unknown) => bounded(error instanceof Error ? error.message : String(error)); diff --git a/test/runner-coordinator.test.ts b/test/runner-coordinator.test.ts new file mode 100644 index 0000000..4fbb7b3 --- /dev/null +++ b/test/runner-coordinator.test.ts @@ -0,0 +1,272 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { Store } from '../runner/store.ts'; +import { RunnerCoordinator, type PreparedAttempt, type RunnerDeps, type StartRequest } from '../runner/coordinator.ts'; +import type { InvocationHandle, InvocationInput, InvocationResult, StopReason } from '../agents/contract.ts'; +import type { PlanIdentity } from '../core/identity.ts'; +import type { Plan, PlanContext } from '../core/plan.ts'; + +const oid = (n: number) => n.toString(16).padStart(40, '0'); +const plan = (summary = 'Example'): Plan => ({ schema_version: 1, revision: 1, issue: 1, summary, questions: [], items: [{ id: 'P1', title: 'Change', intent: 'Improve', files: [{ path: 'a', kind: 'edit', renamed_from: null, change: 'Change' }], acceptance: [{ type: 'check', text: 'Works' }], depends_on: [] }] }); +const ctx = (identity: PlanIdentity): PlanContext => ({ identity, issue: 1, baseEntries: [{ path: 'a', kind: 'file' }], pathKey: p => p, allowedCommands: [] }); +const A = { repositoryId: 'repo', taskId: 'task-a', planId: 'plan' }, B = { repositoryId: 'repo', taskId: 'task-b', planId: 'plan' }; +const dirs: string[] = [], stores: Store[] = [], coordinators: RunnerCoordinator[] = []; +afterEach(async () => { + for (const c of coordinators.splice(0)) await c.close().catch(() => undefined); + for (const s of stores.splice(0)) s.close(); + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +interface Launch { input: InvocationInput; cancels: StopReason[]; settle(over?: Partial): void } +interface Preparation { attemptId: string; signal: AbortSignal; resolve(): void; reject(error: Error): void } +/** A fake D and preparation whose promises the test controls. */ +function fakeD(options: { prepareIgnoresAbort?: boolean } = {}) { + const launches: Launch[] = [], preparations: Preparation[] = []; + let cleaned = 0, startError: Error | undefined; + const prepared = (attemptId: string): PreparedAttempt => ({ clone: { id: `clone-${attemptId}`, taskId: 'task', directory: '/tmp/x', head: oid(2) }, vendor: 'claude', approvedArgv: [] }); + const deps: RunnerDeps = { + prepare: (attempt, signal) => new Promise((resolve, reject) => { + preparations.push({ attemptId: attempt.id, signal, resolve: () => resolve(prepared(attempt.id)), reject }); + if (!options.prepareIgnoresAbort) signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }), + cleanupPreparation: async () => { cleaned++; }, + start: input => { + if (startError) throw startError; + let settle!: (r: InvocationResult) => void; + const settled = new Promise(resolve => { settle = resolve; }); + const launch: Launch = { input, cancels: [], settle: over => settle({ attemptId: input.attemptId, context: input.context, exitCode: 0, signal: null, stdout: 'done', stderr: '', ...over }) }; + launches.push(launch); + const handle: InvocationHandle = { attemptId: input.attemptId, settled, cancel: reason => { launch.cancels.push(reason); } }; + return handle; + }, + validate: (_attempt, result) => { if (result.stdout === 'bad') throw new Error('schema mismatch'); return { text: result.stdout }; }, + }; + return { deps, launches, preparations, cleaned: () => cleaned, failStart: (error: Error) => { startError = error; } }; +} +function setup(options: { prepareIgnoresAbort?: boolean; limits?: { writable: number; readOnly: number } } = {}) { + const dir = mkdtempSync(join(tmpdir(), 'codeboost-coordinator-')); dirs.push(dir); + const store = new Store(join(dir, 'state.sqlite')); stores.push(store); + for (const identity of [A, B]) { + store.createPlan(JSON.stringify(plan()), 'json', ctx(identity), oid(1), oid(2)); + store.transitionTask(identity, store.getTask(identity).stateVersion, 'queued'); + } + const d = fakeD(options); + const runner = new RunnerCoordinator(store, d.deps, options.limits); coordinators.push(runner); + return { store, runner, ...d }; +} +const request = (store: Store, identity: PlanIdentity, extra: Partial = {}): StartRequest => ({ + expectedStateVersion: store.getTask(identity).stateVersion, kind: 'execute', item: 'P1', + expectedContext: store.currentContext(identity), deadline: Date.now() + 60_000, ...extra, +}); +const tick = () => new Promise(resolve => setImmediate(resolve)); +async function until(check: () => boolean, label: string) { + for (let i = 0; i < 500; i++) { if (check()) return; await tick(); } + throw new Error(`Timed out waiting for ${label}`); +} + +describe('admission and slots', () => { + it('runs an attempt to completion and frees its slot', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + expect(store.getAttempt(A, attempt.id).state).toBe('running'); + launches[0]!.settle(); + await runner.settled(A); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'completed', result: { text: 'done' } }); + expect(runner.isActive(A)).toBe(false); + }); + it('lets exactly one of two same-tick admissions take the single writable slot', () => { + const { store, runner } = setup(); + runner.start(A, request(store, A)); + expect(() => runner.start(B, request(store, B))).toThrow(/No free runner slot/); + expect(store.getAttempts(B)).toHaveLength(0); + }); + it('releases the reservation when the Store refuses admission', () => { + const { store, runner } = setup(); + expect(() => runner.start(A, request(store, A, { expectedStateVersion: -1 }))).toThrow(/Stale task state/); + expect(runner.isActive(A)).toBe(false); + expect(() => runner.start(B, request(store, B))).not.toThrow(); + }); +}); + +describe('stops and settlement', () => { + it('keeps the slot after cancel until D settles, and keeps the first reason', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + expect(runner.stop(A, attempt.id, 'cancelled')).toBe(true); + expect(runner.stop(A, attempt.id, 'stale')).toBe(false); + expect(launches[0]!.cancels[0]).toBe('cancelled'); + expect(runner.status(A)).toMatchObject({ active: true, stopRequested: { reason: 'cancelled', saved: true } }); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'running', firstReason: 'cancelled' }); + expect(() => runner.start(B, request(store, B))).toThrow(/No free runner slot/); + launches[0]!.settle({ exitCode: 1, stopReason: 'cancelled' }); + await runner.settled(A); + expect(store.getAttempt(A, attempt.id).state).toBe('cancelled'); + expect(() => runner.start(B, request(store, B))).not.toThrow(); + }); + it('refuses a retry while the old attempt is unsettled, even after the clock jumps', async () => { + let clock = Date.now(); + const { store, runner, launches, preparations, deps } = setup(); + deps.now = () => clock; + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + clock += 24 * 60 * 60 * 1000; + expect(() => runner.retry(A, attempt.id, request(store, A))).toThrow(/already active/); + launches[0]!.settle({ exitCode: null, stopReason: 'timeout' }); + await runner.settled(A); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'failed', diagnostic: 'Timed out.' }); + clock = Date.now(); + expect(runner.retry(A, attempt.id, request(store, A)).state).toBe('pending'); + }); + it('ends stale without calling D when the context changes during preparation', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); + store.setAssignment(A, store.getTask(A).stateVersion, 'reassigned', 'hash'); + preparations[0]!.resolve(); + await runner.settled(A); + expect(launches).toHaveLength(0); + expect(store.getAttempt(A, attempt.id).state).toBe('stale'); + }); + it('does not launch when a stop lands while preparation finishes', async () => { + const { store, runner, launches, preparations, cleaned } = setup({ prepareIgnoresAbort: true }); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); + runner.stop(A, attempt.id, 'cancelled'); + preparations[0]!.resolve(); + await runner.settled(A); + expect(launches).toHaveLength(0); + expect(cleaned()).toBe(1); + expect(store.getAttempt(A, attempt.id).state).toBe('cancelled'); + }); + it('fails with the launch error when D start throws and no stop is recorded', async () => { + const { store, runner, preparations, failStart } = setup(); + failStart(new Error('docker unavailable')); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await runner.settled(A); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'failed', diagnostic: 'Launch failed: docker unavailable' }); + }); + it('fails an invalid clean result with its validation reason', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + launches[0]!.settle({ stdout: 'bad' }); + await runner.settled(A); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'failed', diagnostic: 'Invalid output: schema mismatch' }); + }); +}); + +describe('storage failures', () => { + it('uses the in-memory first reason when its write failed, and shows it as unsaved', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + vi.spyOn(store, 'recordFirstReason').mockImplementation(() => { throw Object.assign(new Error('disk'), { code: 'ERR_SQLITE_ERROR' }); }); + runner.stop(A, attempt.id, 'cancelled'); + expect(runner.status(A).stopRequested).toEqual({ attemptId: attempt.id, reason: 'cancelled', saved: false }); + launches[0]!.settle(); + await runner.settled(A); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'cancelled', firstReason: 'cancelled' }); + }); + it('holds the slot under an unresolved marker when the terminal write fails', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + vi.spyOn(store, 'settleAttempt').mockImplementation(() => { throw Object.assign(new Error('disk'), { code: 'ERR_SQLITE_ERROR' }); }); + launches[0]!.settle(); + await runner.settled(A); + expect(runner.status(A).unresolved).toEqual({ attemptId: attempt.id, reason: 'result-not-saved' }); + expect(() => runner.start(A, request(store, A))).toThrow(/Needs restart/); + expect(() => runner.start(B, request(store, B))).toThrow(/No free runner slot/); + }); + it('cancels and settles the handle, then holds a marker, when pending -> running cannot be saved', async () => { + const { store, runner, launches, preparations } = setup(); + vi.spyOn(store, 'markRunning').mockImplementation(() => { throw Object.assign(new Error('disk'), { code: 'ERR_SQLITE_ERROR' }); }); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + expect(launches[0]!.cancels).toEqual(['capture-failure']); + expect(runner.isActive(A)).toBe(true); + launches[0]!.settle({ exitCode: null, stopReason: 'capture-failure' }); + await until(() => !runner.isActive(A), 'settlement'); + expect(runner.status(A).unresolved).toEqual({ attemptId: attempt.id, reason: 'start-not-saved' }); + }); +}); + +describe('cancel task, limits and shutdown', () => { + it('stops the running attempt on cancel task and closes the task when it settles, even with a valid result', async () => { + const { store, runner, launches, preparations } = setup(); + runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + expect(runner.cancelTask(A, store.getTask(A).stateVersion, randomUUID())).toBe('stopping'); + expect(launches[0]!.cancels).toEqual(['cancelled']); + launches[0]!.settle(); + await runner.settled(A); + expect(store.getTask(A).status).toBe('cancelled'); + }); + it('stops preparation at the task budget and moves the task to needs human without calling D', async () => { + const { store, runner, launches } = setup(); + const attempt = runner.start(A, request(store, A, { budgetMs: 30 })); + await runner.settled(A); + expect(launches).toHaveLength(0); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'cancelled', firstReason: 'time-limit' }); + expect(store.getTask(A).status).toBe('needs human'); + }); + it('fails preparation at the attempt deadline without a first reason', async () => { + const { store, runner, launches } = setup(); + const attempt = runner.start(A, request(store, A, { deadline: Date.now() + 30 })); + await runner.settled(A); + expect(launches).toHaveLength(0); + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'failed', firstReason: null, diagnostic: 'Timed out.' }); + expect(store.getTask(A).status).toBe('running'); + }); + it('rejects new work once shutdown starts and waits for D to settle', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + let closed = false; + const closing = runner.close().then(() => { closed = true; }); + expect(() => runner.start(B, request(store, B))).toThrow(/shutting down/); + expect(launches[0]!.cancels).toEqual(['shutdown']); + for (let i = 0; i < 20; i++) await tick(); + expect(closed).toBe(false); + launches[0]!.settle({ exitCode: null, stopReason: 'shutdown' }); + await closing; + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'cancelled', diagnostic: 'Stopped by shutdown' }); + }); + it('lets a D timeout that came before shutdown win', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + const closing = runner.close(); + launches[0]!.settle({ exitCode: null, stopReason: 'timeout' }); + await closing; + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'failed', firstReason: 'shutdown', diagnostic: 'Timed out.' }); + }); + it('keeps an existing stop reason when shutdown arrives', async () => { + const { store, runner, launches, preparations } = setup(); + const attempt = runner.start(A, request(store, A)); + await until(() => preparations.length === 1, 'preparation'); preparations[0]!.resolve(); + await until(() => launches.length === 1, 'launch'); + runner.stop(A, attempt.id, 'stale'); + const closing = runner.close(); + launches[0]!.settle(); + await closing; + expect(store.getAttempt(A, attempt.id)).toMatchObject({ state: 'stale', firstReason: 'stale' }); + }); +});