diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 07ee58fdc..121d5293e 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -87,6 +87,20 @@ report installed live acceptance. Startup shares the configured `HACK_COMPOSE_STARTUP_TIMEOUT_MS` budget across input preparation and native review, with a native maximum of 300000 milliseconds. +Each startup compiler call is bounded by the time the startup budget has left, +capped at 60 seconds, rather than by a fixed 10-second clock. Compiler calls outside +startup (the down hooks' input check after Ctrl-C or a stop request, and checks after +ready) have no startup budget and take the 60-second cap, so a slow host does not +fail a graceful stop (HACK-1458). A start that fails or is cancelled after host +processes began and before its startup intent exists stops those processes, or ends +the launch it had begun through its own session token, before it returns. A +teardown it cannot prove keeps the host process records retained. If the graph fails +before Ready after reserving the startup intent, host teardown additionally requires +the original producer and readers to settle and the original graph group to be +absent. Startup reacquires unchanged host inputs under the hard-stop signal, then +authenticates an exact Removed receipt and absent resource observations before +stopping the owned host session. A live, foreign, unknown or unreserved receipt, +changed input, or hard cancellation keeps this attempt retained. Startup refusals identify a fixed owning stage and, when available, a closed compiler or native error code. They omit arbitrary errors, child output, paths and values. If startup fails before publishing a receipt and the later retirement diff --git a/docs/reference/native-compose-adoption.md b/docs/reference/native-compose-adoption.md index 91f4d4391..f14b0b473 100644 --- a/docs/reference/native-compose-adoption.md +++ b/docs/reference/native-compose-adoption.md @@ -136,7 +136,14 @@ stop recovery can clear it. A retained operation that reaches its caller deadlin refuses with its own fixed message ("reached its caller deadline"), not the source-or-binding-changed message, which only an observed drift issues (HACK-1340). `hack up`, `hack down` and `config adopt --stop` pass the Compose -startup budget as that deadline. These rechecks do not freeze external editors or +startup budget as that deadline. Every adopted project command (`up --detach`, +`restart`, `down`, `ps`, `logs`, `exec`) starts one such deadline before it opens the +store, and each store inspection phase takes the time it has left, capped at 60 +seconds, instead of a fixed 15-second clock. Publication file moves, rollback +restoration and scoped routing revalidation retain that live remaining budget; +copying their operation context cannot restore time already spent. When that deadline expires the command +returns `E_STARTUP_TIMEOUT` with the "reached its caller deadline" message, not +`E_CONFIG_INVALID` (HACK-1458). These rechecks do not freeze external editors or the engine atomically. Public output remains field/count/status metadata; capabilities and authored probe arguments are private. Prior versions 1–4 keep their existing execution behavior, and older upgraded owners refuse version 5 diff --git a/src/backends/native-authored-host-processes.ts b/src/backends/native-authored-host-processes.ts index f71e92f50..f67a5ae30 100644 --- a/src/backends/native-authored-host-processes.ts +++ b/src/backends/native-authored-host-processes.ts @@ -98,6 +98,9 @@ async function unchanged(saved: Awaited>) { return refuse(); } } +/** An abandoned launch's groups get 5 s to exit after their session ends. */ +const GROUP_SETTLEMENT_INTERVAL_MS = 50; +const GROUP_SETTLEMENT_ATTEMPTS = 100; function absentGroup(group: number): void { try { process.kill(-group, 0); @@ -109,6 +112,20 @@ function absentGroup(group: number): void { } refuse(); } +/** Bounded wait for an owned group to exit after its session was ended. */ +async function settledGroup(group: number): Promise { + for (let attempt = 1; ; attempt++) { + try { + absentGroup(group); + return; + } catch (error) { + if (attempt >= GROUP_SETTLEMENT_ATTEMPTS) { + throw error; + } + } + await Bun.sleep(GROUP_SETTLEMENT_INTERVAL_MS); + } +} function entry(value: unknown): LifecycleStateEntry | null { if (value === null) { return null; @@ -539,6 +556,7 @@ export async function createNativeAuthoredHostProcesses(input: { let active = true; let began = false; let launchIntent = false; + let abandoned = false; let stopped = false; let ready: Awaited> | undefined; let stateFile: Awaited> | undefined; @@ -560,6 +578,8 @@ export async function createNativeAuthoredHostProcesses(input: { await recheckDirectories(directories); await unchanged(saved); }; + /** Groups this owner launched, recorded before their launch completes. */ + const launchedGroups = new Set(); const deliveries = new Set< Awaited> >(); @@ -630,6 +650,7 @@ export async function createNativeAuthoredHostProcesses(input: { if (!(value.processGroupId && value.panePid)) { return refuse(); } + launchedGroups.add(value.processGroupId); await delivery.started(); await check(); opts.remaining(); @@ -640,6 +661,36 @@ export async function createNativeAuthoredHostProcesses(input: { }; }, }); + /** + * HACK-1458: a start that fails after its launch intent (a cancel, the + * startup deadline or a refused check) ends what it launched through the + * session ownership token it created, rather than leaving the groups + * running behind an intent that recovery never converts into authority. + * Only a recorded, verified-absent launch clears the owner bookkeeping; + * anything unproven stays retained. + */ + const abandonLaunch = async () => { + try { + const launched = await currentEntry(opts.scope, composeProject); + await controller.abort(); + if ((await currentEntry(opts.scope, composeProject)) !== null) { + return; + } + const groups = new Set(launchedGroups); + for (const process of launched?.processes ?? []) { + groups.add(process.processGroupId ?? refuse()); + } + if (controller.hasStarted() && groups.size === 0) { + return; + } + for (const group of groups) { + await settledGroup(group); + } + abandoned = true; + } catch { + // Unproven teardown keeps the launch intent and owner retained. + } + }; const assertReady = async () => { await check(); if (!ready || stopped) { @@ -704,74 +755,79 @@ export async function createNativeAuthoredHostProcesses(input: { opts.remaining(); } launchIntent = true; - await writeExclusive( - join(root, `${opts.run}.host-process-intent.json`), - JSON.stringify({ - version: 1, - kind: "native-authored-host-process-intent", - run: opts.run, - owner: saved.pin, - }) - ); - await synchronizeDirectories(directories); - await check(); - opts.remaining(); - if (processes.length !== 0) { - await controller.startMany({ processes }); - } - await controller.finalize(); - savedEntry = await currentEntry(opts.scope, composeProject); - if ((savedEntry !== null) !== controller.hasStarted()) { - return refuse(); - } - if ( - savedEntry && - (savedEntry.definitionHash !== opts.semanticHash || - savedEntry.projectName !== projectName || - savedEntry.processes.some( - (process) => !owner.names.includes(process.name) - )) - ) { - return refuse(); + try { + await writeExclusive( + join(root, `${opts.run}.host-process-intent.json`), + JSON.stringify({ + version: 1, + kind: "native-authored-host-process-intent", + run: opts.run, + owner: saved.pin, + }) + ); + await synchronizeDirectories(directories); + await check(); + opts.remaining(); + if (processes.length !== 0) { + await controller.startMany({ processes }); + } + await controller.finalize(); + savedEntry = await currentEntry(opts.scope, composeProject); + if ((savedEntry !== null) !== controller.hasStarted()) { + return refuse(); + } + if ( + savedEntry && + (savedEntry.definitionHash !== opts.semanticHash || + savedEntry.projectName !== projectName || + savedEntry.processes.some( + (process) => !owner.names.includes(process.name) + )) + ) { + return refuse(); + } + await check(); + opts.remaining(); + await writeExclusive( + join(root, `${opts.run}.host-process-state.json`), + JSON.stringify({ + version: 1, + kind: "native-authored-host-process-state", + run: opts.run, + owner: saved.pin, + entry: savedEntry, + }) + ); + await writeExclusive( + join(root, `${opts.run}.host-process-ready.json`), + JSON.stringify({ + version: 1, + kind: "native-authored-host-process-ready", + run: opts.run, + project: owner.project, + branch: owner.branch, + semantic_hash: owner.semantic_hash, + owner: saved.pin, + pid: owner.pid, + uid: owner.uid, + }) + ); + await synchronizeDirectories(directories); + ready = await capture( + join(root, `${opts.run}.host-process-ready.json`) + ); + stateFile = await capture( + join(root, `${opts.run}.host-process-state.json`) + ); + await assertReady(); + } catch (error) { + await abandonLaunch(); + throw error; } - await check(); - opts.remaining(); - await writeExclusive( - join(root, `${opts.run}.host-process-state.json`), - JSON.stringify({ - version: 1, - kind: "native-authored-host-process-state", - run: opts.run, - owner: saved.pin, - entry: savedEntry, - }) - ); - await writeExclusive( - join(root, `${opts.run}.host-process-ready.json`), - JSON.stringify({ - version: 1, - kind: "native-authored-host-process-ready", - run: opts.run, - project: owner.project, - branch: owner.branch, - semantic_hash: owner.semantic_hash, - owner: saved.pin, - pid: owner.pid, - uid: owner.uid, - }) - ); - await synchronizeDirectories(directories); - ready = await capture( - join(root, `${opts.run}.host-process-ready.json`) - ); - stateFile = await capture( - join(root, `${opts.run}.host-process-state.json`) - ); - await assertReady(); }, assertReady, async stop() { - if (!launchIntent) { + if (!launchIntent || abandoned) { stopped = true; return; } diff --git a/src/backends/native-authored-project-start.ts b/src/backends/native-authored-project-start.ts index a0fc2e8c2..dda192ba7 100644 --- a/src/backends/native-authored-project-start.ts +++ b/src/backends/native-authored-project-start.ts @@ -6,7 +6,11 @@ import { } from "../lib/compose-env-fingerprint.ts"; import { deepFreeze } from "../lib/guards.ts"; import { NativeComposeHostHookError } from "../lib/native-compose-host-contract.ts"; -import { NativeConfigCompilerError } from "../lib/native-config-compiler.ts"; +import { + NATIVE_CONFIG_COMPILER_MAX_TIMEOUT_MS, + NativeConfigCompilerError, + nativeConfigCompilerTimeoutWithin, +} from "../lib/native-config-compiler.ts"; import { acquireNativeExecutionInputs } from "../lib/native-execution-inputs.ts"; import { assertNativeFiniteHookBindings, @@ -571,6 +575,27 @@ async function prepareHostSelection(opts: { return { hooks, lifecycle, persistent, owner, processes }; } +/** A canceled startup input cannot authorize teardown after a pre-Ready failure. */ +async function refreshPreReadyHostInputs(opts: { + readonly hostProcesses: NativeAuthoredHostProcesses | undefined; + readonly ready: Attempt["ready"]; + readonly processSettled: boolean; + readonly refresh: () => Promise; +}): Promise { + if (!opts.hostProcesses || opts.ready !== undefined) { + return; + } + // No down.before phase refreshed these inputs. Do not acquire teardown + // authority until the original producer and its readers have settled. + // The caller still authenticates Removed immediately before host stop. + if (!opts.processSettled) { + throw new Error( + "Native foreground settlement is unconfirmed; values omitted." + ); + } + await opts.refresh(); +} + async function runPreparedLifecycle(ctx: { readonly options: Options; readonly admitted: Attempt; @@ -581,6 +606,7 @@ async function runPreparedLifecycle(ctx: { readonly hookOwner: NativeAuthoredHookOwner | undefined; readonly hostProcesses: NativeAuthoredHostProcesses | undefined; readonly hostRetired: () => void; + readonly refreshHostInputs: () => Promise; readonly phase: (name: NativeHookPhase) => Promise; readonly graph: AbortController; readonly hard: AbortController; @@ -720,6 +746,12 @@ async function runPreparedLifecycle(ctx: { attempt: ctx.admitted, }; if (ctx.hookOwner) { + await refreshPreReadyHostInputs({ + hostProcesses: ctx.hostProcesses, + ready: ctx.admitted.ready, + processSettled, + refresh: ctx.refreshHostInputs, + }); // Keep frontend bindings recoverable until every hook has known completion. await confirmRemoved(retirement); if (ctx.hostProcesses) { @@ -823,14 +855,17 @@ async function stopUnstartedHostProcesses(opts: { readonly processes: NativeAuthoredHostProcesses | undefined; readonly attempt: Attempt | undefined; readonly outcome: Outcome; + /** Re-acquires teardown inputs that do not depend on the startup signal. */ + readonly refresh?: () => Promise; }): Promise<{ readonly processes: NativeAuthoredHostProcesses | undefined; readonly outcome: Outcome; }> { if (!opts.processes || opts.attempt) { - return opts; + return { processes: opts.processes, outcome: opts.outcome }; } try { + await opts.refresh?.(); await opts.processes.stop(); await opts.processes.close(); return { processes: undefined, outcome: opts.outcome }; @@ -914,6 +949,18 @@ export async function serveNativeAuthoredProject( } return time; }; + // HACK-1458: compiler calls never keep the compiler's fixed 10 s default. + // Startup calls share the startup deadline. Calls that are not part of + // startup (down-hook acquisition, and checks after publication) have no + // startup deadline, so each takes the transport's largest budget: a slow + // host must not fail a graceful stop, which would retain the live owner + // until a hard cancellation. + const startupCompilerTimeoutMs = () => + published + ? NATIVE_CONFIG_COMPILER_MAX_TIMEOUT_MS + : nativeConfigCompilerTimeoutWithin(deadline - performance.now()); + const settlementCompilerTimeoutMs = () => + NATIVE_CONFIG_COMPILER_MAX_TIMEOUT_MS; const abort = () => { if (!published) { controller.abort(); @@ -947,6 +994,7 @@ export async function serveNativeAuthoredProject( let nativeCauseCode: string | undefined; let hookOwner: NativeAuthoredHookOwner | undefined; let hostProcesses: NativeAuthoredHostProcesses | undefined; + let teardownInputs: (() => Promise) | undefined; let storageTool: NativeAuthoredStorageTool | undefined; let hookStop: | { readonly close: (force?: boolean) => Promise } @@ -981,6 +1029,10 @@ export async function serveNativeAuthoredProject( httpsPort: opts.httpsPort ?? null, }, signal, + compilerTimeoutMs: + signal === controller.signal + ? startupCompilerTimeoutMs + : settlementCompilerTimeoutMs, }); let inputs = await acquire(controller.signal); if (nativeAuthoredStorageToolRequired(inputs.result.plan)) { @@ -1002,6 +1054,21 @@ export async function serveNativeAuthoredProject( const { hooks, persistent } = selectedHost; hookOwner = selectedHost.owner; hostProcesses = selectedHost.processes; + // HACK-1458: inputs acquired for startup check freshness under the + // startup signal and deadline, so after a cancel or an expired + // deadline host process teardown could never pass its own freshness + // check. Teardown re-acquires them under the hard signal, like the + // down hooks, and still refuses a changed selection. + teardownInputs = async () => { + const current = await acquire(hardSignal.signal); + if ( + hookSelection(current) !== identity || + current.result.semantic_hash !== inputs.result.semantic_hash + ) { + throw new Error("Native host source changed; values omitted."); + } + inputs = current; + }; const phase = hookPhaseRunner({ owner: hookOwner, hooks, @@ -1144,6 +1211,7 @@ export async function serveNativeAuthoredProject( storageTool, hookOwner, hostProcesses, + refreshHostInputs: teardownInputs, hostRetired: () => { hostProcesses = undefined; }, @@ -1183,6 +1251,7 @@ export async function serveNativeAuthoredProject( processes: hostProcesses, attempt, outcome, + refresh: teardownInputs, }); hostProcesses = host.processes; outcome = host.outcome; diff --git a/src/lib/native-compose-adoption-command.ts b/src/lib/native-compose-adoption-command.ts index b8065bc84..11083c537 100644 --- a/src/lib/native-compose-adoption-command.ts +++ b/src/lib/native-compose-adoption-command.ts @@ -8,6 +8,7 @@ import { legacyComposeOrderedRefusal } from "./native-compose-adoption-diagnosti import { runLegacyComposeRetainedOperation } from "./native-compose-adoption-execution.ts"; import { LegacyComposeAdoptedGenerationError, + legacyAdoptionProbePhaseMs, openLegacyComposeAdoptedGenerationStore, } from "./native-compose-adoption-generation.ts"; import { inspectLegacyComposeAdoptionSelection } from "./native-compose-adoption-marker.ts"; @@ -43,6 +44,37 @@ function unsupported(): never { "Adopted Compose supports retained-container start, restart, stop, ps, logs and exec only. Recreation, run and changed selections are unsupported; original data is retained. Values omitted.", }); } +/** + * Closed public refusal of an adopted command. A caller-deadline expiry is a + * timeout, not a config fault (HACK-1458); the fixed store message is kept + * for closed diagnostics. + */ +function throwAdoptedCommandRefusal(error: unknown): never { + if (error instanceof HackCliError || error instanceof CliUsageError) { + throw error; + } + const ordered = legacyComposeOrderedRefusal(error); + throw new HackCliError({ + code: + error instanceof LegacyComposeAdoptedGenerationError && + error.code === "E_LEGACY_ADOPTION_DEADLINE" + ? "E_STARTUP_TIMEOUT" + : "E_CONFIG_INVALID", + message: + error instanceof LegacyComposeAdoptedGenerationError + ? error.message + : LEGACY_COMPOSE_ADOPTED_COMMAND_MESSAGES.refused, + ...(ordered ? { detail: { legacy_adoption_refusal: ordered } } : {}), + }); +} +/** Time left before the command deadline; an expired deadline refuses as one. */ +function remainingMs(deadline: number): number { + const remaining = Math.floor(deadline - Date.now()); + if (remaining <= 0) { + throw new LegacyComposeAdoptedGenerationError("E_LEGACY_ADOPTION_DEADLINE"); + } + return remaining; +} function cancelled(signal: AbortSignal) { if (signal.aborted) { throw new HackCliError({ @@ -215,6 +247,7 @@ async function observe(opts: { readonly binding: LegacyComposeVerifiedBinding; readonly signal: AbortSignal; readonly timeoutMs: number; + readonly deadline: number; }) { const { options, binding, signal, timeoutMs } = opts; cancelled(signal); @@ -223,6 +256,10 @@ async function observe(opts: { const states = await inspectLegacyComposeContainerStates({ binding, signal, + timeoutMs: legacyAdoptionProbePhaseMs({ + deadline: opts.deadline, + now: Date.now(), + }), }); const report = containers.map((container) => ({ service: container.service, @@ -290,13 +327,17 @@ export async function tryLegacyComposeAdoptedCommand( process.on("SIGINT", cancel); process.on("SIGTERM", cancel); try { + // One caller deadline spans the whole command: store inspection, the + // retained operation and its final proof (HACK-1458). const signal = controller.signal, timeoutMs = resolveComposeStartupTimeoutMs(), + deadline = Date.now() + timeoutMs, store = await openLegacyComposeAdoptedGenerationStore({ projectRoot, requestedBranch: options.instance, mode: "saved", signal, + deadline, }); try { if (options.recover) { @@ -311,7 +352,6 @@ export async function tryLegacyComposeAdoptedCommand( const requested = requestedServices(options); const operation = mutationOperation(options.operation); if (operation) { - const deadline = Date.now() + timeoutMs; return await store.withMutation({ generation, operation, @@ -354,7 +394,7 @@ export async function tryLegacyComposeAdoptedCommand( stdout: "ignore", stderr: "ignore", forwardSignals: true, - timeoutMs, + timeoutMs: remainingMs(deadline), } ); }, @@ -369,29 +409,14 @@ export async function tryLegacyComposeAdoptedCommand( binding: privateInput.binding, signal, timeoutMs, + deadline, }), }); } finally { await store.close(); } } catch (error: unknown) { - if (error instanceof HackCliError || error instanceof CliUsageError) { - throw error; - } - throw new HackCliError({ - code: "E_CONFIG_INVALID", - message: - error instanceof LegacyComposeAdoptedGenerationError - ? error.message - : LEGACY_COMPOSE_ADOPTED_COMMAND_MESSAGES.refused, - ...(legacyComposeOrderedRefusal(error) - ? { - detail: { - legacy_adoption_refusal: legacyComposeOrderedRefusal(error), - }, - } - : {}), - }); + throwAdoptedCommandRefusal(error); } finally { process.off("SIGINT", cancel); process.off("SIGTERM", cancel); diff --git a/src/lib/native-compose-adoption-generation.ts b/src/lib/native-compose-adoption-generation.ts index e1785b525..34e7c1197 100644 --- a/src/lib/native-compose-adoption-generation.ts +++ b/src/lib/native-compose-adoption-generation.ts @@ -85,7 +85,10 @@ import { type LegacyComposeContainerState, } from "./native-compose-adoption-runtime.ts"; import { holdSavedLegacyComposeSourceBind } from "./native-compose-adoption-source-bind.ts"; -import { createNativeComposeProbe } from "./native-compose-ownership.ts"; +import { + createNativeComposeProbe, + nativeComposeProbeFailure, +} from "./native-compose-ownership.ts"; import { createNativeComposePrivateMutationLock, type HeldDirectory, @@ -274,7 +277,62 @@ function cancelled(signal?: AbortSignal) { refuse("E_LEGACY_ADOPTION_CANCELLED"); } } -function translate(error: unknown, signal?: AbortSignal): never { +/** + * Default aggregate clock of one store inspection phase (one probe owner) when + * the caller names neither a phase bound nor a deadline. + */ +export const LEGACY_ADOPTION_PROBE_PHASE_DEFAULT_MS = 15_000; +/** Largest aggregate clock one probe owner accepts. */ +export const LEGACY_ADOPTION_PROBE_PHASE_MAX_MS = 60_000; + +/** + * Phase budget of a store opened with a caller deadline (HACK-1458): each new + * inspection phase gets the caller's remaining time, capped by the explicit + * phase bound (if any) and {@link LEGACY_ADOPTION_PROBE_PHASE_MAX_MS}. Without a + * deadline the explicit bound passes through unchanged (undefined keeps each + * owner's own default). An expired deadline refuses as + * `E_LEGACY_ADOPTION_DEADLINE` before another probe starts. + */ +export function legacyAdoptionProbePhaseMs(opts: { + readonly timeoutMs?: number; + readonly deadline?: number; + readonly now: number; +}): number | undefined { + if (opts.deadline === undefined) { + return opts.timeoutMs; + } + const remaining = Math.floor(opts.deadline - opts.now); + if (remaining <= 0) { + refuse("E_LEGACY_ADOPTION_DEADLINE"); + } + return Math.min( + opts.timeoutMs ?? LEGACY_ADOPTION_PROBE_PHASE_MAX_MS, + LEGACY_ADOPTION_PROBE_PHASE_MAX_MS, + remaining + ); +} + +/** + * A store with a caller deadline reports an unclassified failure as a + * deadline, not as invalid state, when a probe phase timed out or the deadline + * has passed: phase clocks are derived from that deadline (HACK-1458). + */ +function deadlineExpiry(error: unknown, deadline?: number): boolean { + return ( + deadline !== undefined && + (nativeComposeProbeFailure(error) === "timeout" || Date.now() >= deadline) + ); +} + +function acceptedDeadline(value: unknown): boolean { + return value === undefined || Number.isFinite(value); +} + +function translate( + error: unknown, + signal?: AbortSignal, + deadline?: number +): never { try { cancelled(signal); if (error instanceof LegacyComposeAdoptedGenerationError) { @@ -287,7 +345,9 @@ function translate(error: unknown, signal?: AbortSignal): never { refuse("E_LEGACY_ADOPTION_BUSY"); } const translated = new LegacyComposeAdoptedGenerationError( - "E_LEGACY_ADOPTION_STATE" + deadlineExpiry(error, deadline) + ? "E_LEGACY_ADOPTION_DEADLINE" + : "E_LEGACY_ADOPTION_STATE" ); const diagnostic = legacyComposeOrderedRefusal(error); if (diagnostic) { @@ -994,7 +1054,13 @@ async function assertRetainedRouteState( refuse(); } const observedManifest = JSON.stringify(first.manifest); - const current: Context = { ...ctx }; + // A context copy must not snapshot the caller's remaining proof budget. + const current: Context = { + ...ctx, + get timeoutMs() { + return ctx.timeoutMs; + }, + }; let active = true; const assertActive = () => { cancelled(current.signal); @@ -2229,7 +2295,11 @@ async function completePublication( refuse(); } const routingDeadline = - Date.now() + Math.min(ctx.timeoutMs ?? 15_000, 60_000); + Date.now() + + Math.min( + ctx.timeoutMs ?? LEGACY_ADOPTION_PROBE_PHASE_DEFAULT_MS, + LEGACY_ADOPTION_PROBE_PHASE_MAX_MS + ); stage = "publication-inputs"; const loaded = await readInputs(ctx, publication.generation); // Publishing pins the bound override for the adoption's life. An earlier @@ -2261,6 +2331,9 @@ async function completePublication( ); const transaction: Context = { ...ctx, + get timeoutMs() { + return ctx.timeoutMs; + }, check: async () => { await ctx.check(); await recheckDirectories([held]); @@ -2367,7 +2440,11 @@ async function completeRollback(ctx: Context, state: Receipt) { refuse(); } const routingDeadline = - Date.now() + Math.min(ctx.timeoutMs ?? 15_000, 60_000); + Date.now() + + Math.min( + ctx.timeoutMs ?? LEGACY_ADOPTION_PROBE_PHASE_DEFAULT_MS, + LEGACY_ADOPTION_PROBE_PHASE_MAX_MS + ); const loaded = await readInputs( ctx, publication.generation, @@ -2382,6 +2459,9 @@ async function completeRollback(ctx: Context, state: Receipt) { ); const transaction: Context = { ...ctx, + get timeoutMs() { + return ctx.timeoutMs; + }, check: async () => { await ctx.check(); await recheckDirectories([held]); @@ -2601,7 +2681,11 @@ function boundedMutationContext(ctx: Context, deadline: number) { ...ctx, signal: controller.signal, get timeoutMs() { - return Math.min(ctx.timeoutMs ?? 15_000, 60_000, remaining()); + return Math.min( + ctx.timeoutMs ?? LEGACY_ADOPTION_PROBE_PHASE_DEFAULT_MS, + LEGACY_ADOPTION_PROBE_PHASE_MAX_MS, + remaining() + ); }, check: async () => { remaining(); @@ -3270,10 +3354,18 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { readonly requestedBranch?: string; readonly mode?: "prepare" | "saved"; readonly signal?: AbortSignal; + /** Explicit bound of each inspection phase; see {@link legacyAdoptionProbePhaseMs}. */ readonly timeoutMs?: number; + /** + * Caller deadline (epoch ms, `Date.now()` clock). Every inspection phase is + * bounded by its remaining time and its expiry refuses as + * `E_LEGACY_ADOPTION_DEADLINE`, never as invalid state. + */ + readonly deadline?: number; }): Promise { const directories: HeldDirectory[] = []; let signal: AbortSignal | undefined; + let deadline: number | undefined; try { if ( !isRecord(input) || @@ -3283,6 +3375,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { (input.signal !== undefined && !(input.signal instanceof AbortSignal)) || (input.mode !== undefined && !["prepare", "saved"].includes(input.mode)) || + !acceptedDeadline(input.deadline) || (input.requestedBranch !== undefined && (typeof input.requestedBranch !== "string" || !input.requestedBranch.trim() || @@ -3293,6 +3386,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { const root = resolve(input.projectRoot), mode = input.mode ?? "prepare"; signal = input.signal; + deadline = input.deadline; const timeoutMs = input.timeoutMs, capturedRoute = route(), routingRoot = join(resolveGlobalHackDir(), "compose-routing"); @@ -3364,7 +3458,13 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { generationsRoot, receiptPath, signal, - timeoutMs, + get timeoutMs() { + return legacyAdoptionProbePhaseMs({ + timeoutMs, + deadline, + now: Date.now(), + }); + }, requestedBranch: input.requestedBranch, check, sourceBind, @@ -3475,7 +3575,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { ); }); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async loadPrepared(opts = {}) { @@ -3498,7 +3598,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { loaded.manifest.adoption_generation_version ); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async loadActive(opts = {}) { @@ -3526,7 +3626,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { loaded.manifest.adoption_generation_version ); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async publish(opts) { @@ -3564,7 +3664,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { await completePublication(ctx, pending, binary); }); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async rollback() { @@ -3592,7 +3692,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { await completeRollback(ctx, pending); }); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async pendingSwitchEnvOverrideChanged() { @@ -3653,7 +3753,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { } }); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async withLease(opts) { @@ -3718,7 +3818,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { return value; }); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async withMutation(opts) { @@ -3728,7 +3828,7 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { async () => await mutateRetainedContainers(ctx, known, captured) ); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async withPreparationStop(opts) { @@ -3743,14 +3843,14 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { await mutateRetainedContainers(ctx, known, captured, true) ); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async recoverInterruptedLock() { try { await lock.recoverInterruptedLock(); } catch (error: unknown) { - translate(error, signal); + translate(error, signal, deadline); } }, async close() { @@ -3768,6 +3868,6 @@ export async function openLegacyComposeAdoptedGenerationStore(input: { return Object.freeze(result); } catch (error: unknown) { await Promise.all(directories.map((directory) => directory.file.close())); - translate(error, signal); + translate(error, signal, deadline); } } diff --git a/src/lib/native-config-compiler.ts b/src/lib/native-config-compiler.ts index 842be1057..568d282f5 100644 --- a/src/lib/native-config-compiler.ts +++ b/src/lib/native-config-compiler.ts @@ -61,6 +61,24 @@ export const NATIVE_CONFIG_INPUT_LIMIT = 1024 * 1024; const OUTPUT_LIMIT = 8 * 1024 * 1024; const STDERR_LIMIT = 64 * 1024; const DEFAULT_TIMEOUT_MS = 10_000; +/** Smallest per-invocation budget the compiler transport accepts. */ +export const NATIVE_CONFIG_COMPILER_MIN_TIMEOUT_MS = 1; +/** Largest per-invocation budget the compiler transport accepts. */ +export const NATIVE_CONFIG_COMPILER_MAX_TIMEOUT_MS = 60_000; + +/** + * Budget of one compiler invocation for a caller with `remainingMs` left on its + * own deadline (HACK-1458): the remaining time, clamped to the transport's + * accepted range. A caller deadline longer than the ceiling still bounds each + * invocation by the ceiling, and a nearly spent one cannot request less than + * the floor; the caller's own deadline remains its authority. + */ +export function nativeConfigCompilerTimeoutWithin(remainingMs: number): number { + return Math.min( + NATIVE_CONFIG_COMPILER_MAX_TIMEOUT_MS, + Math.max(NATIVE_CONFIG_COMPILER_MIN_TIMEOUT_MS, Math.floor(remainingMs)) + ); +} const HASH_PATTERN = /^[a-f0-9]{64}$/; const OVERLAY_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; const RESOLVE_REQUEST_LIMIT = 20 * 1024 * 1024; @@ -843,7 +861,11 @@ function compilerRequest(opts: { throw failure("E_COMPILER_PATH", "Compiler path must be absolute."); } const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; - if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 60_000) { + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs < NATIVE_CONFIG_COMPILER_MIN_TIMEOUT_MS || + timeoutMs > NATIVE_CONFIG_COMPILER_MAX_TIMEOUT_MS + ) { throw failure( "E_COMPILER_BUDGET", "Compiler timeout is outside the supported budget." diff --git a/src/lib/native-execution-inputs.ts b/src/lib/native-execution-inputs.ts index 730b53e9c..b066ccbf6 100644 --- a/src/lib/native-execution-inputs.ts +++ b/src/lib/native-execution-inputs.ts @@ -89,6 +89,12 @@ type NativeExecutionInputSelection = { */ readonly runtimeMetadata?: NativeAuthoredRuntimeMetadataSelection; readonly signal?: AbortSignal; + /** + * Budget of each compiler invocation, including the later freshness and + * value checks of the returned inputs; read again for every invocation. + * Undefined keeps the compiler's own default (HACK-1458). + */ + readonly compilerTimeoutMs?: () => number | undefined; }; function projectName(plan: Readonly>): string { @@ -126,6 +132,7 @@ async function acquireInputs(input: { explicitDomain: opts.explicitDomain, compilerBranch: opts.compilerBranch, signal, + compilerTimeoutMs: opts.compilerTimeoutMs, }; const prepare = () => prepareNativeProjectSelection({ ...selection, requireEnvPlanning: true }); diff --git a/src/lib/native-project-validation.ts b/src/lib/native-project-validation.ts index d98d98fb4..e8bfae783 100644 --- a/src/lib/native-project-validation.ts +++ b/src/lib/native-project-validation.ts @@ -32,6 +32,12 @@ export type NativeProjectSelection = { /** Exact native source namespace context; supplied branches never trigger implicit Git/domain discovery. */ readonly compilerBranch?: string; readonly signal?: AbortSignal; + /** + * Budget of each compiler invocation, read again for every invocation so a + * caller can bound them by its remaining deadline (HACK-1458). Undefined, or + * a getter returning undefined, keeps the compiler's own default. + */ + readonly compilerTimeoutMs?: () => number | undefined; }; /** Offline project selection; local policy is read only after Rust validates it. */ @@ -117,6 +123,7 @@ export async function planPreparedNativeProject(opts: { } const result = await planNativeConfig({ ...prepared.selection, + timeoutMs: prepared.selection.compilerTimeoutMs?.(), signal: opts.signal, input: prepared.input, ...prepared.locals, @@ -195,6 +202,7 @@ export async function prepareNativeProjectSelection( const compiled = await compileNativeConfig({ input: project.input, profiles: opts.profiles, + timeoutMs: opts.compilerTimeoutMs?.(), signal: opts.signal, requireLocalResolution: true, requireEnvPlanning: opts.requireEnvPlanning, @@ -260,6 +268,7 @@ export async function prepareNativeProjectSelection( }; let result = await resolveNativeConfig({ ...resolveInputs, + timeoutMs: opts.compilerTimeoutMs?.(), probeRoutingInputs: opts.compilerBranch === undefined, }); checkResolvedIdentity(result); @@ -275,6 +284,7 @@ export async function prepareNativeProjectSelection( result = await resolveNativeConfig({ ...resolveInputs, ...routingInputs, + timeoutMs: opts.compilerTimeoutMs?.(), requireRoutingPlanning: true, }); } diff --git a/tests/helpers/lifecycle-mux-fixture.ts b/tests/helpers/lifecycle-mux-fixture.ts index bd9a19e59..d75d1541b 100644 --- a/tests/helpers/lifecycle-mux-fixture.ts +++ b/tests/helpers/lifecycle-mux-fixture.ts @@ -1,4 +1,4 @@ -import { chmod, writeFile } from "node:fs/promises"; +import { chmod, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; /** A closed tmux transport stand-in. It starts the shipping wrapper/client, not @@ -31,7 +31,7 @@ case "list-windows": case "new-window": { if(!state)process.exit(91); const child=Bun.spawn(["/bin/sh","-c",args.at(-1)],{cwd:selected("-c"),detached:true,stdin:"ignore",stdout:"ignore",stderr:"ignore"}); - child.unref();state.windows[selected("-n")]={pid:child.pid};await writeFile(file,JSON.stringify(state));break; + child.unref();const started=Bun.spawnSync(["/bin/ps","-o","lstart=","-p",String(child.pid)],{env:{LC_ALL:"C",TZ:"UTC"}}).stdout.toString().trim();state.windows[selected("-n")]={pid:child.pid,started};await writeFile(file,JSON.stringify(state));break; } case "list-panes": { if(!state)process.exit(91);const name=selected("-t").split(":").at(-1);if(state.windows[name])console.log(state.windows[name].pid);break; @@ -48,3 +48,101 @@ default:process.exit(93); await chmod(path, 0o700); return path; } + +/** A process's start time as the stand-in records it; null when it has exited. */ +function leaderStartedAt(pid: number): string | null { + const started = Bun.spawnSync( + ["/bin/ps", "-o", "lstart=", "-p", String(pid)], + { + env: { LC_ALL: "C", TZ: "UTC" }, + } + ) + .stdout.toString() + .trim(); + return started === "" ? null : started; +} + +/** + * Whether a recorded group is still the window this fixture started. A live + * leader must have the recorded start time. With no live leader, a group that + * still exists keeps its id from being reused, so it can only be ours. A row + * with no recorded start time proves nothing and is left alone. + */ +function stillOurGroup(row: { + readonly pid: number; + readonly started: string; +}): boolean { + const started = leaderStartedAt(row.pid); + return started === null || started === row.started; +} + +function groupPresent(group: number): boolean { + try { + process.kill(-group, 0); + return true; + } catch (error) { + if ((error as { readonly code?: unknown }).code === "ESRCH") { + return false; + } + throw error; + } +} + +/** Time a stand-in window group gets to handle SIGTERM before SIGKILL. */ +const WINDOW_GROUP_TERM_GRACE_MS = 2000; + +/** + * End the window groups the stand-in mux recorded in `stateFile`. Test cleanup + * for a failed case only: a retained real start would leave them for `hack + * down` recovery, and nothing would otherwise stop these captured children. + * The file is rewritten as a whole, so an unreadable one names no groups yet. + * A window that exited stays in the file, so its id may belong to someone else + * by now: each signal first checks the group is still the recorded one. + */ +export async function endLifecycleMuxFixtureGroups( + stateFile: string +): Promise { + type Row = { readonly pid?: unknown; readonly started?: unknown }; + let windows: Record; + try { + windows = ( + JSON.parse(await readFile(stateFile, "utf8")) as { + readonly windows: Record; + } + ).windows; + } catch { + return; + } + const rows = Object.values(windows ?? {}).flatMap((row) => + typeof row.pid === "number" && + row.pid > 1 && + typeof row.started === "string" && + row.started !== "" + ? [{ pid: row.pid, started: row.started }] + : [] + ); + const signalAll = (signal: NodeJS.Signals) => { + for (const row of rows) { + if (!stillOurGroup(row)) { + continue; + } + try { + process.kill(-row.pid, signal); + } catch (error) { + // The group ended after the check. The others still get their signal. + if ((error as { readonly code?: unknown }).code !== "ESRCH") { + throw error; + } + } + } + }; + signalAll("SIGTERM"); + const deadline = performance.now() + WINDOW_GROUP_TERM_GRACE_MS; + while ( + rows.some((row) => stillOurGroup(row) && groupPresent(row.pid)) && + performance.now() < deadline + ) { + await Bun.sleep(20); + } + signalAll("SIGKILL"); +} diff --git a/tests/helpers/native-fixture-cases.ts b/tests/helpers/native-fixture-cases.ts index 76ad6ebaa..f295610b1 100644 --- a/tests/helpers/native-fixture-cases.ts +++ b/tests/helpers/native-fixture-cases.ts @@ -36,11 +36,23 @@ export type NativeFixtureCase = { ) => Readonly>; /** Record a failure seen in a callback that must not throw; the case then fails. */ readonly fail: (error: unknown) => void; + /** + * End test-owned processes when this case fails or does not settle. A + * passing case never runs it, so a leak the product caused still fails + * settlement. A body that outlives its budget triggers it again when it ends, + * because that body can start more processes after the case has failed. + */ + readonly onFailure: (cleanup: () => void | Promise) => void; }; export type NativeFixtureCases = { /** The guarded case running the caller. Throws outside a guarded case. */ readonly current: () => NativeFixtureCase; + /** Run one guarded case body directly and reject with its outcome. */ + readonly run: ( + body: () => void | Promise, + timeoutMs?: number + ) => Promise; /** * Register a guarded case. `timeoutMs` bounds the body; the Bun timeout adds * the settlement budget and a margin, so cleanup always finishes inside it. @@ -133,6 +145,18 @@ export function createNativeFixtureCases(opts: { const roots: string[] = []; const saved = new Map(); const observed: unknown[] = []; + const failureCleanups: (() => void | Promise)[] = []; + const runFailureCleanups = async () => { + for (const cleanup of failureCleanups) { + try { + await cleanup(); + } catch (error) { + console.error( + `${opts.label} failure cleanup refused: ${message(error)}` + ); + } + } + }; const settlement = createNativeAuthoredSettlement(); const context: NativeFixtureCase = { root: (path) => { @@ -150,22 +174,31 @@ export function createNativeFixtureCases(opts: { fail: (error) => { observed.push(error); }, + onFailure: (cleanup) => { + failureCleanups.push(cleanup); + }, }; let failure: { readonly error: unknown } | undefined; + let bodyRunning = true; + const running = (async () => { + await storage.run(context, body); + })().finally(() => { + bodyRunning = false; + }); try { - await withinBudget( - opts.label, - (async () => { - await storage.run(context, body); - })(), - timeoutMs - ); + await withinBudget(opts.label, running, timeoutMs); if (observed.length > 0) { throw observed[0]; } } catch (error) { failure = { error }; } + if (failure !== undefined) { + await runFailureCleanups(); + if (bodyRunning) { + running.catch(() => undefined).then(runFailureCleanups); + } + } // Settle this case's own targets even after a failure, so the next case // normally starts without them; a timeout here fails only this case. let unsettled: { readonly error: unknown } | undefined; @@ -173,6 +206,9 @@ export function createNativeFixtureCases(opts: { await settlement.settle({ budgetMs: opts.settlementBudgetMs }); } catch (error) { unsettled = { error }; + if (failure === undefined) { + await runFailureCleanups(); + } } for (const [key, value] of saved) { restoreEnv(key, value); @@ -186,6 +222,7 @@ export function createNativeFixtureCases(opts: { }; return { current, + run: (body, timeoutMs = opts.defaultTimeoutMs) => runCase(body, timeoutMs), test: (name, body, timeoutMs = opts.defaultTimeoutMs) => { bunTest( name, diff --git a/tests/helpers/optional-fixture-node.ts b/tests/helpers/optional-fixture-node.ts new file mode 100644 index 000000000..18abec6b0 --- /dev/null +++ b/tests/helpers/optional-fixture-node.ts @@ -0,0 +1,64 @@ +import { open, realpath } from "node:fs/promises"; +import { basename } from "node:path"; + +/** Optional fixture acceleration must never install a runtime or execute a + * version-manager shim. Its descendants/pipes have no fixture owner. Callers + * can put a direct installed Node on PATH; all other cases retain Bun. + */ +export async function optionalFixtureNode( + path = process.env.PATH ?? "" +): Promise { + const selected = Bun.which("node", { PATH: path }); + if (!selected) { + return null; + } + try { + const executable = await realpath(selected); + if (basename(executable) !== "node") { + return null; + } + const handle = await open(executable, "r"); + const header = Buffer.alloc(4); + try { + await handle.read(header, 0, 4, 0); + } finally { + await handle.close(); + } + // ELF and Mach-O (including universal) executables. A script called node + // remains a shim even if its path happens to end with that name. + if ( + !new Set([ + "7f454c46", + "cffaedfe", + "feedfacf", + "cefaedfe", + "feedface", + "cafebabe", + "bebafeca", + "cafebabf", + "bfbafeca", + ]).has(header.toString("hex")) + ) { + return null; + } + const child = Bun.spawn( + [ + executable, + "--input-type=module", + "-e", + 'import * as fs from "node:fs/promises"; if(Number(process.versions.node.split(".")[0]) < 18) process.exit(1); await fs.stat(process.execPath);', + ], + { + env: { PATH: "/usr/bin:/bin", LANG: "C" }, + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + timeout: 5000, + killSignal: "SIGKILL", + } + ); + return (await child.exited) === 0 ? executable : null; + } catch { + return null; + } +} diff --git a/tests/helpers/retained-routing-adoption.ts b/tests/helpers/retained-routing-adoption.ts index f70aedb37..b93936a0a 100644 --- a/tests/helpers/retained-routing-adoption.ts +++ b/tests/helpers/retained-routing-adoption.ts @@ -21,6 +21,10 @@ import { mapLegacyNativeRetainedRouting } from "../../src/lib/native-config-impo import type { NativeRoutingResolution } from "../../src/lib/native-routing-plan-protocol.ts"; import * as shell from "../../src/lib/shell.ts"; import { restoreEnv } from "./env.ts"; +import { optionalFixtureNode } from "./optional-fixture-node.ts"; +import { installStaticFixtureNode } from "./static-fixture-node.ts"; + +let fixtureNode: Promise | undefined; export const ROUTING_CANARY = "synthetic-retained-sql-row"; /** Closed set: the product's 15 s default proof deadline or its 60 s cap. */ @@ -121,8 +125,12 @@ export function retainedRoutingFixtureLifetime(deadline: number) { /** Private-store model only. Docker metadata and child effects are injected; * this does not qualify Go formatting, compiler semantics, TLS, SQL or a daemon. */ export async function retainedRoutingFixture( - lifetime: ReturnType + lifetime: ReturnType, + options: { readonly lightweightCompiler?: boolean } = {} ) { + const node = options.lightweightCompiler + ? await (fixtureNode ??= optionalFixtureNode()) + : null; const outer = await realpath( await mkdtemp(join(tmpdir(), "retained-routing-owner-")) ); @@ -222,6 +230,7 @@ else { `, { mode: 0o700 } ); + await installStaticFixtureNode({ binary: compiler, node }); const previousHome = process.env.HACK_HOME; process.env.HACK_HOME = home; const model = { @@ -239,6 +248,9 @@ else { effects: string[][] = []; const hooks: { afterProbe?: (args: readonly string[]) => Promise; + beforeProbeOwner?: ( + opts: Parameters[0] + ) => void; afterEffect?: () => Promise; } = {}; const json = (value: unknown) => JSON.stringify(value); @@ -514,9 +526,10 @@ else { }; const originalKill = process.kill; const spies = [ - spyOn(ownership, "createNativeComposeProbe").mockImplementation( - () => probe - ), + spyOn(ownership, "createNativeComposeProbe").mockImplementation((opts) => { + hooks.beforeProbeOwner?.(opts); + return probe; + }), spyOn(engine, "createNativeComposeEngineIdentityObserver").mockReturnValue( null ), @@ -601,11 +614,15 @@ else { * a compound lifecycle test may ask for the 60 s product cap so suite load cannot * expire its rollback proof. Both values are well-known product bounds. */ store: ( - opts: { readonly timeoutMs?: RetainedRoutingStoreTimeoutMs } = {} + opts: { + readonly timeoutMs?: RetainedRoutingStoreTimeoutMs; + readonly deadline?: number; + } = {} ) => openLegacyComposeAdoptedGenerationStore({ projectRoot: root, timeoutMs: opts.timeoutMs ?? 15_000, + deadline: opts.deadline, }), operation: async ( store: Awaited< diff --git a/tests/helpers/static-fixture-node.ts b/tests/helpers/static-fixture-node.ts new file mode 100644 index 000000000..e3fd62311 --- /dev/null +++ b/tests/helpers/static-fixture-node.ts @@ -0,0 +1,50 @@ +import { rename, writeFile } from "node:fs/promises"; + +/** Filesystem/stdin subset consumed by the generated static stand-ins. Their + * parents already exist and writes have no consumed return value. */ +const NODE_FIXTURE_IO = `import * as fixtureFs from "node:fs/promises"; +const Bun = { + file(path) { return { + async exists() { try { return (await fixtureFs.stat(path)).isFile(); } + catch(error) { if(error.code === "ENOENT" || error.code === "ENOTDIR") return false; throw error; } }, + async text() { return await fixtureFs.readFile(path, "utf8"); }, + async json() { return JSON.parse(await fixtureFs.readFile(path, "utf8")); } + }; }, + async write(path, value) { await fixtureFs.writeFile(path, value); }, + stdin: { async text() { const chunks=[]; for await(const chunk of process.stdin) chunks.push(chunk); return Buffer.concat(chunks).toString("utf8"); } } +}; +`; +function quote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + +/** Optional transport for generated stand-ins whose inputs are argv, stdin + * and fixture files, never inherited environment. Resolve Node before fixture + * PATH changes. An absent runtime leaves the original Bun executable untouched. + * The owning production code still controls process groups, IO and deadlines. + */ +export async function installStaticFixtureNode(opts: { + readonly binary: string; + readonly node: string | null; +}): Promise { + if (!opts.node) { + return false; + } + const original = await Bun.file(opts.binary).text(); + const source = `${opts.binary}.node.mjs`; + await writeFile( + source, + NODE_FIXTURE_IO + original.replace(/^#![^\n]*\n/, ""), + { + mode: 0o600, + } + ); + const wrapper = `${opts.binary}.node-wrapper`; + await writeFile( + wrapper, + `#!/bin/sh\nexec /usr/bin/env -i PATH=/usr/bin:/bin ${quote(opts.node)} ${quote(source)} "$@"\n`, + { mode: 0o700 } + ); + await rename(wrapper, opts.binary); + return true; +} diff --git a/tests/native-authored-project-start.test.ts b/tests/native-authored-project-start.test.ts index 664a69d0e..1bce31702 100644 --- a/tests/native-authored-project-start.test.ts +++ b/tests/native-authored-project-start.test.ts @@ -8,6 +8,7 @@ import { readdir, readFile, realpath, + rm, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -40,13 +41,18 @@ import { type ComposeEnvFingerprintRecord, evaluateComposeServiceEnvFreshness, } from "../src/lib/compose-env-fingerprint.ts"; -import { writeLifecycleMuxFixture } from "./helpers/lifecycle-mux-fixture.ts"; +import { + endLifecycleMuxFixtureGroups, + writeLifecycleMuxFixture, +} from "./helpers/lifecycle-mux-fixture.ts"; import { awaitPidFile } from "./helpers/native-authored-settlement.ts"; import { createNativeFixtureCases, NATIVE_FIXTURE_BUN_MARGIN_MS, type NativeFixtureCase, } from "./helpers/native-fixture-cases.ts"; +import { optionalFixtureNode } from "./helpers/optional-fixture-node.ts"; +import { installStaticFixtureNode } from "./helpers/static-fixture-node.ts"; import { CASE_TIMEOUT_CEILING_MS, testBudgetMs, @@ -97,6 +103,12 @@ const DEADLINE_WALL_BOUND_MS = 1000; * shard's per-file limit. Passing cases release or cancel them explicitly. */ const STAND_IN_CAP_MS = CASE_TIMEOUT_CEILING_MS; +/** + * Longest life of a stand-in host process that nothing stopped. A failed case + * ends its groups through `onFailure`; this bounds the rest (a runner killed by + * the shard's limit). It outlasts that limit, so no case ever reaches it. + */ +const HOST_PROCESS_CAP_MS = 900_000; const cases = createNativeFixtureCases({ label: "Startup fixture", defaultTimeoutMs: CASE_TIMEOUT_MS, @@ -256,6 +268,9 @@ type FixtureOptions = { /** Closed native code for `planFailure` (default `native_graph_compile`). */ readonly planFailureCode?: string; readonly failBeforeReceipt?: boolean; + readonly removedBeforeReady?: boolean; + readonly foreignRemovalReview?: boolean; + readonly releaseHostAfterTest?: boolean; /** * Like `failBeforeReceipt`, but serve fails the way authored foreground * control reports a runtime that stopped: `runtime_not_running` wrapped in @@ -295,9 +310,14 @@ type FixtureOptions = { | "foreign-owner"; }; +let compilerFixtureNode: Promise | undefined; + /** Fake compiler/engine transport; real managed inputs, admission, process and storage owners. */ async function fixture(options: FixtureOptions = {}) { const owner = cases.current(); + // Resolve before fixture PATH/env mutation. This only changes the pure + // compiler protocol transport; the original Bun script remains the fallback. + const node = await (compilerFixtureNode ??= optionalFixtureNode()); // Optional effect-free integration uses the real compiler and native planner; // execution/status/cleanup remain this file's separately controlled fake driver. const realCompiler = process.env.HACK_TEST_NATIVE_COMPILER; @@ -343,6 +363,9 @@ async function fixture(options: FixtureOptions = {}) { emptySettled: true, parse: muxWindowPids, }); + // A start that fails after reservation retains these groups on purpose, + // for `hack down` recovery that a failed case never reaches. + owner.onFailure(() => endLifecycleMuxFixtureGroups(join(root, "mux.json"))); } const hook = (name: string) => ({ name, @@ -380,7 +403,11 @@ async function fixture(options: FixtureOptions = {}) { process.execPath, "--no-env-file", "-e", - 'const {appendFileSync}=await import("node:fs");appendFileSync("process-order","started\\n");await Bun.write("process-value",process.env.PROCESS_TOKEN??"missing");const stop=()=>{appendFileSync("process-order","stopped\\n");process.exit(0)};process.on("SIGTERM",stop);process.on("SIGINT",stop);setInterval(()=>{},1000)', + 'const {appendFileSync}=await import("node:fs");appendFileSync("process-order","started\\n");await Bun.write("process-value",process.env.PROCESS_TOKEN??"missing");const stop=()=>{appendFileSync("process-order","stopped\\n");process.exit(0)};process.on("SIGTERM",stop);process.on("SIGINT",stop);' + + (options.releaseHostAfterTest + ? 'const {existsSync}=await import("node:fs");setInterval(()=>{if(existsSync("release-host-test")){appendFileSync("process-order","fixture-released"+String.fromCharCode(10));process.exit(0)}},20)' + : "setInterval(()=>{},1000)") + + `;setTimeout(()=>process.exit(0),${HOST_PROCESS_CAP_MS})`, ], }, env_target: { kind: "host" }, @@ -455,6 +482,7 @@ console.log(JSON.stringify(result)); ` ); await chmod(compiler, 0o700); + await installStaticFixtureNode({ binary: compiler, node }); process.env.HACK_CONFIG_COMPILER_BINARY = options.compilerFailure === true ? compiler : (realCompiler ?? compiler); const bundle = @@ -556,6 +584,17 @@ if(['plan','serve','frontend-plan','frontend-serve'].includes(action)){ if(${options.failBeforeReceipt === true}){console.error(JSON.stringify({code:${JSON.stringify(options.failBeforeReceiptCode ?? "graph_network_intent")},message:${JSON.stringify(CANARY)}}));process.exit(2)} if(${options.stoppedDuringServe === true}){console.error(JSON.stringify({code:'native_graph_foreground',cause_code:'runtime_not_running',message:${JSON.stringify(CANARY)}}));process.exit(2)} const receipt={version:2,kind:'native-graph-runtime',owner:'f'.repeat(32),boot:'00000000-0000-0000-0000-000000000001',review,phase:'ready-observed',readiness:{web:'started'},resources:{'network:default':{kind:'network',key:'default',name:'hkn-'+input.run+'-network-0',id:'1'.repeat(64),image:null,phase:'created',outbound:true},'container:web':{kind:'container',key:'web',name:'hkn-'+input.run+'-container-0',id:'2'.repeat(64),image:'sha256:'+ '3'.repeat(64),phase:'started',networks:['default']}}}; + if(${options.removedBeforeReady === true}){ + if(${JSON.stringify(options.cleanup ?? "removed")}!=='live'){ + receipt.phase='removed';for(const resource of Object.values(receipt.resources))resource.phase='removed'; + } + if(${options.foreignRemovalReview === true}){ + provenance.namespace='9'.repeat(64); + review.review_id=createHash('sha256').update('hack.native-graph-review/v1\\0').update(JSON.stringify(provenance)).digest('hex'); + } + await writeFile(journal,JSON.stringify(receipt)); + console.error(JSON.stringify({code:'graph_network_intent',message:${JSON.stringify(CANARY)}}));process.exit(2); + } if(${options.storage === true && options.liveStop === true}){ const namespace=provenance.namespace, owner='9'.repeat(32); receipt.version=4; @@ -2065,3 +2104,410 @@ for (const [httpsPort, branch, url] of [ } ); } + +/** + * Just past the compiler transport's fixed 10 s per-invocation default, which + * authored compiler calls used to keep whatever time their caller had left. + */ +const SLOW_COMPILE_MS = 10_500; + +/** + * A wrapper around the fixture compiler whose first invocation of `operation` + * (any operation when omitted) sleeps {@link SLOW_COMPILE_MS} before it runs. + * `install` selects it; the case guard restores the fixture's compiler key. + * `slept` is the marker the wrapper writes when it has slept. + */ +async function slowCompilerOnce(opts: { + readonly root: string; + readonly operation?: string; +}): Promise<{ readonly install: () => void; readonly slept: string }> { + const compiler = process.env.HACK_CONFIG_COMPILER_BINARY; + if (!compiler) { + throw new Error("missing fixture compiler"); + } + const slept = join(opts.root, "slow-compile-once"); + const wrapper = join(opts.root, "slow-compiler"); + const selected = opts.operation + ? `process.argv[2]===${JSON.stringify(opts.operation)}` + : "true"; + await writeFile( + wrapper, + `#!${process.execPath} +import {existsSync,writeFileSync} from "node:fs"; +if(${selected}&&!existsSync(${JSON.stringify(slept)})){writeFileSync(${JSON.stringify(slept)},"");await Bun.sleep(${SLOW_COMPILE_MS});} +const child=Bun.spawn([${JSON.stringify(compiler)},...process.argv.slice(2)],{stdin:"inherit",stdout:"inherit",stderr:"inherit"}); +process.exit(await child.exited); +` + ); + await chmod(wrapper, 0o700); + return { + install: () => { + process.env.HACK_CONFIG_COMPILER_BINARY = wrapper; + }, + slept, + }; +} + +macTest( + "HACK-1458: a startup compiler call may outlast the compiler's 10 s default while the startup deadline remains", + async () => { + const selected = await fixture({ noManaged: true }); + const slow = await slowCompilerOnce({ + root: selected.root, + operation: "compile", + }); + slow.install(); + expect( + await serveNativeAuthoredProject({ + ...selected, + run, + startupTimeoutMs: STARTUP_TIMEOUT_MS, + }) + ).toBe(0); + expect(await Bun.file(slow.slept).exists()).toBe(true); + expect(await artifacts(selected.scope)).toEqual([]); + } +); + +// HACK-1458: the loaded hang of "Source5 cancellation after readiness". The +// cancel's down.before acquisition kept the compiler's fixed 10 s default, so +// a slow compiler failed the graceful stop, and the start then retained the +// live owner and host processes until a hard cancellation that never came. +macTest( + "HACK-1458: Source5 cancellation after readiness settles when its down acquisition compiler is slow", + async () => { + const selected = await fixture({ finiteHooks: true, hostProcesses: true }); + const slow = await slowCompilerOnce({ root: selected.root }); + const signal = new AbortController(); + const stages: string[] = []; + expect( + await serveNativeAuthoredProject({ + ...selected, + run, + signal: signal.signal, + startupTimeoutMs: STARTUP_TIMEOUT_MS, + onHookDiagnostic: (event) => { + stages.push(`${event.phase}:${event.boundary}`); + }, + onReady: () => { + // The next compiler invocation is the cancel's down acquisition. + slow.install(); + signal.abort(); + return undefined; + }, + }) + ).toBe(0); + expect(await Bun.file(slow.slept).exists()).toBe(true); + expect(stages).not.toContain("down.before:stop-operation"); + expect(stages).toContain("down.after:complete"); + expect( + await Bun.file(join(selected.scope.projectRoot, "process-order")).text() + ).toBe("started\nstopped\n"); + expect( + await Bun.file(join(selected.scope.projectRoot, "hook-order")).text() + ).toBe("up-before\nup-after\ndown-before\ndown-after\n"); + const files = await artifacts(selected.scope); + expect( + files.some((name) => name.endsWith(".host-process-owner.json")) + ).toBe(false); + expect(files.some((name) => name.endsWith(".hooks.json"))).toBe(false); + } +); + +/** + * Wrap the fixture's native stand-in so its second `frontend-plan` request + * (the one after up.before has run and the host processes have started) + * publishes `held` and then waits until the start kills it. + */ +async function holdSecondFrontendPlan(opts: { + readonly root: string; + readonly binary: string; +}): Promise<{ readonly binary: string; readonly held: string }> { + const plans = join(opts.root, "frontend-plans"); + const held = join(opts.root, "second-frontend-plan-held"); + const binary = join(opts.root, "native-hold-second-plan"); + await writeFile( + binary, + `#!${process.execPath} +import {appendFileSync,readFileSync,writeFileSync} from "node:fs"; +const args=process.argv.slice(2); +if(args.includes("frontend-plan")){appendFileSync(${JSON.stringify(plans)},"plan\\n");if(readFileSync(${JSON.stringify(plans)},"utf8").trim().split("\\n").length===2){writeFileSync(${JSON.stringify(held)},String(process.pid));setInterval(()=>{},1000);await new Promise(()=>{});}} +const child=Bun.spawn([${JSON.stringify(opts.binary)},...args],{stdin:"inherit",stdout:"inherit",stderr:"inherit"}); +process.exit(await child.exited); +` + ); + await chmod(binary, 0o700); + return { binary, held }; +} + +// HACK-1458: a cancel (or the startup deadline, which aborts the same signal) +// after the host processes started and before the startup intent exists must +// stop them. Their teardown freshness check used the startup inputs, bound to +// the aborted startup signal, so it always refused and left the groups running. +macTest( + "HACK-1458: cancellation after host processes start and before reservation stops them", + async () => { + const selected = await fixture({ finiteHooks: true, hostProcesses: true }); + const hold = await holdSecondFrontendPlan({ + root: selected.root, + binary: selected.runtime.binary, + }); + const signal = new AbortController(); + let settled = false; + const pending = serveNativeAuthoredProject({ + ...selected, + runtime: { ...selected.runtime, binary: hold.binary }, + run, + signal: signal.signal, + startupTimeoutMs: STARTUP_TIMEOUT_MS, + }) + .catch((error: unknown) => error) + .finally(() => { + settled = true; + }); + while (!(settled || (await Bun.file(hold.held).exists()))) { + await Bun.sleep(20); + } + expect(await Bun.file(hold.held).exists()).toBe(true); + signal.abort(); + const error = await pending; + expect(error).toBeInstanceOf(NativeAuthoredProjectStartError); + expect(error).toHaveProperty("canceled", true); + expect( + await Bun.file(join(selected.scope.projectRoot, "process-order")).text() + ).toBe("started\nstopped\n"); + const files = await artifacts(selected.scope); + expect(files.some((name) => name.endsWith(".start.json"))).toBe(false); + expect( + files.some((name) => name.endsWith(".host-process-owner.json")) + ).toBe(false); + } +); + +/** + * Delay the stand-in mux's first `list-panes` (issued right after it started + * the host process window) and publish `held` first, so a case can cancel + * while the host process launch is still in progress. + */ +async function holdFirstPaneQuery(root: string): Promise { + const bin = join(root, "bin"); + const held = join(root, "pane-query-held"); + await writeFile( + join(bin, "tmux-stand-in"), + await readFile(join(bin, "tmux")), + { mode: 0o700 } + ); + await writeFile( + join(bin, "tmux"), + `#!${process.execPath} +import {existsSync,writeFileSync} from "node:fs"; +const args=process.argv.slice(2); +if(args[0]==="list-panes"&&!existsSync(${JSON.stringify(held)})){writeFileSync(${JSON.stringify(held)},"");await Bun.sleep(1000);} +const child=Bun.spawn([${JSON.stringify(join(bin, "tmux-stand-in"))},...args],{stdin:"inherit",stdout:"inherit",stderr:"inherit"}); +process.exit(await child.exited); +` + ); + await chmod(join(bin, "tmux"), 0o700); + return held; +} + +/** True once no process in `group` remains, within a 5 s bound. */ +async function groupEnded(group: number): Promise { + for (let attempt = 0; attempt < 100; attempt++) { + try { + process.kill(-group, 0); + } catch (error) { + if ((error as { code?: unknown }).code === "ESRCH") { + return true; + } + throw error; + } + await Bun.sleep(50); + } + return false; +} + +// HACK-1458: a start that failed after the host process launch began but +// before its ready record left the launched group running: teardown required +// a ready record, and recovery never converts a bare launch intent into +// authority. The launch now ends what it started through its session token. +macTest( + "HACK-1458: cancellation during the host process launch ends the launched group", + async () => { + const selected = await fixture({ finiteHooks: true, hostProcesses: true }); + const held = await holdFirstPaneQuery(selected.root); + const signal = new AbortController(); + let settled = false; + const pending = serveNativeAuthoredProject({ + ...selected, + run, + signal: signal.signal, + startupTimeoutMs: STARTUP_TIMEOUT_MS, + }) + .catch((error: unknown) => error) + .finally(() => { + settled = true; + }); + while (!(settled || (await Bun.file(held).exists()))) { + await Bun.sleep(20); + } + expect(await Bun.file(held).exists()).toBe(true); + // The window exists now; record its group before the launch can end it. + const state = JSON.parse( + await readFile(join(selected.root, "mux.json"), "utf8") + ) as { readonly windows: Record }; + const groups = Object.values(state.windows).map((row) => row.pid); + expect(groups).toHaveLength(1); + signal.abort(); + const error = await pending; + expect(error).toBeInstanceOf(NativeAuthoredProjectStartError); + expect(error).toHaveProperty("canceled", true); + for (const group of groups) { + expect(await groupEnded(group)).toBe(true); + } + // The owned session was ended through its token. Its owner record stays + // unless every launched group was recorded and proven absent; a launch + // cancelled this early may not have recorded its command group yet. + expect(await Bun.file(join(selected.root, "mux.json")).exists()).toBe( + false + ); + } +); + +for (const proof of [ + "removed", + "live", + "foreign", + "hard-canceled", + "unconfirmed-group", +] as const) { + macTest( + `pre-Ready host retirement requires settled authenticated removal: ${proof}`, + async () => { + const selected = await fixture({ + finiteHooks: true, + hostProcesses: true, + removedBeforeReady: true, + releaseHostAfterTest: true, + cleanup: proof === "live" ? "live" : "removed", + foreignRemovalReview: proof === "foreign", + }); + const signal = new AbortController(); + const force = new AbortController(); + let ready = false; + const originalKill = process.kill.bind(process); + let group: number | undefined; + let killSpy: ReturnType> | undefined; + try { + const error = await failure( + serveNativeAuthoredProject({ + ...selected, + run, + signal: signal.signal, + forceSignal: force.signal, + startupTimeoutMs: STARTUP_TIMEOUT_MS, + onReady: () => { + ready = true; + return undefined; + }, + onExitDiagnostic: () => { + signal.abort(); + if (proof === "unconfirmed-group") { + killSpy = spyOn(process, "kill").mockImplementation( + (pid, signal) => { + // Refuse only the consumer's first settlement probe. A later + // real ESRCH must not replace its missing onSettled event. + if (pid < 0 && signal === 0 && group === undefined) { + group = pid; + throw Object.assign( + new Error("synthetic group uncertainty"), + { code: "EPERM" } + ); + } + return originalKill(pid, signal); + } + ); + } + if (proof === "hard-canceled") { + force.abort(); + } + }, + }), + proof === "removed" ? "removed" : "retained" + ); + expect(error).toHaveProperty("canceled", true); + expect(ready).toBe(false); + expect( + await Bun.file( + join(selected.scope.projectRoot, "process-order") + ).text() + ).toBe(proof === "removed" ? "started\nstopped\n" : "started\n"); + const files = await artifacts(selected.scope); + expect(files.some((name) => name.endsWith(".start.json"))).toBe( + proof !== "removed" + ); + expect( + files.some((name) => name.endsWith(".host-process-owner.json")) + ).toBe(proof !== "removed"); + } finally { + if (killSpy) { + confirmTestGroupAbsent(group, originalKill); + killSpy.mockRestore(); + } + // Only this test's cooperating stand-in observes this marker. It does not + // turn an unproved production cleanup into an accepted stop. + await Bun.write( + join(selected.scope.projectRoot, "release-host-test"), + "release" + ); + } + } + ); +} + +// HACK-1458 item 4: a failed case must not leave the stand-in groups running. +// Bun runs this body outside `cases.test`, because the guarded case fails here. +bunTest.skipIf(process.platform !== "darwin")( + "HACK-1458: a failed case ends the stand-in mux groups it left running", + async () => { + let group: number | undefined; + let root: string | undefined; + try { + await expect( + cases.run(async () => { + const selected = await fixture({ hostProcesses: true }); + root = selected.root; + const mux = join(selected.root, "bin", "tmux"); + for (const args of [ + ["new-session", "-s", "failed-case", "-c", selected.root], + ["new-window", "-n", "held", "-c", selected.root, "sleep 300"], + ]) { + const child = Bun.spawn([mux, ...args], { + env: process.env, + stdout: "ignore", + stderr: "ignore", + }); + expect(await child.exited).toBe(0); + } + const state = JSON.parse( + await readFile(join(selected.root, "mux.json"), "utf8") + ) as { + readonly windows: { readonly held: { readonly pid: number } }; + }; + group = state.windows.held.pid; + process.kill(-group, 0); + throw new Error("deliberate case failure"); + }, CASE_TIMEOUT_MS) + ).rejects.toThrow("deliberate case failure"); + if (group === undefined) { + throw new Error("the stand-in window was never recorded"); + } + expect(await groupEnded(group)).toBe(true); + } finally { + if (root !== undefined) { + await rm(root, { recursive: true, force: true }); + } + } + }, + CASE_TIMEOUT_MS + SETTLEMENT_BUDGET_MS + NATIVE_FIXTURE_BUN_MARGIN_MS +); diff --git a/tests/native-compose-adoption-generation.test.ts b/tests/native-compose-adoption-generation.test.ts index 506400e28..638888186 100644 --- a/tests/native-compose-adoption-generation.test.ts +++ b/tests/native-compose-adoption-generation.test.ts @@ -5421,3 +5421,91 @@ test("preparation stop refuses changed source, forged claims and config drift be await store.close(); } }); + +/** + * HACK-1458: public adoption dispatch derives each store inspection phase from + * the command's caller deadline. A synthetic clock jumps once, right after the + * dispatch's first engine probe, so these cases need no real waiting: the + * phase that probe opened must survive the jump while the deadline remains, + * and must end as a timeout once the deadline has passed. + * + * The case budget is a ceiling only: prepare and publish spawn hundreds of + * stub-engine probes, so a loaded host needs minutes; a passing case never + * waits on it. + */ +const PUBLIC_DEADLINE_CASE_MS = 300_000; +async function publicDispatchAfterClockJump(opts: { + readonly startupTimeoutMs: number; + readonly jumpMs: number; +}): Promise<{ readonly result: unknown; readonly jumped: boolean }> { + const { store, generation } = await prepared(); + const priorCompiler = process.env.HACK_CONFIG_COMPILER_BINARY; + const priorStartup = process.env.HACK_COMPOSE_STARTUP_TIMEOUT_MS; + const realNow = Date.now; + let clock: { mockRestore: () => void } | undefined; + let offset = 0; + try { + const binary = await compiler(); + process.env.HACK_CONFIG_COMPILER_BINARY = binary; + await store.publish({ generation, binary }); + process.env.HACK_COMPOSE_STARTUP_TIMEOUT_MS = String(opts.startupTimeoutMs); + const commands = join(root, "commands"); + const before = Bun.file(commands).size; + clock = spyOn(Date, "now").mockImplementation(() => { + if (offset === 0 && Bun.file(commands).size > before) { + offset = opts.jumpMs; + } + return realNow() + offset; + }); + const result = await tryLegacyComposeAdoptedCommand({ + cwd: projectRoot, + operation: "up", + detach: true, + }).catch((error: unknown) => error); + return { result, jumped: offset === opts.jumpMs }; + } finally { + clock?.mockRestore(); + restoreEnv("HACK_CONFIG_COMPILER_BINARY", priorCompiler); + restoreEnv("HACK_COMPOSE_STARTUP_TIMEOUT_MS", priorStartup); + await store.close(); + } +} + +boundedTest( + "HACK-1458: public dispatch inspection outlives the former fixed 15 s phase clock while the caller deadline remains", + async () => { + const { result, jumped } = await publicDispatchAfterClockJump({ + startupTimeoutMs: 120_000, + jumpMs: 16_000, + }); + expect(jumped).toBe(true); + expect(result).toBe(0); + expect(await mutationCommands()).toEqual([["container", "start", ID]]); + }, + PUBLIC_DEADLINE_CASE_MS +); + +boundedTest( + "HACK-1458: an expired public dispatch deadline ends inspection as a startup timeout, not invalid config", + async () => { + const { result, jumped } = await publicDispatchAfterClockJump({ + startupTimeoutMs: 5000, + jumpMs: 6000, + }); + expect(jumped).toBe(true); + expect(result).toBeInstanceOf(HackCliError); + if (!(result instanceof HackCliError)) { + throw new Error("missing public refusal"); + } + expect(result.code).toBe("E_STARTUP_TIMEOUT"); + // The fixed store message keeps the closed `legacy-deadline` diagnostic. + expect(result.message).toBe( + new LegacyComposeAdoptedGenerationError("E_LEGACY_ADOPTION_DEADLINE") + .message + ); + expect(result.message).not.toContain(CANARY); + expect(await mutationCommands()).toEqual([]); + expect((await readReceipt()).pendingOperation).toBeNull(); + }, + PUBLIC_DEADLINE_CASE_MS +); diff --git a/tests/native-compose-adoption-routing-generation.test.ts b/tests/native-compose-adoption-routing-generation.test.ts index 6438a2083..7f82fdaa9 100644 --- a/tests/native-compose-adoption-routing-generation.test.ts +++ b/tests/native-compose-adoption-routing-generation.test.ts @@ -7,6 +7,7 @@ import { isRecord } from "../src/lib/guards.ts"; import * as admission from "../src/lib/native-compose-adoption-compiler.ts"; import { LegacyComposeAdoptedGenerationError } from "../src/lib/native-compose-adoption-generation.ts"; import { legacyComposePublicationRefusal } from "../src/lib/native-compose-adoption-publication-diagnostics.ts"; +import * as retainedRouting from "../src/lib/native-compose-adoption-routing.ts"; import { runLegacyComposeRetainedRoutingOperation } from "../src/lib/native-compose-adoption-routing-execution.ts"; import * as privateState from "../src/lib/native-compose-private-state.ts"; import { restoreEnv } from "./helpers/env.ts"; @@ -129,7 +130,11 @@ const test = ( | typeof COMPOUND_TEST_BUDGET_MS; } = {} ) => { - const timeoutMs = opts.timeoutMs ?? 30_000; + // Every default case cold-starts the same store fixture and must also settle + // its cleanup. The former 30 s outer guard interrupted unchanged diagnostics + // under native I/O load and left the lifetime gate closed for later cases. + // This named hang guard does not extend any product proof/caller deadline. + const timeoutMs = opts.timeoutMs ?? SINGLE_OPERATION_TEST_BUDGET_MS; return gatedTest( name, async (restored) => { @@ -138,7 +143,9 @@ const test = ( let failed = false; let failure: unknown; try { - h = await retainedRoutingFixture(lifetime); + h = await retainedRoutingFixture(lifetime, { + lightweightCompiler: true, + }); fixture = h; await run(); } catch (error: unknown) { @@ -1616,6 +1623,125 @@ test("HACK-1391: the callback revocation fence past the caller deadline refuses } }); +for (const phase of ["publication", "rollback"] as const) { + test( + `live caller budget reaches the ${phase} transaction after a durable file move`, + async () => { + let virtualNow = Date.now(); + const deadline = virtualNow + 45_000; + const clock = spyOn(Date, "now").mockImplementation(() => virtualNow); + let store: Awaited> | undefined; + let restoreRename: (() => void) | undefined; + let charged = false; + let issued: number | undefined; + try { + store = await h.store({ timeoutMs: 60_000, deadline }); + const generation = await store.prepare({ binary: h.compiler }); + if (phase === "rollback") { + await store.publish({ generation, binary: h.compiler }); + } + h.hooks.beforeProbeOwner = (opts) => { + if (charged && issued === undefined) { + issued = opts.timeoutMs; + } + }; + const rename = fs.rename; + const renameSpy = spyOn(fs, "rename").mockImplementation( + async (...args) => { + const result = await rename(...args); + const target = String(args[1]); + if ( + !charged && + (phase === "publication" + ? String(args[0]) === join(h.root, ".hack/hack.config.json") && + target.includes("/originals/") + : target.endsWith("/originals/native.rollback")) + ) { + charged = true; + virtualNow += 5000; + } + return result; + } + ); + restoreRename = () => renameSpy.mockRestore(); + if (phase === "publication") { + await store.publish({ generation, binary: h.compiler }); + } else { + await store.rollback(); + } + expect(charged).toBe(true); + expect(issued).toBe(deadline - virtualNow); + expect((await h.receipt()).publication?.phase).toBe( + phase === "publication" ? "active" : "rolled-back" + ); + expect(h.model.sqlRow).toBe(ROUTING_CANARY); + expect(h.effects).toEqual([]); + } finally { + restoreRename?.(); + Reflect.deleteProperty(h.hooks, "beforeProbeOwner"); + clock.mockRestore(); + await store?.close(); + } + }, + { timeoutMs: SINGLE_OPERATION_TEST_BUDGET_MS } + ); +} + +// Publication's sixth saved-manifest read is the scoped routing owner's +// entry revalidation, after its Context copy and before routing proof issuance. +// Charge that read, then observe the real proof's admission budget. +test( + "live caller budget reaches the scoped routing proof after entry revalidation", + async () => { + let virtualNow = Date.now(); + const deadline = virtualNow + 45_000; + const clock = spyOn(Date, "now").mockImplementation(() => virtualNow); + let store: Awaited> | undefined; + const restores: (() => void)[] = []; + let reads = 0; + let charged = false; + let issued: { reads: number; budget: number | undefined } | undefined; + try { + store = await h.store({ timeoutMs: 60_000, deadline }); + const generation = await store.prepare({ binary: h.compiler }); + const read = privateState.readPrivate; + const readSpy = spyOn(privateState, "readPrivate").mockImplementation( + async (...args) => { + const result = await read(...args); + if (args[0].endsWith("/manifest.json") && ++reads === 6) { + charged = true; + virtualNow += 5000; + } + return result; + } + ); + restores.push(() => readSpy.mockRestore()); + const prove = retainedRouting.assertLegacyComposeRetainedRoutingState; + const proofSpy = spyOn( + retainedRouting, + "assertLegacyComposeRetainedRoutingState" + ).mockImplementation(async (opts) => { + issued ??= { reads, budget: opts.timeoutMs }; + return await prove(opts); + }); + restores.push(() => proofSpy.mockRestore()); + await store.publish({ generation, binary: h.compiler }); + expect(charged).toBe(true); + expect(issued).toEqual({ reads: 6, budget: deadline - virtualNow }); + expect((await h.receipt()).publication?.phase).toBe("active"); + expect(h.model.sqlRow).toBe(ROUTING_CANARY); + expect(h.effects).toEqual([]); + } finally { + for (const restore of restores.reverse()) { + restore(); + } + clock.mockRestore(); + await store?.close(); + } + }, + { timeoutMs: SINGLE_OPERATION_TEST_BUDGET_MS } +); + test("HACK-1359: version14 rejects missing and non-finite mutation deadlines without effects", async () => { const { store, generation } = await prepare(); try { diff --git a/tests/native-fixture-cases.test.ts b/tests/native-fixture-cases.test.ts new file mode 100644 index 000000000..2fb64a43d --- /dev/null +++ b/tests/native-fixture-cases.test.ts @@ -0,0 +1,236 @@ +import { afterEach, expect, spyOn, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + endLifecycleMuxFixtureGroups, + writeLifecycleMuxFixture, +} from "./helpers/lifecycle-mux-fixture.ts"; +import { createNativeFixtureCases } from "./helpers/native-fixture-cases.ts"; + +const cases = createNativeFixtureCases({ + label: "Fixture harness", + defaultTimeoutMs: 10_000, + settlementBudgetMs: 300, +}); + +const started: number[] = []; + +/** A detached group this test owns; killed in `afterEach` whatever happens. */ +function detachedGroup(): number { + const child = Bun.spawn(["/bin/sleep", "300"], { + detached: true, + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }); + child.unref(); + started.push(child.pid); + return child.pid; +} + +function groupPresent(group: number): boolean { + try { + process.kill(-group, 0); + return true; + } catch { + return false; + } +} + +async function groupEnded(group: number): Promise { + const deadline = performance.now() + 5000; + while (performance.now() < deadline) { + if (!groupPresent(group)) { + return true; + } + await Bun.sleep(10); + } + return false; +} + +function endGroup(group: number): void { + if (groupPresent(group)) { + process.kill(-group, "SIGKILL"); + } +} + +afterEach(() => { + for (const group of started.splice(0)) { + endGroup(group); + } +}); + +test("a failed case runs its failure cleanup, then settles and keeps its own failure", async () => { + const noise = spyOn(console, "error").mockImplementation(() => undefined); + try { + let group = 0; + await expect( + cases.run(() => { + group = detachedGroup(); + const owner = cases.current(); + owner.settlement.record({ + kind: "process", + name: "held group", + pid: group, + group: true, + }); + owner.onFailure(() => endGroup(group)); + throw new Error("deliberate case failure"); + }) + ).rejects.toThrow("deliberate case failure"); + // Settlement confirmed absence, so no "also unconfirmed" line was reported. + expect(await groupEnded(group)).toBe(true); + expect( + noise.mock.calls.some((call) => + String(call[0]).includes("settlement also unconfirmed") + ) + ).toBe(false); + } finally { + noise.mockRestore(); + } +}); + +test("a passing case that leaves a group still fails settlement, and only then is it ended", async () => { + const noise = spyOn(console, "error").mockImplementation(() => undefined); + try { + let group = 0; + let presentAtCleanup = false; + await expect( + cases.run(() => { + group = detachedGroup(); + const owner = cases.current(); + owner.settlement.record({ + kind: "process", + name: "leaked group", + pid: group, + group: true, + }); + owner.onFailure(() => { + presentAtCleanup = groupPresent(group); + endGroup(group); + }); + }) + ).rejects.toThrow("settlement unconfirmed"); + expect(presentAtCleanup).toBe(true); + expect(await groupEnded(group)).toBe(true); + } finally { + noise.mockRestore(); + } +}); + +test("a passing case that settles never runs its failure cleanup", async () => { + let ran = false; + await cases.run(() => { + cases.current().onFailure(() => { + ran = true; + }); + }); + expect(ran).toBe(false); +}); + +test("a body that outlives its budget runs the failure cleanup again when it ends", async () => { + const noise = spyOn(console, "error").mockImplementation(() => undefined); + try { + let runs = 0; + const release = Promise.withResolvers(); + const ended = Promise.withResolvers(); + await expect( + cases.run(async () => { + cases.current().onFailure(() => { + runs += 1; + if (runs === 2) { + ended.resolve(); + } + }); + await release.promise; + }, 20) + ).rejects.toThrow("exceeded its 20 ms budget"); + expect(runs).toBe(1); + release.resolve(); + await ended.promise; + expect(runs).toBe(2); + } finally { + noise.mockRestore(); + } +}); + +test("the stand-in mux cleanup ends recorded window groups and ignores a missing state", async () => { + const root = await mkdtemp(join(tmpdir(), "lifecycle-mux-cleanup-")); + try { + const mux = await writeLifecycleMuxFixture(root); + const state = join(root, "mux.json"); + await endLifecycleMuxFixtureGroups(state); + for (const args of [ + ["new-session", "-s", "cleanup", "-c", root], + // Ignores SIGTERM, so only the SIGKILL escalation can end this group. + ["new-window", "-n", "held", "-c", root, "trap '' TERM; sleep 300"], + ]) { + const child = Bun.spawn([mux, ...args], { + env: { ...process.env, HACK_TEST_MUX_STATE: state }, + stdout: "ignore", + stderr: "ignore", + }); + expect(await child.exited).toBe(0); + } + const { windows } = JSON.parse(await readFile(state, "utf8")) as { + readonly windows: { readonly held: { readonly pid: number } }; + }; + started.push(windows.held.pid); + // Let the shell install its trap before the cleanup signals it. + await Bun.sleep(200); + expect(groupPresent(windows.held.pid)).toBe(true); + await endLifecycleMuxFixtureGroups(state); + expect(await groupEnded(windows.held.pid)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}, 30_000); + +test("the stand-in mux cleanup signals only its own groups and gets past one that exited", async () => { + const root = await mkdtemp(join(tmpdir(), "lifecycle-mux-reuse-")); + try { + const mux = await writeLifecycleMuxFixture(root); + const state = join(root, "mux.json"); + for (const args of [ + ["new-session", "-s", "reuse", "-c", root], + ["new-window", "-n", "live", "-c", root, "sleep 300"], + ]) { + const child = Bun.spawn([mux, ...args], { + env: { ...process.env, HACK_TEST_MUX_STATE: state }, + stdout: "ignore", + stderr: "ignore", + }); + expect(await child.exited).toBe(0); + } + const saved = JSON.parse(await readFile(state, "utf8")) as { + windows: Record< + string, + { readonly pid: number; readonly started?: string } + >; + }; + const live = saved.windows.live?.pid as number; + started.push(live); + const reused = detachedGroup(); + const unproven = detachedGroup(); + const exited = Bun.spawn(["/usr/bin/true"], { + stdout: "ignore", + stderr: "ignore", + }); + await exited.exited; + // `gone` comes first: stopping there would leave the live window running. + saved.windows = { + gone: { pid: exited.pid, started: "Thu Jan 1 00:00:00 1970" }, + reused: { pid: reused, started: "Thu Jan 1 00:00:00 1970" }, + unproven: { pid: unproven }, + ...saved.windows, + }; + await writeFile(state, JSON.stringify(saved)); + await endLifecycleMuxFixtureGroups(state); + expect(await groupEnded(live)).toBe(true); + expect(groupPresent(reused)).toBe(true); + expect(groupPresent(unproven)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/tests/optional-fixture-node.test.ts b/tests/optional-fixture-node.test.ts new file mode 100644 index 000000000..1d6578745 --- /dev/null +++ b/tests/optional-fixture-node.test.ts @@ -0,0 +1,96 @@ +import { expect, test } from "bun:test"; +import { + chmod, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { optionalFixtureNode } from "./helpers/optional-fixture-node.ts"; + +test("optional fixture runtime skips a shim that could retain capture pipes", async () => { + const root = await mkdtemp(join(tmpdir(), "optional-node-shim-")); + try { + const marker = join(root, "executed"); + const shim = join(root, "node"); + await writeFile( + shim, + `#!/bin/sh\nprintf reached > '${marker}'\n/bin/sleep 30 &\nexit 0\n` + ); + await chmod(shim, 0o700); + expect(await optionalFixtureNode(root)).toBeNull(); + await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("optional fixture runtime does not execute a version-manager target behind node", async () => { + const root = await mkdtemp(join(tmpdir(), "optional-node-manager-")); + try { + const marker = join(root, "executed"); + const manager = join(root, "mise"); + await writeFile( + manager, + `#!/bin/sh\nprintf reached > '${marker}'\nexit 0\n` + ); + await chmod(manager, 0o700); + await symlink(manager, join(root, "node")); + expect(await optionalFixtureNode(root)).toBeNull(); + await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("missing optional fixture runtime leaves compiled Bun selected", async () => { + expect( + await optionalFixtureNode("/nonexistent-hack-fixture-node") + ).toBeNull(); +}); + +test("optional fixture discovery ignores ambient Node preload configuration", async () => { + const expected = await optionalFixtureNode(); + const root = await mkdtemp(join(tmpdir(), "optional-node-preload-")); + try { + const marker = join(root, "executed"); + const preload = join(root, "preload.cjs"); + await writeFile( + preload, + `if (!process.versions.bun) { require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'reached'); process.exit(87); }` + ); + const probe = join(root, "probe.ts"); + await writeFile( + probe, + `import {optionalFixtureNode} from ${JSON.stringify(join(import.meta.dir, "helpers/optional-fixture-node.ts"))}; console.log(JSON.stringify(await optionalFixtureNode()));` + ); + // Bun's implicit child environment is captured at startup. Put the poison + // in that startup snapshot, rather than only editing this runner's env. + const child = Bun.spawn([process.execPath, "--no-env-file", probe], { + env: { + PATH: process.env.PATH ?? "", + NODE_OPTIONS: `--require=${preload}`, + NODE_PATH: root, + }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + timeout: 10_000, + killSignal: "SIGKILL", + }); + const [output, errors, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(errors).toBe(""); + expect(code).toBe(0); + expect(JSON.parse(output)).toBe(expected); + await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}, 15_000); diff --git a/tests/static-fixture-node.test.ts b/tests/static-fixture-node.test.ts new file mode 100644 index 000000000..257e6e481 --- /dev/null +++ b/tests/static-fixture-node.test.ts @@ -0,0 +1,81 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { optionalFixtureNode } from "./helpers/optional-fixture-node.ts"; +import { installStaticFixtureNode } from "./helpers/static-fixture-node.ts"; + +const node = await optionalFixtureNode(); + +test("missing optional transport leaves the original generated executable byte-identical", async () => { + const root = await mkdtemp(join(tmpdir(), "static-fixture-original-")); + try { + const binary = join(root, "compiler"); + const source = "#!/bin/sh\nexit 97\n"; + await writeFile(binary, source, { mode: 0o700 }); + expect(await installStaticFixtureNode({ binary, node: null })).toBe(false); + expect(await readFile(binary, "utf8")).toBe(source); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test.skipIf(node === null)( + "static transport preserves UTF-8 stdin, file JSON and quoted argv without ambient Node loading", + async () => { + const root = await mkdtemp(join(tmpdir(), "static fixture's-")); + try { + const binary = join(root, "fixture'compiler"); + const data = join(root, "data"); + const missing = join(root, "missing"); + const marker = join(root, "preload-executed"); + const preload = join(root, "preload.cjs"); + await writeFile( + preload, + `require('node:fs').writeFileSync(${JSON.stringify(marker)},'reached');process.exit(87);` + ); + await writeFile( + binary, + `#!${process.execPath}\nconst raw=await Bun.stdin.text();await Bun.write(${JSON.stringify(data)},raw);console.log(JSON.stringify({args:process.argv.slice(2),text:await Bun.file(${JSON.stringify(data)}).text(),data:await Bun.file(${JSON.stringify(data)}).json(),missing:await Bun.file(${JSON.stringify(missing)}).exists()}));\n`, + { mode: 0o700 } + ); + expect(await installStaticFixtureNode({ binary, node })).toBe(true); + const value = { text: "Δ 😀", array: [1, 2] }; + const text = JSON.stringify(value); + const args = ["a b", "'quoted'", "Δ"]; + const child = Bun.spawn([binary, ...args], { + env: { + PATH: "/usr/bin:/bin", + NODE_OPTIONS: `--require=${JSON.stringify(preload)}`, + NODE_PATH: root, + NODE_V8_COVERAGE: join(root, "coverage"), + }, + stdin: new TextEncoder().encode(text), + stdout: "pipe", + stderr: "pipe", + timeout: 10_000, + killSignal: "SIGKILL", + }); + const [output, errors, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(code).toBe(0); + expect(errors).toBe(""); + expect(JSON.parse(output)).toEqual({ + args, + text, + data: value, + missing: false, + }); + await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(stat(join(root, "coverage"))).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + 15_000 +);