From 0b408466ac9973d195c190ee037659ccf0a6a74b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 13:18:42 -0400 Subject: [PATCH 01/15] test(e2e): report closed evidence for failing network fixture commands native-config-networks fails intermittently with only "Fixture command must exit 0; captures retained", and its private captures are not uploaded, so the failing command and product refusal cannot be identified from hosted logs. Each failing fixture command now logs a fixed diagnostic: command class (hack verb or docker verb/action), checkout role, sequence, expected and actual exit, and the product's closed error code and effect-refusal stage/reason when present. A failed phase also logs a census of exact owned containers (service, storage carrier, fixture) by closed engine state. No argv values, output text or paths are printed. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../e2e/native-config-networks-acceptance.ts | 276 ++++++++++++++++-- tests/native-networks-fixture.test.ts | 135 +++++++++ 2 files changed, 394 insertions(+), 17 deletions(-) diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index 633cb0449..a9718e813 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -39,6 +39,7 @@ const GENERATION = "io.hack.native-config.generation"; const STORAGE = "io.hack.native-config.storage"; const GLOBAL_PROJECT = "hack-dev-proxy"; const FIXTURE = "hack.e2e.native-config-networks"; +const CARRIER_OWNER = "io.hack.storage-witness.owner"; const OUTPUT_LIMIT = 2 * 1024 * 1024; const COMMAND_TIMEOUT = 180_000; // Witness-bearing exec/up verification runs three fresh helper carriers per proof; on the @@ -102,6 +103,163 @@ export function nativeNetworkFixtureRefusalDiagnostic(opts: { dockerInvoked: opts.dockerInvoked, })}`; } +const HACK_VERBS = [ + "up", + "down", + "exec", + "run", + "restart", + "ps", + "config", +] as const; +const DOCKER_VERBS = [ + "container", + "network", + "volume", + "image", + "info", + "compose", + "exec", +] as const; +const DOCKER_ACTIONS = [ + "create", + "start", + "stop", + "rm", + "inspect", + "ls", + "version", +] as const; +export const NATIVE_NETWORK_FIXTURE_ROLES = [ + "primary", + "sibling-a", + "sibling-b", + "fixture", +] as const; +type FixtureRole = (typeof NATIVE_NETWORK_FIXTURE_ROLES)[number]; +const CONTAINER_STATES = [ + "created", + "running", + "paused", + "restarting", + "removing", + "exited", + "dead", +] as const; +const CLOSED_CODE = /^E_[A-Z0-9_]{1,62}$/; +const CLOSED_TOKEN = /^[a-z][a-z0-9-]{0,62}$/; + +/** Fixed command class only: executable role, one closed verb and one closed action. */ +export function nativeNetworkFixtureStep( + argv: readonly string[], + executables: { readonly cli: string; readonly engine: string } +): string { + const [executable, first, second] = argv; + if (executable === executables.cli) { + const verb = HACK_VERBS.find((value) => value === first) ?? "other"; + return `hack:${verb}${argv.includes("--recover") ? "+recover" : ""}`; + } + if (executable === executables.engine) { + const verb = DOCKER_VERBS.find((value) => value === first) ?? "other"; + const action = DOCKER_ACTIONS.find((value) => value === second); + return action ? `docker:${verb}:${action}` : `docker:${verb}`; + } + return "other"; +} + +function closedFailure(result: Pick): { + readonly code: string | null; + readonly stage: string | null; + readonly reason: string | null; +} { + let code: string | null = null; + let stage: string | null = null; + let reason: string | null = null; + try { + const value: unknown = JSON.parse(result.stdout); + const error = isRecord(value) ? own(value, "error") : undefined; + if (isRecord(error)) { + const candidate = own(error, "code"); + code = + typeof candidate === "string" && CLOSED_CODE.test(candidate) + ? candidate + : null; + const detail = own(error, "detail"); + const refusal = isRecord(detail) + ? own(detail, "native_compose_effect_refusal") + : undefined; + if (isRecord(refusal)) { + const s = own(refusal, "stage"); + const r = own(refusal, "reason"); + stage = typeof s === "string" && CLOSED_TOKEN.test(s) ? s : null; + reason = typeof r === "string" && CLOSED_TOKEN.test(r) ? r : null; + } + } + } catch { + // Human output carries only its leading closed error code. + const match = /^ERROR (E_[A-Z0-9_]{1,62}):/m.exec(result.stderr); + code = match?.[1] ?? null; + } + return { code, stage, reason }; +} + +/** Fixed failing-command evidence: command class, checkout role, sequence, exit and closed product codes. Never argv, output text, values or paths. */ +export function nativeNetworkFixtureCommandDiagnostic(opts: { + readonly step: string; + readonly role: FixtureRole; + readonly sequence: number; + readonly expected: number; + readonly result: Pick< + CliResult, + "exitCode" | "timedOut" | "stdout" | "stderr" + >; +}): string { + const step = /^(hack|docker):[a-z]{1,16}(?::[a-z]{1,16}|\+recover)?$|^other$/; + if ( + !( + step.test(opts.step) && + NATIVE_NETWORK_FIXTURE_ROLES.includes(opts.role) && + Number.isSafeInteger(opts.sequence) && + Number.isSafeInteger(opts.expected) && + Number.isSafeInteger(opts.result.exitCode) && + typeof opts.result.timedOut === "boolean" && + typeof opts.result.stdout === "string" && + typeof opts.result.stderr === "string" + ) + ) { + return "Network command diagnostic unavailable"; + } + return `Network command diagnostic ${JSON.stringify({ + step: opts.step, + role: opts.role, + sequence: opts.sequence, + expected: opts.expected, + exit: opts.result.exitCode, + timedOut: opts.result.timedOut, + ...closedFailure(opts.result), + })}`; +} + +/** Counts exact owned engine objects by fixed role, kind and closed state; foreign rows are never selected. */ +export function nativeNetworkFixtureCensus( + rows: readonly { + readonly role: FixtureRole; + readonly kind: "service" | "carrier" | "fixture"; + readonly state: string; + }[] +): string { + const counts: Record = {}; + for (const row of rows) { + const state = CONTAINER_STATES.find((value) => value === row.state); + const key = `${NATIVE_NETWORK_FIXTURE_ROLES.includes(row.role) ? row.role : "unknown"}:${["service", "carrier", "fixture"].includes(row.kind) ? row.kind : "unknown"}:${state ?? "unknown"}`; + counts[key] = (counts[key] ?? 0) + 1; + } + return `Network failure census ${JSON.stringify( + Object.fromEntries( + Object.entries(counts).sort(([a], [b]) => a.localeCompare(b)) + ) + )}`; +} function object(text: string): Record { let value: unknown; try { @@ -1166,6 +1324,18 @@ export const nativeConfigNetworksScenario: Scenario = { "Fixture requires absolute Docker and Git executables" ); let cleanupStarted: number | null = null; + // Closed failure evidence: hosted captures stay private, so the log carries only fixed fields. + const roles = new Map(); + const commands = new WeakMap< + CliResult, + { + readonly step: string; + readonly role: FixtureRole; + readonly sequence: number; + } + >(); + let sequence = 0; + let stepExecutables = { cli: "", engine: engine ?? "" }; const execute = async ( argv: readonly string[], cwd = ctx.tempRoot, @@ -1179,15 +1349,37 @@ export const nativeConfigNetworksScenario: Scenario = { remaining > 0, "Network scenario exceeded its whole-run budget" ); - return await runNativeNetworkFixtureCommand({ + sequence += 1; + const command = { + step: nativeNetworkFixtureStep(argv, stepExecutables), + role: roles.get(cwd) ?? ("fixture" as const), + sequence, + }; + const result = await runNativeNetworkFixtureCommand({ argv, cwd, env: { ...env, ...extra }, captures, timeoutMs: Math.min(COMMAND_TIMEOUT, remaining), }); + commands.set(result, command); + return result; + }; + const ok = (result: CliResult, expected = 0): void => { + const command = commands.get(result); + if (command && !(result.exitCode === expected && !result.timedOut)) { + ctx.log( + nativeNetworkFixtureCommandDiagnostic({ + ...command, + expected, + result, + }) + ); + } + resultOk(result, expected); }; const manifest = await qualifyArtifacts(ctx, execute); + stepExecutables = { cli: manifest.cli.path, engine: engine ?? "" }; const composePlugin = await provisionNativeNetworkFixtureComposePlugin({ path: manifest.composePlugin.path, expectedHash: manifest.composePlugin.sha256, @@ -1203,7 +1395,7 @@ export const nativeConfigNetworksScenario: Scenario = { "version", "--short", ]); - resultOk(pluginVersion); + ok(pluginVersion); const composeVersion = pluginVersion.stdout.trim(); requireValue( /^v?\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?$/.test(composeVersion), @@ -1211,7 +1403,7 @@ export const nativeConfigNetworksScenario: Scenario = { ); const docker: Docker = async (args) => { const result = await execute([engine, ...args]); - resultOk(result); + ok(result); return result.stdout.trim(); }; const raw = ( @@ -1224,7 +1416,7 @@ export const nativeConfigNetworksScenario: Scenario = { args: readonly string[] ): Promise => { const result = await raw(checkout, args); - resultOk(result); + ok(result); return result; }; requireValue( @@ -1257,6 +1449,7 @@ export const nativeConfigNetworksScenario: Scenario = { const name = `e2e-net-${token.slice(0, 12)}`; const root = join(await realpath(ctx.tempRoot), name); await mkdir(join(root, ".hack"), { recursive: true, mode: 0o700 }); + roles.set(root, "primary"); const baseText = `${JSON.stringify(authored({ name, image, marker: "primary" }))}\n`; await Bun.write(join(root, ".hack", "hack.project.json"), baseText); await Bun.write( @@ -1280,7 +1473,7 @@ export const nativeConfigNetworksScenario: Scenario = { "Fixture native networks", ], ]) { - resultOk(await execute([git, ...args], root)); + ok(await execute([git, ...args], root)); } const checkouts: Checkout[] = [ { @@ -1291,11 +1484,9 @@ export const nativeConfigNetworksScenario: Scenario = { text: baseText, }, ]; - for (const branch of ["sibling-a", "sibling-b"]) { + for (const branch of ["sibling-a", "sibling-b"] as const) { const linked = join(ctx.tempRoot, `${name}-${branch}`); - resultOk( - await execute([git, "worktree", "add", "-qb", branch, linked], root) - ); + ok(await execute([git, "worktree", "add", "-qb", branch, linked], root)); const text = `${JSON.stringify(authored({ name, image, marker: branch }))}\n`; const checkout = { root: await realpath(linked), @@ -1304,6 +1495,7 @@ export const nativeConfigNetworksScenario: Scenario = { path: join(linked, ".hack", "hack.project.json"), text, }; + roles.set(checkout.root, branch); await Bun.write(checkout.path, text); checkouts.push(checkout); } @@ -1365,9 +1557,7 @@ export const nativeConfigNetworksScenario: Scenario = { service: Name, script: string ): Promise => { - resultOk( - await raw(checkout, ["exec", service, "--", "bun", "-e", script]) - ); + ok(await raw(checkout, ["exec", service, "--", "bun", "-e", script])); }; const markers = async (checkout: Checkout): Promise => { for (const [service, target, alias] of [ @@ -1423,7 +1613,7 @@ export const nativeConfigNetworksScenario: Scenario = { receipt, }); const result = await raw(checkout, args, { PATH: `${shim}:${env.PATH}` }); - resultOk(result, 1); + ok(result, 1); const fragmentPresent = result.combined.includes(fragment); const dockerInvoked = await Bun.file(receipt).exists(); if (!fragmentPresent || dockerInvoked) { @@ -1580,6 +1770,57 @@ export const nativeConfigNetworksScenario: Scenario = { "Proxy cleanup requires exact no-port/tmpfs/readonly-socket fixture identity" ); }; + // Owned objects only, selected by exact saved identity or fixture token; logs counts by closed state. + const census = async (): Promise => { + try { + const rows: Parameters[0][number][] = + []; + const states = async (filter: string): Promise => + ( + await docker([ + "container", + "ls", + "-a", + "--no-trunc", + "--filter", + filter, + "--format", + "{{.State}}", + ]) + ) + .split(/\s+/) + .filter(Boolean); + for (const checkout of checkouts) { + if (!startedRoots.has(checkout.root)) { + continue; + } + const store = await openNativeComposeGenerationStore({ + projectRoot: checkout.root, + instance: null, + mode: "saved", + }); + const identity = store.identity; + await store.close(); + const role = roles.get(checkout.root) ?? "fixture"; + for (const state of await states( + `label=${PROJECT}=${identity.composeProject}` + )) { + rows.push({ role, kind: "service", state }); + } + for (const state of await states( + `label=${CARRIER_OWNER}=${identity.ownerToken}` + )) { + rows.push({ role, kind: "carrier", state }); + } + } + for (const state of await states(`label=${FIXTURE}=${token}`)) { + rows.push({ role: "fixture", kind: "fixture", state }); + } + ctx.log(nativeNetworkFixtureCensus(rows)); + } catch { + ctx.log("Network failure census unavailable"); + } + }; const runPhases = async (): Promise => { const store = await openNativeComposeGenerationStore({ projectRoot: primary.root, @@ -1598,7 +1839,7 @@ export const nativeConfigNetworksScenario: Scenario = { receipt: createReceipt, }); startedRoots.add(primary.root); - resultOk( + ok( await raw(primary, ["up", "--detach", "--json"], { PATH: `${shim}:${env.PATH}`, }), @@ -1629,7 +1870,7 @@ export const nativeConfigNetworksScenario: Scenario = { "Controlled create must expose the actual empty NetworkID recovery window" ); } - resultOk(await raw(primary, ["up", "--detach", "--json"]), 1); + ok(await raw(primary, ["up", "--detach", "--json"]), 1); await cli(primary, ["down", "--recover", "--json"]); await resourceAbsent(primary); ctx.log( @@ -1805,7 +2046,7 @@ export const nativeConfigNetworksScenario: Scenario = { "Foreign endpoint must be running on the pinned bridge before admission" ); const foreign = await raw(primary, ["up", "--detach", "--json"]); - resultOk(foreign, 1); + ok(foreign, 1); const foreignState = await saved(primary.root); const foreignSnapshot = await observe(primary); requireValue( @@ -1859,7 +2100,7 @@ export const nativeConfigNetworksScenario: Scenario = { internal: true, }; const beforeTamper = await saved(primary.root); - resultOk(await raw(primary, ["up", "--detach", "--json"]), 1); + ok(await raw(primary, ["up", "--detach", "--json"]), 1); requireValue( (await inventory(docker, "container", original.identity.composeProject)) .length === 0 && @@ -2083,6 +2324,7 @@ export const nativeConfigNetworksScenario: Scenario = { await runPhases(); } catch (error: unknown) { failure = error; + await census(); } cleanupStarted = Date.now(); diff --git a/tests/native-networks-fixture.test.ts b/tests/native-networks-fixture.test.ts index 6386d6d87..7be0ad710 100644 --- a/tests/native-networks-fixture.test.ts +++ b/tests/native-networks-fixture.test.ts @@ -17,6 +17,8 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { nativeNetworkFixtureAttachmentMatches, + nativeNetworkFixtureCensus, + nativeNetworkFixtureCommandDiagnostic, nativeNetworkFixtureCreateArgs, nativeNetworkFixtureHasNoPublication, nativeNetworkFixtureInventory, @@ -26,6 +28,7 @@ import { nativeNetworkFixtureRefusalDiagnostic, nativeNetworkFixtureShim, nativeNetworkFixtureStateRefused, + nativeNetworkFixtureStep, nativeNetworkFixtureVolumeMatches, nativeNetworkFixtureVolumeSelectionMatches, provisionNativeNetworkFixtureComposePlugin, @@ -141,6 +144,138 @@ test("refusal diagnostics never print untrusted labels or non-boolean values", ( } }); +test("failing command evidence names only its closed class, role and product codes", () => { + const secret = "/private/fixture/secret-path"; + const executables = { cli: "/pinned/hack", engine: "/pinned/docker" }; + expect( + nativeNetworkFixtureStep( + ["/pinned/hack", "down", "--recover", "--json"], + executables + ) + ).toBe("hack:down+recover"); + expect( + nativeNetworkFixtureStep( + ["/pinned/hack", "exec", "web", "--", "bun", "-e", secret], + executables + ) + ).toBe("hack:exec"); + expect( + nativeNetworkFixtureStep( + ["/pinned/docker", "container", "start", secret], + executables + ) + ).toBe("docker:container:start"); + expect( + nativeNetworkFixtureStep(["/pinned/docker", "exec", secret], executables) + ).toBe("docker:exec"); + expect(nativeNetworkFixtureStep([secret, "up"], executables)).toBe("other"); + expect(nativeNetworkFixtureStep(["/pinned/hack", secret], executables)).toBe( + "hack:other" + ); + + const json = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:down+recover", + role: "primary", + sequence: 412, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + stdout: JSON.stringify({ + ok: false, + error: { + code: "E_CONFIG_INVALID", + message: `values ${secret}`, + detail: { + native_compose_effect_refusal: { + stage: "storage-helper-start", + reason: "probe-timeout", + }, + }, + }, + }), + stderr: secret, + }, + }); + expect(json).toBe( + 'Network command diagnostic {"step":"hack:down+recover","role":"primary","sequence":412,"expected":0,"exit":1,"timedOut":false,"code":"E_CONFIG_INVALID","stage":"storage-helper-start","reason":"probe-timeout"}' + ); + const human = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:exec", + role: "sibling-b", + sequence: 7, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + stdout: secret, + stderr: `\nERROR E_COMPOSE_FAILED: Native Compose operation failed ${secret}\n`, + }, + }); + expect(human).toContain('"code":"E_COMPOSE_FAILED","stage":null'); + for (const result of [json, human]) { + expect(result).not.toContain(secret); + expect(result).not.toContain("values"); + } +}); + +test("failing command evidence drops non-closed codes and malformed fields", () => { + const secret = "secret-value-must-not-appear"; + const forged = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:up", + role: "primary", + sequence: 1, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + stdout: JSON.stringify({ + ok: false, + error: { + code: `E_${secret}`, + detail: { + native_compose_effect_refusal: { + stage: `/${secret}`, + reason: { value: secret }, + }, + }, + }, + }), + stderr: "", + }, + }); + expect(forged).toContain('"code":null,"stage":null,"reason":null'); + expect(forged).not.toContain(secret); + for (const value of [ + { step: secret, role: "primary" }, + { step: "hack:up", role: secret }, + { step: "hack:up --json", role: "primary" }, + ]) { + const result = nativeNetworkFixtureCommandDiagnostic({ + ...(value as { step: string; role: "primary" }), + sequence: 1, + expected: 0, + result: { exitCode: 1, timedOut: false, stdout: "", stderr: "" }, + }); + expect(result).toBe("Network command diagnostic unavailable"); + } +}); + +test("failure census counts owned roles and kinds by closed engine state only", () => { + expect( + nativeNetworkFixtureCensus([ + { role: "primary", kind: "service", state: "running" }, + { role: "primary", kind: "service", state: "running" }, + { role: "primary", kind: "carrier", state: "exited" }, + { role: "fixture", kind: "fixture", state: "created" }, + { role: "sibling-a", kind: "service", state: "secret-state" }, + ]) + ).toBe( + 'Network failure census {"fixture:fixture:created":1,"primary:carrier:exited":1,"primary:service:running":2,"sibling-a:service:unknown":1}' + ); + expect(nativeNetworkFixtureCensus([])).toBe("Network failure census {}"); +}); + async function withPluginFixture( run: (opts: { path: string; From 94f47eb6c4439e5b04fcf84b3cf1d99d5f6bfd19 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 13:23:26 -0400 Subject: [PATCH 02/15] test(e2e): temporarily sample native-config-networks on every shard Hosted sampling for HACK-1281 only; reverted before review. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- tests/e2e/run.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 7fdfb7ff4..71e266f81 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -204,7 +204,8 @@ function printShardPlan(opts: { async function main(): Promise { const args = parseArgs(Bun.argv.slice(2)); - const shardNames = args.shard ? shardScenarioNames(args.shard) : null; + // TEMPORARY HACK-1281 sampling: every hosted shard runs native-config-networks only. Reverted before review. + const shardNames = args.shard ? (["native-config-networks"] as const) : null; if (args.shard && shardNames?.length === 0) { process.stdout.write( `Shard ${args.shard.index}/${args.shard.count} selects no scenarios\n` From d2eb9bcc5f344ab70c5139f78bbb634a72877813 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 14:06:20 -0400 Subject: [PATCH 03/15] test(e2e): add duration and cleanup cause to network fixture evidence The failing-command diagnostic now carries its duration, and a cleanup failure after an original failure logs its fixed assertion sentence (or withholds it) plus a second owned-object census. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../e2e/native-config-networks-acceptance.ts | 16 +++++++- tests/native-networks-fixture.test.ts | 38 ++++++++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index a9718e813..8d2021fec 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -211,7 +211,7 @@ export function nativeNetworkFixtureCommandDiagnostic(opts: { readonly expected: number; readonly result: Pick< CliResult, - "exitCode" | "timedOut" | "stdout" | "stderr" + "exitCode" | "timedOut" | "stdout" | "stderr" | "durationMs" >; }): string { const step = /^(hack|docker):[a-z]{1,16}(?::[a-z]{1,16}|\+recover)?$|^other$/; @@ -223,6 +223,7 @@ export function nativeNetworkFixtureCommandDiagnostic(opts: { Number.isSafeInteger(opts.expected) && Number.isSafeInteger(opts.result.exitCode) && typeof opts.result.timedOut === "boolean" && + Number.isSafeInteger(opts.result.durationMs) && typeof opts.result.stdout === "string" && typeof opts.result.stderr === "string" ) @@ -236,10 +237,21 @@ export function nativeNetworkFixtureCommandDiagnostic(opts: { expected: opts.expected, exit: opts.result.exitCode, timedOut: opts.result.timedOut, + durationMs: opts.result.durationMs, ...closedFailure(opts.result), })}`; } +/** A fixture-authored assertion is a fixed sentence; anything else (paths, values, product text) is withheld. */ +export function nativeNetworkFixtureFailureLabel(error: unknown): string { + const message = error instanceof Error ? error.message : null; + return typeof message === "string" && + message.length <= 160 && + /^[A-Z](?:[A-Za-z ,;'-]|(?<=[a-z])\/(?=[a-z]))*$/.test(message) + ? `Network failure ${JSON.stringify(message)}` + : "Network failure withheld"; +} + /** Counts exact owned engine objects by fixed role, kind and closed state; foreign rows are never selected. */ export function nativeNetworkFixtureCensus( rows: readonly { @@ -2441,6 +2453,8 @@ export const nativeConfigNetworksScenario: Scenario = { ctx.log( "Cleanup also failed; original failure retained in private captures" ); + ctx.log(nativeNetworkFixtureFailureLabel(cleanupError)); + await census(); } } if (failure !== undefined) { diff --git a/tests/native-networks-fixture.test.ts b/tests/native-networks-fixture.test.ts index 7be0ad710..dc6f111d4 100644 --- a/tests/native-networks-fixture.test.ts +++ b/tests/native-networks-fixture.test.ts @@ -20,6 +20,7 @@ import { nativeNetworkFixtureCensus, nativeNetworkFixtureCommandDiagnostic, nativeNetworkFixtureCreateArgs, + nativeNetworkFixtureFailureLabel, nativeNetworkFixtureHasNoPublication, nativeNetworkFixtureInventory, nativeNetworkFixtureNetworkMatches, @@ -181,6 +182,7 @@ test("failing command evidence names only its closed class, role and product cod result: { exitCode: 1, timedOut: false, + durationMs: 2345, stdout: JSON.stringify({ ok: false, error: { @@ -198,7 +200,7 @@ test("failing command evidence names only its closed class, role and product cod }, }); expect(json).toBe( - 'Network command diagnostic {"step":"hack:down+recover","role":"primary","sequence":412,"expected":0,"exit":1,"timedOut":false,"code":"E_CONFIG_INVALID","stage":"storage-helper-start","reason":"probe-timeout"}' + 'Network command diagnostic {"step":"hack:down+recover","role":"primary","sequence":412,"expected":0,"exit":1,"timedOut":false,"durationMs":2345,"code":"E_CONFIG_INVALID","stage":"storage-helper-start","reason":"probe-timeout"}' ); const human = nativeNetworkFixtureCommandDiagnostic({ step: "hack:exec", @@ -208,6 +210,7 @@ test("failing command evidence names only its closed class, role and product cod result: { exitCode: 1, timedOut: false, + durationMs: 7, stdout: secret, stderr: `\nERROR E_COMPOSE_FAILED: Native Compose operation failed ${secret}\n`, }, @@ -229,6 +232,7 @@ test("failing command evidence drops non-closed codes and malformed fields", () result: { exitCode: 1, timedOut: false, + durationMs: 1, stdout: JSON.stringify({ ok: false, error: { @@ -255,12 +259,42 @@ test("failing command evidence drops non-closed codes and malformed fields", () ...(value as { step: string; role: "primary" }), sequence: 1, expected: 0, - result: { exitCode: 1, timedOut: false, stdout: "", stderr: "" }, + result: { + exitCode: 1, + timedOut: false, + durationMs: 1, + stdout: "", + stderr: "", + }, }); expect(result).toBe("Network command diagnostic unavailable"); } }); +test("fixture failure label prints only fixed assertion sentences", () => { + expect( + nativeNetworkFixtureFailureLabel( + new Error( + "Product saved down must retire all exact fixture containers/bridges and clear intent" + ) + ) + ).toBe( + 'Network failure "Product saved down must retire all exact fixture containers/bridges and clear intent"' + ); + for (const error of [ + new Error("Refused /private/fixture/root"), + new Error("Value 42 omitted"), + new Error("lowercase start"), + new Error(`Long ${"x".repeat(200)}`), + "Fixture string throw", + { message: "Fixture object" }, + ]) { + expect(nativeNetworkFixtureFailureLabel(error)).toBe( + "Network failure withheld" + ); + } +}); + test("failure census counts owned roles and kinds by closed engine state only", () => { expect( nativeNetworkFixtureCensus([ From 0ca3bb1e8a692c8e26e3ed548cf70694c78a0ddb Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 14:06:21 -0400 Subject: [PATCH 04/15] test(e2e): type the temporary networks sampling selection Hosted sampling for HACK-1281 only; reverted before review. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- tests/e2e/run.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 71e266f81..4b304d271 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -205,7 +205,9 @@ function printShardPlan(opts: { async function main(): Promise { const args = parseArgs(Bun.argv.slice(2)); // TEMPORARY HACK-1281 sampling: every hosted shard runs native-config-networks only. Reverted before review. - const shardNames = args.shard ? (["native-config-networks"] as const) : null; + const shardNames: readonly string[] | null = args.shard + ? ["native-config-networks"] + : null; if (args.shard && shardNames?.length === 0) { process.stdout.write( `Shard ${args.shard.index}/${args.shard.count} selects no scenarios\n` From 1fee1e128fe9eebb3bc0aef0121c24098857c425 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 14:48:21 -0400 Subject: [PATCH 05/15] test(e2e): repeat temporary networks sampling three times per shard Hosted sampling for HACK-1281 only; reverted before review. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- tests/e2e/run.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 4b304d271..29ceb6ab9 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -249,12 +249,20 @@ async function main(): Promise { const dockerEnabled = (process.env.HACK_E2E_DOCKER ?? "") === "1"; const requireDocker = (process.env.HACK_E2E_REQUIRE_DOCKER ?? "") === "1"; const keepTempDirs = (process.env.HACK_E2E_KEEP ?? "") === "1"; - const outcomes = await runScenarios({ - scenarios: ALL_SCENARIOS, - dockerEnabled, - only: shardNames ?? args.only, - keepTempDirs, - }); + // TEMPORARY HACK-1281 sampling: three sequential iterations per shard, stopping at a failure. + const outcomes: Awaited> = []; + for (let iteration = 0; iteration < (args.shard ? 3 : 1); iteration++) { + const batch = await runScenarios({ + scenarios: ALL_SCENARIOS, + dockerEnabled, + only: shardNames ?? args.only, + keepTempDirs, + }); + outcomes.push(...batch); + if (batch.some((outcome) => outcome.status === "fail")) { + break; + } + } const dockerSkipped = outcomes.some( (outcome) => outcome.tier === "docker" && outcome.status === "skip" From 461921096a7b48f67bec0c113ecca39100fd6f3f Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 14:57:22 -0400 Subject: [PATCH 06/15] test(e2e): separate canonical, one-off and stray containers in network census A non-canonical project container (for example a Compose recreate temporary) blocks every native command, including down --recover, with an ownership refusal. The failure census now reports such containers apart from canonical services by closed kind only. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../e2e/native-config-networks-acceptance.ts | 60 +++++++++++++++---- tests/native-networks-fixture.test.ts | 38 ++++++++++++ 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index 8d2021fec..2d96467b4 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -252,18 +252,43 @@ export function nativeNetworkFixtureFailureLabel(error: unknown): string { : "Network failure withheld"; } +const CENSUS_KINDS = [ + "service", + "oneoff", + "stray", + "carrier", + "fixture", +] as const; + +/** A project-labelled container is a canonical service, a Compose one-off, or a stray (for example a recreate temporary). */ +export function nativeNetworkFixtureProjectKind(opts: { + readonly project: string; + readonly name: string; + readonly service: string; + readonly oneoff: string; +}): "service" | "oneoff" | "stray" { + if (opts.oneoff === "True") { + return "oneoff"; + } + return opts.oneoff === "False" && + NAMES.includes(opts.service as Name) && + opts.name === `${opts.project}-${opts.service}-1` + ? "service" + : "stray"; +} + /** Counts exact owned engine objects by fixed role, kind and closed state; foreign rows are never selected. */ export function nativeNetworkFixtureCensus( rows: readonly { readonly role: FixtureRole; - readonly kind: "service" | "carrier" | "fixture"; + readonly kind: (typeof CENSUS_KINDS)[number]; readonly state: string; }[] ): string { const counts: Record = {}; for (const row of rows) { const state = CONTAINER_STATES.find((value) => value === row.state); - const key = `${NATIVE_NETWORK_FIXTURE_ROLES.includes(row.role) ? row.role : "unknown"}:${["service", "carrier", "fixture"].includes(row.kind) ? row.kind : "unknown"}:${state ?? "unknown"}`; + const key = `${NATIVE_NETWORK_FIXTURE_ROLES.includes(row.role) ? row.role : "unknown"}:${CENSUS_KINDS.includes(row.kind) ? row.kind : "unknown"}:${state ?? "unknown"}`; counts[key] = (counts[key] ?? 0) + 1; } return `Network failure census ${JSON.stringify( @@ -1787,7 +1812,7 @@ export const nativeConfigNetworksScenario: Scenario = { try { const rows: Parameters[0][number][] = []; - const states = async (filter: string): Promise => + const listed = async (filter: string): Promise => ( await docker([ "container", @@ -1797,11 +1822,14 @@ export const nativeConfigNetworksScenario: Scenario = { "--filter", filter, "--format", - "{{.State}}", + `{{.State}}|{{.Names}}|{{.Label "${SERVICE}"}}|{{.Label "com.docker.compose.oneoff"}}`, ]) ) - .split(/\s+/) - .filter(Boolean); + .split("\n") + .filter(Boolean) + .map((line) => line.split("|")); + const states = async (filter: string): Promise => + (await listed(filter)).map(([state]) => state ?? "unknown"); for (const checkout of checkouts) { if (!startedRoots.has(checkout.root)) { continue; @@ -1814,10 +1842,22 @@ export const nativeConfigNetworksScenario: Scenario = { const identity = store.identity; await store.close(); const role = roles.get(checkout.root) ?? "fixture"; - for (const state of await states( - `label=${PROJECT}=${identity.composeProject}` - )) { - rows.push({ role, kind: "service", state }); + for (const [ + state = "unknown", + name = "", + service = "", + oneoff = "", + ] of await listed(`label=${PROJECT}=${identity.composeProject}`)) { + rows.push({ + role, + kind: nativeNetworkFixtureProjectKind({ + project: identity.composeProject, + name, + service, + oneoff, + }), + state, + }); } for (const state of await states( `label=${CARRIER_OWNER}=${identity.ownerToken}` diff --git a/tests/native-networks-fixture.test.ts b/tests/native-networks-fixture.test.ts index dc6f111d4..29308d13b 100644 --- a/tests/native-networks-fixture.test.ts +++ b/tests/native-networks-fixture.test.ts @@ -25,6 +25,7 @@ import { nativeNetworkFixtureInventory, nativeNetworkFixtureNetworkMatches, nativeNetworkFixturePreservedContainerMatches, + nativeNetworkFixtureProjectKind, nativeNetworkFixtureProtocolMatches, nativeNetworkFixtureRefusalDiagnostic, nativeNetworkFixtureShim, @@ -295,6 +296,43 @@ test("fixture failure label prints only fixed assertion sentences", () => { } }); +test("failure census separates canonical services from one-offs and strays", () => { + const project = "hack-nc-0123456789abcdef0123456789abcdef"; + expect( + nativeNetworkFixtureProjectKind({ + project, + name: `${project}-web-1`, + service: "web", + oneoff: "False", + }) + ).toBe("service"); + expect( + nativeNetworkFixtureProjectKind({ + project, + name: `0123456789ab_${project}-web-1`, + service: "web", + oneoff: "False", + }) + ).toBe("stray"); + expect( + nativeNetworkFixtureProjectKind({ + project, + name: `${project}-run-0123`, + service: "web", + oneoff: "True", + }) + ).toBe("oneoff"); + for (const value of [ + { name: `${project}-other-1`, service: "other", oneoff: "False" }, + { name: `${project}-web-2`, service: "web", oneoff: "False" }, + { name: `${project}-web-1`, service: "web", oneoff: "" }, + ]) { + expect(nativeNetworkFixtureProjectKind({ project, ...value })).toBe( + "stray" + ); + } +}); + test("failure census counts owned roles and kinds by closed engine state only", () => { expect( nativeNetworkFixtureCensus([ From 3617ac8cc8d04342449f16d0501499f80fdd0ffa Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 20 Oct 2026 09:30:10 -0400 Subject: [PATCH 07/15] fix(native): keep the closed refusal boundary on unclassified failures A native command that failed with a plain error (for example a storage carrier refusal during a witness-bearing exec) mapped to the fixed E_COMPOSE_FAILED text without the closed stage/reason its owner had issued, and a command without --json never printed that boundary at all. Hosted native-config-networks failures therefore could not say which owner refused. The generic E_COMPOSE_FAILED translation now carries detail.native_compose_effect_refusal when the registry holds an issued diagnostic; the code and text are unchanged. Without --json, a coded error prints one more fixed line, "Refusal boundary: (); values omitted.", only for a known stage and reason pair read as own data. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- docs/reference/native-compose-generation.md | 7 +- src/cli/run.ts | 10 ++ src/lib/native-compose-command.ts | 6 ++ src/lib/native-compose-effect-diagnostics.ts | 32 ++++++ tests/cli-closed-error-output.test.ts | 56 ++++++++++ .../native-compose-effect-diagnostics.test.ts | 100 ++++++++++++++++++ 6 files changed, 210 insertions(+), 1 deletion(-) diff --git a/docs/reference/native-compose-generation.md b/docs/reference/native-compose-generation.md index 01d2373c7..a6bb9af0d 100644 --- a/docs/reference/native-compose-generation.md +++ b/docs/reference/native-compose-generation.md @@ -73,7 +73,12 @@ are never diagnostic fields. Copied error properties do not acquire provenance. The diagnostic does not grant recovery, clear intent or change the original error code, checks, child lifetime or deadline. Missing detail leaves the cause unknown; returned incomplete outcomes retain their existing reporting contract. Historical -failures without this detail remain unexplained. +failures without this detail remain unexplained. An unclassified native failure +(fixed `E_COMPOSE_FAILED` text, for example a storage-carrier refusal during +`exec`) carries the same detail when its owner issued one. Without `--json`, the +CLI prints the same pair after the error line as +`Refusal boundary: (); values omitted.`; detail with any other +shape prints nothing extra. `publishRunProjection` derives a one-off document from the verified saved artifact inside the mutation lock. It accepts a target service, never caller-supplied JSON. diff --git a/src/cli/run.ts b/src/cli/run.ts index cf2d4eaa2..0717802fa 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -170,6 +170,16 @@ export async function handleRunCliError(opts: { if (error instanceof HackCliError) { logger.error({ message: `${error.code}: ${error.message}` }); + if (error.detail !== undefined) { + // Loaded only for a detailed error, off the ordinary CLI start path. + const { nativeComposeEffectRefusalSummary } = await import( + "../lib/native-compose-effect-diagnostics.ts" + ); + const summary = nativeComposeEffectRefusalSummary(error.detail); + if (summary) { + logger.error({ message: summary }); + } + } logDebugStack(error); return 1; } diff --git a/src/lib/native-compose-command.ts b/src/lib/native-compose-command.ts index 224e0f359..2969349aa 100644 --- a/src/lib/native-compose-command.ts +++ b/src/lib/native-compose-command.ts @@ -2466,9 +2466,15 @@ function throwNativeComposeCommandError(error: unknown): never { if (error instanceof HackCliError || error instanceof CliUsageError) { throw error; } + // An unclassified failure (for example a storage-carrier refusal) keeps the + // closed boundary its owner issued; the code and fixed text are unchanged. + const diagnostic = nativeComposeEffectRefusal(error); throw new HackCliError({ code: "E_COMPOSE_FAILED", message: "Native Compose operation failed; saved ownership and persistent data are retained. Values omitted.", + ...(diagnostic + ? { detail: { native_compose_effect_refusal: diagnostic } } + : {}), }); } diff --git a/src/lib/native-compose-effect-diagnostics.ts b/src/lib/native-compose-effect-diagnostics.ts index 9d0913f6b..7e4a5a841 100644 --- a/src/lib/native-compose-effect-diagnostics.ts +++ b/src/lib/native-compose-effect-diagnostics.ts @@ -255,6 +255,38 @@ export function nativeComposeEffectReason( return "unclassified"; } +function ownData(value: unknown, key: string): unknown { + if (typeof value !== "object" || value === null) { + return undefined; + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + return descriptor && Object.hasOwn(descriptor, "value") + ? descriptor.value + : undefined; +} + +/** + * One fixed human line for the closed boundary carried in public CLI error + * detail, so a command without `--json` (such as `exec`) still names it. + * Anything but a known stage and reason pair yields null; accessors are not read. + */ +export function nativeComposeEffectRefusalSummary( + detail: unknown +): string | null { + try { + const refusal = ownData(detail, "native_compose_effect_refusal"); + const stage = stages.find((value) => value === ownData(refusal, "stage")); + const reason = reasons.find( + (value) => value === ownData(refusal, "reason") + ); + return stage && reason + ? `Refusal boundary: ${stage} (${reason}); values omitted.` + : null; + } catch { + return null; + } +} + /** Keep the original issued boundary through a deliberate fixed-code translation. */ export function copyNativeComposeEffectRefusal( source: unknown, diff --git a/tests/cli-closed-error-output.test.ts b/tests/cli-closed-error-output.test.ts index 07b0ed038..94e4fae07 100644 --- a/tests/cli-closed-error-output.test.ts +++ b/tests/cli-closed-error-output.test.ts @@ -94,6 +94,62 @@ test("an unexpected exception keeps its stack", async () => { expect(output).toMatch(STACK_FRAME); }); +test("a coded native refusal also names its closed boundary without --json", async () => { + const coded = new HackCliError({ + code: "E_COMPOSE_FAILED", + message: "Native Compose operation failed. Values omitted.", + detail: { + native_compose_effect_refusal: { + stage: "storage-helper-start", + reason: "probe-timeout", + }, + }, + }); + expect(await handleRunCliError({ error: coded, jsonRequested: false })).toBe( + 1 + ); + expect(output).toBe( + "ERROR: E_COMPOSE_FAILED: Native Compose operation failed. Values omitted.\nERROR: Refusal boundary: storage-helper-start (probe-timeout); values omitted.\n" + ); +}); + +test("unknown or accessor-backed boundary detail prints nothing extra", async () => { + const canary = "private-boundary-canary"; + for (const detail of [ + { + native_compose_effect_refusal: { stage: canary, reason: "probe-timeout" }, + }, + { + native_compose_effect_refusal: { + stage: "storage-target", + reason: canary, + }, + }, + { native_compose_effect_refusal: "storage-target" }, + { + native_compose_effect_refusal: { + get stage() { + return "storage-target"; + }, + reason: "unclassified", + }, + }, + { other: { stage: "storage-target", reason: "unclassified" } }, + canary, + ]) { + output = ""; + const coded = new HackCliError({ + code: "E_COMPOSE_FAILED", + message: "Closed refusal.", + detail, + }); + expect( + await handleRunCliError({ error: coded, jsonRequested: false }) + ).toBe(1); + expect(output).toBe("ERROR: E_COMPOSE_FAILED: Closed refusal.\n"); + } +}); + test("--json keeps one envelope for a closed refusal", async () => { const error = nativeRefusal(); expect(await handleRunCliError({ error, jsonRequested: true })).toBe(1); diff --git a/tests/native-compose-effect-diagnostics.test.ts b/tests/native-compose-effect-diagnostics.test.ts index 0f6ed455a..b3afdb0d5 100644 --- a/tests/native-compose-effect-diagnostics.test.ts +++ b/tests/native-compose-effect-diagnostics.test.ts @@ -12,6 +12,7 @@ import { copyNativeComposeEffectRefusal, nativeComposeEffectReason, nativeComposeEffectRefusal, + nativeComposeEffectRefusalSummary, retainNativeComposeEffectRefusal, } from "../src/lib/native-compose-effect-diagnostics.ts"; import * as generation from "../src/lib/native-compose-generation.ts"; @@ -229,6 +230,105 @@ test("reason classification preserves an original issued probe refusal without c expect(nativeComposeEffectReason(copied)).toBe("unclassified"); }); +test.each([ + "issued", + "unissued", +] as const)("an unclassified native failure keeps only an %s boundary under its fixed E_COMPOSE_FAILED text", async (kind) => { + const root = await realpath( + await mkdtemp(join(tmpdir(), "native-effect-fallback-")) + ); + const oldBackend = process.env.HACK_RUNTIME_BACKEND; + // A storage carrier refuses with a plain Error whose text is never public. + const error = new Error("private-carrier-canary"); + const diagnostic = { + stage: "storage-helper-policy", + reason: "unclassified", + } as const; + if (kind === "issued") { + retainNativeComposeEffectRefusal(error, diagnostic); + } + const opened = spyOn( + generation, + "openNativeComposeGenerationStore" + ).mockImplementation(() => Promise.reject(error)); + try { + process.env.HACK_RUNTIME_BACKEND = "compose"; + await mkdir(join(root, ".hack")); + await Bun.write( + join(root, ".hack/hack.project.json"), + JSON.stringify({ schema_version: 1, name: "fixture" }) + ); + let caught: unknown; + try { + await tryNativeComposeCommand({ + cwd: root, + operation: "down", + json: true, + }); + } catch (value) { + caught = value; + } + expect(opened).toHaveBeenCalledTimes(1); + const result = errorResultFromUnknown({ error: caught }); + expect(result).toEqual({ + ok: false, + error: { + code: "E_COMPOSE_FAILED", + message: + "Native Compose operation failed; saved ownership and persistent data are retained. Values omitted.", + ...(kind === "issued" + ? { detail: { native_compose_effect_refusal: diagnostic } } + : {}), + }, + }); + expect(JSON.stringify(result)).not.toContain("private-carrier-canary"); + } finally { + opened.mockRestore(); + restoreEnv("HACK_RUNTIME_BACKEND", oldBackend); + await rm(root, { recursive: true, force: true }); + } +}); + +test("the human boundary summary names only a known stage and reason pair", () => { + expect( + nativeComposeEffectRefusalSummary({ + native_compose_effect_refusal: { + stage: "storage-helper-start", + reason: "probe-timeout", + }, + }) + ).toBe( + "Refusal boundary: storage-helper-start (probe-timeout); values omitted." + ); + const canary = "private-summary-canary"; + for (const detail of [ + undefined, + null, + canary, + { + native_compose_effect_refusal: { stage: canary, reason: "unclassified" }, + }, + { + native_compose_effect_refusal: { + stage: "storage-target", + reason: canary, + }, + }, + { + native_compose_effect_refusal: new Proxy( + {}, + { + getOwnPropertyDescriptor: () => { + throw new Error(canary); + }, + } + ), + }, + ]) { + expect(nativeComposeEffectRefusalSummary(detail)).toBeNull(); + } +}); + test.each([ "guard-fresh-before", "guard-ownership", From 9f1bacda7c399b09b17d00bf5b925027ec714d35 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 20 Oct 2026 09:30:11 -0400 Subject: [PATCH 08/15] test(e2e): classify probe, human-code and saved-state network failure evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The failing-command diagnostic read a human error code only from stderr as "ERROR E_X:", which no logger backend prints (clack writes "■ E_X: " to stdout), so every failing hack exec reported code null. It now reads the leading closed code and the closed refusal boundary line from either stream, ANSI stripped, for the clack, gum and console formats. In-container probe scripts now exit with closed codes for every failure (rejected or timed-out fetch, unreadable or unwritable marker), so a hack exec that exits 1 is the product, not an uncaught probe error. The failure census also logs each started checkout's saved stopped/pending operation and storage/hook pending flags. No values, text or paths. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../e2e/native-config-networks-acceptance.ts | 215 +++++++++++++++--- tests/native-networks-fixture.test.ts | 183 ++++++++++++++- 2 files changed, 361 insertions(+), 37 deletions(-) diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index 2d96467b4..51d35af02 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -167,40 +167,121 @@ export function nativeNetworkFixtureStep( return "other"; } -function closedFailure(result: Pick): { +type ClosedFailure = { readonly code: string | null; readonly stage: string | null; readonly reason: string | null; -} { - let code: string | null = null; - let stage: string | null = null; - let reason: string | null = null; +}; + +function closedToken(value: unknown, pattern: RegExp): string | null { + return typeof value === "string" && pattern.test(value) ? value : null; +} + +/** The JSON envelope's closed code and issued boundary; null when stdout is not JSON. */ +function jsonClosedFailure(stdout: string): ClosedFailure | null { + let value: unknown; try { - const value: unknown = JSON.parse(result.stdout); - const error = isRecord(value) ? own(value, "error") : undefined; - if (isRecord(error)) { - const candidate = own(error, "code"); - code = - typeof candidate === "string" && CLOSED_CODE.test(candidate) - ? candidate - : null; - const detail = own(error, "detail"); - const refusal = isRecord(detail) - ? own(detail, "native_compose_effect_refusal") - : undefined; - if (isRecord(refusal)) { - const s = own(refusal, "stage"); - const r = own(refusal, "reason"); - stage = typeof s === "string" && CLOSED_TOKEN.test(s) ? s : null; - reason = typeof r === "string" && CLOSED_TOKEN.test(r) ? r : null; - } - } + value = JSON.parse(stdout); } catch { - // Human output carries only its leading closed error code. - const match = /^ERROR (E_[A-Z0-9_]{1,62}):/m.exec(result.stderr); - code = match?.[1] ?? null; + return null; + } + const error = isRecord(value) ? own(value, "error") : undefined; + if (!isRecord(error)) { + return { code: null, stage: null, reason: null }; + } + const detail = own(error, "detail"); + const refusal = isRecord(detail) + ? own(detail, "native_compose_effect_refusal") + : undefined; + return { + code: closedToken(own(error, "code"), CLOSED_CODE), + stage: isRecord(refusal) + ? closedToken(own(refusal, "stage"), CLOSED_TOKEN) + : null, + reason: isRecord(refusal) + ? closedToken(own(refusal, "reason"), CLOSED_TOKEN) + : null, + }; +} + +function closedFailure( + result: Pick +): ClosedFailure { + const json = jsonClosedFailure(result.stdout); + if (json) { + return json; } - return { code, stage, reason }; + const boundary = humanBoundary(result.stdout) ?? humanBoundary(result.stderr); + return { + code: humanClosedCode(result.stdout) ?? humanClosedCode(result.stderr), + stage: boundary?.stage ?? null, + reason: boundary?.reason ?? null, + }; +} + +const HUMAN_BOUNDARY = + /^[^\w\n]*(?:ERROR:?[ \t]+)?Refusal boundary: ([a-z][a-z0-9-]{0,62}) \(([a-z][a-z0-9-]{0,62})\); values omitted\.$/m; + +/** The closed refusal boundary line a human CLI error prints after its code. */ +export function humanBoundary( + text: string +): { readonly stage: string; readonly reason: string } | null { + const match = HUMAN_BOUNDARY.exec(text.replace(ANSI, "")); + return match?.[1] && match[2] ? { stage: match[1], reason: match[2] } : null; +} + +const ANSI = /\u001b\[[0-9;]*m/g; +// A human error line opens with the logger's glyph or level, then `E_CODE: `: +// clack writes `■ E_CODE: ` to stdout, gum `ERROR E_CODE: ` and console +// `ERROR: E_CODE: ` to stderr. Only the closed code is kept. +const HUMAN_CODE = /^[^\w\n]*(?:ERROR:?[ \t]+)?(E_[A-Z0-9_]{1,62}): /m; + +/** The leading closed error code of human CLI output; nothing else of the text. */ +export function humanClosedCode(text: string): string | null { + return HUMAN_CODE.exec(text.replace(ANSI, ""))?.[1] ?? null; +} + +/** + * Closed exit codes of the in-container probe scripts. Every script failure + * exits with one of them, so a failing `hack exec` that exits 1 is the product + * (or the runtime), never an uncaught probe error. + */ +export const NATIVE_NETWORK_PROBE_EXIT = { + environment: 23, + response: 24, + marker: 25, + rejected: 26, + timeout: 27, + unreadable: 28, + unwritable: 29, +} as const; +const PROBE = NATIVE_NETWORK_PROBE_EXIT; + +/** Fetches the endpoint origin and its alias; both must answer the exact marker. */ +export function nativeNetworkFixturePeerProbe(opts: { + readonly origin: string; + readonly alias: string; + readonly expected: string; + readonly timeoutMs?: number; +}): string { + const timeout = opts.timeoutMs ?? 4000; + return `if(process.env.PEER_ORIGIN!==${JSON.stringify(opts.origin)})process.exit(${PROBE.environment});for(const url of [process.env.PEER_ORIGIN,${JSON.stringify(opts.alias)}]){let ok=false;try{const r=await fetch(url,{signal:AbortSignal.timeout(${timeout})});ok=r.ok&&await r.text()===${JSON.stringify(opts.expected)}}catch(e){process.exit(e?.name==="TimeoutError"?${PROBE.timeout}:${PROBE.rejected})}if(!ok)process.exit(${PROBE.response})}`; +} + +/** Reads the retained marker; a missing or unreadable file has its own code. */ +export function nativeNetworkFixtureMarkerRead(opts: { + readonly path: string; + readonly marker: string; +}): string { + return `let t;try{t=await Bun.file(${JSON.stringify(opts.path)}).text()}catch{process.exit(${PROBE.unreadable})}if(t!==${JSON.stringify(opts.marker)})process.exit(${PROBE.marker})`; +} + +/** Writes the retained marker. */ +export function nativeNetworkFixtureMarkerWrite(opts: { + readonly path: string; + readonly marker: string; +}): string { + return `try{await Bun.write(${JSON.stringify(opts.path)},${JSON.stringify(opts.marker)})}catch{process.exit(${PROBE.unwritable})}`; } /** Fixed failing-command evidence: command class, checkout role, sequence, exit and closed product codes. Never argv, output text, values or paths. */ @@ -297,6 +378,41 @@ export function nativeNetworkFixtureCensus( ) )}`; } +const PENDING_OPERATIONS = ["up", "restart", "run", "down"] as const; + +/** Saved intent per started checkout: closed booleans and the pending operation only. */ +export function nativeNetworkFixtureSavedEvidence( + rows: readonly { + readonly role: FixtureRole; + readonly stopped: unknown; + readonly pending: unknown; + readonly storageWitnessesPending: unknown; + readonly beforeHooksPending: unknown; + }[] +): string { + const flag = (value: unknown) => + typeof value === "boolean" ? value : "unknown"; + const entries = rows.map((row) => { + const operation = + isRecord(row.pending) && typeof row.pending.operation === "string" + ? row.pending.operation + : null; + return [ + NATIVE_NETWORK_FIXTURE_ROLES.includes(row.role) ? row.role : "unknown", + { + stopped: flag(row.stopped), + pending: + row.pending === null + ? "none" + : (PENDING_OPERATIONS.find((value) => value === operation) ?? + "unknown"), + storageWitnessesPending: flag(row.storageWitnessesPending), + beforeHooksPending: flag(row.beforeHooksPending), + }, + ] as const; + }); + return `Network failure saved state ${JSON.stringify(Object.fromEntries(entries))}`; +} function object(text: string): Record { let value: unknown; try { @@ -1603,17 +1719,23 @@ export const nativeConfigNetworksScenario: Scenario = { ["vault", "reader", "reader-alias"], ["reader", "vault", "vault-alias"], ] as const) { - const expected = JSON.stringify(`${checkout.marker}:${target}`); await exec( checkout, service, - `if(process.env.PEER_ORIGIN!==${JSON.stringify(`http://${target}:3000`)})process.exit(23);for(const url of [process.env.PEER_ORIGIN,${JSON.stringify(`http://${alias}:3000/`)}]){const r=await fetch(url,{signal:AbortSignal.timeout(4000)});if(!r.ok||await r.text()!==${expected})process.exit(24)}` + nativeNetworkFixturePeerProbe({ + origin: `http://${target}:3000`, + alias: `http://${alias}:3000/`, + expected: `${checkout.marker}:${target}`, + }) ); } await exec( checkout, "web", - `if(await Bun.file("/state/marker").text()!==${JSON.stringify(checkout.marker)})process.exit(25)` + nativeNetworkFixtureMarkerRead({ + path: "/state/marker", + marker: checkout.marker, + }) ); }; const siblings = checkouts.slice(1); @@ -1830,6 +1952,9 @@ export const nativeConfigNetworksScenario: Scenario = { .map((line) => line.split("|")); const states = async (filter: string): Promise => (await listed(filter)).map(([state]) => state ?? "unknown"); + const savedRows: Parameters< + typeof nativeNetworkFixtureSavedEvidence + >[0][number][] = []; for (const checkout of checkouts) { if (!startedRoots.has(checkout.root)) { continue; @@ -1840,8 +1965,28 @@ export const nativeConfigNetworksScenario: Scenario = { mode: "saved", }); const identity = store.identity; - await store.close(); const role = roles.get(checkout.root) ?? "fixture"; + try { + // storageWitnessesPending is nonenumerable; read each field explicitly. + const state = await store.loadCurrent(); + savedRows.push({ + role, + stopped: state.stopped, + pending: state.pending, + storageWitnessesPending: state.storageWitnessesPending, + beforeHooksPending: state.beforeHooksPending, + }); + } catch { + savedRows.push({ + role, + stopped: null, + pending: undefined, + storageWitnessesPending: null, + beforeHooksPending: null, + }); + } finally { + await store.close(); + } for (const [ state = "unknown", name = "", @@ -1869,6 +2014,7 @@ export const nativeConfigNetworksScenario: Scenario = { rows.push({ role: "fixture", kind: "fixture", state }); } ctx.log(nativeNetworkFixtureCensus(rows)); + ctx.log(nativeNetworkFixtureSavedEvidence(savedRows)); } catch { ctx.log("Network failure census unavailable"); } @@ -1935,7 +2081,10 @@ export const nativeConfigNetworksScenario: Scenario = { await exec( checkout, "web", - `await Bun.write("/state/marker",${JSON.stringify(checkout.marker)})` + nativeNetworkFixtureMarkerWrite({ + path: "/state/marker", + marker: checkout.marker, + }) ); await markers(checkout); await cli(checkout, ["ps", "--json"]); diff --git a/tests/native-networks-fixture.test.ts b/tests/native-networks-fixture.test.ts index 29308d13b..546addaa3 100644 --- a/tests/native-networks-fixture.test.ts +++ b/tests/native-networks-fixture.test.ts @@ -16,6 +16,9 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { + humanBoundary, + humanClosedCode, + NATIVE_NETWORK_PROBE_EXIT, nativeNetworkFixtureAttachmentMatches, nativeNetworkFixtureCensus, nativeNetworkFixtureCommandDiagnostic, @@ -23,11 +26,15 @@ import { nativeNetworkFixtureFailureLabel, nativeNetworkFixtureHasNoPublication, nativeNetworkFixtureInventory, + nativeNetworkFixtureMarkerRead, + nativeNetworkFixtureMarkerWrite, nativeNetworkFixtureNetworkMatches, + nativeNetworkFixturePeerProbe, nativeNetworkFixturePreservedContainerMatches, nativeNetworkFixtureProjectKind, nativeNetworkFixtureProtocolMatches, nativeNetworkFixtureRefusalDiagnostic, + nativeNetworkFixtureSavedEvidence, nativeNetworkFixtureShim, nativeNetworkFixtureStateRefused, nativeNetworkFixtureStep, @@ -212,17 +219,185 @@ test("failing command evidence names only its closed class, role and product cod exitCode: 1, timedOut: false, durationMs: 7, - stdout: secret, - stderr: `\nERROR E_COMPOSE_FAILED: Native Compose operation failed ${secret}\n`, + // The clack backend (no gum on the hosted runner) writes errors to stdout. + stdout: `\u2502\n\u25a0 E_COMPOSE_FAILED: Native Compose operation failed ${secret}\n\u2502\n\u25a0 Refusal boundary: storage-helper-policy (unclassified); values omitted.\n`, + stderr: secret, + }, + }); + expect(human).toContain( + '"code":"E_COMPOSE_FAILED","stage":"storage-helper-policy","reason":"unclassified"' + ); + const unbounded = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:exec", + role: "sibling-b", + sequence: 8, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 7, + stdout: `\u25a0 E_CONFIG_INVALID: Refused ${secret}\n`, + stderr: "", }, }); - expect(human).toContain('"code":"E_COMPOSE_FAILED","stage":null'); - for (const result of [json, human]) { + expect(unbounded).toContain( + '"code":"E_CONFIG_INVALID","stage":null,"reason":null' + ); + for (const result of [json, human, unbounded]) { expect(result).not.toContain(secret); expect(result).not.toContain("values"); } }); +test("human error output yields only its leading closed code for every logger backend", () => { + const secret = "/private/fixture/secret-path"; + for (const [text, code] of [ + // clack (hosted runner, NO_COLOR) on stdout + [`│\n■ E_CONFIG_INVALID: Refused ${secret}\n`, "E_CONFIG_INVALID"], + // clack with color + [ + `\u001b[90m│\u001b[39m\n\u001b[31m■\u001b[39m E_COMPOSE_FAILED: ${secret}\n`, + "E_COMPOSE_FAILED", + ], + // gum and console backends on stderr + [`ERROR E_NATIVE_COMPOSE_STATE: ${secret}`, "E_NATIVE_COMPOSE_STATE"], + [`ERROR: E_LIFECYCLE_FAILED: ${secret}\n`, "E_LIFECYCLE_FAILED"], + ] as const) { + expect(humanClosedCode(text)).toBe(code); + } + for (const text of [ + "", + secret, + "ERROR MissingProjectContextError: E_FAKE: nested", + `value E_INLINE: ${secret}`, + `■ e_lower: ${secret}`, + `■ E_${"X".repeat(70)}: ${secret}`, + ]) { + expect(humanClosedCode(text)).toBeNull(); + } + expect( + humanBoundary( + "ERROR: Refusal boundary: storage-target (probe-timeout); values omitted.\n" + ) + ).toEqual({ stage: "storage-target", reason: "probe-timeout" }); + for (const text of [ + `■ Refusal boundary: ${secret} (probe-timeout); values omitted.`, + "■ Refusal boundary: storage-target (Probe Timeout); values omitted.", + "■ Refusal boundary: storage-target (probe-timeout)", + "value Refusal boundary: storage-target (probe-timeout); values omitted.", + ]) { + expect(humanBoundary(text)).toBeNull(); + } +}); + +test("probe scripts exit with a closed code for every in-container failure", async () => { + const root = await mkdtemp(join(tmpdir(), "hack-network-probe-")); + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/hang") { + return new Promise(() => undefined); + } + return new Response(path === "/wrong" ? "other" : "primary:peer"); + }, + }); + const closed = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response(""), + }); + const refused = `http://127.0.0.1:${closed.port}/`; + closed.stop(true); + const origin = `http://127.0.0.1:${server.port}`; + const run = async (script: string, peer = origin): Promise => { + const child = Bun.spawn([process.execPath, "-e", script], { + cwd: root, + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", PEER_ORIGIN: peer }, + stdout: "ignore", + stderr: "ignore", + }); + return await child.exited; + }; + try { + const probe = (alias: string, timeoutMs = 4000) => + nativeNetworkFixturePeerProbe({ + origin, + alias, + expected: "primary:peer", + timeoutMs, + }); + expect(await run(probe(`${origin}/alias`))).toBe(0); + expect(await run(probe(`${origin}/alias`), `${origin}/x`)).toBe( + NATIVE_NETWORK_PROBE_EXIT.environment + ); + expect(await run(probe(`${origin}/wrong`))).toBe( + NATIVE_NETWORK_PROBE_EXIT.response + ); + expect(await run(probe(refused))).toBe(NATIVE_NETWORK_PROBE_EXIT.rejected); + expect(await run(probe(`${origin}/hang`, 200))).toBe( + NATIVE_NETWORK_PROBE_EXIT.timeout + ); + + const marker = join(root, "marker"); + const read = nativeNetworkFixtureMarkerRead({ path: marker, marker: "m" }); + expect(await run(read)).toBe(NATIVE_NETWORK_PROBE_EXIT.unreadable); + expect( + await run(nativeNetworkFixtureMarkerWrite({ path: marker, marker: "m" })) + ).toBe(0); + expect(await run(read)).toBe(0); + expect( + await run(nativeNetworkFixtureMarkerRead({ path: marker, marker: "n" })) + ).toBe(NATIVE_NETWORK_PROBE_EXIT.marker); + expect( + await run( + nativeNetworkFixtureMarkerWrite({ + // Bun.write creates directories; a regular-file parent cannot be one. + path: join(marker, "child"), + marker: "m", + }) + ) + ).toBe(NATIVE_NETWORK_PROBE_EXIT.unwritable); + // No probe code collides with an ordinary product failure exit. + expect(Object.values(NATIVE_NETWORK_PROBE_EXIT)).not.toContain(1); + } finally { + server.stop(true); + await rm(root, { recursive: true, force: true }); + } +}); + +test("saved-state evidence reports closed flags and pending operation only", () => { + const secret = "secret-generation-token"; + expect( + nativeNetworkFixtureSavedEvidence([ + { + role: "primary", + stopped: false, + pending: null, + storageWitnessesPending: false, + beforeHooksPending: false, + }, + { + role: "sibling-b", + stopped: false, + pending: { operation: "up", token: secret, generationId: secret }, + storageWitnessesPending: true, + beforeHooksPending: false, + }, + { + role: "sibling-a", + stopped: secret, + pending: { operation: secret }, + storageWitnessesPending: undefined, + beforeHooksPending: null, + }, + ]) + ).toBe( + 'Network failure saved state {"primary":{"stopped":false,"pending":"none","storageWitnessesPending":false,"beforeHooksPending":false},"sibling-b":{"stopped":false,"pending":"up","storageWitnessesPending":true,"beforeHooksPending":false},"sibling-a":{"stopped":"unknown","pending":"unknown","storageWitnessesPending":"unknown","beforeHooksPending":"unknown"}}' + ); +}); + test("failing command evidence drops non-closed codes and malformed fields", () => { const secret = "secret-value-must-not-appear"; const forged = nativeNetworkFixtureCommandDiagnostic({ From 566687c60de9fb89fe64d3cfd9633daa6d6b9f02 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 20 Oct 2026 09:30:12 -0400 Subject: [PATCH 09/15] test(e2e): bound temporary network sampling and add load on two shards Hosted sampling for HACK-1281 only; reverted before review. A third iteration crossed the 150-minute job budget, so a new iteration starts only within 55 minutes. Shards 1 and 2 add two CPU spinners and an fsync loop. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- tests/e2e/run.ts | 56 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 29ceb6ab9..99bef09eb 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -249,18 +249,52 @@ async function main(): Promise { const dockerEnabled = (process.env.HACK_E2E_DOCKER ?? "") === "1"; const requireDocker = (process.env.HACK_E2E_REQUIRE_DOCKER ?? "") === "1"; const keepTempDirs = (process.env.HACK_E2E_KEEP ?? "") === "1"; - // TEMPORARY HACK-1281 sampling: three sequential iterations per shard, stopping at a failure. + // TEMPORARY HACK-1281 sampling: sequential iterations while one more fits the 150-minute + // job budget (a third iteration once crossed it), stopping at a failure. Shards 1 and 2 + // add background CPU and fsync load to widen any timing window; 3 and 4 run unloaded. const outcomes: Awaited> = []; - for (let iteration = 0; iteration < (args.shard ? 3 : 1); iteration++) { - const batch = await runScenarios({ - scenarios: ALL_SCENARIOS, - dockerEnabled, - only: shardNames ?? args.only, - keepTempDirs, - }); - outcomes.push(...batch); - if (batch.some((outcome) => outcome.status === "fail")) { - break; + const samplingStarted = Date.now(); + const load: ReturnType[] = []; + if (args.shard && args.shard.index <= 2) { + process.stdout.write("HACK-1281 sampling: background load enabled\n"); + for (let spinner = 0; spinner < 2; spinner++) { + load.push( + Bun.spawn([process.execPath, "-e", "for(;;){}"], { + stdout: "ignore", + stderr: "ignore", + }) + ); + } + load.push( + Bun.spawn( + [ + "sh", + "-c", + 'f="${TMPDIR:-/tmp}/hack-1281-load-$$"; trap \'rm -f "$f"\' EXIT; while :; do dd if=/dev/zero of="$f" bs=1M count=128 conv=fsync status=none; done', + ], + { stdout: "ignore", stderr: "ignore" } + ) + ); + } + try { + for (let iteration = 0; iteration < (args.shard ? 3 : 1); iteration++) { + if (iteration > 0 && Date.now() - samplingStarted > 55 * 60_000) { + break; + } + const batch = await runScenarios({ + scenarios: ALL_SCENARIOS, + dockerEnabled, + only: shardNames ?? args.only, + keepTempDirs, + }); + outcomes.push(...batch); + if (batch.some((outcome) => outcome.status === "fail")) { + break; + } + } + } finally { + for (const child of load) { + child.kill("SIGTERM"); } } From 3edd1c65e3b13a99c74a7357d81deb44356cfce9 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 20 Oct 2026 09:34:37 -0400 Subject: [PATCH 10/15] fix(native): classify timed-out and over-budget storage carrier commands A storage carrier command refusal collapsed into one plain error, so its issued boundary reason was always "unclassified": a timed-out or cancelled child, an exhausted publication budget or a settlement that outlasted its three-second post-exit budget looked identical to a real identity mismatch. The command runner now keeps a closed cause with the same fixed refusal (probe-timeout, probe-cancel or probe-budget), and every carrier boundary reports it before the generic reason. No check, deadline, retention or recovery rule changes: an over-budget settlement still leaves the original command unsettled, as the new focused test replays with a slow parent sync. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../native-compose-storage-witness-docker.ts | 82 +++++++++++++++---- ...ive-compose-storage-command-record.test.ts | 78 +++++++++++++++--- 2 files changed, 133 insertions(+), 27 deletions(-) diff --git a/src/lib/native-compose-storage-witness-docker.ts b/src/lib/native-compose-storage-witness-docker.ts index c8e062188..ea8ce0e1f 100644 --- a/src/lib/native-compose-storage-witness-docker.ts +++ b/src/lib/native-compose-storage-witness-docker.ts @@ -810,7 +810,7 @@ async function groupAbsent( return refuse(); } if (now() >= deadline) { - return refuse(); + return refuseCommand("probe-budget"); } await new Promise((resolve) => setTimeout(resolve, 20)); } @@ -903,6 +903,42 @@ async function commandBoot( } /** No callback can reject before the shared owner awaits exit. A rejected owner or * unproven group retains the journal; daemon effects are never inferred from exit. */ +type StorageCommandRefusal = "probe-timeout" | "probe-cancel" | "probe-budget"; +const storageCommandRefusals = new WeakMap(); +/** The same fixed refusal, keeping only its closed command cause (a timed-out or + * cancelled child, or an exhausted publication/settlement budget) for the effect + * diagnostic. It grants nothing and changes no check, deadline or retention. */ +function refuseCommand(reason: StorageCommandRefusal | undefined): never { + const error = new Error( + "Native storage xattr proof refused; values omitted." + ); + if (reason) { + storageCommandRefusals.set(error, reason); + } + throw error; +} +function storageCommandRefusal( + error: unknown +): StorageCommandRefusal | undefined { + return typeof error === "object" && error !== null + ? storageCommandRefusals.get(error) + : undefined; +} +function exitRefusal( + exit: RunExitEvent | null +): StorageCommandRefusal | undefined { + if (exit?.timedOut) { + return "probe-timeout"; + } + return exit?.cancelled ? "probe-cancel" : undefined; +} +/** Effect reason for a carrier boundary: a classified command cause first. */ +export function nativeComposeStorageCarrierReason( + error: unknown +): NativeComposeEffectRefusal["reason"] { + return storageCommandRefusal(error) ?? nativeComposeEffectReason(error); +} + export async function runNativeComposeStorageDockerCommand(supplied: { readonly context: Pick; readonly files: Pick< @@ -958,6 +994,7 @@ export async function runNativeComposeStorageDockerCommand(supplied: { exit: RunExitEvent | null = null; let armed: NativeComposeStorageArmedCommand | null = null; let publicationRefused = false; + let publicationCause: StorageCommandRefusal | undefined; const cancellation = new AbortController(); try { await opts.assertAdmitted(); @@ -965,19 +1002,24 @@ export async function runNativeComposeStorageDockerCommand(supplied: { ? opts.timeoutMs : Math.min(opts.timeoutMs, opts.context.deadline - Date.now()); if (!(timeoutMs > 0)) { - return refuse(); + return refuseCommand("probe-budget"); } const deadline = Date.now() + timeoutMs; let boundaryDeadline = performance.now() + timeoutMs; let recordingSettlement = false; const record = opts.originalCommand; const guard = () => { + if (performance.now() >= boundaryDeadline) { + return refuseCommand("probe-budget"); + } + if (publicationRefused) { + return refuseCommand(publicationCause); + } if ( - performance.now() >= boundaryDeadline || - publicationRefused || - (!(recordingSettlement || opts.cleanup) && opts.context.signal.aborted) + !(recordingSettlement || opts.cleanup) && + opts.context.signal.aborted ) { - return refuse(); + return refuseCommand("probe-cancel"); } }; const bounded = async (work: Promise): Promise => { @@ -990,6 +1032,7 @@ export async function runNativeComposeStorageDockerCommand(supplied: { timer = setTimeout( () => { publicationRefused = true; + publicationCause ??= "probe-budget"; if (record) { invalidateNativeComposeStorageCommandOwner(record.owner); } @@ -1006,13 +1049,14 @@ export async function runNativeComposeStorageDockerCommand(supplied: { ]); guard(); return result; - } catch { + } catch (error) { publicationRefused = true; + publicationCause ??= storageCommandRefusal(error); if (record) { invalidateNativeComposeStorageCommandOwner(record.owner); } cancellation.abort(); - return refuse(); + return refuseCommand(publicationCause); } finally { if (timer !== undefined) { clearTimeout(timer); @@ -1156,8 +1200,9 @@ export async function runNativeComposeStorageDockerCommand(supplied: { if (!event.resumeSuspended?.()) { return refuse(); } - } catch { + } catch (error) { publicationRefused = true; + publicationCause ??= storageCommandRefusal(error); invalidateNativeComposeStorageCommandOwner(record.owner); cancellation.abort(); } @@ -1173,7 +1218,10 @@ export async function runNativeComposeStorageDockerCommand(supplied: { await groupAbsent(pid, record ? boundaryDeadline : undefined); settled = true; if (record) { - if (publicationRefused || !armed || !exit) { + if (publicationRefused) { + return refuseCommand(publicationCause); + } + if (!(armed && exit)) { return refuse(); } if ( @@ -1208,13 +1256,13 @@ export async function runNativeComposeStorageDockerCommand(supplied: { !(exit as RunExitEvent).cancelled ) ) { - return refuse(); + return refuseCommand(exitRefusal(exit)); } return { exitCode: code, stdout: io.read().stdout }; } catch (error) { if (opts.originalCommand) { invalidateNativeComposeStorageCommandOwner(opts.originalCommand.owner); - return refuse(); + return refuseCommand(storageCommandRefusal(error)); } throw error; } finally { @@ -1334,7 +1382,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage, - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -1375,7 +1423,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage: "storage-target", - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -1535,7 +1583,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage, - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -1689,7 +1737,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage: "storage-helper-policy", - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -2026,7 +2074,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage, - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } diff --git a/tests/native-compose-storage-command-record.test.ts b/tests/native-compose-storage-command-record.test.ts index 2bc06d105..34fd4c31e 100644 --- a/tests/native-compose-storage-command-record.test.ts +++ b/tests/native-compose-storage-command-record.test.ts @@ -31,7 +31,10 @@ import { publishNativeComposeStorageCommandChild, settleNativeComposeStorageCommand, } from "../src/lib/native-compose-storage-command-record.ts"; -import { runNativeComposeStorageDockerCommand } from "../src/lib/native-compose-storage-witness-docker.ts"; +import { + nativeComposeStorageCarrierReason, + runNativeComposeStorageDockerCommand, +} from "../src/lib/native-compose-storage-witness-docker.ts"; import * as shell from "../src/lib/shell.ts"; const hash = "a".repeat(64); @@ -389,7 +392,8 @@ test.each(["timedOut", "cancelled"] as const)( async function transport( f: Awaited>, marker: string, - timeoutMs = 3000 + timeoutMs = 3000, + script = 'printf complete > "$1"; printf value' ) { try { return await runNativeComposeStorageDockerCommand({ @@ -398,13 +402,7 @@ async function transport( deadline: Date.now() + timeoutMs, }, files: f.files, - args: [ - "/bin/sh", - "-c", - 'printf complete > "$1"; printf value', - "owned", - marker, - ], + args: ["/bin/sh", "-c", script, "owned", marker], timeoutMs, beforeSpawn: () => {}, assertAdmitted: async () => {}, @@ -520,7 +518,11 @@ test( const f = await fixture(), marker = join(f.root, "effect"); f.failChild(); - await expect(transport(f, marker)).rejects.toThrow("values omitted"); + const caught = await transport(f, marker).catch((error: unknown) => error); + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toContain("values omitted"); + // An ordinary refusal keeps no command cause. + expect(nativeComposeStorageCarrierReason(caught)).toBe("unclassified"); expect(existsSync(marker)).toBe(false); expect((await f.read()).commands[0]?.child?.pid).toBeGreaterThan(1); expect((await f.read()).commands[0]?.settlement).toBeNull(); @@ -567,6 +569,62 @@ test( }), 30_000 ); +test( + "a settlement that outlasts its post-exit budget refuses as probe-budget and stays unsettled", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"), + sync = f.directory.file.sync.bind(f.directory.file); + // Only syncs after the target ran are slow: arm and child publication are not. + const observer = spyOn(f.directory.file, "sync").mockImplementation( + async () => { + if (existsSync(marker)) { + await Bun.sleep(3500); + } + await sync(); + } + ); + try { + const caught = await transport(f, marker).catch( + (error: unknown) => error + ); + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toContain("values omitted"); + expect(nativeComposeStorageCarrierReason(caught)).toBe("probe-budget"); + expect(await readFile(marker, "utf8")).toBe("complete"); + expect((await f.read()).commands[0]?.settlement).toBeNull(); + expect(nativeComposeStorageCommandOwnerConfirmed(f.owner)).toBe(false); + retained.add(f.root); + } finally { + if (confirmedTestLifetime()) { + observer.mockRestore(); + } + } + }), + 30_000 +); +test( + "a recorded child that outlives its command budget refuses as probe-timeout", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"); + const caught = await transport( + f, + marker, + 1500, + 'printf started > "$1"; exec sleep 10' + ).catch((error: unknown) => error); + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toContain("values omitted"); + expect(nativeComposeStorageCarrierReason(caught)).toBe("probe-timeout"); + expect(await readFile(marker, "utf8")).toBe("started"); + expect((await f.read()).commands[0]?.settlement).toMatchObject({ + timedOut: true, + groupAbsent: true, + }); + }), + 30_000 +); test( "missing original exit callback cannot be replaced by capture bytes or group absence", ownedCase(async () => { From 4c9aeef232fe26ee8e03ffa8702739636ca159b9 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 20 Oct 2026 14:12:16 -0400 Subject: [PATCH 11/15] test(e2e): revert temporary native-config-networks sampling Removes the hosted HACK-1281 sampling from tests/e2e/run.ts (every shard running only native-config-networks, repeated iterations and background CPU/fsync load on shards 1 and 2). Shard selection and a single runScenarios pass are restored exactly as on the merge base. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- tests/e2e/run.ts | 59 ++++++------------------------------------------ 1 file changed, 7 insertions(+), 52 deletions(-) diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 99bef09eb..7fdfb7ff4 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -204,10 +204,7 @@ function printShardPlan(opts: { async function main(): Promise { const args = parseArgs(Bun.argv.slice(2)); - // TEMPORARY HACK-1281 sampling: every hosted shard runs native-config-networks only. Reverted before review. - const shardNames: readonly string[] | null = args.shard - ? ["native-config-networks"] - : null; + const shardNames = args.shard ? shardScenarioNames(args.shard) : null; if (args.shard && shardNames?.length === 0) { process.stdout.write( `Shard ${args.shard.index}/${args.shard.count} selects no scenarios\n` @@ -249,54 +246,12 @@ async function main(): Promise { const dockerEnabled = (process.env.HACK_E2E_DOCKER ?? "") === "1"; const requireDocker = (process.env.HACK_E2E_REQUIRE_DOCKER ?? "") === "1"; const keepTempDirs = (process.env.HACK_E2E_KEEP ?? "") === "1"; - // TEMPORARY HACK-1281 sampling: sequential iterations while one more fits the 150-minute - // job budget (a third iteration once crossed it), stopping at a failure. Shards 1 and 2 - // add background CPU and fsync load to widen any timing window; 3 and 4 run unloaded. - const outcomes: Awaited> = []; - const samplingStarted = Date.now(); - const load: ReturnType[] = []; - if (args.shard && args.shard.index <= 2) { - process.stdout.write("HACK-1281 sampling: background load enabled\n"); - for (let spinner = 0; spinner < 2; spinner++) { - load.push( - Bun.spawn([process.execPath, "-e", "for(;;){}"], { - stdout: "ignore", - stderr: "ignore", - }) - ); - } - load.push( - Bun.spawn( - [ - "sh", - "-c", - 'f="${TMPDIR:-/tmp}/hack-1281-load-$$"; trap \'rm -f "$f"\' EXIT; while :; do dd if=/dev/zero of="$f" bs=1M count=128 conv=fsync status=none; done', - ], - { stdout: "ignore", stderr: "ignore" } - ) - ); - } - try { - for (let iteration = 0; iteration < (args.shard ? 3 : 1); iteration++) { - if (iteration > 0 && Date.now() - samplingStarted > 55 * 60_000) { - break; - } - const batch = await runScenarios({ - scenarios: ALL_SCENARIOS, - dockerEnabled, - only: shardNames ?? args.only, - keepTempDirs, - }); - outcomes.push(...batch); - if (batch.some((outcome) => outcome.status === "fail")) { - break; - } - } - } finally { - for (const child of load) { - child.kill("SIGTERM"); - } - } + const outcomes = await runScenarios({ + scenarios: ALL_SCENARIOS, + dockerEnabled, + only: shardNames ?? args.only, + keepTempDirs, + }); const dockerSkipped = outcomes.some( (outcome) => outcome.tier === "docker" && outcome.status === "skip" From 79b0714e00093a29a3b77e15cf02365503b27ed4 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 21 Oct 2026 13:17:57 -0400 Subject: [PATCH 12/15] test(e2e): name incomplete stops and killed commands in network failure evidence A failing `down --recover` that stays incomplete issues no refusal boundary, so its diagnostic printed E_COMPOSE_FAILED with a null stage and reason. A command the runner killed at its budget printed no diagnostic at all. The command diagnostic now adds a closed `message` label for an exact fixed product sentence (stop-incomplete, operation-failed, native-state, -busy, -uncertain, -stale); any other or changed text yields null. A command the runner could not return logs its class, role, sequence, duration and a closed runner outcome (timeout, output-bound, group-unconfirmed, capture, other). No output, argv or path is printed. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../e2e/native-config-networks-acceptance.ts | 126 ++++++++++++++++-- tests/native-networks-fixture.test.ts | 112 +++++++++++++++- 2 files changed, 229 insertions(+), 9 deletions(-) diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index 51d35af02..1b23d1049 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -19,6 +19,7 @@ import type { } from "../../packages/config-compiler/generated/native-config.ts"; import { isRecord } from "../../src/lib/guards.ts"; import { + NativeComposeGenerationError, type NativeComposeIdentity, openNativeComposeGenerationStore, readNativeComposeNetworkTopology, @@ -171,8 +172,40 @@ type ClosedFailure = { readonly code: string | null; readonly stage: string | null; readonly reason: string | null; + readonly message: string | null; }; +/** + * Exact fixed product sentences, each named by a closed label. A failure that + * issues no refusal boundary (for example an incomplete saved stop) is still + * classified; any other text, or a changed sentence, yields null. + */ +const FIXED_MESSAGES: ReadonlyMap = new Map([ + [ + "Native Compose stop is incomplete; saved ownership and persistent data are retained.", + "stop-incomplete", + ], + [ + "Native Compose operation failed; saved ownership and persistent data are retained. Values omitted.", + "operation-failed", + ], + ...( + [ + ["E_NATIVE_COMPOSE_STATE", "native-state"], + ["E_NATIVE_COMPOSE_BUSY", "native-busy"], + ["E_NATIVE_COMPOSE_UNCERTAIN", "native-uncertain"], + ["E_NATIVE_COMPOSE_STALE", "native-stale"], + ] as const + ).map( + ([code, label]) => + [new NativeComposeGenerationError(code).message, label] as const + ), +]); + +function fixedMessage(value: unknown): string | null { + return typeof value === "string" ? (FIXED_MESSAGES.get(value) ?? null) : null; +} + function closedToken(value: unknown, pattern: RegExp): string | null { return typeof value === "string" && pattern.test(value) ? value : null; } @@ -187,7 +220,7 @@ function jsonClosedFailure(stdout: string): ClosedFailure | null { } const error = isRecord(value) ? own(value, "error") : undefined; if (!isRecord(error)) { - return { code: null, stage: null, reason: null }; + return { code: null, stage: null, reason: null, message: null }; } const detail = own(error, "detail"); const refusal = isRecord(detail) @@ -201,6 +234,7 @@ function jsonClosedFailure(stdout: string): ClosedFailure | null { reason: isRecord(refusal) ? closedToken(own(refusal, "reason"), CLOSED_TOKEN) : null, + message: fixedMessage(own(error, "message")), }; } @@ -216,9 +250,18 @@ function closedFailure( code: humanClosedCode(result.stdout) ?? humanClosedCode(result.stderr), stage: boundary?.stage ?? null, reason: boundary?.reason ?? null, + message: + humanFixedMessage(result.stdout) ?? humanFixedMessage(result.stderr), }; } +const HUMAN_MESSAGE = /^[^\w\n]*(?:ERROR:?[ \t]+)?E_[A-Z0-9_]{1,62}: (.+)$/m; + +/** The closed label of a human error line's exact fixed sentence; never its text. */ +export function humanFixedMessage(text: string): string | null { + return fixedMessage(HUMAN_MESSAGE.exec(text.replace(ANSI, ""))?.[1]); +} + const HUMAN_BOUNDARY = /^[^\w\n]*(?:ERROR:?[ \t]+)?Refusal boundary: ([a-z][a-z0-9-]{0,62}) \(([a-z][a-z0-9-]{0,62})\); values omitted\.$/m; @@ -323,6 +366,59 @@ export function nativeNetworkFixtureCommandDiagnostic(opts: { })}`; } +const RUNNER_FAILURES = [ + [ + "Fixture command timed out; uncertain engine outcome and captures retained", + "timeout", + ], + [ + "Fixture command exceeded its output bound; bounded captures retained", + "output-bound", + ], + [ + "Fixture could not confirm owned group cleanup; captures retained", + "group-unconfirmed", + ], + [ + "Fixture capture failed after owned command reap; captures retained", + "capture", + ], +] as const; + +/** + * A command the runner could not return (a timeout kills it) still names its + * closed class, role, sequence, duration and runner outcome; never its output. + */ +export function nativeNetworkFixtureRunnerDiagnostic(opts: { + readonly step: string; + readonly role: FixtureRole; + readonly sequence: number; + readonly durationMs: number; + readonly error: unknown; +}): string { + const step = /^(hack|docker):[a-z]{1,16}(?::[a-z]{1,16}|\+recover)?$|^other$/; + if ( + !( + step.test(opts.step) && + NATIVE_NETWORK_FIXTURE_ROLES.includes(opts.role) && + Number.isSafeInteger(opts.sequence) && + Number.isSafeInteger(opts.durationMs) + ) + ) { + return "Network command diagnostic unavailable"; + } + const message = opts.error instanceof Error ? opts.error.message : null; + const outcome = + RUNNER_FAILURES.find(([text]) => text === message)?.[1] ?? "other"; + return `Network command diagnostic ${JSON.stringify({ + step: opts.step, + role: opts.role, + sequence: opts.sequence, + durationMs: opts.durationMs, + runner: outcome, + })}`; +} + /** A fixture-authored assertion is a fixed sentence; anything else (paths, values, product text) is withheld. */ export function nativeNetworkFixtureFailureLabel(error: unknown): string { const message = error instanceof Error ? error.message : null; @@ -1508,13 +1604,27 @@ export const nativeConfigNetworksScenario: Scenario = { role: roles.get(cwd) ?? ("fixture" as const), sequence, }; - const result = await runNativeNetworkFixtureCommand({ - argv, - cwd, - env: { ...env, ...extra }, - captures, - timeoutMs: Math.min(COMMAND_TIMEOUT, remaining), - }); + const begun = Date.now(); + let result: CliResult; + try { + result = await runNativeNetworkFixtureCommand({ + argv, + cwd, + env: { ...env, ...extra }, + captures, + timeoutMs: Math.min(COMMAND_TIMEOUT, remaining), + }); + } catch (error: unknown) { + // A killed or unreturned command has no result for ok(); name it here. + ctx.log( + nativeNetworkFixtureRunnerDiagnostic({ + ...command, + durationMs: Date.now() - begun, + error, + }) + ); + throw error; + } commands.set(result, command); return result; }; diff --git a/tests/native-networks-fixture.test.ts b/tests/native-networks-fixture.test.ts index 546addaa3..b831b34d6 100644 --- a/tests/native-networks-fixture.test.ts +++ b/tests/native-networks-fixture.test.ts @@ -34,6 +34,7 @@ import { nativeNetworkFixtureProjectKind, nativeNetworkFixtureProtocolMatches, nativeNetworkFixtureRefusalDiagnostic, + nativeNetworkFixtureRunnerDiagnostic, nativeNetworkFixtureSavedEvidence, nativeNetworkFixtureShim, nativeNetworkFixtureStateRefused, @@ -208,7 +209,7 @@ test("failing command evidence names only its closed class, role and product cod }, }); expect(json).toBe( - 'Network command diagnostic {"step":"hack:down+recover","role":"primary","sequence":412,"expected":0,"exit":1,"timedOut":false,"durationMs":2345,"code":"E_CONFIG_INVALID","stage":"storage-helper-start","reason":"probe-timeout"}' + 'Network command diagnostic {"step":"hack:down+recover","role":"primary","sequence":412,"expected":0,"exit":1,"timedOut":false,"durationMs":2345,"code":"E_CONFIG_INVALID","stage":"storage-helper-start","reason":"probe-timeout","message":null}' ); const human = nativeNetworkFixtureCommandDiagnostic({ step: "hack:exec", @@ -398,6 +399,115 @@ test("saved-state evidence reports closed flags and pending operation only", () ); }); +test("an incomplete saved stop without an issued boundary is still named by its fixed sentence", () => { + const secret = "/private/fixture/secret-path"; + const stop = + "Native Compose stop is incomplete; saved ownership and persistent data are retained."; + const diagnostic = (stdout: string, stderr = "") => + nativeNetworkFixtureCommandDiagnostic({ + step: "hack:down+recover", + role: "sibling-a", + sequence: 942, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 2750, + stdout, + stderr, + }, + }); + // The sampled HACK-1281 cleanup failure: E_COMPOSE_FAILED with no stage or reason. + const json = diagnostic( + JSON.stringify({ + ok: false, + error: { code: "E_COMPOSE_FAILED", message: stop }, + }) + ); + expect(json).toContain( + '"code":"E_COMPOSE_FAILED","stage":null,"reason":null,"message":"stop-incomplete"' + ); + expect(diagnostic(`│\n■ E_COMPOSE_FAILED: ${stop}\n`, secret)).toContain( + '"message":"stop-incomplete"' + ); + expect( + diagnostic( + JSON.stringify({ + ok: false, + error: { + code: "E_CONFIG_INVALID", + message: + "Native Compose operation has an uncertain outcome; inspect the saved generation or explicitly stop its owned resources before retrying.", + }, + }) + ) + ).toContain('"message":"native-uncertain"'); + // Only the exact sentence is classified; appended values are withheld. + for (const changed of [`${stop} ${secret}`, stop.toUpperCase(), ""]) { + const result = diagnostic( + JSON.stringify({ + ok: false, + error: { code: "E_COMPOSE_FAILED", message: changed }, + }) + ); + expect(result).toContain('"message":null'); + expect(result).not.toContain(secret); + } + expect(diagnostic(`■ E_COMPOSE_FAILED: ${stop} ${secret}\n`)).toContain( + '"message":null' + ); +}); + +test("a command the runner killed or could not return still names its class and runner outcome", async () => { + const root = await mkdtemp(join(tmpdir(), "native-network-runner-")); + try { + const caught = await runNativeNetworkFixtureCommand({ + argv: ["/bin/sh", "-c", "sleep 5"], + cwd: root, + env: {}, + captures: join(root, "captures"), + timeoutMs: 150, + }).catch((error: unknown) => error); + const timeout = nativeNetworkFixtureRunnerDiagnostic({ + step: "hack:exec", + role: "sibling-a", + sequence: 941, + durationMs: 180_000, + error: caught, + }); + expect(timeout).toBe( + 'Network command diagnostic {"step":"hack:exec","role":"sibling-a","sequence":941,"durationMs":180000,"runner":"timeout"}' + ); + const secret = "/private/fixture/secret-path"; + expect( + nativeNetworkFixtureRunnerDiagnostic({ + step: "hack:exec", + role: "primary", + sequence: 1, + durationMs: 1, + error: new Error(secret), + }) + ).toBe( + 'Network command diagnostic {"step":"hack:exec","role":"primary","sequence":1,"durationMs":1,"runner":"other"}' + ); + for (const value of [ + { step: secret, role: "primary" }, + { step: "hack:exec", role: secret }, + ]) { + expect( + nativeNetworkFixtureRunnerDiagnostic({ + ...(value as { step: string; role: "primary" }), + sequence: 1, + durationMs: 1, + error: caught, + }) + ).toBe("Network command diagnostic unavailable"); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test("failing command evidence drops non-closed codes and malformed fields", () => { const secret = "secret-value-must-not-appear"; const forged = nativeNetworkFixtureCommandDiagnostic({ From fbf5cf680ab70ae025485aafc8cf1ca943a4acdb Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 21 Oct 2026 13:20:31 -0400 Subject: [PATCH 13/15] test(e2e): temporarily replay CLI stalls in native-config-networks Hosted sampling for HACK-1281 only; reverted before review. Every shard runs only native-config-networks under a host-stall monitor (event-loop lag and a product-shaped `ps` identity read every two seconds; numbers only). Shards 1 and 2 stop the CLI process 1.2 s of every 2 s during the first 20 saved exec commands after the created-recovery phase, replaying a host stall. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../e2e/native-config-networks-acceptance.ts | 68 ++++++++++++++++ tests/e2e/run.ts | 78 +++++++++++++++++-- 2 files changed, 140 insertions(+), 6 deletions(-) diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index 1b23d1049..420118ad2 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -725,6 +725,8 @@ export async function runNativeNetworkFixtureCommand(opts: { readonly outputLimit?: number; /** Focused fault observation. A thrown capture error must kill/reap before fixture cleanup. */ readonly afterCaptureWrite?: () => void; + /** TEMPORARY HACK-1281 replay: stop only the launched process for pauseMs every restMs. */ + readonly stall?: { readonly pauseMs: number; readonly restMs: number }; }): Promise { const timeout = opts.timeoutMs ?? COMMAND_TIMEOUT; const limit = opts.outputLimit ?? OUTPUT_LIMIT; @@ -820,14 +822,45 @@ export async function runNativeNetworkFixtureCommand(opts: { timedOut = true; kill(); }, timeout); + // TEMPORARY HACK-1281 replay: a host stall of the CLI process only. + let stalling = opts.stall !== undefined && launched.pid !== undefined; + void (async () => { + const pid = launched.pid; + const stall = opts.stall; + if (!(stall && pid !== undefined)) { + return; + } + while (stalling) { + await Bun.sleep(stall.restMs); + if (!stalling) { + return; + } + try { + process.kill(pid, "SIGSTOP"); + } catch { + return; + } + await Bun.sleep(stall.pauseMs); + try { + process.kill(pid, "SIGCONT"); + } catch { + return; + } + } + })(); const exited = new Promise((resolveExit, reject) => { launched.once("error", reject); launched.once("exit", (code) => { + stalling = false; // Reap descendants of this just-launched detached group; no stale PID reclamation. kill(); resolveExit(code ?? 128); }); }); + const stopStalling = () => { + stalling = false; + }; + exited.then(stopStalling, stopStalling); const stdoutCapture = capture(launched.stdout, output); const stderrCapture = capture(launched.stderr, errors); let exitCode: number; @@ -1585,6 +1618,12 @@ export const nativeConfigNetworksScenario: Scenario = { >(); let sequence = 0; let stepExecutables = { cli: "", engine: engine ?? "" }; + // TEMPORARY HACK-1281 replay controls. + const stallInjection = ["1/4", "2/4"].includes( + process.env.HACK_E2E_SHARD ?? "" + ); + let stallsArmed = false; + let stalledCommands = 0; const execute = async ( argv: readonly string[], cwd = ctx.tempRoot, @@ -1604,6 +1643,21 @@ export const nativeConfigNetworksScenario: Scenario = { role: roles.get(cwd) ?? ("fixture" as const), sequence, }; + // TEMPORARY HACK-1281 replay: shards 1 and 2 stall the CLI process of the + // first 20 saved `exec` commands after the created-recovery phase. + const stall = + stallInjection && + stallsArmed && + cleanupStarted === null && + stalledCommands < 20 && + stepExecutables.cli !== "" && + argv[0] === stepExecutables.cli && + argv[1] === "exec" + ? { pauseMs: 1200, restMs: 800 } + : undefined; + if (stall) { + stalledCommands += 1; + } const begun = Date.now(); let result: CliResult; try { @@ -1613,6 +1667,7 @@ export const nativeConfigNetworksScenario: Scenario = { env: { ...env, ...extra }, captures, timeoutMs: Math.min(COMMAND_TIMEOUT, remaining), + ...(stall ? { stall } : {}), }); } catch (error: unknown) { // A killed or unreturned command has no result for ok(); name it here. @@ -1625,6 +1680,12 @@ export const nativeConfigNetworksScenario: Scenario = { ); throw error; } + if (stall) { + // TEMPORARY HACK-1281 replay: closed numbers only. + ctx.log( + `HACK-1281 replay stalled exec ${JSON.stringify({ index: stalledCommands, exit: result.exitCode, durationMs: result.durationMs })}` + ); + } commands.set(result, command); return result; }; @@ -2184,6 +2245,13 @@ export const nativeConfigNetworksScenario: Scenario = { ctx.log( "Actual created-before-start empty network identity recovered through owned saved down" ); + // TEMPORARY HACK-1281 replay. + stallsArmed = stallInjection; + if (stallInjection) { + ctx.log( + "HACK-1281 replay: stalling the CLI 1.2 s of every 2 s in the next 20 saved exec commands" + ); + } for (const checkout of checkouts) { startedRoots.add(checkout.root); await cli(checkout, ["up", "--detach", "--json"]); diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 7fdfb7ff4..7c7600ac9 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -246,12 +246,22 @@ async function main(): Promise { const dockerEnabled = (process.env.HACK_E2E_DOCKER ?? "") === "1"; const requireDocker = (process.env.HACK_E2E_REQUIRE_DOCKER ?? "") === "1"; const keepTempDirs = (process.env.HACK_E2E_KEEP ?? "") === "1"; - const outcomes = await runScenarios({ - scenarios: ALL_SCENARIOS, - dockerEnabled, - only: shardNames ?? args.only, - keepTempDirs, - }); + // TEMPORARY HACK-1281 sampling: every shard runs only native-config-networks + // (shards 1-2 inject CLI stalls) under a natural host-stall monitor. + const monitor = startHack1281StallMonitor(); + let outcomes: Awaited>; + try { + outcomes = await runScenarios({ + scenarios: ALL_SCENARIOS, + dockerEnabled, + only: args.shard ? ["native-config-networks"] : (shardNames ?? args.only), + keepTempDirs, + }); + } finally { + process.stdout.write( + `HACK-1281 stall monitor ${JSON.stringify(await monitor())}\n` + ); + } const dockerSkipped = outcomes.some( (outcome) => outcome.tier === "docker" && outcome.status === "skip" @@ -263,6 +273,62 @@ async function main(): Promise { : 0; } +/** TEMPORARY HACK-1281 sampling: event-loop lag and the duration of a product-shaped + * process identity read (`sh -c 'ulimit; exec ps'`) every two seconds. Numbers only. */ +function startHack1281StallMonitor(): () => Promise> { + let last = performance.now(); + const lags: number[] = []; + const reads: number[] = []; + const lag = setInterval(() => { + const now = performance.now(); + lags.push(now - last - 100); + last = now; + }, 100); + let pending: Promise = Promise.resolve(); + const probe = setInterval(() => { + pending = (async () => { + const started = performance.now(); + const child = Bun.spawn( + [ + "/bin/sh", + "-c", + 'ulimit -f 8; exec "$@"', + "hack-1281-monitor", + "/bin/ps", + "-p", + String(process.pid), + "-o", + "uid=,pgid=,state=,lstart=,comm=", + ], + { stdout: "ignore", stderr: "ignore" } + ); + await child.exited; + reads.push(performance.now() - started); + })(); + }, 2000); + const summary = (values: number[], prefix: string) => { + const sorted = [...values].sort((a, b) => a - b); + const at = (p: number) => + Math.round( + sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0 + ); + return { + [`${prefix}Count`]: sorted.length, + [`${prefix}P99Ms`]: at(0.99), + [`${prefix}MaxMs`]: Math.round(sorted.at(-1) ?? 0), + [`${prefix}Over250`]: sorted.filter((value) => value >= 250).length, + [`${prefix}Over1000`]: sorted.filter((value) => value >= 1000).length, + [`${prefix}Over3000`]: sorted.filter((value) => value >= 3000).length, + }; + }; + return async () => { + clearInterval(lag); + clearInterval(probe); + await pending; + return { ...summary(lags, "lag"), ...summary(reads, "read") }; + }; +} + if (import.meta.main) { await main(); } From fc4095b8cf6801aaec5b26ea30aa0fe6ca83961e Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 21 Oct 2026 13:21:21 -0400 Subject: [PATCH 14/15] fix(native): keep storage carrier commands alive through host stalls A recorded storage carrier command read process identity with a fixed one-second bound and settled within three seconds after exit. A host stall of the CLI longer than that resumes with the timer already due, so a healthy command refused after its daemon effect (probe-timeout or probe-budget). The refusal leaves the carrier intent uncertain: the next startup, exec or restart refuses, and `down --recover` stays incomplete by design. This is the native-config-networks failure signature in HACK-1281 (a product command exits 1, then cleanup also fails, with a created carrier and storage witness work pending). Identity reads now share the enclosing handshake or settlement budget, and the post-exit settlement budget is a finite 15 seconds. No check, record, retention, recovery rule or authority changes. An exhausted budget still refuses as probe-budget; the race also observes the abandoned work so its rejection cannot escape as an unhandled process error. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../native-compose-storage-xattr-carrier.md | 14 ++- .../native-compose-storage-witness-docker.ts | 17 ++- ...ive-compose-storage-command-record.test.ts | 106 +++++++++++++++++- 3 files changed, 128 insertions(+), 9 deletions(-) diff --git a/docs/reference/native-compose-storage-xattr-carrier.md b/docs/reference/native-compose-storage-xattr-carrier.md index e9a000efb..9aeb0876f 100644 --- a/docs/reference/native-compose-storage-xattr-carrier.md +++ b/docs/reference/native-compose-storage-xattr-carrier.md @@ -381,7 +381,8 @@ publishes and synchronizes that identity, then repeats identity, boot, executabl and admission checks before a one-use continuation of the captured child. A missing or failed publication cancels that original invocation; an armed record alone proves neither spawn nor absence. The command's original budget includes -the handshake. No guessed PID or retry is used to rescue a fast command. +the handshake, and each process identity read shares that remaining budget rather +than a tighter fixed bound. No guessed PID or retry is used to rescue a fast command. Live handshake and settlement budgets use a captured monotonic deadline; the saved wall deadline is a record field only. Executable device/inode identities use exact decimal strings from bigint metadata, including sealed macOS binaries. @@ -392,9 +393,14 @@ The carrier uses held regular capture files, not pipes. Version 2 therefore name its capture evidence `held-files-quiescent`, never EOF. The original exit callback, strict ESRCH group absence, held file/path identity, both synchronized bounded captures and their hashes precede settlement publication. EPERM/EIO and unknown -exit or capture disposition refuse. Settlement publication shares the existing -three-second post-command group-observation budget; timeout/cancellation remain -refused results and cannot grant a subsequent daemon command. +exit or capture disposition refuse. Settlement publication shares one finite +15-second post-command budget with group observation and its identity reads; +timeout/cancellation remain refused results and cannot grant a subsequent daemon +command. These budgets must outlast an ordinary multi-second host stall: a stalled +owner resumes with its timer already due, and a refusal after the daemon effect +leaves durable uncertainty that only explicit recovery can report. Earlier +one-second identity reads and a three-second settlement budget refused healthy +commands that way on loaded hosts. Only the original in-memory writer can advance its exact sequence. Publication failure or deadline permanently invalidates that writer and retains its private diff --git a/src/lib/native-compose-storage-witness-docker.ts b/src/lib/native-compose-storage-witness-docker.ts index ea8ce0e1f..edfe267f2 100644 --- a/src/lib/native-compose-storage-witness-docker.ts +++ b/src/lib/native-compose-storage-witness-docker.ts @@ -105,6 +105,15 @@ const PROCESS = const TICKS = /^[1-9][0-9]*$/; const BOOT = /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/; const CAPTURE_LIMIT = 4096; +/** + * Post-exit settlement budget of a recorded command (group absence, boot and + * executable identity, capture sync and settlement publication). It and the + * identity reads must outlast an ordinary multi-second host stall: a stalled + * owner resumes with its timer already due, so a sub-second or three-second + * bound refused healthy commands after their daemon effect and left durable + * uncertainty (HACK-1281). It stays finite and grants nothing. + */ +const SETTLEMENT_BUDGET_MS = 15_000; // A late/unconfirmed publication cannot release its original held directory FDs. const unsettledCommandFiles = new Set(); type Selection = { readonly name: string; readonly storage: string }; @@ -1024,6 +1033,9 @@ export async function runNativeComposeStorageDockerCommand(supplied: { }; const bounded = async (work: Promise): Promise => { let timer: ReturnType | undefined; + // An exhausted budget refuses below before the race observes the work; + // its own rejection must not escape as an unhandled process error. + work.catch(() => undefined); try { guard(); const result = await Promise.race([ @@ -1080,7 +1092,8 @@ export async function runNativeComposeStorageDockerCommand(supplied: { guard(); return Promise.resolve(); }, - timeoutMs: Math.min(1000, boundaryDeadline - performance.now()), + // Shares the enclosing handshake or settlement budget; no tighter bound. + timeoutMs: boundaryDeadline - performance.now(), }); if (result.exitCode !== 0) { return refuse(); @@ -1213,7 +1226,7 @@ export async function runNativeComposeStorageDockerCommand(supplied: { }, } ); - boundaryDeadline = performance.now() + 3000; + boundaryDeadline = performance.now() + SETTLEMENT_BUDGET_MS; recordingSettlement = true; await groupAbsent(pid, record ? boundaryDeadline : undefined); settled = true; diff --git a/tests/native-compose-storage-command-record.test.ts b/tests/native-compose-storage-command-record.test.ts index 34fd4c31e..7a306e547 100644 --- a/tests/native-compose-storage-command-record.test.ts +++ b/tests/native-compose-storage-command-record.test.ts @@ -569,21 +569,120 @@ test( }), 30_000 ); +// HACK-1281 replay: a stalled owner resumes with its timer already due. A +// one-second identity read refused this healthy command after its stopped child +// was published, leaving the durable uncertainty that blocks saved recovery. test( - "a settlement that outlasts its post-exit budget refuses as probe-budget and stays unsettled", + "a host stall in a child identity read stays within the command budget", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"), + run = shell.run; + let stalled = 0; + const observer = spyOn(shell, "run").mockImplementation((args, opts) => { + const pid = args[args.indexOf("-p") + 1]; + if ( + stalled === 0 && + args[4] === "/bin/ps" && + pid !== undefined && + pid !== String(process.pid) + ) { + stalled++; + // The read itself stalls past the former one-second bound. + return run( + [ + args[0] ?? "", + args[1] ?? "", + `sleep 1.2; ${args[2]}`, + ...args.slice(3), + ], + opts + ); + } + return run(args, opts); + }); + try { + expect(await transport(f, marker, 15_000)).toEqual({ + exitCode: 0, + stdout: "value", + }); + expect(stalled).toBe(1); + expect(await readFile(marker, "utf8")).toBe("complete"); + expect((await f.read()).commands[0]?.settlement).toMatchObject({ + exitCode: 0, + timedOut: false, + groupAbsent: true, + }); + } finally { + if (confirmedTestLifetime()) { + observer.mockRestore(); + } + } + }), + 30_000 +); +test( + "a multi-second host stall after exit still settles the original command", ownedCase(async () => { const f = await fixture(), marker = join(f.root, "effect"), sync = f.directory.file.sync.bind(f.directory.file); - // Only syncs after the target ran are slow: arm and child publication are not. + let stalled = false; + // One post-exit sync stalls past the former three-second settlement budget. const observer = spyOn(f.directory.file, "sync").mockImplementation( async () => { - if (existsSync(marker)) { + if (existsSync(marker) && !stalled) { + stalled = true; await Bun.sleep(3500); } await sync(); } ); + try { + expect(await transport(f, marker)).toEqual({ + exitCode: 0, + stdout: "value", + }); + expect(stalled).toBe(true); + expect((await f.read()).commands[0]?.settlement).toMatchObject({ + exitCode: 0, + groupAbsent: true, + }); + expect(nativeComposeStorageCommandOwnerConfirmed(f.owner)).toBe(true); + } finally { + if (confirmedTestLifetime()) { + observer.mockRestore(); + } + } + }), + 30_000 +); +test( + "a settlement that outlasts its post-exit budget refuses as probe-budget and stays unsettled", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"), + kill = process.kill, + monotonic = performance.now.bind(performance); + let elapsed = 0; + // The post-exit group observation sees 16 s of monotonic time pass: past + // the finite settlement budget, without a real 16 s wait. + const clock = spyOn(performance, "now").mockImplementation( + () => monotonic() + elapsed + ); + const observer = spyOn(process, "kill").mockImplementation( + (pid, signal) => { + if ( + signal === 0 && + typeof pid === "number" && + pid < 0 && + existsSync(marker) + ) { + elapsed = 16_000; + } + return kill(pid, signal); + } + ); try { const caught = await transport(f, marker).catch( (error: unknown) => error @@ -597,6 +696,7 @@ test( retained.add(f.root); } finally { if (confirmedTestLifetime()) { + clock.mockRestore(); observer.mockRestore(); } } From 055eccec6ef28eda7beaa077e4d2e735fe9fd9a4 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 21 Oct 2026 14:35:38 -0400 Subject: [PATCH 15/15] test(e2e): revert temporary native-config-networks stall replay Reverts fbf5cf68. Hosted results are recorded in the PR: on the pre-fix carrier (run 39643040488) both stalled shards failed with the HACK-1281 signature; on fc4095b8 (run 39644645624) both stalled shards passed all 20 stalled exec commands and all four shards passed. tests/e2e/run.ts and the scenario's stall hooks are back to their reviewed state. Refs HACK-1281 Co-Authored-By: Claude Opus 5.5 --- .../e2e/native-config-networks-acceptance.ts | 68 ---------------- tests/e2e/run.ts | 78 ++----------------- 2 files changed, 6 insertions(+), 140 deletions(-) diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index 420118ad2..1b23d1049 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -725,8 +725,6 @@ export async function runNativeNetworkFixtureCommand(opts: { readonly outputLimit?: number; /** Focused fault observation. A thrown capture error must kill/reap before fixture cleanup. */ readonly afterCaptureWrite?: () => void; - /** TEMPORARY HACK-1281 replay: stop only the launched process for pauseMs every restMs. */ - readonly stall?: { readonly pauseMs: number; readonly restMs: number }; }): Promise { const timeout = opts.timeoutMs ?? COMMAND_TIMEOUT; const limit = opts.outputLimit ?? OUTPUT_LIMIT; @@ -822,45 +820,14 @@ export async function runNativeNetworkFixtureCommand(opts: { timedOut = true; kill(); }, timeout); - // TEMPORARY HACK-1281 replay: a host stall of the CLI process only. - let stalling = opts.stall !== undefined && launched.pid !== undefined; - void (async () => { - const pid = launched.pid; - const stall = opts.stall; - if (!(stall && pid !== undefined)) { - return; - } - while (stalling) { - await Bun.sleep(stall.restMs); - if (!stalling) { - return; - } - try { - process.kill(pid, "SIGSTOP"); - } catch { - return; - } - await Bun.sleep(stall.pauseMs); - try { - process.kill(pid, "SIGCONT"); - } catch { - return; - } - } - })(); const exited = new Promise((resolveExit, reject) => { launched.once("error", reject); launched.once("exit", (code) => { - stalling = false; // Reap descendants of this just-launched detached group; no stale PID reclamation. kill(); resolveExit(code ?? 128); }); }); - const stopStalling = () => { - stalling = false; - }; - exited.then(stopStalling, stopStalling); const stdoutCapture = capture(launched.stdout, output); const stderrCapture = capture(launched.stderr, errors); let exitCode: number; @@ -1618,12 +1585,6 @@ export const nativeConfigNetworksScenario: Scenario = { >(); let sequence = 0; let stepExecutables = { cli: "", engine: engine ?? "" }; - // TEMPORARY HACK-1281 replay controls. - const stallInjection = ["1/4", "2/4"].includes( - process.env.HACK_E2E_SHARD ?? "" - ); - let stallsArmed = false; - let stalledCommands = 0; const execute = async ( argv: readonly string[], cwd = ctx.tempRoot, @@ -1643,21 +1604,6 @@ export const nativeConfigNetworksScenario: Scenario = { role: roles.get(cwd) ?? ("fixture" as const), sequence, }; - // TEMPORARY HACK-1281 replay: shards 1 and 2 stall the CLI process of the - // first 20 saved `exec` commands after the created-recovery phase. - const stall = - stallInjection && - stallsArmed && - cleanupStarted === null && - stalledCommands < 20 && - stepExecutables.cli !== "" && - argv[0] === stepExecutables.cli && - argv[1] === "exec" - ? { pauseMs: 1200, restMs: 800 } - : undefined; - if (stall) { - stalledCommands += 1; - } const begun = Date.now(); let result: CliResult; try { @@ -1667,7 +1613,6 @@ export const nativeConfigNetworksScenario: Scenario = { env: { ...env, ...extra }, captures, timeoutMs: Math.min(COMMAND_TIMEOUT, remaining), - ...(stall ? { stall } : {}), }); } catch (error: unknown) { // A killed or unreturned command has no result for ok(); name it here. @@ -1680,12 +1625,6 @@ export const nativeConfigNetworksScenario: Scenario = { ); throw error; } - if (stall) { - // TEMPORARY HACK-1281 replay: closed numbers only. - ctx.log( - `HACK-1281 replay stalled exec ${JSON.stringify({ index: stalledCommands, exit: result.exitCode, durationMs: result.durationMs })}` - ); - } commands.set(result, command); return result; }; @@ -2245,13 +2184,6 @@ export const nativeConfigNetworksScenario: Scenario = { ctx.log( "Actual created-before-start empty network identity recovered through owned saved down" ); - // TEMPORARY HACK-1281 replay. - stallsArmed = stallInjection; - if (stallInjection) { - ctx.log( - "HACK-1281 replay: stalling the CLI 1.2 s of every 2 s in the next 20 saved exec commands" - ); - } for (const checkout of checkouts) { startedRoots.add(checkout.root); await cli(checkout, ["up", "--detach", "--json"]); diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 7c7600ac9..7fdfb7ff4 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -246,22 +246,12 @@ async function main(): Promise { const dockerEnabled = (process.env.HACK_E2E_DOCKER ?? "") === "1"; const requireDocker = (process.env.HACK_E2E_REQUIRE_DOCKER ?? "") === "1"; const keepTempDirs = (process.env.HACK_E2E_KEEP ?? "") === "1"; - // TEMPORARY HACK-1281 sampling: every shard runs only native-config-networks - // (shards 1-2 inject CLI stalls) under a natural host-stall monitor. - const monitor = startHack1281StallMonitor(); - let outcomes: Awaited>; - try { - outcomes = await runScenarios({ - scenarios: ALL_SCENARIOS, - dockerEnabled, - only: args.shard ? ["native-config-networks"] : (shardNames ?? args.only), - keepTempDirs, - }); - } finally { - process.stdout.write( - `HACK-1281 stall monitor ${JSON.stringify(await monitor())}\n` - ); - } + const outcomes = await runScenarios({ + scenarios: ALL_SCENARIOS, + dockerEnabled, + only: shardNames ?? args.only, + keepTempDirs, + }); const dockerSkipped = outcomes.some( (outcome) => outcome.tier === "docker" && outcome.status === "skip" @@ -273,62 +263,6 @@ async function main(): Promise { : 0; } -/** TEMPORARY HACK-1281 sampling: event-loop lag and the duration of a product-shaped - * process identity read (`sh -c 'ulimit; exec ps'`) every two seconds. Numbers only. */ -function startHack1281StallMonitor(): () => Promise> { - let last = performance.now(); - const lags: number[] = []; - const reads: number[] = []; - const lag = setInterval(() => { - const now = performance.now(); - lags.push(now - last - 100); - last = now; - }, 100); - let pending: Promise = Promise.resolve(); - const probe = setInterval(() => { - pending = (async () => { - const started = performance.now(); - const child = Bun.spawn( - [ - "/bin/sh", - "-c", - 'ulimit -f 8; exec "$@"', - "hack-1281-monitor", - "/bin/ps", - "-p", - String(process.pid), - "-o", - "uid=,pgid=,state=,lstart=,comm=", - ], - { stdout: "ignore", stderr: "ignore" } - ); - await child.exited; - reads.push(performance.now() - started); - })(); - }, 2000); - const summary = (values: number[], prefix: string) => { - const sorted = [...values].sort((a, b) => a - b); - const at = (p: number) => - Math.round( - sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] ?? 0 - ); - return { - [`${prefix}Count`]: sorted.length, - [`${prefix}P99Ms`]: at(0.99), - [`${prefix}MaxMs`]: Math.round(sorted.at(-1) ?? 0), - [`${prefix}Over250`]: sorted.filter((value) => value >= 250).length, - [`${prefix}Over1000`]: sorted.filter((value) => value >= 1000).length, - [`${prefix}Over3000`]: sorted.filter((value) => value >= 3000).length, - }; - }; - return async () => { - clearInterval(lag); - clearInterval(probe); - await pending; - return { ...summary(lags, "lag"), ...summary(reads, "read") }; - }; -} - if (import.meta.main) { await main(); }