From e649d420b330b7c13acdab3ae6fbb2c0fca22533 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 20 Oct 2026 12:53:59 -0400 Subject: [PATCH] fix(native): keep on-failure restart-backoff and recheck-placeholder members unready until the readiness deadline The startup Compose readiness check refused two Docker on-failure restart races that #350 left open: - Case 2 (captured in run 39436132066, after-compose observation 3): the retry container was inspected running, then exited, and Docker released its endpoint for the restart backoff before the same scan read the bridge. The member was absent with no placeholder, so startup refused with live-endpoint-membership. - Case 1: the retry container was an ID member in the first scan, then restarted, and the recheck read its new ep- placeholder. Startup refused with network-member-id. Startup now keeps both unready, but only for a selected, non-oneoff on-failure container read running or restarting, on the single owned default bridge. A recheck placeholder is admitted only for the same container that the first scan saw by ID or as absent. A placeholder that persists across both scans, and every foreign, malformed, strict or saved shape, still refuses as before. waitNativeComposeStartupOwned bounds this with the existing readiness deadline. If the deadline passes while the last observation was unready only because of these running shapes, one settled check refuses a member that is still missing or placeholdered. Refs HACK-1279 Co-Authored-By: Claude Opus 5.5 --- src/lib/native-compose-command.ts | 20 +- src/lib/native-compose-ownership.ts | 237 ++++++++++++++--- tests/e2e/README.md | 25 +- .../native-process-policy-initial-replay.ts | 12 +- tests/native-compose-ownership.test.ts | 245 ++++++++++++++++- ...ative-process-policy-initial-trace.test.ts | 250 +++++++++++++++++- 6 files changed, 704 insertions(+), 85 deletions(-) diff --git a/src/lib/native-compose-command.ts b/src/lib/native-compose-command.ts index 224e0f359..87c510faf 100644 --- a/src/lib/native-compose-command.ts +++ b/src/lib/native-compose-command.ts @@ -99,8 +99,8 @@ import { NativeComposeOwnershipError, type NativeComposeOwnershipObservation, type NativeComposeOwnershipOptions, - observeNativeComposeStartupOwned, observeSavedNativeComposeOwned, + waitNativeComposeStartupOwned, } from "./native-compose-ownership.ts"; import { measureNativeComposePhase, @@ -137,7 +137,6 @@ import { requireNativeComposeBackend, selectNativeComposeProject, } from "./native-compose-selection.ts"; -import { waitNativeComposeReady } from "./native-compose-wait-ready.ts"; import { NativeConfigCompilerError } from "./native-config-compiler.ts"; import { nativeFilePlanningRequired } from "./native-file-plan-protocol.ts"; import { run } from "./shell.ts"; @@ -473,20 +472,11 @@ async function waitReady(opts: { observed: NativeComposeOwnershipObservation ) => void; }): Promise { - const services = nativeComposeOnFailureServices(opts.document); - return await waitNativeComposeReady({ + return await waitNativeComposeStartupOwned({ + ownership: opts.ownership, + restartingServices: nativeComposeOnFailureServices(opts.document), deadline: opts.deadline, - signal: opts.ownership.signal, - observe: async () => { - const state = await observeNativeComposeStartupOwned( - opts.ownership, - services - ); - if (state) { - opts.observeStorage(state); - } - return state; - }, + observed: opts.observeStorage, ready: (state) => ready(opts.document, state, opts.generation), }); } diff --git a/src/lib/native-compose-ownership.ts b/src/lib/native-compose-ownership.ts index da2a537a0..7359f1eda 100644 --- a/src/lib/native-compose-ownership.ts +++ b/src/lib/native-compose-ownership.ts @@ -1,6 +1,7 @@ import { DEFAULT_INGRESS_NETWORK } from "../constants.ts"; import { isRecord } from "./guards.ts"; import { nativeComposeVolumeCreatedAt } from "./native-compose-retained-storage.ts"; +import { waitNativeComposeReady } from "./native-compose-wait-ready.ts"; import { beginNativeCpuChild } from "./native-cpu-diagnostics.ts"; import { findExecutableInPath } from "./shell.ts"; @@ -416,28 +417,99 @@ export async function assertNativeComposeOwned( .observation; } -/** Startup readiness only: an exact owned on-failure restart without bridge membership remains unready. Never use this observation as effect authority. */ -export async function observeNativeComposeStartupOwned( +/** + * How startup readiness treats an exact owned on-failure container that was inspected + * `running` while its bridge membership was missing or an `ep-` placeholder. `transient` + * keeps it unready while its restart can still be in flight. `settled` is the check after + * the readiness deadline: it refuses that shape. Both still admit a container inspected + * `restarting` exactly as before. + */ +export type NativeComposeStartupMembership = "settled" | "transient"; + +function startupSelection( input: NativeComposeOwnershipOptions, restartingServices: readonly string[] -): Promise { +): ReadonlySet { const selected = new Set(restartingServices); requireValue( input.recovery === undefined && selected.size === restartingServices.length && restartingServices.every((name) => input.expectedServices.includes(name)) ); - const result = await queryNativeComposeOwned(input, "startup", selected); + return selected; +} + +/** Startup readiness only: an exact owned on-failure restart without bridge membership remains unready. Never use this observation as effect authority. */ +export async function observeNativeComposeStartupOwned( + input: NativeComposeOwnershipOptions, + restartingServices: readonly string[], + opts: { readonly membership?: NativeComposeStartupMembership } = {} +): Promise { + const result = await queryNativeComposeOwned( + input, + "startup", + startupSelection(input, restartingServices), + (opts.membership ?? "transient") === "transient" + ); return result.restarting ? null : result.observation; } +/** + * Startup readiness wait under the caller's readiness deadline. Each observation is a + * fresh `transient` startup query. If the deadline passes while the last observation was + * unready only because a `running` on-failure member was missing or placeholdered, one + * `settled` query follows: it refuses with the unchanged topology predicate when that + * member is still absent, so a permanently missing member cannot stay unready. The + * settled query never turns a timeout into readiness; this returns `null` either way. + * The settled query runs after the deadline, so it is bounded by the probe timeout. + * Never use the observation as effect authority. + */ +export async function waitNativeComposeStartupOwned(opts: { + readonly ownership: NativeComposeOwnershipOptions; + readonly restartingServices: readonly string[]; + readonly deadline: number; + readonly ready: (state: NativeComposeOwnershipObservation) => boolean; + readonly observed?: (state: NativeComposeOwnershipObservation) => void; +}): Promise { + const selected = startupSelection(opts.ownership, opts.restartingServices); + let transient = false; + const state = await waitNativeComposeReady({ + deadline: opts.deadline, + signal: opts.ownership.signal, + observe: async () => { + const result = await queryNativeComposeOwned( + opts.ownership, + "startup", + selected, + true + ); + transient = result.restarting && result.transient; + if (result.restarting) { + return null; + } + opts.observed?.(result.observation); + return result.observation; + }, + ready: opts.ready, + }); + if (state !== null || !transient || opts.ownership.signal?.aborted) { + return state; + } + await queryNativeComposeOwned(opts.ownership, "startup", selected, false); + return null; +} + async function queryNativeComposeOwned( input: NativeComposeOwnershipOptions, mode: "saved" | "startup" | "strict", - restartingServices: ReadonlySet + restartingServices: ReadonlySet, + /** Startup only: admit the `running` restart-race shapes; see {@link NativeComposeStartupMembership}. */ + transientMembership = false ): Promise<{ readonly observation: NativeComposeOwnershipObservation; readonly restarting: boolean; + /** An admitted container depended on a shape that a `settled` query refuses. */ + readonly transient: boolean; }> { try { requireValue(mode === "strict" || input.recovery === undefined); @@ -537,14 +609,16 @@ async function queryNativeComposeOwned( scan: "first", mode, restartingServices, + transientMembership, }); } - const firstRestarting = validateTopology( + const first = validateTopology({ opts, observations, mode, - restartingServices - ); + restartingServices, + transientMembership, + }); for (const volume of opts.expectedVolumes ?? []) { if (volume.mustExist || volume.createdAt !== undefined) { requireValue( @@ -577,17 +651,20 @@ async function queryNativeComposeOwned( probe, observations: rechecked, scan: "recheck", + first: observations, mode, restartingServices, + transientMembership, }); } - const secondRestarting = validateTopology( + const second = validateTopology({ opts, - rechecked, + observations: rechecked, mode, - restartingServices - ); - const restarting = new Set([...firstRestarting, ...secondRestarting]); + restartingServices, + transientMembership, + }); + const restarting = new Set([...first.restarting, ...second.restarting]); for (const id of restarting) { const first = observations.containers.find((value) => value.id === id); const second = rechecked.containers.find((value) => value.id === id); @@ -625,6 +702,10 @@ async function queryNativeComposeOwned( networks: rechecked.networks, }, restarting: restarting.size > 0, + transient: + first.transient || + second.transient || + [...rechecked.placeholders.values()].some((ids) => ids.size > 0), }; } catch (error: unknown) { if (error instanceof NativeComposeOwnershipError) { @@ -654,10 +735,13 @@ async function collectInspections(input: { readonly selection: NativeComposeOwnershipOptions; readonly probe: (args: readonly string[]) => Promise; readonly observations: MutableObservation; - /** Only the first startup scan may admit a restarting endpoint placeholder; the recheck must resolve it. */ + /** Placeholder admission differs by scan; see {@link networkMembers}. */ readonly scan: "first" | "recheck"; + /** The same query's first-scan observations; the recheck scan only. */ + readonly first?: MutableObservation; readonly mode: "saved" | "startup" | "strict"; readonly restartingServices: ReadonlySet; + readonly transientMembership: boolean; }): Promise { const { kind, resources, selection: opts, probe, observations } = input; const { containers, volumes, networks } = observations; @@ -767,8 +851,10 @@ async function collectInspections(input: { members: row.containers, network: resource.name, observations, - admitRestartingPlaceholders: - input.scan === "first" && input.mode === "startup", + startup: input.mode === "startup", + scan: input.scan, + first: input.first, + transientMembership: input.transientMembership, restartingServices: input.restartingServices, }); observations.members.set(resource.name, members); @@ -780,23 +866,41 @@ async function collectInspections(input: { } /** - * Member keys are immutable container IDs. While a restarting container's endpoint - * exists without its sandbox, Docker keys it `ep-` instead; that placeholder - * is never a member identity. Only the first startup scan tolerates one such key per - * selected on-failure container whose same-scan inspect reports state `restarting` or - * `running` and an endpoint on this network. `running` is admitted because the scan - * inspects containers before networks and Docker does not lock them together: a - * container observed running can exit and begin its on-failure restart before the - * bridge is read. The live-endpoint admission keeps every admitted container unready; - * the recheck scan must observe only container IDs, so an unresolved, created/exited, - * oneoff or unselected placeholder refuses with the unchanged `network-member-id` - * predicate. Key and name values are compared only, never retained or disclosed. + * Member keys are immutable container IDs. While a container's endpoint exists without + * its sandbox, Docker keys it `ep-` instead; that placeholder is never a + * member identity. + * + * Docker on-failure restart semantics (the reason any placeholder or missing member is + * tolerated): when a container whose policy allows another attempt exits non-zero, the + * daemon increments its restart count, releases its network endpoints as part of exit + * cleanup, and waits a backoff (100 ms, doubling per quick failure) before starting the + * same container ID again. During that backoff the bridge lists no key for it at all. + * The restart then recreates the endpoint before the sandbox joins, so for a moment the + * bridge keys it `ep-`; after the join the key is the container ID again. + * Each query reads containers, then volumes, then networks, and Docker does not lock + * these reads together, so the container read and the bridge read can straddle any of + * those transitions. + * + * Only a startup read admits a placeholder, and only for one selected on-failure, + * non-oneoff container with that exact name, an endpoint on this network, and no + * container-ID membership. The first scan admits a container inspected `restarting`, or + * `running` while membership is `transient` (inspected before it exited). The recheck + * admits one only while membership is `transient`, for a container inspected `running` + * or `restarting` that the first scan saw by the same ID and name without a placeholder: + * it was a member or absent there, then exited and began restarting between the scans. + * A placeholder that persists from the first scan into the recheck, and every + * created/exited, oneoff, unselected, foreign or malformed key, refuses with the + * unchanged `network-member-id` predicate. Admission keeps the container unready, never + * ready. Key and name values are compared only, never retained or disclosed. */ function networkMembers(opts: { readonly members: Record; readonly network: string; readonly observations: MutableObservation; - readonly admitRestartingPlaceholders: boolean; + readonly startup: boolean; + readonly scan: "first" | "recheck"; + readonly first: MutableObservation | undefined; + readonly transientMembership: boolean; readonly restartingServices: ReadonlySet; }): { readonly members: readonly string[]; @@ -809,7 +913,8 @@ function networkMembers(opts: { continue; } requireValue( - opts.admitRestartingPlaceholders && + opts.startup && + (opts.scan === "first" || opts.transientMembership) && ENDPOINT_PLACEHOLDER.test(key) && isRecord(member) && typeof member.Name === "string", @@ -823,7 +928,8 @@ function networkMembers(opts: { requireValue( owners.length === 1 && owner !== undefined && - (owner.state === "restarting" || owner.state === "running") && + (owner.state === "restarting" || + (owner.state === "running" && opts.transientMembership)) && !owner.oneoff && opts.restartingServices.has(owner.service) && !members.includes(owner.id) && @@ -831,7 +937,13 @@ function networkMembers(opts: { Object.hasOwn( opts.observations.endpoints.get(owner.id) ?? {}, opts.network - ), + ) && + (opts.scan === "first" || + restartedBetweenScans({ + first: opts.first, + owner, + network: opts.network, + })), "topology", "network-member-id" ); @@ -840,6 +952,28 @@ function networkMembers(opts: { return { members: members.sort(), placeholders: claimed }; } +/** The first scan saw this exact container, live, as an ID member or absent, never as a placeholder. */ +function restartedBetweenScans(opts: { + readonly first: MutableObservation | undefined; + readonly owner: NativeComposeContainerObservation; + readonly network: string; +}): boolean { + const { first, owner, network } = opts; + const previous = first?.containers.find( + (container) => container.id === owner.id + ); + return ( + first !== undefined && + previous !== undefined && + previous.name === owner.name && + previous.service === owner.service && + !previous.oneoff && + (previous.state === "running" || previous.state === "restarting") && + first.members.has(network) && + first.placeholders.get(network)?.has(owner.id) !== true + ); +} + function containerObservation( row: Record, opts: NativeComposeOwnershipOptions @@ -1010,6 +1144,17 @@ function endpointContainerName(opts: { return predecessors.length === 1 ? canonical : container.name; } +/** + * A live container must be a container-ID member of each owned bridge it is attached + * to. Startup admits (keeps unready) one that is not, only on the single owned outbound + * default bridge, only for a selected on-failure, non-oneoff container, and only while + * its restart can explain the gap: inspected `restarting` (the backoff), or inspected + * `running` while membership is `transient`. A `running` container can be read just + * before it exits; Docker then releases its endpoint for the restart backoff before the + * bridge read (no key) or recreates it before the sandbox joins (an `ep-` key; see + * {@link networkMembers}). The caller's readiness deadline bounds `transient`: the + * `settled` check after it refuses a `running` container still missing here. + */ function requireLiveMember(opts: { readonly selection: NativeComposeOwnershipOptions; readonly observations: MutableObservation; @@ -1020,6 +1165,7 @@ function requireLiveMember(opts: { readonly endpoint: Record; readonly mode: "saved" | "startup" | "strict"; readonly restartingServices: ReadonlySet; + readonly transientMembership: boolean; }): boolean { const { selection, @@ -1031,6 +1177,7 @@ function requireLiveMember(opts: { endpoint, mode, restartingServices, + transientMembership, } = opts; if ( attachment.externalId !== undefined || @@ -1042,9 +1189,7 @@ function requireLiveMember(opts: { requireValue( mode === "startup" && (container.state === "restarting" || - (container.state === "running" && - observations.placeholders.get(attachment.name)?.has(container.id) === - true)) && + (container.state === "running" && transientMembership)) && !container.oneoff && restartingServices.has(container.service) && policy.networks.length === 1 && @@ -1059,13 +1204,21 @@ function requireLiveMember(opts: { return true; } -function validateTopology( - opts: NativeComposeOwnershipOptions, - observations: MutableObservation, - mode: "saved" | "startup" | "strict", - restartingServices: ReadonlySet -): ReadonlySet { +function validateTopology(input: { + readonly opts: NativeComposeOwnershipOptions; + readonly observations: MutableObservation; + readonly mode: "saved" | "startup" | "strict"; + readonly restartingServices: ReadonlySet; + readonly transientMembership: boolean; +}): { + /** Containers admitted without container-ID membership; they keep startup unready. */ + readonly restarting: ReadonlySet; + /** Some admitted container was inspected `running`, which only `transient` membership admits. */ + readonly transient: boolean; +} { + const { opts, observations, mode, restartingServices } = input; const restarting = new Set(); + let transient = false; const policies = workloadNetworks(opts); const owned = new Map( observations.networks.map((network) => [network.name, network.id]) @@ -1156,13 +1309,15 @@ function validateTopology( endpoint, mode, restartingServices, + transientMembership: input.transientMembership, }) ) { restarting.add(container.id); + transient ||= container.state === "running"; } } } - return restarting; + return { restarting, transient }; } function topologySnapshot( diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 4902e0cfd..9845e5f2e 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -401,13 +401,24 @@ for the bridge the `network-member-id` predicate inspected: counts of `hex64`, service names observed in the `restarting` state. The keys themselves are never emitted. This evidence classifies the flake; the diagnostics do not change the guard. -The startup readiness guard itself admits one `ep-` placeholder only in the first scan -of a query, and only for a selected on-failure container that the same scan inspected -as `restarting` or `running`. `running` is included because containers are inspected -before the bridge, so the retry container can exit and begin its restart between the -two reads. That container stays unready, and the recheck scan must see its container -ID. A placeholder that persists, or one that names any other container, still refuses -with `network-member-id`. +The startup readiness guard follows Docker's on-failure restart: on a failing exit the +daemon releases the container's endpoint, waits a backoff, then recreates the endpoint +(keyed `ep-` until the sandbox joins) for the same container ID. Containers +are inspected before the bridge, and Docker does not lock the reads together. So for a +selected, non-oneoff on-failure container inspected `restarting` or `running`, startup +stays unready (never ready, never effect authority) when its bridge membership is +missing (the backoff, HACK-1279 case 2) or is one `ep-` placeholder in the first scan +(HACK-1271). A placeholder first seen in the recheck scan is admitted only for the same +container that the first scan saw by ID or as absent (HACK-1279 case 1). A placeholder +that persists from the first scan into the recheck, or one that names any other +container, still refuses with `network-member-id`. Strict and saved reads admit none of +these shapes. + +The readiness deadline bounds the `running` shapes. If it passes while the last +observation was unready only because of them, one settled check follows. That check +refuses a member that is still missing or placeholdered with the original topology +predicate, so a permanently missing member cannot stay unready. The replay harness +exposes this check as mode `settled`. ## Native config routing qualification diff --git a/tests/e2e/native-process-policy-initial-replay.ts b/tests/e2e/native-process-policy-initial-replay.ts index 12cf81e82..18daa785c 100644 --- a/tests/e2e/native-process-policy-initial-replay.ts +++ b/tests/e2e/native-process-policy-initial-replay.ts @@ -115,12 +115,20 @@ export function summarizeConsumedProcessPolicyReplies( }; } +const REPLAY_CALLS = { + startup: 'await observeNativeComposeStartupOwned(selection,["retry"])', + settled: + 'await observeNativeComposeStartupOwned(selection,["retry"],{membership:"settled"})', + strict: "await assertNativeComposeOwned(selection)", +} as const; + /** Run the real ownership policy against original recorded replies, with no engine access. */ export async function replayProcessPolicyInitialOwnership(opts: { readonly directory: string; readonly queries: readonly ProcessPolicyInitialTraceQuery[]; readonly selection: NativeComposeOwnershipOptions; - readonly mode: "startup" | "strict"; + /** `settled` is the startup check after the readiness deadline: no transient running-member admission. */ + readonly mode: "settled" | "startup" | "strict"; readonly timeoutMs?: number; }): Promise { await mkdir(opts.directory, { mode: 0o700 }); @@ -154,7 +162,7 @@ await Bun.write(Bun.stdout,row.stdout);process.exit(row.exitCode); import {assertNativeComposeOwned,observeNativeComposeStartupOwned,NativeComposeOwnershipError,nativeComposeOwnershipRefusal,nativeComposeTopologyPredicate} from ${JSON.stringify(join(import.meta.dir, "../../src/lib/native-compose-ownership.ts"))}; const {selection}=await Bun.file(${JSON.stringify(inputs)}).json(); let outcome="refused",code=null,reason=null,topologyPredicate=null; -try {const value=${opts.mode === "startup" ? 'await observeNativeComposeStartupOwned(selection,["retry"])' : "await assertNativeComposeOwned(selection)"};outcome=value===null?"unready":"owned";} +try {const value=${REPLAY_CALLS[opts.mode]};outcome=value===null?"unready":"owned";} catch(error){code=error instanceof NativeComposeOwnershipError?error.code:null;reason=nativeComposeOwnershipRefusal(error)??null;topologyPredicate=nativeComposeTopologyPredicate(error)??null;} const consumed=(await Bun.file(${JSON.stringify(cursor)}).exists())?Number(await Bun.file(${JSON.stringify(cursor)}).text()):0; process.stdout.write(JSON.stringify({outcome,code,reason,topologyPredicate,consumed,protocolMatched:!(await Bun.file(${JSON.stringify(mismatch)}).exists())})); diff --git a/tests/native-compose-ownership.test.ts b/tests/native-compose-ownership.test.ts index cccc73db8..51d711a47 100644 --- a/tests/native-compose-ownership.test.ts +++ b/tests/native-compose-ownership.test.ts @@ -20,6 +20,7 @@ import { nativeComposeTopologyPredicate, observeNativeComposeStartupOwned, observeSavedNativeComposeOwned, + waitNativeComposeStartupOwned, } from "../src/lib/native-compose-ownership.ts"; import { restoreEnv } from "./helpers/env.ts"; @@ -55,6 +56,11 @@ type Fixture = { container?: Record[]; volume?: Record[]; network?: Record[]; + /** Inspect rows served after the first inspect of that kind (the recheck scan onward). */ + recheck?: { + container?: Record[]; + network?: Record[]; + }; mode?: string; }; let root: string; @@ -96,11 +102,11 @@ if (action === "ls") { for (const row of rows) console.log(JSON.stringify(row)); } else { const selected = args.slice(args.indexOf("--format") + 2); - const rows = fixture[kind] ?? []; if (fixture.mode === "vanished") process.exit(1); const counter = root + "/inspects-" + kind; const count = existsSync(counter) ? Number(readFileSync(counter, "utf8")) : 0; writeFileSync(counter, String(count + 1)); + const rows = (count > 0 ? fixture.recheck?.[kind] : undefined) ?? fixture[kind] ?? []; if (fixture.mode === "endpoint-drift" && kind === "container" && count > 0) { rows[0].networks[Object.keys(rows[0].networks)[0]].NetworkID = "f".repeat(64); } @@ -599,7 +605,17 @@ test("a placeholder for a running on-failure container inspected before its rest expect(nativeComposeOwnershipRefusal(unresolved)).toBe("topology"); expect(nativeComposeTopologyPredicate(unresolved)).toBe("network-member-id"); expect(String(unresolved)).not.toContain(PLACEHOLDER_KEY); - // A running container with no placeholder and no membership remains fenced. + // The settled check after the readiness deadline refuses the running placeholder. + fixture.mode = "restart-placeholder-transition"; + await prepare(fixture); + const settled = await observeNativeComposeStartupOwned( + placeholderOptions, + ["retry"], + { membership: "settled" } + ).catch((error: unknown) => error); + expect(nativeComposeTopologyPredicate(settled)).toBe("network-member-id"); + // A running container with no placeholder and no membership (its restart backoff, + // HACK-1279) stays unready, and the settled check refuses it. const memberless = restartingPlaceholder(); const memberlessRetry = memberless.container?.[3]; const memberlessNetwork = memberless.network?.[0]; @@ -611,13 +627,26 @@ test("a placeholder for a running on-failure container inspected before its rest PLACEHOLDER_KEY ]; await prepare(memberless); - const fenced = await observeNativeComposeStartupOwned(placeholderOptions, [ - "retry", - ]).catch((error: unknown) => error); + expect( + await observeNativeComposeStartupOwned(placeholderOptions, ["retry"]) + ).toBeNull(); + await prepare(memberless); + const fenced = await observeNativeComposeStartupOwned( + placeholderOptions, + ["retry"], + { membership: "settled" } + ).catch((error: unknown) => error); expect(fenced).toBeInstanceOf(NativeComposeOwnershipError); expect(nativeComposeTopologyPredicate(fenced)).toBe( "live-endpoint-membership" ); + await prepare(memberless); + await expectRefusal( + placeholderOptions, + "E_NATIVE_COMPOSE_OWNERSHIP", + "topology", + "live-endpoint-membership" + ); expect(await Bun.file(join(root, "mutated")).exists()).toBe(false); }); @@ -677,6 +706,131 @@ test("endpoint placeholders for non-restarting, unknown, duplicate or memberless expect(await Bun.file(join(root, "mutated")).exists()).toBe(false); }); +/** + * HACK-1279 case 1: `retry` was a container-ID member in the first scan, then exited and + * restarted; the recheck reads its new endpoint before the sandbox joins. + */ +function recheckPlaceholder(recheckState: "restarting" | "running"): Fixture { + const fixture = restartingPlaceholder(); + const retry = fixture.container?.[3]; + const network = fixture.network?.[0]; + if (!(retry && network)) { + throw new Error("Missing owned placeholder fixture"); + } + retry.state = "running"; + retry.exitCode = 0; + const recheck = structuredClone(fixture); + const members = network.containers as Record; + members[RETRY_ID] = members[PLACEHOLDER_KEY]; + delete members[PLACEHOLDER_KEY]; + const recheckRetry = recheck.container?.[3]; + if (!recheckRetry) { + throw new Error("Missing owned placeholder fixture"); + } + recheckRetry.state = recheckState; + recheckRetry.exitCode = recheckState === "restarting" ? 17 : 0; + fixture.recheck = { container: recheck.container, network: recheck.network }; + return fixture; +} + +test("a placeholder first seen in the recheck scan stays unready only for the exact restarted container", async () => { + for (const state of ["running", "restarting"] as const) { + await prepare(recheckPlaceholder(state)); + const before = await inspectCount(); + expect( + await observeNativeComposeStartupOwned(placeholderOptions, ["retry"]) + ).toBeNull(); + expect((await inspectCount()) - before).toBe(6); + // Strict, saved and settled reads never admit it. + await prepare(recheckPlaceholder(state)); + await expectRefusal( + placeholderOptions, + "E_NATIVE_COMPOSE_OWNERSHIP", + "topology", + "network-member-id" + ); + await prepare(recheckPlaceholder(state)); + await expect( + observeSavedNativeComposeOwned(placeholderOptions) + ).rejects.toBeInstanceOf(NativeComposeOwnershipError); + await prepare(recheckPlaceholder(state)); + const settled = await observeNativeComposeStartupOwned( + placeholderOptions, + ["retry"], + { membership: "settled" } + ).catch((error: unknown) => error); + expect(nativeComposeTopologyPredicate(settled)).toBe("network-member-id"); + } + for (const change of [ + "created", + "exited", + "unknown-name", + "missing-name", + "non-record", + "also-member", + "duplicate", + "other-prefix", + "no-endpoint", + "unselected", + "oneoff", + "first-exited", + "first-placeholder", + ]) { + const fixture = recheckPlaceholder("running"); + const network = fixture.recheck?.network?.[0]; + const retry = fixture.recheck?.container?.[3]; + const firstRetry = fixture.container?.[3]; + const firstNetwork = fixture.network?.[0]; + if (!(network && retry && firstRetry && firstNetwork)) { + throw new Error("Missing owned placeholder fixture"); + } + const members = network.containers as Record; + const firstMembers = firstNetwork.containers as Record; + if (change === "created" || change === "exited") { + retry.state = change; + } else if (change === "unknown-name") { + members[PLACEHOLDER_KEY] = { Name: `${PROJECT}-foreign-1` }; + } else if (change === "missing-name") { + members[PLACEHOLDER_KEY] = {}; + } else if (change === "non-record") { + members[PLACEHOLDER_KEY] = `${PROJECT}-retry-1`; + } else if (change === "also-member") { + members[RETRY_ID] = { Name: `${PROJECT}-retry-1` }; + } else if (change === "duplicate") { + members[`ep-${"8".repeat(64)}`] = { Name: `${PROJECT}-retry-1` }; + } else if (change === "other-prefix") { + delete members[PLACEHOLDER_KEY]; + members[`endpoint-${"9".repeat(64)}`] = { Name: `${PROJECT}-retry-1` }; + } else if (change === "no-endpoint") { + retry.networks = {}; + } else if (change === "oneoff") { + retry.oneoff = "True"; + } else if (change === "first-exited") { + // The first scan must have seen the same container live (on-failure backoff + // reports `restarting`, never `exited`). + firstRetry.state = "exited"; + firstRetry.exitCode = 17; + delete firstMembers[RETRY_ID]; + } else if (change === "first-placeholder") { + // A placeholder that persists from the first scan into the recheck. + delete firstMembers[RETRY_ID]; + firstMembers[PLACEHOLDER_KEY] = { Name: `${PROJECT}-retry-1` }; + } + await prepare(fixture); + const error = await observeNativeComposeStartupOwned( + placeholderOptions, + change === "unselected" ? ["web"] : ["retry"] + ).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(NativeComposeOwnershipError); + expect(nativeComposeOwnershipRefusal(error)).toBe("topology"); + expect([change, nativeComposeTopologyPredicate(error)]).toEqual([ + change, + "network-member-id", + ]); + } + expect(await Bun.file(join(root, "mutated")).exists()).toBe(false); +}); + test("restart observation keeps foreign and incomplete endpoint shapes fenced", async () => { for (const change of [ "wrong-service", @@ -744,16 +898,89 @@ test("restart observation keeps foreign and incomplete endpoint shapes fenced", ], }; } + const restarting = change === "wrong-service" ? ["install"] : ["web"]; + // A selected `running` container absent from its bridge is the restart-backoff race + // (HACK-1279): unready while transient, refused once settled. Every other shape + // refuses in both. + await prepare(fixture); + if (change === "running") { + expect( + await observeNativeComposeStartupOwned(selection, restarting) + ).toBeNull(); + } else { + await expect( + observeNativeComposeStartupOwned(selection, restarting) + ).rejects.toBeInstanceOf(NativeComposeOwnershipError); + } await prepare(fixture); await expect( - observeNativeComposeStartupOwned( - selection, - change === "wrong-service" ? ["install"] : ["web"] - ) + observeNativeComposeStartupOwned(selection, restarting, { + membership: "settled", + }) ).rejects.toBeInstanceOf(NativeComposeOwnershipError); } expect(await Bun.file(join(root, "mutated")).exists()).toBe(false); }); + +/** HACK-1279: a restart race may keep startup unready only until the caller's readiness deadline. */ +test("startup wait bounds a missing running member by the readiness deadline", async () => { + const backoff = owned(); + const container = backoff.container?.[0]; + const network = backoff.network?.[0]; + if (!(container && network)) { + throw new Error("Missing owned retry fixture"); + } + // Inspected `running`, then absent from the bridge: the restart backoff race. + network.containers = {}; + const wait = (deadline: number) => + waitNativeComposeStartupOwned({ + ownership: options, + restartingServices: ["web"], + deadline, + ready: () => true, + }); + // Permanently missing: unready until the deadline, then the settled check refuses. + await prepare(backoff); + const missing = await wait(Date.now() + 250).catch((error: unknown) => error); + expect(missing).toBeInstanceOf(NativeComposeOwnershipError); + expect(nativeComposeOwnershipRefusal(missing)).toBe("topology"); + expect(nativeComposeTopologyPredicate(missing)).toBe( + "live-endpoint-membership" + ); + // The member rejoins: the next observation is owned and ready before the deadline. + backoff.mode = "restart-member-transition"; + await prepare(backoff); + const observed: unknown[] = []; + expect( + ( + await waitNativeComposeStartupOwned({ + ownership: options, + restartingServices: ["web"], + deadline: Date.now() + 30_000, + ready: () => true, + observed: (state) => observed.push(state), + }) + )?.containers[0] + ).toMatchObject({ id: ID, state: "running" }); + expect(observed).toHaveLength(1); + // A container inspected `restarting` past the deadline times out unready, as before. + const restarting = owned(); + const restartingContainer = restarting.container?.[0]; + const restartingNetwork = restarting.network?.[0]; + if (!(restartingContainer && restartingNetwork)) { + throw new Error("Missing owned retry fixture"); + } + restartingContainer.state = "restarting"; + restartingNetwork.containers = {}; + await prepare(restarting); + expect(await wait(Date.now() + 250)).toBeNull(); + // A deadline that passes before the first observation reads nothing. + await prepare(backoff); + const before = await inspectCount(); + expect(await wait(Date.now() - 1)).toBeNull(); + expect(await inspectCount()).toBe(before); + expect(await Bun.file(join(root, "mutated")).exists()).toBe(false); +}); test("owned resources yield only bounded readiness observations and exact read-only queries", async () => { await prepare(owned()); expect(await assertNativeComposeOwned(options)).toEqual({ diff --git a/tests/native-process-policy-initial-trace.test.ts b/tests/native-process-policy-initial-trace.test.ts index 3beee72c0..9439adfc3 100644 --- a/tests/native-process-policy-initial-trace.test.ts +++ b/tests/native-process-policy-initial-trace.test.ts @@ -863,9 +863,13 @@ test("startup diagnostic classifies refused member keys instead of hiding the ca } }); -/** The captured docker-e2e after-compose shape: three container-ID members and one `ep-` placeholder for the restarting retry service. */ -test("trace replay re-observes a restarting or just-restarted endpoint placeholder member and refuses an unresolved one", async () => { - const root = await mkdtemp(join(tmpdir(), "process-initial-placeholder-")); +/** One retry-service scan: its bridge key (`null` when absent from the bridge) and its inspected state. */ +type RetryScan = { + readonly retryKey: string | null; + readonly retryState: "running" | "restarting"; +}; +/** The process-policy fixture fleet: three settled services plus the on-failure `retry` service, as one after-compose query of 12 recorded replies. */ +function restartFleet() { const services = ["forced", "graceful", "reaper", "retry"] as const; const ids = { forced: "1".repeat(64), @@ -901,14 +905,16 @@ test("trace replay re-observes a restarting or just-restarted endpoint placehold }, }, }); - const members = (retryKey: string) => + const members = (retryKey: string | null) => Object.fromEntries( - services.map((service) => [ - service === "retry" ? retryKey : ids[service], - { Name: `${project}-${service}-1` }, - ]) + services + .filter((service) => service !== "retry" || retryKey !== null) + .map((service) => [ + service === "retry" ? retryKey : ids[service], + { Name: `${project}-${service}-1` }, + ]) ); - const network = (retryKey: string) => + const network = (retryKey: string | null) => JSON.stringify({ id: networkId, name: `${project}_default`, @@ -968,8 +974,8 @@ test("trace replay re-observes a restarting or just-restarted endpoint placehold project, }); const queries = ( - first: { retryKey: string; retryState: "running" | "restarting" }, - recheck: { retryKey: string; retryState: "running" | "restarting" } + first: RetryScan, + recheck: RetryScan ): ProcessPolicyInitialTraceQuery[] => { const calls: [string[], string][] = [ [list("container"), containerList], @@ -996,6 +1002,13 @@ test("trace replay re-observes a restarting or just-restarted endpoint placehold exitCode: 0, })); }; + return { fleet, ids, placeholder, queries }; +} + +/** The captured docker-e2e after-compose shape: three container-ID members and one `ep-` placeholder for the restarting retry service. */ +test("trace replay re-observes a restarting or just-restarted endpoint placeholder member and refuses an unresolved one", async () => { + const root = await mkdtemp(join(tmpdir(), "process-initial-placeholder-")); + const { fleet, ids, placeholder, queries } = restartFleet(); const resolved = queries( { retryKey: placeholder, retryState: "restarting" }, { retryKey: ids.retry, retryState: "running" } @@ -1146,3 +1159,218 @@ test("trace replay re-observes a restarting or just-restarted endpoint placehold await rm(root, { recursive: true, force: true }); } }, 30_000); + +/** + * HACK-1279 capture: run 39436132066, job 118673527798, scenario + * `native-config-process-policy`, after-compose observation 3. These are the capsule's + * fixed fields; the capsule carries no key, ID or label value. Startup and strict both + * refused at the first scan's topology check, the bridge held three container-ID members + * and no placeholder, and no service was inspected as `restarting`. Only `retry` + * (on-failure, max 2) restarts in this fixture, so the absent member is `retry`. + */ +const HACK_1279_CAPSULE = { + observationIndex: 3, + startupReason: "topology", + strictReason: "topology", + startupTopologyPredicate: "live-endpoint-membership", + strictTopologyPredicate: "live-endpoint-membership", + startupMemberKeyClasses: { hex64: 3, epPrefixed: 0, other: 0 }, + strictMemberKeyClasses: { hex64: 3, epPrefixed: 0, other: 0 }, + startupRestartingServices: [], + strictRestartingServices: [], +} as const; +/** The capsule's refusal as a replay result; the original query stopped after the first scan's six replies. */ +const HACK_1279_REFUSAL = { + outcome: "refused", + code: "E_NATIVE_COMPOSE_OWNERSHIP", + reason: HACK_1279_CAPSULE.strictReason, + topologyPredicate: HACK_1279_CAPSULE.strictTopologyPredicate, + consumed: 6, + protocolMatched: true, + memberKeyClasses: HACK_1279_CAPSULE.strictMemberKeyClasses, + restartingServices: HACK_1279_CAPSULE.strictRestartingServices, +} as const; +type ReplayMode = "settled" | "startup" | "strict"; + +test("trace replay keeps an on-failure member absent during its restart backoff unready instead of refusing (HACK-1279 case 2)", async () => { + const root = await mkdtemp(join(tmpdir(), "process-initial-backoff-")); + const { fleet, ids, placeholder, queries } = restartFleet(); + const replay = ( + name: string, + mode: ReplayMode, + recorded: ProcessPolicyInitialTraceQuery[] + ) => + replayProcessPolicyInitialOwnership({ + directory: join(root, name), + queries: recorded, + selection: fleet, + mode, + }); + // The capture: `retry` was inspected `running`, then exited, and Docker released its + // endpoint before the same scan read the bridge. + const captured = { retryKey: null, retryState: "running" } as const; + try { + expect(HACK_1279_CAPSULE.observationIndex).toBe(3); + // The original query recorded no recheck. Every recheck Docker can produce next is + // unready: the endpoint rejoined, the backoff still running, the new endpoint not yet + // joined to its sandbox, or the container read `running` again before the bridge. + const rechecks = [ + { + name: "rejoined", + recheck: { retryKey: ids.retry, retryState: "running" }, + memberKeyClasses: { hex64: 4, epPrefixed: 0, other: 0 }, + restartingServices: [], + }, + { + name: "backoff", + recheck: { retryKey: null, retryState: "restarting" }, + memberKeyClasses: { hex64: 3, epPrefixed: 0, other: 0 }, + restartingServices: ["retry"], + }, + { + name: "joining", + recheck: { retryKey: placeholder, retryState: "restarting" }, + memberKeyClasses: { hex64: 3, epPrefixed: 1, other: 0 }, + restartingServices: ["retry"], + }, + { + name: "absent", + recheck: { retryKey: null, retryState: "running" }, + memberKeyClasses: { hex64: 3, epPrefixed: 0, other: 0 }, + restartingServices: [], + }, + ] as const; + for (const item of rechecks) { + const recorded = queries(captured, item.recheck); + expect(await replay(`startup-${item.name}`, "startup", recorded)).toEqual( + { + outcome: "unready", + code: null, + reason: null, + topologyPredicate: null, + consumed: recorded.length, + protocolMatched: true, + memberKeyClasses: item.memberKeyClasses, + restartingServices: item.restartingServices, + } + ); + // Strict effect authority still refuses exactly as captured. + expect(await replay(`strict-${item.name}`, "strict", recorded)).toEqual( + HACK_1279_REFUSAL + ); + } + // Once the readiness deadline has passed, the settled check refuses a member that is + // still absent with the captured startup refusal. + expect( + await replay("settled", "settled", queries(captured, captured)) + ).toEqual({ + ...HACK_1279_REFUSAL, + reason: HACK_1279_CAPSULE.startupReason, + topologyPredicate: HACK_1279_CAPSULE.startupTopologyPredicate, + memberKeyClasses: HACK_1279_CAPSULE.startupMemberKeyClasses, + restartingServices: HACK_1279_CAPSULE.startupRestartingServices, + }); + // A member that has rejoined by the settled check is owned. + const rejoined = queries( + { retryKey: ids.retry, retryState: "running" }, + { retryKey: ids.retry, retryState: "running" } + ); + expect(await replay("settled-rejoined", "settled", rejoined)).toMatchObject( + { outcome: "owned", consumed: rejoined.length } + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}, 60_000); + +test("trace replay keeps a member whose placeholder first appears in the recheck scan unready (HACK-1279 case 1)", async () => { + const root = await mkdtemp(join(tmpdir(), "process-initial-recheck-")); + const { fleet, ids, placeholder, queries } = restartFleet(); + const replay = ( + name: string, + mode: ReplayMode, + recorded: ProcessPolicyInitialTraceQuery[] + ) => + replayProcessPolicyInitialOwnership({ + directory: join(root, name), + queries: recorded, + selection: fleet, + mode, + }); + const refusedAtRecheck = (restartingServices: readonly "retry"[]) => ({ + outcome: "refused" as const, + code: "E_NATIVE_COMPOSE_OWNERSHIP" as const, + reason: "topology" as const, + topologyPredicate: "network-member-id" as const, + consumed: 9, + protocolMatched: true, + memberKeyClasses: { hex64: 3, epPrefixed: 1, other: 0 }, + restartingServices, + }); + try { + // `retry` was a container-ID member in the first scan, then exited and restarted; the + // recheck read its new endpoint before its sandbox joined. + const shapes = [ + { + name: "running", + first: { retryKey: ids.retry, retryState: "running" }, + recheck: { retryKey: placeholder, retryState: "running" }, + }, + { + name: "restarting", + first: { retryKey: ids.retry, retryState: "running" }, + recheck: { retryKey: placeholder, retryState: "restarting" }, + }, + // Case 2 progressing within one query: absent in the first scan, joining in the recheck. + { + name: "backoff-joining", + first: { retryKey: null, retryState: "running" }, + recheck: { retryKey: placeholder, retryState: "running" }, + }, + ] as const; + for (const { name, first, recheck } of shapes) { + const recorded = queries(first, recheck); + const restartingServices: readonly "retry"[] = + recheck.retryState === "restarting" ? ["retry"] : []; + expect(await replay(`startup-${name}`, "startup", recorded)).toEqual({ + outcome: "unready", + code: null, + reason: null, + topologyPredicate: null, + consumed: recorded.length, + protocolMatched: true, + memberKeyClasses: { hex64: 3, epPrefixed: 1, other: 0 }, + restartingServices, + }); + // The settled check never admits a recheck placeholder or a running absent member. + expect(await replay(`settled-${name}`, "settled", recorded)).toEqual( + first.retryKey === null + ? HACK_1279_REFUSAL + : refusedAtRecheck(restartingServices) + ); + } + // Strict effect authority refuses the recheck placeholder. + expect( + await replay( + "strict", + "strict", + queries(shapes[0].first, shapes[0].recheck) + ) + ).toEqual(refusedAtRecheck([])); + // A placeholder that persists from the first scan into the recheck still refuses. + for (const retryState of ["running", "restarting"] as const) { + expect( + await replay( + `persistent-${retryState}`, + "startup", + queries( + { retryKey: placeholder, retryState }, + { retryKey: placeholder, retryState } + ) + ) + ).toEqual(refusedAtRecheck(retryState === "restarting" ? ["retry"] : [])); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}, 60_000);