fix: hermetic test fixtures, load-sized budgets and a coded env lock refusal

docs/cli.md

@@ -325,7 +325,12 @@ names the phase that was hit, its budget, and the variable that raises it; with
 `depends_on` waiting, not lifecycle hooks or an end-to-end application readiness
 deadline. A restart repair attempt has its own budgets and is reported
 separately. Foreground `up` stays a single unbounded Compose invocation; `down`
-retains its separate 90-second limit.
+retains its separate 90-second limit. A native `down` reads that limit from
+`HACK_COMPOSE_STARTUP_TIMEOUT_MS`, and each check that it still owns its containers
+may use that time too, at least 15 and at most 60 seconds. The last check before
+`compose down` must also leave about a fifth of that limit, at most 5 seconds, for the
+stop itself; otherwise `down` refuses before it starts Compose, and `hack down --recover`
+completes the stop.
 
 On timeout, Hack terminates the owned Compose process group of that phase. After an
 image-phase timeout no container was started by that attempt, but containers may

docs/env.md

@@ -715,9 +715,15 @@ message in the envelope:
 - `E_PATH_CONFLICT`: a path is the wrong kind of entry, such as a file where Hack needs
   a directory, a directory where Hack needs a file (including a `.hack/.gitignore` that
   is a directory or a symlink), or a `.hack/.internal` that `hack env hydrate` finds to
-  be a symlink (dangling or not) or a file. A file name or path that is too long for
-  the filesystem is reported with this code too; use a shorter `--env` overlay name or
+  be a symlink (dangling or not). A file name or path that is too long for the
+  filesystem is reported with this code too; use a shorter `--env` overlay name or
   move the project to a shorter path.
+- `E_CONFIG_INVALID`: `hack env hydrate` could not check its project inputs. A
+  `.hack/.internal` that is a file gives this code with `Cannot inspect Hack project
+  inputs. Check filesystem permissions and paths before retrying. Values omitted.`:
+  Hack cannot tell whether the private state it keeps inside `.hack/.internal`
+  exists, so it stops before hydrating instead of treating it as absent. Replace
+  the file with a directory (or remove it), then retry.
 - `E_LOCKED`: the project env key lock (`<git common dir>/.hack.secret.key.lock`, taken
   by the first `--secret` value) stayed held for the whole 10-second wait. Nothing was
   written. Another hack command may be creating the key; retry when it finishes. If no
@@ -775,8 +781,10 @@ command is running, remove both and retry. `hack env hydrate` reports the same m
 when it cannot get the env file's lock.
 
 `.hack/.internal/` is ignored by the managed `.hack/.gitignore`, so lock files never show
-in `git status`. If `.hack/.internal` is a symlink or not a directory, these commands
-refuse instead of creating the lock through it. If you cannot write the directory that
+in `git status`. If `.hack/.internal` is a symlink, dangling or not, these commands
+refuse with one `E_PATH_CONFLICT` line instead of creating the lock through it. If it is a
+regular file, they refuse with one `E_CONFIG_INVALID` line (`Cannot inspect Hack project
+inputs...`) and write nothing. If you cannot write the directory that
 holds the env file (for example a read-only checkout), no lock is taken: no command can
 replace the file there, so a command that changes nothing still succeeds and one that
 would change the file is refused as before.

docs/reference/native-compose-storage-xattr-carrier.md

@@ -437,7 +437,11 @@ Budgets are layered and all finite:
   helpers on loaded hosted runners as `storage-helper-start`/`probe-timeout`
   while most of the 90-second budget remained (HACK-1369). A timeout that ends
   before the carrier's deadline still reports `probe-timeout`; one that reaches
-  it reports `storage-budget`. A recorded command's 15-second settlement budget
+  it reports `storage-budget`. A hung volume create or helper create/start is
+  bounded only by the carrier's remaining budget, so it always reports
+  `storage-budget` (for a start, `storage-helper-start`/`storage-budget`), never
+  `probe-timeout`: the remedy is the storage budget the summary names
+  (HACK-1459). A recorded command's 15-second settlement budget
   follows its exit.
 - An unrecorded command (an identity read or a legacy root/seed command) waits
   at most three seconds for its process group to become absent after exit,

src/lib/env-layer-lock.ts

@@ -1,7 +1,7 @@
 import { constants } from "node:fs";
 import { access, lstat } from "node:fs/promises";
 import { basename, dirname, join } from "node:path";
-import { ClosedCliError } from "./cli-result.ts";
+import { ClosedCliError, HackCliError } from "./cli-result.ts";
 import { closedFileWriteError, ensureDirForWrite } from "./fs.ts";
 import { isRecord } from "./guards.ts";
 import {
@@ -65,14 +65,18 @@ function envLayerLockTimeoutMessage(opts: {
 }
 
 /**
- * A `.internal` entry next to the layer that is a symlink or not a directory.
- * The lock is never created through it.
+ * A `.internal` entry next to the layer that is a symlink (dangling or not) or
+ * not a directory. The lock is never created through it. Closed and
+ * value-free, with the stable `E_PATH_CONFLICT` code that `hack env hydrate`
+ * gives the same entry, so the CLI prints one `E_PATH_CONFLICT: ...` line and
+ * `--json` reports that code instead of `E_UNEXPECTED` (HACK-1459).
  */
-export class EnvLayerLockRefusedError extends ClosedCliError {
+export class EnvLayerLockRefusedError extends HackCliError {
   constructor(opts: { readonly path: string }) {
-    super(
-      `Refused to update ${opts.path}: the .internal entry beside it is a symlink or not a directory, so its write lock cannot be taken safely. Replace it with a regular directory and retry. Nothing was written.`
-    );
+    super({
+      code: "E_PATH_CONFLICT",
+      message: `Refused to update ${opts.path}: the .internal entry beside it is a symlink or not a directory, so its write lock cannot be taken safely. Replace it with a regular directory and retry. Nothing was written.`,
+    });
     this.name = "EnvLayerLockRefusedError";
   }
 }

src/lib/native-compose-command.ts

@@ -108,6 +108,9 @@ import {
   NativeComposeOwnershipError,
   type NativeComposeOwnershipObservation,
   type NativeComposeOwnershipOptions,
+  nativeComposePreSpawnOwnershipTimeoutMs,
+  nativeComposeStopChildReserveMs,
+  nativeComposeStopOwnershipTimeoutMs,
   observeSavedNativeComposeOwned,
   waitNativeComposeStartupOwned,
 } from "./native-compose-ownership.ts";
@@ -437,6 +440,13 @@ async function ownershipSelection(opts: {
     ...(opts.operation === "down" && opts.recover
       ? { recovery: "down" as const }
       : {}),
+    ...(opts.operation === "down"
+      ? {
+          timeoutMs: nativeComposeStopOwnershipTimeoutMs(
+            resolveComposeStartupTimeoutMs()
+          ),
+        }
+      : {}),
     signal: opts.signal,
   };
 }
@@ -810,8 +820,8 @@ async function savedCommand(opts: {
               >;
             } = { files: null };
             try {
-              const stopDeadline =
-                Date.now() + resolveComposeStartupTimeoutMs();
+              const stopBudgetMs = resolveComposeStartupTimeoutMs();
+              const stopDeadline = Date.now() + stopBudgetMs;
               const code = await runNativeComposeOwnedFileChild({
                 command: [
                   ...composeArgs(generation),
@@ -820,14 +830,27 @@ async function savedCommand(opts: {
                 ],
                 signal,
                 deadline: () => stopDeadline,
+                minimumChildMs: nativeComposeStopChildReserveMs(stopBudgetMs),
                 arm: async () => {
                   stopping.files = await prepareNativeComposeSavedFileStop({
                     mutation,
                     store,
                     saved,
                   });
                 },
-                assertOwned,
+                assertOwned: async (admittedMs) => {
+                  await prepared.assertOwned({
+                    timeoutMs: nativeComposePreSpawnOwnershipTimeoutMs({
+                      stopBudgetMs,
+                      admittedMs,
+                    }),
+                  });
+                  await assertNativeComposeSavedFileEngines({
+                    saved,
+                    store,
+                    signal,
+                  });
+                },
                 options: {
                   cwd: base.cwd,
                   env: base.env,
@@ -1048,8 +1071,11 @@ async function prepareSavedStop(opts: {
   const hasDownHooks = Object.hasOwn(document, "x-hack-native-down-hooks");
   let engineStopped = false;
   let storage: readonly NativeComposeRetainedVolume[] | null = null;
-  const assertOwned = async () => {
-    const observed = await assertNativeComposeOwned(selection);
+  const assertOwned = async (bound?: { readonly timeoutMs: number }) => {
+    const observed = await assertNativeComposeOwned({
+      ...selection,
+      ...bound,
+    });
     storage = observed.volumes;
     if (
       engineStopped &&

src/lib/native-compose-file-command.ts

@@ -57,7 +57,11 @@ type Saved = {
  * Cancellation never grants a later retry authority over an earlier armed child. */
 export async function runNativeComposeOwnedFileChild(opts: {
   readonly arm: () => Promise<void>;
-  readonly assertOwned: () => Promise<void>;
+  /**
+   * Final guard. `admittedMs` is the time it may use and still leave the child
+   * {@link minimumChildMs}; a guard that takes longer is refused with no child.
+   */
+  readonly assertOwned: (admittedMs: number) => Promise<void>;
   readonly command: readonly string[];
   readonly signal: AbortSignal;
   /**
@@ -66,6 +70,8 @@ export async function runNativeComposeOwnedFileChild(opts: {
    * only by storage proof time that the guard excludes from the start budget.
    */
   readonly deadline: () => number;
+  /** Least time before the deadline that admits the child; defaults to 0. */
+  readonly minimumChildMs?: number;
   readonly options: Omit<
     RunOptions,
     "timeoutMs" | "beforeSpawn" | "onSpawn" | "onExit"
@@ -82,15 +88,17 @@ export async function runNativeComposeOwnedFileChild(opts: {
     options.unsetEnvKeys = [...opts.options.unsetEnvKeys];
   }
   const hooks = opts.hooks;
+  const minimumChildMs = opts.minimumChildMs ?? 0;
+  const admittedMs = () => deadline() - minimumChildMs - Date.now();
   const requireAdmission = () => {
-    if (signal.aborted || Date.now() >= deadline()) {
+    if (signal.aborted || admittedMs() <= 0) {
       throw new NativeComposeGenerationError("E_NATIVE_COMPOSE_UNCERTAIN");
     }
   };
   requireAdmission();
   await arm();
   requireAdmission();
-  await assertOwned();
+  await assertOwned(admittedMs());
   requireAdmission();
   return await run(command, {
     ...options,

src/lib/native-compose-ownership.ts

@@ -16,6 +16,8 @@ const OUTPUT_LIMIT = 8 * 1024 * 1024;
 const STDERR_LIMIT = 16 * 1024;
 const ARGUMENT_LIMIT = 16 * 1024;
 const DEFAULT_TIMEOUT_MS = 15_000;
+/** The most {@link createNativeComposeProbe} accepts for one acquisition. */
+const MAX_PROBE_TIMEOUT_MS = 60_000;
 const PROJECT_LABEL = "com.docker.compose.project";
 const PREFIX = "io.hack.native-config";
 
@@ -1376,6 +1378,60 @@ function batches(resources: readonly Inventory[]): Inventory[][] {
   return result;
 }
 
+/**
+ * Aggregate bound of one ownership query inside a native `down`, from the stop's
+ * budget (`HACK_COMPOSE_STARTUP_TIMEOUT_MS`). A fixed 15 s bound refused a healthy
+ * but slow engine as `probe-timeout` while most of that budget remained
+ * (HACK-1459). A shorter budget never shortens the query below the 15 s default,
+ * and a hung engine is still refused within the probe's 60 s ceiling.
+ */
+export function nativeComposeStopOwnershipTimeoutMs(
+  stopBudgetMs: number
+): number {
+  return Math.min(
+    MAX_PROBE_TIMEOUT_MS,
+    Math.max(DEFAULT_TIMEOUT_MS, stopBudgetMs)
+  );
+}
+
+/** The most stop time {@link nativeComposeStopChildReserveMs} keeps for the child. */
+const MAX_STOP_CHILD_RESERVE_MS = 5000;
+
+/** The shortest budget accepted, which has no stop time to set aside. */
+const MIN_STOP_BUDGET_MS = 1000;
+
+/**
+ * Stop time kept for `compose down` itself: a fifth of the stop's budget above its
+ * 1 s minimum, at most 5 s. An engine that answers the last ownership check just
+ * before the deadline would otherwise start a child that is killed partway, with an
+ * incomplete stop. A short budget keeps nearly all of its time for that check.
+ */
+export function nativeComposeStopChildReserveMs(stopBudgetMs: number): number {
+  return Math.min(
+    MAX_STOP_CHILD_RESERVE_MS,
+    Math.floor(Math.max(0, stopBudgetMs - MIN_STOP_BUDGET_MS) / 5)
+  );
+}
+
+/**
+ * Bound of the ownership query just before `compose down` starts: the smaller of
+ * the probe timeout and the time admitted before the stop deadline, less the
+ * child's reserve. With no time left it is the probe's 1 ms minimum, which refuses
+ * as `probe-timeout` before any child runs.
+ */
+export function nativeComposePreSpawnOwnershipTimeoutMs(opts: {
+  readonly stopBudgetMs: number;
+  readonly admittedMs: number;
+}): number {
+  return Math.max(
+    1,
+    Math.min(
+      nativeComposeStopOwnershipTimeoutMs(opts.stopBudgetMs),
+      Math.floor(opts.admittedMs)
+    )
+  );
+}
+
 /**
  * One bounded aggregate Docker acquisition; callers supply only fixed read commands.
  * Lifecycle polling must issue a fresh owner for each capture, rather than reuse
@@ -1393,7 +1449,11 @@ export function createNativeComposeProbe(
 ): (args: readonly string[]) => Promise<string> {
   const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   if (
-    !(Number.isSafeInteger(timeoutMs) && timeoutMs >= 1 && timeoutMs <= 60_000)
+    !(
+      Number.isSafeInteger(timeoutMs) &&
+      timeoutMs >= 1 &&
+      timeoutMs <= MAX_PROBE_TIMEOUT_MS
+    )
   ) {
     probeRefused("E_NATIVE_COMPOSE_OWNERSHIP", "operation");
   }

tests/branches-effective-branch.test.ts

@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 
 import { resolveEffectiveBranch } from "../src/lib/branches.ts";
+import { fixtureGitEnv } from "./helpers/fixture-git.ts";
 
 const tempDirs = new Set<string>();
 
@@ -16,6 +17,7 @@ afterEach(async () => {
 
 async function runGit(args: readonly string[], cwd: string): Promise<string> {
   const proc = Bun.spawn({
+    env: fixtureGitEnv(),
     cmd: ["git", ...args],
     cwd,
     stdout: "pipe",

tests/checkout-write-guard.test.ts

@@ -18,6 +18,7 @@ import {
 } from "./helpers/checkout-write-guard.ts";
 import { expectClosedThrow } from "./helpers/closed-refusal.ts";
 import { fixtureChildren } from "./helpers/fixture-children.ts";
+import { fixtureGitEnv } from "./helpers/fixture-git.ts";
 import { requireFixtureRoot } from "./helpers/fixture-root.ts";
 import { privateCliEnv } from "./helpers/private-cli-env.ts";
 
@@ -63,14 +64,12 @@ const fixture = (): string =>
 async function run(opts: {
   readonly argv: readonly string[];
   readonly cwd: string;
-  readonly env?: Readonly<Record<string, string>>;
+  /** The complete child environment; defaults to the private CLI env. */
+  readonly env?: Record<string, string>;
 }): Promise<{ readonly exitCode: number; readonly output: string }> {
   const child = Bun.spawn([...opts.argv], {
     cwd: opts.cwd,
-    env: privateCliEnv({
-      root: fixture(),
-      env: { NO_COLOR: "1", ...opts.env },
-    }),
+    env: opts.env ?? privateCliEnv({ root: fixture(), env: { NO_COLOR: "1" } }),
     stdin: "ignore",
     stdout: "pipe",
     stderr: "pipe",
@@ -93,7 +92,11 @@ async function gitCheckout(
 ): Promise<string> {
   const checkout = join(fixture(), name);
   await mkdir(checkout);
-  const init = await run({ argv: ["git", "init", "-q"], cwd: checkout });
+  const init = await run({
+    env: fixtureGitEnv(),
+    argv: ["git", "init", "-q"],
+    cwd: checkout,
+  });
   expect(init.exitCode).toBe(0);
   const files = Object.keys(committed);
   if (files.length === 0) {
@@ -103,11 +106,13 @@ async function gitCheckout(
     await writeFile(join(checkout, file), committed[file] ?? "");
   }
   const add = await run({
+    env: fixtureGitEnv(),
     argv: ["git", "add", "--", ...files],
     cwd: checkout,
   });
   expect(add.exitCode).toBe(0);
   const commit = await run({
+    env: fixtureGitEnv(),
     argv: [
       "git",
       "-c",
@@ -451,7 +456,10 @@ test(
     const result = await run({
       argv: [process.execPath, "test"],
       cwd: checkout,
-      env: { PATH: `${bin}:${process.env.PATH ?? ""}` },
+      env: privateCliEnv({
+        root: fixture(),
+        env: { NO_COLOR: "1", PATH: `${bin}:${process.env.PATH ?? ""}` },
+      }),
     });
     expect(result.output).not.toContain("checkout write guard");
     expect(result.output).not.toContain("dangling process");

tests/dependency-cache.test.ts

@@ -8,6 +8,7 @@ import {
   resolveDependencyCacheOverride,
 } from "../src/lib/dependency-cache.ts";
 import { restoreEnv } from "./helpers/env.ts";
+import { fixtureGitEnv } from "./helpers/fixture-git.ts";
 
 const tempDirs: string[] = [];
 
@@ -184,6 +185,7 @@ test("compatible linked worktrees share the same declared runtime cache", async
   const project = await createProject();
   const git = async (args: string[]) => {
     const child = Bun.spawn(["git", ...args], {
+      env: fixtureGitEnv(),
       cwd: project.projectRoot,
       stdout: "ignore",
       stderr: "pipe",