train b33 integration gate (do not merge)

docs/architecture.md

@@ -70,8 +70,10 @@ graph LR
     - `.internal/extra-hosts.json` — CLI-managed, not hand-edited (use
       `hack internal extra-hosts set/unset/list`)
     - `.branch/compose.<branch>.override.yml` (branch builds)
-  - Everything under `.hack/.internal` is a hack-managed artifact; do not hand-edit generated files
-    there.
+  - Everything under `.hack/.internal` is a hack-managed, machine-local artifact; do not hand-edit
+    or commit generated files there. Newer Hack versions rewrite them (for example
+    `compose.override.yml`), so a tracked copy leaves the working tree dirty; `hack doctor`
+    reports tracked files and `hack doctor --fix` untracks them.
 
 
 ## Internal DNS + TLS (containers)
@@ -185,6 +187,25 @@ sequenceDiagram
 NDJSON streaming (`hack logs --json`) emits `start`, `log`, `error`, and `end` events for MCP/TUI
 consumers.
 
+Snapshots (`hack logs --no-follow`) use Loki by default (`logs.snapshotBackend`). Without
+`--since`, a Loki snapshot looks back 24 hours and returns the newest `--tail` entries. If Loki
+returns no entries and Loki was not requested explicitly, `hack logs` writes a one-line notice
+to stderr and answers from `docker compose logs` instead. Nothing from Loki is written first,
+so `--json` output still has one `start`/`end` pair. When Loki was requested explicitly
+(`--loki`, `--services`, `--query`, `--since`, `--until`), the empty result stands and the
+notice suggests `hack logs --compose`. Follow mode (`hack logs` without `--no-follow`)
+does not fall back.
+
+Short-lived containers: Alloy finds containers by polling Docker every 5 seconds. It includes
+exited containers, so a one-shot Compose service (a migration or seed job) is shipped after it
+stops, as long as Docker keeps the container. A container that starts, exits and is removed
+between two polls (for example `docker compose run --rm`) never reaches Loki; use
+`hack logs --compose` or the command's own output for those. Existing installs pick up this
+discovery config after `hack global install` rewrites `~/.hack/logging/alloy.alloy` and the
+Alloy container restarts (`hack global down` then `hack global up`). On its first start with
+this config, Alloy also reads the retained logs of exited Compose containers. Loki rejects
+entries older than its sample age limit (one week by default).
+
 Retention:
 - Loki retention is set in the global Loki config (`~/.hack/logging/loki.yaml`), default `168h` in the template.
 - Per-project overrides live in `hack.config.json` under `logs.retention_period` and apply when `hack down` prunes logs.

docs/guides/candidate-install.md

@@ -96,9 +96,14 @@ Compose testing shares state with stable Hack:
 - The candidate reads and writes the same project registry, global configuration
   and project `.hack/` files as stable `hack`. Stable v4 uses an older registry
   lock protocol, so do not run stable `hack` and the candidate against the same
-  home at the same time. Stop one before you start the other.
+  home at the same time. Stop one before you start the other, including the
+  stable `hackd` daemon; see
+  [Stop stable hackd before a Compose run](#stop-stable-hackd-before-a-compose-run).
 - `hack-next` does not set up the shared global services for you. Install or
-  change them with stable `hack global` commands.
+  change them with stable `hack global` commands before the candidate run.
+- Both backends write generated files under the project's `.hack/.internal/`.
+  Keep that directory out of Git; see
+  [Keep `.hack/.internal/` out of Git](#keep-hackinternal-out-of-git).
 - To test Compose in isolation instead, set `HACK_HOME` to a separate private
   directory. That home has no global services, so routing, container DNS and CA
   trust are not tested.
@@ -118,6 +123,54 @@ The bundled `hack-v5` entrypoint follows the same rule for manual bundles:
 unset selects native and requires `HACK_NATIVE_HOME`; `compose` needs no native
 home and uses your own Hack home.
 
+### Stop stable hackd before a Compose run
+
+A Compose candidate run uses the same `~/.hack` as stable Hack, so the stable
+`hackd` daemon must not run while the candidate does. Stopping it once is not
+enough: stable `hack global up` (v4.2.0) starts `hackd` again whenever it is
+stopped, because the stable `daemon.autoStart` setting is on by default. Other
+stable commands that talk to `hackd` can also start it again.
+
+Use this order:
+
+```sh
+hack global up                  # stable: global services up; also restarts hackd
+hack daemon stop                # stable: stop hackd after global up, not before
+pgrep -fl "hack daemon start"   # must print nothing
+HACK_RUNTIME_BACKEND=compose "$HOME/.hack-next/bin/hack-next" up --path /absolute/project
+```
+
+1. Run stable `hack global up` first, so the global Caddy and CoreDNS services run.
+2. Then run stable `hack daemon stop`.
+3. Then check with `pgrep -fl "hack daemon start"`. If it lists a process, a
+   stable `hackd` is still running, for example one that its launchd service
+   restarted. Do not start the candidate until the check prints nothing.
+
+Run no stable `hack` command against the same `~/.hack` while the candidate run
+is active, including `hack global`, `hack status`, `hack doctor` and project
+commands. Any of them can start `hackd` again or take the older registry lock.
+When you finish, stop the candidate graph with
+`HACK_RUNTIME_BACKEND=compose "$HOME/.hack-next/bin/hack-next" down --path /absolute/project`,
+then use stable `hack` again. Stable `hack global up` or `hack daemon start`
+starts `hackd` again when you need it.
+
+### Keep `.hack/.internal/` out of Git
+
+Do not commit a project's `.hack/.internal/` directory. It holds generated,
+machine-local files, such as `.hack/.internal/compose.override.yml`, and Hack
+rewrites them when project commands such as `up` run. Newer generators, including
+candidate bundles, can write different content into those files than older ones.
+If a project tracks `.hack/.internal/`, every run of a newer generator leaves its
+working tree dirty, and switching between stable Hack and the candidate can make
+the files change back and forth.
+
+Hack's committed `.hack/.gitignore` already ignores `.internal/`, but an ignore
+rule does not untrack files that are already committed. Before a candidate run,
+run stable `hack doctor` in the project. Its generated-files check lists any
+tracked `.hack/.internal/` paths. `hack doctor --fix` removes them from the Git
+index with `git rm --cached` and keeps the files on disk; commit that removal.
+See [ignore rules](../env.md#ignore-rules-committed-hackgitignore).
+
 ## Upgrade and return to a retained candidate
 
 Shared MCP, config compiler and storage witness packages need the matching

docs/guides/native-candidate.md

@@ -117,7 +117,12 @@ selects the native backend and the adjacent executor, even if an inherited
 with your own Hack home and existing global routing services. It needs no
 `HACK_NATIVE_HOME` and removes the native selectors. Other values refuse with exit
 code 64. The [candidate installer guide](https://github.com/hack-dance/hack/blob/next/docs/guides/candidate-install.md#test-both-backends)
-describes Compose testing and its shared-state limits. It preserves arguments, exit codes and
+describes Compose testing and its shared-state limits: run stable `hack global up`,
+then stable `hack daemon stop`, confirm `pgrep -fl "hack daemon start"` prints
+nothing, and run no stable `hack` command while the candidate uses your Hack home.
+Stable `hack global up` (v4.2.0) starts a stopped `hackd` again. Do not commit the
+project's generated `.hack/.internal/` directory; newer generators rewrite files
+such as `compose.override.yml`. The entrypoint preserves arguments, exit codes and
 signals through `exec`, and never replaces the installed `hack`. Project-specific
 native adaptation, dependency and routing selections remain explicit; this entrypoint
 does not prepare a provider or migrate existing projects. An unversioned local

docs/guides/v5-dogfooding.md

@@ -40,7 +40,12 @@ These commands do not establish that an arbitrary v4 project can run natively.
    `HACK_RUNTIME_BACKEND=compose hack-next` runs the same candidate on Compose with
    your own Hack home, so global routing, DNS and CA trust are checked too. See
    [Test both backends](candidate-install.md#test-both-backends), including why
-   stable `hack` must not run against the same home at the same time.
+   stable `hack` must not run against the same home at the same time. For a Compose
+   run, use the order in
+   [Stop stable hackd before a Compose run](candidate-install.md#stop-stable-hackd-before-a-compose-run):
+   stable `hack global up`, then `hack daemon stop`, then confirm
+   `pgrep -fl "hack daemon start"` prints nothing. Keep the project's
+   `.hack/.internal/` directory out of Git.
 3. Exercise the project's supported command form: foreground start, readiness,
    status, finite logs, noninteractive exec, retaining stop and restart. Exercise
    one-off run only where admitted. For an authored project, check generated public

src/backends/log-backend.ts

@@ -1,10 +1,11 @@
+import type { LokiSnapshotOutcome } from "../lib/logs.ts";
 import {
   dockerComposeLogsJson,
   dockerComposeLogsPlain,
   dockerComposeLogsPretty,
 } from "../ui/docker-logs.ts";
 import type { LogStreamContext } from "../ui/log-stream.ts";
-import { canReachLoki, lokiLogs } from "../ui/loki-logs.ts";
+import { canReachLoki, lokiLogs, lokiSnapshot } from "../ui/loki-logs.ts";
 
 export type LogBackendName = "compose" | "loki";
 export type LogOutputFormat = "plain" | "pretty" | "json";
@@ -50,7 +51,15 @@ export interface LogBackendAvailability {
 
 export type ComposeLogBackend = LogBackend<ComposeLogOptions>;
 export type LokiLogBackend = LogBackend<LokiLogOptions> &
-  LogBackendAvailability;
+  LogBackendAvailability & {
+    /**
+     * Bounded, non-follow snapshot. With `deferEmpty`, an empty result writes
+     * nothing and reports `deferred` so the caller can use another backend.
+     */
+    snapshot(
+      opts: Omit<LokiLogOptions, "follow"> & { readonly deferEmpty: boolean }
+    ): Promise<LokiSnapshotOutcome>;
+  };
 
 export const composeLogBackend: ComposeLogBackend = {
   name: "compose",
@@ -117,4 +126,19 @@ export const lokiLogBackend: LokiLogBackend = {
       end: opts.end,
     });
   },
+  async snapshot(opts) {
+    return await lokiSnapshot({
+      baseUrl: opts.baseUrl,
+      query: opts.query,
+      follow: false,
+      tail: opts.tail,
+      pretty: opts.format === "pretty",
+      json: opts.format === "json" ? true : undefined,
+      showProjectPrefix: opts.showProjectPrefix,
+      streamContext: opts.streamContext,
+      start: opts.start,
+      end: opts.end,
+      deferEmpty: opts.deferEmpty,
+    });
+  },
 };

src/commands/project.ts

@@ -223,6 +223,7 @@ import {
   buildLogSelector,
   resolveShouldTryLoki,
   resolveUseLoki,
+  runLokiSnapshotWithFallback,
 } from "../lib/logs.ts";
 import { refuseNativeComposeBackendFeatures } from "../lib/native-compose-backend-features.ts";
 import { tryNativeComposeCommand } from "../lib/native-compose-command.ts";
@@ -8082,6 +8083,9 @@ async function runLogsWithLoki(opts: {
   readonly queryOpt: string | undefined;
   readonly since: string | undefined;
   readonly until: string | undefined;
+  readonly wantsLokiExplicit: boolean;
+  /** Compose snapshot used when an implicit Loki snapshot comes back empty. */
+  readonly runCompose: () => Promise<number>;
 }): Promise<number> {
   if (isSlimExecutionMode()) {
     process.stderr.write(
@@ -8124,16 +8128,30 @@ async function runLogsWithLoki(opts: {
     services,
   });
 
-  return await lokiLogBackend.run({
+  const lokiOptions = {
     baseUrl: opts.baseUrl,
     query,
-    follow: opts.follow,
     tail: opts.tail,
     format: opts.format,
     showProjectPrefix: true,
     streamContext,
     start: opts.timeRange.start ?? undefined,
     end: opts.timeRange.end ?? undefined,
+  };
+  if (opts.follow) {
+    return await lokiLogBackend.run({ ...lokiOptions, follow: true });
+  }
+
+  return await runLokiSnapshotWithFallback({
+    wantsLokiExplicit: opts.wantsLokiExplicit,
+    query,
+    service: opts.positionalService,
+    runLoki: async ({ deferEmpty }) =>
+      await lokiLogBackend.snapshot({ ...lokiOptions, deferEmpty }),
+    runCompose: opts.runCompose,
+    writeNotice: (line) => {
+      process.stderr.write(`${line}\n`);
+    },
   });
 }
 
@@ -8446,6 +8464,22 @@ async function handleLogs({
     lokiReachable,
   });
 
+  const runCompose = async (): Promise<number> =>
+    await runLogsWithCompose({
+      project,
+      projectNameForPrefix,
+      composeProject: branch ? projectNameForPrefix : undefined,
+      branch,
+      json,
+      follow,
+      tail,
+      service: typeof service === "string" ? service : undefined,
+      profiles,
+      format,
+      since: args.options.since,
+      until: args.options.until,
+    });
+
   if (useLoki) {
     return await runLogsWithLoki({
       baseUrl,
@@ -8462,24 +8496,13 @@ async function handleLogs({
       queryOpt: args.options.query,
       since: args.options.since,
       until: args.options.until,
+      wantsLokiExplicit,
+      runCompose,
     });
   }
 
   // Fallback to docker compose logs when Loki isn't available.
-  return await runLogsWithCompose({
-    project,
-    projectNameForPrefix,
-    composeProject: branch ? projectNameForPrefix : undefined,
-    branch,
-    json,
-    follow,
-    tail,
-    service: typeof service === "string" ? service : undefined,
-    profiles,
-    format,
-    since: args.options.since,
-    until: args.options.until,
-  });
+  return await runCompose();
 }
 
 /** An omitted option stays omitted so native dispatch can distinguish it from an explicit empty list. */