From f87ee54c6a27dc9733d9a7392ae8299d33c39531 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 22 Oct 2026 20:40:11 -0400 Subject: [PATCH 01/55] fix(env): refuse --service and unusable layers in legacy env set/unset In a legacy .hack/.env project, `hack env set` and `hack env unset` ignored --service and changed the global value. They now refuse an invalid scope with the same closed message as env-layer projects, and refuse any other non-global scope with a usage error that names the remedy (migrate to env layers, or omit --service). The env-format decision of set/unset no longer treats a layer entry that is not a usable file (a dangling symlink, a link to a directory, a directory) as absent: it refuses with E_PATH_CONFLICT before stdin is read or anything is written, instead of falling back to .hack/.env. A symlink to a regular layer counts as an env layer. Reads of an invalid layer (env list, explain, materialize) now raise a closed one-line error instead of a plain Error with a stack, and an invalid scope name in a layer is no longer echoed into that line. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 13 + src/cli/env-arguments.ts | 21 ++ src/commands/env.ts | 27 +- src/lib/project-env-config.ts | 65 +++- tests/env-legacy-service-scope.test.ts | 460 +++++++++++++++++++++++++ 5 files changed, 579 insertions(+), 7 deletions(-) create mode 100644 tests/env-legacy-service-scope.test.ts diff --git a/docs/env.md b/docs/env.md index 45f1c4d07..6ef4b916b 100644 --- a/docs/env.md +++ b/docs/env.md @@ -926,6 +926,19 @@ That migrates the repo into: For modern env repos, `hack doctor` also warns when materialized compatibility output in `.hack/.env` or `.hack/.env.state.json` is stale and points to `hack env materialize`. +The legacy `.hack/.env` holds one value per key, with no per-service values. In a legacy +project, `hack env set` and `hack env unset` change only that global value, so they refuse +`--service ` with a usage error instead of ignoring it (an invalid scope is refused +the same way in both formats). `--service global`, or no `--service`, still works. Migrate to +env layers first to set or unset a service-scoped value. + +`hack env set` and `hack env unset` choose between the env layers and the legacy file by +looking for `.hack/hack.env.*.yaml` entries, `hack.env.default.yaml` included. A regular +file, or a symlink to one, counts as an env layer. Any other entry with a layer name, such +as a dangling symlink or a directory, is refused with `E_PATH_CONFLICT` and nothing is +written; it never silently selects the legacy `.hack/.env`. Replace or remove the entry, +then retry. + ## Compatibility notes The modern YAML env model is the canonical repo format today. diff --git a/src/cli/env-arguments.ts b/src/cli/env-arguments.ts index 1423e2fd0..c31b4f500 100644 --- a/src/cli/env-arguments.ts +++ b/src/cli/env-arguments.ts @@ -34,6 +34,27 @@ export function parseEnvScopeOption(input: { } } +/** + * Refusal of a non-global `--service` for `hack env set`/`unset` in a project + * that still uses the legacy `.hack/.env` file. That file holds one value per + * key, so honoring the scope is impossible and ignoring it would change the + * global value instead. + */ +export const LEGACY_ENV_SERVICE_SCOPE_MESSAGE = + '--service needs env layers, but this project uses the legacy .hack/.env file, which has no per-service values. Migrate to env layers with `hack doctor --migrate-env-config` (see "Legacy repos and migration" in docs/env.md), or omit --service to change the global value.'; + +/** + * `hack env set`/`unset --service` for a legacy `.hack/.env` project: only the + * global scope (omitted, empty or `global`) is accepted. Pass the scope after + * an invalid one has been refused, and call it before stdin is read or any + * file or secret store is touched. + */ +export function assertLegacyEnvScope(input: { readonly scope: string }): void { + if (input.scope !== "global") { + throw new CliUsageError(LEGACY_ENV_SERVICE_SCOPE_MESSAGE); + } +} + /** * Resolve the `--scope`/`--service` pair of host execution commands. * Returns `null` when neither names a scope (the global scope). diff --git a/src/commands/env.ts b/src/commands/env.ts index b360ff4e2..d7f67d901 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -10,6 +10,7 @@ import { withHandler, } from "../cli/command.ts"; import { + assertLegacyEnvScope, parseApplyServiceOption, parseEnvGetArguments, parseEnvScopeOption, @@ -71,6 +72,7 @@ import { sanitizeProjectSlug, } from "../lib/project.ts"; import { + assertValidProjectEnvScopeName, discoverComposeServiceNames, ENV_KEY_INVALID_MESSAGE, hydrateProjectEnvLocalLayer, @@ -78,6 +80,7 @@ import { migrateLegacyProjectEnv, parseProjectEnvTarget, projectEnvConfigExists, + projectEnvConfigExistsForMutation, readMaterializedProjectEnv, resolveProjectEnvConfig, resolveProjectEnvValue, @@ -1871,6 +1874,11 @@ const handleEnvSet: CommandHandlerFor = async ({ hasPositionalValue: valueFromSpec !== null, }); } + // Refuse an invalid scope in either env format before the project is read, + // with the same closed one-line refusal as `env add`. + const scope = assertValidProjectEnvScopeName({ + scopeName: args.options.service, + }); const project = await resolveProjectForEnv({ ctx, @@ -1882,7 +1890,9 @@ const handleEnvSet: CommandHandlerFor = async ({ envOption: args.options.env, }); - const modernExists = await projectEnvConfigExists({ + // A layer entry that is not a usable file is refused here instead of + // silently selecting the legacy .hack/.env. + const modernExists = await projectEnvConfigExistsForMutation({ projectDir: project.projectDir, }); if (modernExists) { @@ -1898,6 +1908,9 @@ const handleEnvSet: CommandHandlerFor = async ({ }); } + // The legacy .hack/.env has no per-service values: refuse a scope rather + // than write the global value, before stdin is read. + assertLegacyEnvScope({ scope }); const storeSecret = args.options.secret === true; const runtimeConfig = await readHackEnvRuntimeConfig({ projectDir: project.projectDir, @@ -2298,12 +2311,18 @@ const handleEnvUnset: CommandHandlerFor = async ({ envOption: args.options.env, }); - const modernExists = await projectEnvConfigExists({ + const modernExists = await projectEnvConfigExistsForMutation({ projectDir: project.projectDir, }); if (!modernExists) { - // Validate the key before resolving a secret store, so a refused key - // never reaches a backend. + // The legacy .hack/.env has no per-service values: refuse a scope rather + // than remove the global value. Validate the scope and key before + // resolving a secret store, so a refused argument never reaches a backend. + assertLegacyEnvScope({ + scope: assertValidProjectEnvScopeName({ + scopeName: args.options.service, + }), + }); const key = await resolveEnvKey({ key: (args.positionals.key ?? "").trim(), }); diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 63a29c6fb..ad168493b 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -26,7 +26,7 @@ import { HACK_DIR_GITIGNORE_END_MARKER, HACK_DIR_GITIGNORE_ENTRIES, } from "../templates.ts"; -import { ClosedCliError } from "./cli-result.ts"; +import { ClosedCliError, HackCliError } from "./cli-result.ts"; import { parseDotEnv, serializeDotEnv } from "./env.ts"; import { AtomicWriteConflictError, @@ -799,6 +799,59 @@ export async function projectEnvConfigExists(opts: { }); } +/** + * Closed refusal for a `.hack/hack.env.*.yaml` entry that is neither a regular + * file nor a symlink to one: a dangling symlink, a link to a directory, a + * directory or another special file. The message names the path and a fixed + * remedy only; the CLI prints it as one `E_PATH_CONFLICT: message` line. + */ +export class ProjectEnvLayerPathConflictError extends HackCliError { + readonly path: string; + + constructor(opts: { readonly path: string }) { + super({ + code: "E_PATH_CONFLICT", + message: `Refusing to choose the env format: ${opts.path} is not a regular file (for example a dangling symlink or a directory). Replace it with a regular env layer file or remove it, then retry.`, + }); + this.name = "ProjectEnvLayerPathConflictError"; + this.path = opts.path; + } +} + +/** + * The env-format decision of a command that writes either the env layers or + * the legacy `.hack/.env` (`hack env set`, `hack env unset`). + * + * {@link projectEnvConfigExists} reads a dangling default layer, and any + * symlinked overlay, as absent, so such a project would silently fall back to + * the legacy file. Here every entry named like an env layer must be a regular + * file or a symlink that resolves to one, and either counts as an env layer; + * any other entry is refused with {@link ProjectEnvLayerPathConflictError} + * before anything is read from stdin or written. + */ +export async function projectEnvConfigExistsForMutation(opts: { + readonly projectDir: string; +}): Promise { + const entries = await readdir(opts.projectDir, { withFileTypes: true }); + let linkedLayer = false; + for (const entry of entries) { + if ( + entry.isFile() || + !entry.name.startsWith(PROJECT_ENV_CONFIG_FILENAME_PREFIX) || + !entry.name.endsWith(PROJECT_ENV_CONFIG_FILENAME_SUFFIX) + ) { + continue; + } + const path = resolve(opts.projectDir, entry.name); + const target = await stat(path).catch(() => null); + if (!target?.isFile()) { + throw new ProjectEnvLayerPathConflictError({ path }); + } + linkedLayer = true; + } + return linkedLayer || (await projectEnvConfigExists(opts)); +} + export async function listProjectEnvOverlayNames(opts: { readonly projectDir: string; }): Promise { @@ -959,7 +1012,10 @@ function parseProjectEnvConfig(opts: { if (!isValidProjectEnvScopeName({ scopeName: scope })) { return { ok: false, - error: `Invalid env scope: ${scope}`, + // The name is not echoed: it is file content and may hold a pasted + // value or a line break, and this error is printed as one line. + error: + 'Invalid env scope name; scopes must be "global" or match ^[a-z0-9][a-z0-9._-]*$.', }; } if (!isRecord(scopeRaw)) { @@ -1166,7 +1222,10 @@ async function readProjectEnvLayers(opts: { } for (const read of reads) { if (read?.parseError) { - throw new Error(`Failed to parse ${read.path}: ${read.parseError}`); + // The same closed, value-free line as the write path's refusal. + throw new ClosedCliError( + `Failed to parse ${read.path}: ${read.parseError}` + ); } } const envLayers = reads.map((read) => (read?.exists ? read.config : null)); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts new file mode 100644 index 000000000..c76478590 --- /dev/null +++ b/tests/env-legacy-service-scope.test.ts @@ -0,0 +1,460 @@ +import { afterEach, expect, test } from "bun:test"; +import { + lstat, + mkdir, + mkdtemp, + readdir, + readFile, + readlink, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { LEGACY_ENV_SERVICE_SCOPE_MESSAGE } from "../src/cli/env-arguments.ts"; +import { + PROJECT_COMPOSE_FILENAME, + PROJECT_CONFIG_FILENAME, +} from "../src/constants.ts"; +import { ClosedCliError, HackCliError } from "../src/lib/cli-result.ts"; +import { + ENV_SCOPE_INVALID_MESSAGE, + ProjectEnvLayerPathConflictError, + projectEnvConfigExistsForMutation, + resolveProjectEnvConfig, +} from "../src/lib/project-env-config.ts"; +import { privateCliEnv } from "./helpers/private-cli-env.ts"; + +/** + * HACK-1370: `hack env set`/`unset` in a legacy `.hack/.env` project refuse a + * `--service` scope instead of silently changing the global value; a layer + * entry that is not a usable file refuses (E_PATH_CONFLICT) instead of + * selecting the legacy file; and reads of an invalid layer print one closed + * line, never a stack. + * + * Isolation: every child gets a private HOME, global config and Docker client + * config inside a temp directory, and an `encrypted_file` secret backend in + * that directory with an env-provided key and the keychain fallback disabled, + * so no keychain, Docker socket or user file is reached even if a guard under + * test is removed. Every value is a synthetic test string. + */ + +const REPO_ROOT = resolve(import.meta.dir, ".."); +const ENTRYPOINT = resolve(REPO_ROOT, "index.ts"); +const NEW_VALUE = ["synthetic", "legacy", "scope", "7e41"].join("-"); +const KEPT_VALUE = ["synthetic", "kept", "c0d2"].join("-"); +const BAD_SCOPE_ARGUMENT = "Bad Scope synthetic-91fa!"; +const LAYER = `version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global:\n KEPT_SETTING: ${KEPT_VALUE}\n`; + +const tempDirs = new Set(); + +afterEach(async () => { + for (const tempDir of tempDirs) { + await rm(tempDir, { recursive: true, force: true }); + } + tempDirs.clear(); +}); + +type Repo = { + readonly root: string; + readonly projectRoot: string; + readonly projectDir: string; + readonly storePath: string; +}; + +async function createLegacyRepo(): Promise { + const root = await mkdtemp(join(tmpdir(), "hack-env-legacy-scope-")); + tempDirs.add(root); + const projectRoot = resolve(root, "repo"); + const projectDir = resolve(projectRoot, ".hack"); + const storePath = resolve(root, "secrets.enc.json"); + await mkdir(projectDir, { recursive: true }); + await writeFile( + resolve(projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n" + ); + await writeFile( + resolve(projectDir, PROJECT_CONFIG_FILENAME), + `${JSON.stringify( + { + name: "env-legacy-scope", + dev_host: "env-legacy-scope.hack", + controlPlane: { + secrets: { + backend: "encrypted_file", + encryptedFile: { path: storePath }, + }, + }, + }, + null, + 2 + )}\n` + ); + return { root, projectRoot, projectDir, storePath }; +} + +type CliRun = { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +}; + +async function runCli(opts: { + readonly repo: Repo; + readonly args: readonly string[]; + readonly stdin?: string; + readonly logger?: string; +}): Promise { + const proc = Bun.spawn([process.execPath, ENTRYPOINT, ...opts.args], { + cwd: opts.repo.projectRoot, + env: privateCliEnv({ + root: opts.repo.root, + env: { + HACK_GLOBAL_CONFIG_PATH: join(opts.repo.root, "hack-config.json"), + HACK_HOME: join(opts.repo.root, "hack-home"), + HACK_LOGGER: opts.logger ?? "console", + HACK_NO_INTERACTIVE: "1", + HACK_SECRETS_FILE_KEY: ["legacy", "scope", "test", "key"].join("-"), + HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + }, + }), + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + proc.stdin.write(opts.stdin ?? ""); + await proc.stdin.end(); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { exitCode, stdout, stderr }; +} + +/** Output lines that carry text, ignoring the clack frame (`│`, `■`, ...). */ +function textLines(run: CliRun): string[] { + return `${run.stdout}\n${run.stderr}` + .split("\n") + .filter((line) => /[A-Za-z]/.test(line)); +} + +/** A refusal: exit 1, the message on the first text line, no stack or value. */ +function expectRefusal( + run: CliRun, + expected: string, + opts: { readonly lines: number } +): void { + expect(run.exitCode).toBe(1); + const lines = textLines(run); + expect(lines).toHaveLength(opts.lines); + expect(lines[0]).toContain(expected); + const output = `${run.stdout}${run.stderr}`; + expect(output).not.toMatch(/\bat\s.*\.ts:\d+/); + expect(output).not.toContain(REPO_ROOT); + expect(output).not.toContain(NEW_VALUE); + expect(output).not.toContain(KEPT_VALUE); +} + +async function hackDirEntries(repo: Repo): Promise { + return (await readdir(repo.projectDir)).sort(); +} + +const LEGACY_BASE_ENTRIES = [ + PROJECT_COMPOSE_FILENAME, + PROJECT_CONFIG_FILENAME, +].sort(); + +test("legacy env set refuses a service scope and writes nothing", async () => { + const repo = await createLegacyRepo(); + for (const extra of [[], ["--secret"]] as const) { + const run = await runCli({ + repo, + args: [ + "env", + "set", + "NEW_SETTING", + "--service", + "api", + ...extra, + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${NEW_VALUE}\n`, + }); + // The usage refusal plus the one-line pointer to the command's help. + expectRefusal(run, LEGACY_ENV_SERVICE_SCOPE_MESSAGE, { lines: 2 }); + expect(textLines(run)[1]).toBe("See `hack env set --help` for usage."); + } + expect(await hackDirEntries(repo)).toEqual(LEGACY_BASE_ENTRIES); + // Neither the plaintext path nor the secret store was reached. + expect(await lstat(repo.storePath).catch(() => null)).toBeNull(); +}); + +test("legacy env set refuses an invalid scope with the value-free scope message", async () => { + const repo = await createLegacyRepo(); + const run = await runCli({ + repo, + args: [ + "env", + "set", + "NEW_SETTING", + "--service", + BAD_SCOPE_ARGUMENT, + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${NEW_VALUE}\n`, + }); + // The same closed one-line refusal as in an env-layer project. + expectRefusal(run, ENV_SCOPE_INVALID_MESSAGE, { lines: 1 }); + expect(`${run.stdout}${run.stderr}`).not.toContain("synthetic-91fa"); + expect(await hackDirEntries(repo)).toEqual(LEGACY_BASE_ENTRIES); +}); + +test("legacy env set still writes the global value without --service or with --service global", async () => { + const repo = await createLegacyRepo(); + for (const scopeArgs of [[], ["--service", "global"]] as const) { + const run = await runCli({ + repo, + args: [ + "env", + "set", + "NEW_SETTING", + ...scopeArgs, + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${NEW_VALUE}\n`, + }); + expect(run.exitCode).toBe(0); + expect(`${run.stdout}${run.stderr}`).not.toContain(NEW_VALUE); + } + const dotenv = await readFile(resolve(repo.projectDir, ".env"), "utf8"); + expect( + dotenv.split("\n").filter((line) => line.startsWith("NEW_SETTING=")) + ).toHaveLength(1); + // Still legacy: no env layer was created. + expect( + (await hackDirEntries(repo)).filter((name) => name.endsWith(".yaml")) + ).toEqual([]); +}); + +test("legacy env unset refuses a service scope and keeps the global value", async () => { + const repo = await createLegacyRepo(); + const envFile = resolve(repo.projectDir, ".env"); + const dotenv = `KEPT_SETTING=${KEPT_VALUE}\n`; + await writeFile(envFile, dotenv); + + for (const scope of ["api", BAD_SCOPE_ARGUMENT]) { + const run = await runCli({ + repo, + args: [ + "env", + "unset", + "KEPT_SETTING", + "--service", + scope, + "--path", + repo.projectRoot, + ], + }); + expectRefusal( + run, + scope === "api" + ? LEGACY_ENV_SERVICE_SCOPE_MESSAGE + : ENV_SCOPE_INVALID_MESSAGE, + // Only the usage refusal adds a pointer to the command's help. + { lines: scope === "api" ? 2 : 1 } + ); + expect(`${run.stdout}${run.stderr}`).not.toContain("synthetic-91fa"); + } + expect(await readFile(envFile, "utf8")).toBe(dotenv); + expect(await lstat(repo.storePath).catch(() => null)).toBeNull(); +}); + +type LayerEntry = { + readonly name: string; + readonly kind: "dangling-link" | "directory" | "link-to-directory"; +}; + +const UNUSABLE_LAYER_ENTRIES: readonly LayerEntry[] = [ + { name: "hack.env.default.yaml", kind: "dangling-link" }, + { name: "hack.env.staging.yaml", kind: "dangling-link" }, + { name: "hack.env.local.yaml", kind: "link-to-directory" }, + { name: "hack.env.default.yaml", kind: "directory" }, +]; + +async function placeLayerEntry(repo: Repo, entry: LayerEntry): Promise { + const path = resolve(repo.projectDir, entry.name); + if (entry.kind === "directory") { + await mkdir(path); + } else if (entry.kind === "link-to-directory") { + await mkdir(resolve(repo.root, "elsewhere"), { recursive: true }); + await symlink(resolve(repo.root, "elsewhere"), path); + } else { + await symlink(resolve(repo.projectDir, "missing-layer.yaml"), path); + } + return path; +} + +for (const entry of UNUSABLE_LAYER_ENTRIES) { + test(`env set and unset refuse a ${entry.kind} ${entry.name} instead of falling back to .hack/.env`, async () => { + const repo = await createLegacyRepo(); + const path = await placeLayerEntry(repo, entry); + const before = await hackDirEntries(repo); + const refusal = `E_PATH_CONFLICT: Refusing to choose the env format: ${path} is not a regular file`; + + const set = await runCli({ + repo, + args: [ + "env", + "set", + "NEW_SETTING", + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${NEW_VALUE}\n`, + }); + expectRefusal(set, refusal, { lines: 1 }); + + const unset = await runCli({ + repo, + args: ["env", "unset", "NEW_SETTING", "--path", repo.projectRoot], + }); + expectRefusal(unset, refusal, { lines: 1 }); + // Nothing was written and the entry is left as it was. + expect(await hackDirEntries(repo)).toEqual(before); + if (entry.kind === "dangling-link") { + expect(await readlink(path)).toBe( + resolve(repo.projectDir, "missing-layer.yaml") + ); + } + expect(await lstat(repo.storePath).catch(() => null)).toBeNull(); + }, 60_000); +} + +test("a symlinked overlay that resolves to a regular layer selects the env layers", async () => { + const repo = await createLegacyRepo(); + const target = resolve(repo.root, "shared-staging.yaml"); + await writeFile( + target, + LAYER.replace("environment: default", "environment: staging") + ); + await symlink(target, resolve(repo.projectDir, "hack.env.staging.yaml")); + expect( + await projectEnvConfigExistsForMutation({ projectDir: repo.projectDir }) + ).toBe(true); + + const run = await runCli({ + repo, + args: [ + "env", + "set", + "NEW_SETTING", + "--service", + "api", + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${NEW_VALUE}\n`, + }); + expect(run.exitCode).toBe(0); + expect(`${run.stdout}${run.stderr}`).not.toContain(NEW_VALUE); + // Written to the default env layer, never to the legacy file. + expect( + await lstat(resolve(repo.projectDir, ".env")).catch(() => null) + ).toBeNull(); + const layer = await readFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + "utf8" + ); + expect(layer).toContain("api:"); + expect(layer).toContain("NEW_SETTING"); +}); + +test("the mutation format check accepts regular layers and refuses unusable entries as closed path conflicts", async () => { + const legacy = await createLegacyRepo(); + expect( + await projectEnvConfigExistsForMutation({ projectDir: legacy.projectDir }) + ).toBe(false); + + const modern = await createLegacyRepo(); + await writeFile(resolve(modern.projectDir, "hack.env.default.yaml"), LAYER); + expect( + await projectEnvConfigExistsForMutation({ projectDir: modern.projectDir }) + ).toBe(true); + + for (const entry of UNUSABLE_LAYER_ENTRIES) { + const repo = await createLegacyRepo(); + const path = await placeLayerEntry(repo, entry); + const error = await projectEnvConfigExistsForMutation({ + projectDir: repo.projectDir, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ProjectEnvLayerPathConflictError); + expect(error).toBeInstanceOf(HackCliError); + const conflict = error as ProjectEnvLayerPathConflictError; + expect(conflict.code).toBe("E_PATH_CONFLICT"); + expect(conflict.path).toBe(path); + expect(conflict.message.split("\n")).toHaveLength(1); + } +}); + +const INVALID_LAYERS = [ + { name: "invalid YAML", text: `${LAYER}version: [\n` }, + { + name: "a scope name with a line break", + text: `${LAYER} "bad\\nscope synthetic-91fa": {}\n`, + }, +] as const; + +for (const logger of ["console", "clack", "gum"] as const) { + test(`env list and explain on an invalid layer print one line with HACK_LOGGER=${logger}`, async () => { + const repo = await createLegacyRepo(); + const file = resolve(repo.projectDir, "hack.env.default.yaml"); + for (const invalid of INVALID_LAYERS) { + await writeFile(file, invalid.text); + for (const args of [ + ["env", "list"], + ["env", "explain", "KEPT_SETTING"], + ]) { + const run = await runCli({ + repo, + args: [...args, "--path", repo.projectRoot], + logger, + }); + expectRefusal(run, `Failed to parse ${file}:`, { lines: 1 }); + expect(`${run.stdout}${run.stderr}`).not.toContain("synthetic-91fa"); + } + expect(await readFile(file, "utf8")).toBe(invalid.text); + } + }, 60_000); +} + +test("an invalid layer read is a closed error with a value-free scope reason", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + INVALID_LAYERS[1].text + ); + const error = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: null, + serviceNames: ["api"], + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ClosedCliError); + const message = (error as Error).message; + expect(message).toContain("Invalid env scope name"); + expect(message).not.toContain("synthetic-91fa"); + expect(message.split("\n")).toHaveLength(1); +}); From 456bdad5c44610ab2736e7a1d3f4d0253c0e913d Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 00:01:33 -0400 Subject: [PATCH 02/55] fix(env): make the legacy --service refusal remedy migrate .env-only projects `hack env set`/`unset --service` in a legacy project points to `hack doctor --migrate-env-config`, but that command did nothing for a project with only `.hack/.env` (no `.hack/hack.env.json`), so the user looped between the refusal and doctor. - `migrateLegacyProjectEnv` takes `migrateUndeclaredDotenv`: without a contract, each `.hack/.env` entry becomes a plaintext global value in `hack.env.default.yaml` and each differing overlay entry a global value in `hack.env..yaml` (an empty default layer when no legacy file exists). Secret-backend values are not migrated, and hack.config.json and its secret store are left unchanged and never offered for cleanup. Only doctor sets it; implicit migrations by other commands still need the contract. - `hack doctor --migrate-env-config` uses it, and runs the file-only env migration before the Docker gate so an unreachable Docker no longer skips it. - docs/env.md documents the .env-only migration and the remedy. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 16 ++- src/commands/doctor.ts | 23 ++-- src/lib/project-env-config.ts | 158 ++++++++++++++++++++++-- tests/doctor-fix-noninteractive.test.ts | 43 ++++++- tests/env-legacy-service-scope.test.ts | 134 ++++++++++++++++++++ 5 files changed, 354 insertions(+), 20 deletions(-) diff --git a/docs/env.md b/docs/env.md index 9305f122b..e5778a45e 100644 --- a/docs/env.md +++ b/docs/env.md @@ -979,6 +979,19 @@ That migrates the repo into: - `.hack/hack.env..yaml` - `.hack.secret.key` when needed +A repo that has no `.hack/hack.env.json` is migrated as well. Its values live only in +`.hack/.env` and `.hack/.env.`, so each `.hack/.env` entry becomes a plaintext +global value in `hack.env.default.yaml`, and each overlay entry that differs from +`.hack/.env` becomes a global value in `hack.env..yaml`. A repo with no legacy env +file gets an empty `hack.env.default.yaml`. Without `hack.env.json`, nothing lists values +kept in a secret backend, so none are migrated: set them again with +`hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and +any secret backend it configures, is left unchanged. The migration changes project files +only, so it runs even when Docker is not reachable; the other `--migrate-env-config` +repairs still need Docker. As with any migration, `hack doctor` then offers to remove the +legacy `.hack/.env*` files that Compose no longer references (a non-interactive run +accepts). + `hack doctor` also warns when a repo still depends on the old format. For modern env repos, `hack doctor` also warns when materialized compatibility output in `.hack/.env` or `.hack/.env.state.json` is stale and points to `hack env materialize`. @@ -987,7 +1000,8 @@ The legacy `.hack/.env` holds one value per key, with no per-service values. In project, `hack env set` and `hack env unset` change only that global value, so they refuse `--service ` with a usage error instead of ignoring it (an invalid scope is refused the same way in both formats). `--service global`, or no `--service`, still works. Migrate to -env layers first to set or unset a service-scoped value. +env layers first with `hack doctor --migrate-env-config`, which also converts a project that +has only `.hack/.env` (see above), then set or unset the service-scoped value. `hack env set` and `hack env unset` choose between the env layers and the legacy file by looking for `.hack/hack.env.*.yaml` entries, `hack.env.default.yaml` included. A regular diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 0465db3d0..abc6e908a 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -43,6 +43,7 @@ import { HACK_PROJECT_DIR_PRIMARY, LEGACY_GRAFANA_HOST, PROJECT_ENV_CONTRACT_FILENAME, + PROJECT_ENV_FILENAME, PROJECT_ENV_KEY_FILENAME, } from "../constants.ts"; import { resolveGatewayConfig } from "../control-plane/extensions/gateway/config.ts"; @@ -3598,6 +3599,11 @@ async function runDoctorFix(opts: { await maybeEnsureHackDirGitignore({ startDir: opts.startDir }); await maybeRemovePlaintextComposeEnvOverride({ startDir: opts.startDir }); await maybeRepairHackd(); + // Env migration changes project files only, so it runs before the Docker + // gate: an unreachable Docker never skips a requested migration. + if (opts.migrateEnvConfig) { + await maybeMigrateProjectEnvConfig({ startDir: opts.startDir }); + } const dockerOk = await dockerInfoOk(); if (!dockerOk) { @@ -3634,9 +3640,6 @@ async function runDoctorFix(opts: { await maybeExportCaddyCaCert({ paths }); await maybeRepairMacHostTlsTrust(); await maybeMigrateDnsmasq(); - if (opts.migrateEnvConfig) { - await maybeMigrateProjectEnvConfig({ startDir: opts.startDir }); - } renderSkippedDoctorFixSteps(); } @@ -3723,13 +3726,14 @@ async function maybeMigrateProjectEnvConfig(opts: { return; } + // A project without the hack.env.json contract keeps its values only in + // .hack/.env and .hack/.env.; it is migrated from those files. + // `hack env set`/`unset --service` refusals in such a project point here. const contractPath = resolve( project.projectDir, PROJECT_ENV_CONTRACT_FILENAME ); - if (!(await pathExists(contractPath))) { - return; - } + const hasContract = await pathExists(contractPath); const cfg = await readProjectConfig(project); const projectName = ( @@ -3743,7 +3747,9 @@ async function maybeMigrateProjectEnvConfig(opts: { note( [ "Detected legacy project env files.", - `- source: ${contractPath} and current local env state`, + hasContract + ? `- source: ${contractPath} and current local env state` + : `- source: ${resolve(project.projectDir, PROJECT_ENV_FILENAME)} and its overlays (no ${PROJECT_ENV_CONTRACT_FILENAME}; values become plaintext global values, and secret-backend values are not migrated)`, `- target: ${resolve(project.projectDir, "hack.env.default.yaml")}`, "- runtime: direct env injection by default; .hack/.env becomes compatibility-only", ].join("\n"), @@ -3764,6 +3770,7 @@ async function maybeMigrateProjectEnvConfig(opts: { projectName, serviceNames, materialize: false, + migrateUndeclaredDotenv: true, }); if (migrated.wroteFiles.length === 0) { note("Legacy env migration found nothing to write.", "doctor"); @@ -4014,7 +4021,7 @@ async function buildEnvMigrationPlanStep(opts: { return "Prompt to migrate legacy env config (.hack/hack.env.json) to hack.env.*.yaml."; } - return "Skip env migration because no legacy project env config was found."; + return "Prompt to migrate legacy env values (.hack/.env and its overlays, no .hack/hack.env.json) to hack.env.*.yaml as plaintext global values."; } /** diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 0c810ca18..28c1ced06 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -50,7 +50,11 @@ import { resolveGitWorktreeDir, } from "./git-worktree.ts"; import { getRecord, getString, isRecord } from "./guards.ts"; -import { readHackEnvContract, resolveHackEnv } from "./hack-env.ts"; +import { + readHackEnvContract, + resolveEnvFilePath, + resolveHackEnv, +} from "./hack-env.ts"; import { isOwnedLegacyAdoptionManagedEnvAdmission, type LegacyAdoptionManagedEnvAdmission, @@ -4735,10 +4739,25 @@ function removeLegacyComposeEnvFileEntries(opts: { return kept.length > 0 ? kept : undefined; } +/** Outcome of {@link migrateLegacyProjectEnv}. */ +export type LegacyProjectEnvMigration = { + readonly wroteFiles: readonly string[]; + readonly migratedOverlays: readonly string[]; + readonly legacyDetected: boolean; + readonly updatedProjectConfig: boolean; + readonly cleanupCandidates: readonly string[]; + readonly blockedCleanupCandidates: readonly string[]; + readonly composeEnvFileReferences: readonly LegacyComposeEnvFileReference[]; +}; + /** * Migrates a legacy v2 `.env`-based project env layout to the v3 * `hack.env.*.yaml` config system. * + * Without a `.hack/hack.env.json` contract nothing is migrated + * (`legacyDetected: false`) unless `migrateUndeclaredDotenv` is set; see + * {@link migrateUndeclaredLegacyDotenv} for that explicit-request path. + * * @deprecated v2→v3 migration path. TODO(remove: v3.2) together with its * callers in commands/project.ts, commands/doctor.ts, commands/session.ts, * and commands/env.ts (`maybeMigrateLegacyProjectEnv`). @@ -4749,19 +4768,21 @@ export async function migrateLegacyProjectEnv(opts: { readonly projectName: string; readonly serviceNames: readonly string[]; readonly materialize: boolean; -}): Promise<{ - readonly wroteFiles: readonly string[]; - readonly migratedOverlays: readonly string[]; - readonly legacyDetected: boolean; - readonly updatedProjectConfig: boolean; - readonly cleanupCandidates: readonly string[]; - readonly blockedCleanupCandidates: readonly string[]; - readonly composeEnvFileReferences: readonly LegacyComposeEnvFileReference[]; -}> { + /** + * Also migrate a project that has no contract: the legacy `.hack/.env` + * and `.hack/.env.` files become plaintext global values. Only an + * explicit migration request (`hack doctor --migrate-env-config`) sets it; + * the implicit migrations run by other commands keep requiring a contract. + */ + readonly migrateUndeclaredDotenv?: boolean; +}): Promise { await assertLegacyProjectDirectory({ projectDir: opts.projectDir }); await assertLegacyProjectInputFamily({ projectRoot: opts.projectRoot }); const contract = await readHackEnvContract({ projectDir: opts.projectDir }); if (!contract.exists) { + if (opts.migrateUndeclaredDotenv === true) { + return await migrateUndeclaredLegacyDotenv(opts); + } return { wroteFiles: [], migratedOverlays: [], @@ -4884,6 +4905,123 @@ export async function migrateLegacyProjectEnv(opts: { }; } +/** + * Migrates a legacy project that has no `.hack/hack.env.json` contract, so + * its values live only in `.hack/.env` and `.hack/.env.`: every + * base entry becomes a plaintext global value in `hack.env.default.yaml`, and + * every overlay entry that differs from the base becomes a global value in + * `hack.env..yaml`. With no legacy file at all it still writes an + * empty default layer, so the project switches to env layers. + * + * Without a contract nothing names secret-backend values, so none are read or + * migrated, and `hack.config.json` (including any secret backend it + * configures, and that backend's store) is left unchanged and never offered + * for cleanup. Values are written to the layer files only, never returned or + * printed. + */ +async function migrateUndeclaredLegacyDotenv(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly materialize: boolean; +}): Promise { + const overlayNames = await discoverLegacyOverlayNames({ + projectDir: opts.projectDir, + }); + const baseValues = await readLegacyDotenvFile({ + path: resolveEnvFilePath({ projectDir: opts.projectDir, envName: null }), + }); + + const wroteFiles: string[] = []; + const defaultPath = resolveProjectEnvConfigPath({ + projectDir: opts.projectDir, + envName: null, + }); + if ( + await writeProjectEnvConfigFile({ + path: defaultPath, + config: buildMigratedProjectEnvConfig({ + environment: "default", + values: { global: { ...baseValues } }, + }), + }) + ) { + wroteFiles.push(defaultPath); + } + + for (const overlayName of overlayNames) { + const overlayValues = await readLegacyDotenvFile({ + path: resolveEnvFilePath({ + projectDir: opts.projectDir, + envName: overlayName, + }), + }); + const changedValues = Object.fromEntries( + Object.entries(overlayValues).filter( + ([key, value]) => baseValues[key] !== value + ) + ); + if (Object.keys(changedValues).length === 0) { + continue; + } + const overlayPath = resolveProjectEnvConfigPath({ + projectDir: opts.projectDir, + envName: overlayName, + }); + if ( + await writeProjectEnvConfigFile({ + path: overlayPath, + config: buildMigratedProjectEnvConfig({ + environment: overlayName, + values: { global: changedValues }, + }), + }) + ) { + wroteFiles.push(overlayPath); + } + } + + if (opts.materialize) { + await materializeProjectEnv({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + envName: undefined, + serviceNames: opts.serviceNames, + }); + } + + const composeEnvFileReferences = await inspectLegacyComposeEnvFileReferences({ + composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), + projectDir: opts.projectDir, + }); + const cleanupCandidates = await collectLegacyProjectEnvCleanupCandidates({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + overlayNames, + materialize: opts.materialize, + configCleanupCandidates: [], + composeEnvFileReferences, + }); + + return { + wroteFiles, + migratedOverlays: overlayNames, + legacyDetected: true, + updatedProjectConfig: false, + cleanupCandidates: cleanupCandidates.allowed, + blockedCleanupCandidates: cleanupCandidates.blocked, + composeEnvFileReferences, + }; +} + +/** Entries of a legacy dotenv file; an absent file has none. */ +async function readLegacyDotenvFile(opts: { + readonly path: string; +}): Promise> { + const text = await readTextFile(opts.path); + return text === null ? {} : parseDotEnv(text); +} + export async function removeLegacyProjectEnvArtifacts(opts: { readonly projectRoot: string; readonly paths: readonly string[]; diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 45ded9d5b..0ab670d7c 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -20,6 +20,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { YAML } from "bun"; import { GLOBAL_DAEMON_DIR_NAME, @@ -52,6 +53,8 @@ const noteCalls: string[] = []; * default, so the untrack step stays idle unless a test sets it. */ let trackedGeneratedPaths: string[] = []; +/** Exit code of the mocked `docker info`; 0 (reachable) unless a test sets it. */ +let dockerInfoExitCode = 0; const dnsMock = await registerScopedModuleMock({ importerPath: import.meta.path, @@ -91,7 +94,7 @@ const shellMock = await registerScopedModuleMock({ exec: async (cmd: readonly string[]) => { execCalls.push([...cmd]); if (cmd[0] === "docker" && cmd[1] === "info") { - return { exitCode: 0, stdout: "", stderr: "" }; + return { exitCode: dockerInfoExitCode, stdout: "", stderr: "" }; } if (cmd[0] === "docker" && cmd[1] === "network" && cmd[2] === "inspect") { return { exitCode: 0, stdout: "[]", stderr: "" }; @@ -173,6 +176,7 @@ beforeEach(async () => { execCalls.length = 0; noteCalls.length = 0; trackedGeneratedPaths = []; + dockerInfoExitCode = 0; }); afterEach(async () => { @@ -605,3 +609,40 @@ test("doctor --fix leaves an override behind a symlinked .hack untouched (HACK-1 expect(notes).not.toContain(canary); expect(notes).not.toContain(outside); }, 30_000); + +for (const docker of [ + { label: "reachable", exitCode: 0 }, + { label: "unreachable", exitCode: 1 }, +] as const) { + test(`doctor --migrate-env-config migrates a project with only .hack/.env while Docker is ${docker.label} (HACK-1370)`, async () => { + const projectRoot = await createProjectWithStaleDaemonPid(); + const projectDir = resolve(projectRoot, ".hack"); + const value = ["synthetic", "doctor", "migrate", "4c2d"].join("-"); + await writeFile(resolve(projectDir, ".env"), `LEGACY_SETTING=${value}\n`); + dockerInfoExitCode = docker.exitCode; + + const { runCli } = await import("../src/cli/run.ts"); + const code = await runCli([ + "doctor", + "--migrate-env-config", + "--no-interactive", + "--path", + projectRoot, + ]); + // An unreachable Docker is still reported as a failed check (exit 1), but + // the requested migration, which needs no Docker, has run. + expect(code).toBe(docker.exitCode === 0 ? 0 : 1); + + const layerPath = resolve(projectDir, "hack.env.default.yaml"); + expect(YAML.parse(await readFile(layerPath, "utf8"))).toMatchObject({ + environment: "default", + values: { global: { LEGACY_SETTING: value } }, + }); + const notes = noteCalls.join("\n"); + expect(notes).toContain(`Wrote ${layerPath}`); + expect(notes).not.toContain(value); + expect(notes.includes("Docker is not reachable")).toBe( + docker.exitCode !== 0 + ); + }, 30_000); +} diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index c76478590..f172a4757 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -12,6 +12,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { YAML } from "bun"; import { LEGACY_ENV_SERVICE_SCOPE_MESSAGE } from "../src/cli/env-arguments.ts"; import { @@ -21,6 +22,8 @@ import { import { ClosedCliError, HackCliError } from "../src/lib/cli-result.ts"; import { ENV_SCOPE_INVALID_MESSAGE, + type LegacyProjectEnvMigration, + migrateLegacyProjectEnv, ProjectEnvLayerPathConflictError, projectEnvConfigExistsForMutation, resolveProjectEnvConfig, @@ -458,3 +461,134 @@ test("an invalid layer read is a closed error with a value-free scope reason", a expect(message).not.toContain("synthetic-91fa"); expect(message.split("\n")).toHaveLength(1); }); + +const BASE_ONLY_VALUE = ["synthetic", "base", "5b19"].join("-"); +const STAGING_VALUE = ["synthetic", "staging", "a3e7"].join("-"); + +/** Parsed `values` of one env layer file. */ +async function readLayerValues( + repo: Repo, + name: string +): Promise>> { + const parsed = YAML.parse( + await readFile(resolve(repo.projectDir, name), "utf8") + ) as { values: Record> }; + return parsed.values; +} + +/** The migration `hack doctor --migrate-env-config` runs (HACK-1370). */ +async function migrateLegacyRepo( + repo: Repo, + opts: { readonly migrateUndeclaredDotenv?: boolean } +): Promise { + return await migrateLegacyProjectEnv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + projectName: "env-legacy-scope", + serviceNames: ["api"], + materialize: false, + ...opts, + }); +} + +async function serviceScopedSet(repo: Repo): Promise { + return await runCli({ + repo, + args: [ + "env", + "set", + "NEW_SETTING", + "--service", + "api", + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${NEW_VALUE}\n`, + }); +} + +test("the migration the refusal names converts a project with only .hack/.env, and a service-scoped set then works", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\nSHARED_SETTING=${BASE_ONLY_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `KEPT_SETTING=${KEPT_VALUE}\nSHARED_SETTING=${STAGING_VALUE}\n` + ); + const configPath = resolve(repo.projectDir, PROJECT_CONFIG_FILENAME); + const configBefore = await readFile(configPath, "utf8"); + expectRefusal( + await serviceScopedSet(repo), + LEGACY_ENV_SERVICE_SCOPE_MESSAGE, + { lines: 2 } + ); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + expect(migrated.wroteFiles).toEqual([ + resolve(repo.projectDir, "hack.env.default.yaml"), + resolve(repo.projectDir, "hack.env.staging.yaml"), + ]); + // Every base entry is a global value; the overlay keeps only what differs. + expect(await readLayerValues(repo, "hack.env.default.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE, SHARED_SETTING: BASE_ONLY_VALUE }, + }); + expect(await readLayerValues(repo, "hack.env.staging.yaml")).toEqual({ + global: { SHARED_SETTING: STAGING_VALUE }, + }); + // Without a contract nothing names secret-backend values: the project config + // and its secret backend store are left alone and never offered for cleanup. + expect(migrated.updatedProjectConfig).toBe(false); + expect(await readFile(configPath, "utf8")).toBe(configBefore); + expect(migrated.cleanupCandidates).toEqual([ + resolve(repo.projectDir, ".env"), + resolve(repo.projectDir, ".env.staging"), + ]); + expect(migrated.blockedCleanupCandidates).toEqual([]); + + // The refusal no longer applies: the set lands in the api scope. + const run = await serviceScopedSet(repo); + expect(run.exitCode).toBe(0); + expect(`${run.stdout}${run.stderr}`).not.toContain(NEW_VALUE); + const values = await readLayerValues(repo, "hack.env.default.yaml"); + expect(values.global).toEqual({ + KEPT_SETTING: KEPT_VALUE, + SHARED_SETTING: BASE_ONLY_VALUE, + }); + expect(Object.keys(values.api ?? {})).toEqual(["NEW_SETTING"]); + expect(await lstat(repo.storePath).catch(() => null)).toBeNull(); +}, 60_000); + +test("an explicit migration of a project with no legacy env file switches it to an empty default layer", async () => { + const repo = await createLegacyRepo(); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + expect(migrated.cleanupCandidates).toEqual([]); + expect(await readLayerValues(repo, "hack.env.default.yaml")).toEqual({ + global: {}, + }); + const run = await serviceScopedSet(repo); + expect(run.exitCode).toBe(0); + const values = await readLayerValues(repo, "hack.env.default.yaml"); + expect(Object.keys(values.api ?? {})).toEqual(["NEW_SETTING"]); +}, 60_000); + +test("the implicit migrations other commands run still need the hack.env.json contract", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const before = await hackDirEntries(repo); + const migrated = await migrateLegacyRepo(repo, {}); + expect(migrated.legacyDetected).toBe(false); + expect(migrated.wroteFiles).toEqual([]); + expect(await hackDirEntries(repo)).toEqual(before); +}); From 78b7276259926a183a89b77aa34e419c7b2fc4ac Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 16:12:24 -0400 Subject: [PATCH 03/55] fix(env): keep gitignored values out of tracked layers in the .env-only migration hack doctor --migrate-env-config, for a project with no hack.env.json, now writes the .hack/.env and overlay values to the gitignored machine-local layers (hack.env.local.yaml, hack.env..local.yaml) and the tracked hack.env.default.yaml/overlay layers with no values, so secret-shaped plaintext never lands in a tracked file. Before writing anything it refuses closed, keeping the legacy files: keys that do not match ^[A-Z_][A-Z0-9_]*$ (E_CONFIG_INVALID, counted per file, never shown), legacy .env.default/.env.local/.env..local overlays, existing or unusable layer entries (E_PATH_CONFLICT), and a machine-local target git tracks or cannot report on. Layers are created only where nothing exists, and created layers are removed if a later write fails. Doctor now decides the env format with projectEnvConfigExistsForMutation, so a symlinked layer is never written through and a dangling layer refuses with E_PATH_CONFLICT instead of being replaced. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 35 ++- src/commands/doctor.ts | 32 ++- src/lib/project-env-config.ts | 290 ++++++++++++++++++++---- tests/doctor-fix-noninteractive.test.ts | 78 ++++++- tests/env-legacy-service-scope.test.ts | 271 +++++++++++++++++++++- 5 files changed, 635 insertions(+), 71 deletions(-) diff --git a/docs/env.md b/docs/env.md index 156eafc8a..7a596b943 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1000,12 +1000,35 @@ That migrates the repo into: - `.hack.secret.key` when needed A repo that has no `.hack/hack.env.json` is migrated as well. Its values live only in -`.hack/.env` and `.hack/.env.`, so each `.hack/.env` entry becomes a plaintext -global value in `hack.env.default.yaml`, and each overlay entry that differs from -`.hack/.env` becomes a global value in `hack.env..yaml`. A repo with no legacy env -file gets an empty `hack.env.default.yaml`. Without `hack.env.json`, nothing lists values -kept in a secret backend, so none are migrated: set them again with -`hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and +`.hack/.env` and `.hack/.env.`, which are gitignored and may hold passwords or +API keys, so the migration keeps them out of tracked files: each `.hack/.env` entry +becomes a plaintext global value in the gitignored machine-local layer +`hack.env.local.yaml`, and each overlay entry that differs from `.hack/.env` becomes a +global value in `hack.env..local.yaml`. The tracked `hack.env.default.yaml`, and +`hack.env..yaml` for each migrated overlay, are written with no values, so the +repo switches to env layers. Teammates do not receive these values through git; share +the ones they need with `hack env set` (plain settings) or `hack env set --secret` +(encrypted) after the migration. A repo with no legacy env file gets only an empty +`hack.env.default.yaml`. + +This migration refuses, with a one-line `E_*` error, writes nothing and keeps the legacy +files, when: + +- a key does not match `^[A-Z_][A-Z0-9_]*$`, for example a lowercase name or a line + that starts with `export ` (`E_CONFIG_INVALID`). The error counts the keys per file but + does not show them, because a malformed line may hold a value. Rename or remove them; +- a legacy file is named `.hack/.env.default`, `.hack/.env.local` or + `.hack/.env..local`, whose layer would be the default or a machine-local layer + rather than an overlay (`E_CONFIG_INVALID`). Rename or remove it; +- an entry named like an env layer (`.hack/hack.env.*.yaml`) is a dangling symlink, a + directory or another non-file (`E_PATH_CONFLICT`). A regular layer file, or a symlink + to one, means the repo already uses env layers: `hack doctor` reports that and writes + nothing, never through the link; +- git tracks a machine-local layer it would write, or, in a git checkout, git cannot + report whether it does (`E_PATH_CONFLICT`). + +Without `hack.env.json`, nothing lists values kept in a secret backend, so none are +migrated: set them again with `hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and any secret backend it configures, is left unchanged. The migration changes project files only, so it runs even when Docker is not reachable; the other `--migrate-env-config` repairs still need Docker. As with any migration, `hack doctor` then offers to remove the diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index bd67d81f7..8b0b70638 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -170,7 +170,9 @@ import { inspectProjectEnvMaterialization, type LegacyComposeEnvFileReference, migrateLegacyProjectEnv, + ProjectEnvLayerPathConflictError, projectEnvConfigExists, + projectEnvConfigExistsForMutation, removeLegacyProjectEnvArtifacts, repairLegacyComposeEnvFileReferences, resolveProjectEnvConfig, @@ -3722,8 +3724,12 @@ async function maybeMigrateProjectEnvConfig(opts: { return; } + // The migration writes env layers, so it decides the format like + // `hack env set`: a symlinked layer counts as an env layer (never written + // through), and a dangling or non-file layer entry refuses closed with + // E_PATH_CONFLICT instead of being replaced. if ( - await projectEnvConfigExists({ + await projectEnvConfigExistsForMutation({ projectDir: project.projectDir, }) ) { @@ -3758,8 +3764,10 @@ async function maybeMigrateProjectEnvConfig(opts: { "Detected legacy project env files.", hasContract ? `- source: ${contractPath} and current local env state` - : `- source: ${resolve(project.projectDir, PROJECT_ENV_FILENAME)} and its overlays (no ${PROJECT_ENV_CONTRACT_FILENAME}; values become plaintext global values, and secret-backend values are not migrated)`, - `- target: ${resolve(project.projectDir, "hack.env.default.yaml")}`, + : `- source: ${resolve(project.projectDir, PROJECT_ENV_FILENAME)} and its overlays (no ${PROJECT_ENV_CONTRACT_FILENAME}; secret-backend values are not migrated)`, + hasContract + ? `- target: ${resolve(project.projectDir, "hack.env.default.yaml")}` + : `- target: values go to the gitignored ${resolve(project.projectDir, "hack.env.local.yaml")} (and hack.env..local.yaml); ${resolve(project.projectDir, "hack.env.default.yaml")} is written with no values`, "- runtime: direct env injection by default; .hack/.env becomes compatibility-only", ].join("\n"), "env migration" @@ -4014,11 +4022,19 @@ async function buildEnvMigrationPlanStep(opts: { return "Skip env migration because no project was detected from this path."; } - if ( - await projectEnvConfigExists({ + // The same format decision as the migration itself; a plan never throws. + let modernExists: boolean; + try { + modernExists = await projectEnvConfigExistsForMutation({ projectDir: opts.project.projectDir, - }) - ) { + }); + } catch (error: unknown) { + if (error instanceof ProjectEnvLayerPathConflictError) { + return `Refuse env migration (E_PATH_CONFLICT) because ${error.path} is not a regular env layer file.`; + } + throw error; + } + if (modernExists) { return "Skip env migration because this project already uses hack.env.*.yaml."; } @@ -4030,7 +4046,7 @@ async function buildEnvMigrationPlanStep(opts: { return "Prompt to migrate legacy env config (.hack/hack.env.json) to hack.env.*.yaml."; } - return "Prompt to migrate legacy env values (.hack/.env and its overlays, no .hack/hack.env.json) to hack.env.*.yaml as plaintext global values."; + return "Prompt to migrate legacy env values (.hack/.env and its overlays, no .hack/hack.env.json) to the gitignored machine-local hack.env.local.yaml and hack.env..local.yaml layers."; } /** diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index cd5d536b2..ffacca686 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -4786,7 +4786,8 @@ export async function migrateLegacyProjectEnv(opts: { readonly materialize: boolean; /** * Also migrate a project that has no contract: the legacy `.hack/.env` - * and `.hack/.env.` files become plaintext global values. Only an + * and `.hack/.env.` files become plaintext global values in the + * gitignored machine-local layers, never in a tracked layer. Only an * explicit migration request (`hack doctor --migrate-env-config`) sets it; * the implicit migrations run by other commands keep requiring a contract. */ @@ -4921,19 +4922,153 @@ export async function migrateLegacyProjectEnv(opts: { }; } +/** + * Closed, value-free refusal of the explicit migration of a project without a + * `.hack/hack.env.json` contract ({@link migrateUndeclaredLegacyDotenv}). It is + * raised before any file is written, so the legacy `.hack/.env*` files and any + * existing entries stay as they were. The message names paths, legacy file + * names and counts only, never a key or a value. + */ +export class LegacyDotenvMigrationRefusalError extends HackCliError { + constructor(opts: { + readonly code: "E_CONFIG_INVALID" | "E_PATH_CONFLICT"; + readonly reason: string; + }) { + super({ + code: opts.code, + message: `Refusing to migrate the legacy .hack/.env files: ${opts.reason} Nothing was written, and the legacy files are kept.`, + }); + this.name = "LegacyDotenvMigrationRefusalError"; + } +} + +/** `.hack/.env.state.json` is Hack's own state file, not a dotenv overlay. */ +const LEGACY_DOTENV_STATE_OVERLAY = PROJECT_ENV_STATE_FILENAME.slice( + `${PROJECT_ENV_FILENAME}.`.length +); + +/** + * Legacy overlay names whose env layers would be the default layer or a + * machine-local layer instead of an overlay (`.env.default`, `.env.local`, + * `.env..local`). + */ +function isReservedLegacyOverlayName(name: string): boolean { + return ( + name === "default" || + name === PROJECT_ENV_LOCAL_SEGMENT || + name.endsWith(`.${PROJECT_ENV_LOCAL_SEGMENT}`) + ); +} + +function countLabel(opts: { + readonly count: number; + readonly noun: string; +}): string { + return `${opts.count} ${opts.noun}${opts.count === 1 ? "" : "s"}`; +} + +type PlannedEnvLayerWrite = { + readonly path: string; + readonly config: ProjectEnvConfig; +}; + +/** + * Refuses a machine-local migration target that git tracks, or that sits in a + * git checkout where git cannot say (fail closed), since migrated values + * written there could be committed. Mirrors `hack env hydrate`. + */ +async function assertUntrackedMigrationLocalLayer(opts: { + readonly projectRoot: string; + readonly path: string; +}): Promise { + const tracked = await isGitTrackedProjectEnvPath(opts); + if (tracked === true) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `git tracks the machine-local env layer ${opts.path}, so migrated values would be committed. Untrack it with \`git rm --cached\` and retry.`, + }); + } + if ( + tracked === null && + (await isInsideGitCheckoutMarker({ startDir: opts.projectRoot })) + ) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `this checkout is a git repository, but git could not report whether it tracks ${opts.path}, so migrated values could be committed. Make \`git status\` work in this checkout and retry.`, + }); + } +} + +/** + * Entries of `.hack/.env` (key `null`) and each legacy overlay file. Refuses, + * before anything is written, when a key cannot be an env layer key; the + * refusal counts such keys per file and never shows them. + */ +async function readValidatedLegacyDotenvFiles(opts: { + readonly projectDir: string; + readonly overlayNames: readonly string[]; +}): Promise>> { + const legacyFiles = [ + { envName: null, label: PROJECT_ENV_FILENAME }, + ...opts.overlayNames.map((name) => ({ + envName: name, + label: `${PROJECT_ENV_FILENAME}.${name}`, + })), + ]; + const legacyValues = new Map>(); + const invalidKeyCounts: string[] = []; + for (const file of legacyFiles) { + const values = await readLegacyDotenvFile({ + path: resolveEnvFilePath({ + projectDir: opts.projectDir, + envName: file.envName, + }), + }); + legacyValues.set(file.envName, values); + const invalid = Object.keys(values).filter( + (key) => !PROJECT_ENV_KEY_PATTERN.test(key) + ).length; + if (invalid > 0) { + invalidKeyCounts.push( + `${countLabel({ count: invalid, noun: "key" })} in ${file.label}` + ); + } + } + if (invalidKeyCounts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `env keys must match ^[A-Z_][A-Z0-9_]*$, but ${invalidKeyCounts.join(", ")} do not (for example a lowercase name or an "export " prefix). The keys are not shown because a malformed line may contain a value. Rename or remove them, then retry.`, + }); + } + return legacyValues; +} + /** * Migrates a legacy project that has no `.hack/hack.env.json` contract, so - * its values live only in `.hack/.env` and `.hack/.env.`: every - * base entry becomes a plaintext global value in `hack.env.default.yaml`, and - * every overlay entry that differs from the base becomes a global value in - * `hack.env..yaml`. With no legacy file at all it still writes an - * empty default layer, so the project switches to env layers. + * its values live only in `.hack/.env` and `.hack/.env.`. * - * Without a contract nothing names secret-backend values, so none are read or - * migrated, and `hack.config.json` (including any secret backend it - * configures, and that backend's store) is left unchanged and never offered - * for cleanup. Values are written to the layer files only, never returned or - * printed. + * Contract: + * - Those files are gitignored, so their values stay out of tracked files: + * every base entry becomes a plaintext global value in the machine-local + * layer `hack.env.local.yaml`, and every overlay entry that differs from the + * base becomes a global value in `hack.env..local.yaml`. The tracked + * `hack.env.default.yaml` (and `hack.env..yaml` for each migrated + * overlay) is written with no values, so the project switches to env layers. + * With no legacy file at all only the empty default layer is written. + * - Refused before anything is written ({@link LegacyDotenvMigrationRefusalError}, + * or {@link ProjectEnvLayerPathConflictError} for an unusable layer entry): + * any existing env layer entry, a legacy overlay named like the default or a + * machine-local layer, a key that does not match `^[A-Z_][A-Z0-9_]*$` + * (counted per file, never shown), and a machine-local target that git + * tracks or, inside a git checkout, cannot report on. + * - Layers are created only where nothing exists (compare-and-swap against an + * absent file); if a later write fails, the layers this call created are + * removed again, so the project keeps using the legacy files. + * - Without a contract nothing names secret-backend values, so none are read + * or migrated, and `hack.config.json` (including any secret backend it + * configures, and that backend's store) is left unchanged and never offered + * for cleanup. Values are written to the layer files only, never returned or + * printed. */ async function migrateUndeclaredLegacyDotenv(opts: { readonly projectRoot: string; @@ -4941,60 +5076,123 @@ async function migrateUndeclaredLegacyDotenv(opts: { readonly serviceNames: readonly string[]; readonly materialize: boolean; }): Promise { - const overlayNames = await discoverLegacyOverlayNames({ - projectDir: opts.projectDir, - }); - const baseValues = await readLegacyDotenvFile({ - path: resolveEnvFilePath({ projectDir: opts.projectDir, envName: null }), - }); + // Also refuses a dangling or non-file layer entry (E_PATH_CONFLICT), which + // an atomic write would otherwise replace or write through. + if ( + await projectEnvConfigExistsForMutation({ projectDir: opts.projectDir }) + ) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `env layer files already exist in ${opts.projectDir}.`, + }); + } + const overlayNames = ( + await discoverLegacyOverlayNames({ projectDir: opts.projectDir }) + ).filter((name) => name !== LEGACY_DOTENV_STATE_OVERLAY); + const reservedOverlays = overlayNames.filter(isReservedLegacyOverlayName); + if (reservedOverlays.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${reservedOverlays.map((name) => `${PROJECT_ENV_FILENAME}.${name}`).join(", ")} would become the default or a machine-local env layer rather than an overlay. Rename it to another overlay name or remove it, then retry.`, + }); + } - const wroteFiles: string[] = []; - const defaultPath = resolveProjectEnvConfigPath({ + const legacyValues = await readValidatedLegacyDotenvFiles({ projectDir: opts.projectDir, - envName: null, + overlayNames, }); - if ( - await writeProjectEnvConfigFile({ - path: defaultPath, + + const baseValues = legacyValues.get(null) ?? {}; + const localWrites: PlannedEnvLayerWrite[] = []; + const trackedWrites: PlannedEnvLayerWrite[] = [ + { + path: resolveProjectEnvConfigPath({ + projectDir: opts.projectDir, + envName: null, + }), config: buildMigratedProjectEnvConfig({ environment: "default", - values: { global: { ...baseValues } }, + values: { global: {} }, }), - }) - ) { - wroteFiles.push(defaultPath); - } - - for (const overlayName of overlayNames) { - const overlayValues = await readLegacyDotenvFile({ - path: resolveEnvFilePath({ + }, + ]; + if (Object.keys(baseValues).length > 0) { + localWrites.push({ + path: resolveProjectEnvLocalConfigPath({ projectDir: opts.projectDir, - envName: overlayName, + envName: null, + }), + config: buildMigratedProjectEnvConfig({ + environment: "default", + values: { global: { ...baseValues } }, }), }); + } + for (const overlayName of overlayNames) { const changedValues = Object.fromEntries( - Object.entries(overlayValues).filter( + Object.entries(legacyValues.get(overlayName) ?? {}).filter( ([key, value]) => baseValues[key] !== value ) ); if (Object.keys(changedValues).length === 0) { continue; } - const overlayPath = resolveProjectEnvConfigPath({ + localWrites.push({ + path: resolveProjectEnvLocalConfigPath({ + projectDir: opts.projectDir, + envName: overlayName, + }), + config: buildMigratedProjectEnvConfig({ + environment: overlayName, + values: { global: changedValues }, + }), + }); + trackedWrites.push({ + path: resolveProjectEnvConfigPath({ + projectDir: opts.projectDir, + envName: overlayName, + }), + config: buildMigratedProjectEnvConfig({ + environment: overlayName, + values: { global: {} }, + }), + }); + } + + for (const write of localWrites) { + await assertUntrackedMigrationLocalLayer({ + projectRoot: opts.projectRoot, + path: write.path, + }); + } + if (localWrites.length > 0) { + await ensureHackDirGitignoreForLocalEnvWrite({ projectDir: opts.projectDir, - envName: overlayName, }); - if ( - await writeProjectEnvConfigFile({ - path: overlayPath, - config: buildMigratedProjectEnvConfig({ - environment: overlayName, - values: { global: changedValues }, - }), - }) - ) { - wroteFiles.push(overlayPath); + } + + // Machine-local layers first, so the tracked layers switch the format only + // once the values have a home. + const wroteFiles: string[] = []; + try { + for (const write of [...localWrites, ...trackedWrites]) { + if ( + await writeProjectEnvConfigFile({ + path: write.path, + config: write.config, + expectedExisting: null, + }) + ) { + wroteFiles.push(write.path); + } + } + } catch (error: unknown) { + // Every target was absent before this call, so removing what it created + // leaves the project on the legacy files. + for (const path of [...wroteFiles].reverse()) { + await unlink(path).catch(() => undefined); } + throw error; } if (opts.materialize) { diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 0ab670d7c..b5ff2d611 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -633,16 +633,90 @@ for (const docker of [ // the requested migration, which needs no Docker, has run. expect(code).toBe(docker.exitCode === 0 ? 0 : 1); + // The gitignored value lands in the machine-local layer; the tracked + // default layer only switches the format. + const localPath = resolve(projectDir, "hack.env.local.yaml"); const layerPath = resolve(projectDir, "hack.env.default.yaml"); - expect(YAML.parse(await readFile(layerPath, "utf8"))).toMatchObject({ + expect(YAML.parse(await readFile(localPath, "utf8"))).toMatchObject({ environment: "default", values: { global: { LEGACY_SETTING: value } }, }); + expect(YAML.parse(await readFile(layerPath, "utf8"))).toMatchObject({ + environment: "default", + values: { global: {} }, + }); + expect(await readFile(layerPath, "utf8")).not.toContain(value); const notes = noteCalls.join("\n"); - expect(notes).toContain(`Wrote ${layerPath}`); + expect(notes).toContain(`Wrote ${localPath}, ${layerPath}`); expect(notes).not.toContain(value); expect(notes.includes("Docker is not reachable")).toBe( docker.exitCode !== 0 ); }, 30_000); } + +test("doctor --migrate-env-config treats a symlinked overlay as env layers and never writes through it (HACK-1370)", async () => { + const projectRoot = await createProjectWithStaleDaemonPid(); + const projectDir = resolve(projectRoot, ".hack"); + const owned = ["synthetic", "owned", "77d1"].join("-"); + const legacy = ["synthetic", "legacy", "b9e0"].join("-"); + await writeFile(resolve(projectDir, ".env"), `OWNED=${legacy}\n`); + await writeFile(resolve(projectDir, ".env.staging"), `OWNED=${legacy}-s\n`); + const target = resolve(projectRoot, "..", "outside-staging.yaml"); + const targetText = `version: 1\nenvironment: staging\nsecretsprovider: project_key\nvalues:\n global:\n OWNED: ${owned}\n`; + await writeFile(target, targetText); + const link = resolve(projectDir, "hack.env.staging.yaml"); + await symlink(target, link); + + const { runCli } = await import("../src/cli/run.ts"); + const code = await runCli([ + "doctor", + "--migrate-env-config", + "--no-interactive", + "--path", + projectRoot, + ]); + expect(code).toBe(0); + expect(noteCalls.join("\n")).toContain( + "Project already uses the new env config files." + ); + // The link and its target are untouched; nothing was migrated or cleaned. + expect(await readlink(link)).toBe(target); + expect(await readFile(target, "utf8")).toBe(targetText); + expect(await readFile(resolve(projectDir, ".env"), "utf8")).toContain(legacy); + expect(await readFile(resolve(projectDir, ".env.staging"), "utf8")).toContain( + legacy + ); + for (const name of ["hack.env.default.yaml", "hack.env.local.yaml"]) { + expect(await lstat(resolve(projectDir, name)).catch(() => null)).toBeNull(); + } +}, 30_000); + +test("doctor --migrate-env-config refuses a dangling default layer with E_PATH_CONFLICT and keeps it (HACK-1370)", async () => { + const projectRoot = await createProjectWithStaleDaemonPid(); + const projectDir = resolve(projectRoot, ".hack"); + const legacy = ["synthetic", "legacy", "0a6f"].join("-"); + await writeFile(resolve(projectDir, ".env"), `OWNED=${legacy}\n`); + const link = resolve(projectDir, "hack.env.default.yaml"); + const missing = resolve(projectDir, "missing-layer.yaml"); + await symlink(missing, link); + + const { runCli } = await import("../src/cli/run.ts"); + const code = await runCli([ + "doctor", + "--migrate-env-config", + "--no-interactive", + "--path", + projectRoot, + ]); + expect(code).toBe(1); + // Still the same dangling link; the legacy file is kept and nothing else + // was written. + expect((await lstat(link)).isSymbolicLink()).toBe(true); + expect(await readlink(link)).toBe(missing); + expect(await readFile(resolve(projectDir, ".env"), "utf8")).toContain(legacy); + expect( + await lstat(resolve(projectDir, "hack.env.local.yaml")).catch(() => null) + ).toBeNull(); + expect(noteCalls.join("\n")).not.toContain(legacy); +}, 30_000); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index f172a4757..cfae30831 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -22,6 +22,7 @@ import { import { ClosedCliError, HackCliError } from "../src/lib/cli-result.ts"; import { ENV_SCOPE_INVALID_MESSAGE, + LegacyDotenvMigrationRefusalError, type LegacyProjectEnvMigration, migrateLegacyProjectEnv, ProjectEnvLayerPathConflictError, @@ -34,8 +35,10 @@ import { privateCliEnv } from "./helpers/private-cli-env.ts"; * HACK-1370: `hack env set`/`unset` in a legacy `.hack/.env` project refuse a * `--service` scope instead of silently changing the global value; a layer * entry that is not a usable file refuses (E_PATH_CONFLICT) instead of - * selecting the legacy file; and reads of an invalid layer print one closed - * line, never a stack. + * selecting the legacy file; reads of an invalid layer print one closed + * line, never a stack; and the doctor migration the refusal names keeps the + * gitignored values out of tracked layers and refuses, writing nothing, what + * it cannot migrate safely. * * Isolation: every child gets a private HOME, global config and Docker client * config inside a temp directory, and an `encrypted_file` secret backend in @@ -530,17 +533,38 @@ test("the migration the refusal names converts a project with only .hack/.env, a migrateUndeclaredDotenv: true, }); expect(migrated.legacyDetected).toBe(true); + // Machine-local layers first, then the tracked layers that switch format. expect(migrated.wroteFiles).toEqual([ + resolve(repo.projectDir, "hack.env.local.yaml"), + resolve(repo.projectDir, "hack.env.staging.local.yaml"), resolve(repo.projectDir, "hack.env.default.yaml"), resolve(repo.projectDir, "hack.env.staging.yaml"), ]); - // Every base entry is a global value; the overlay keeps only what differs. - expect(await readLayerValues(repo, "hack.env.default.yaml")).toEqual({ + // Every base entry is a machine-local global value; the overlay keeps only + // what differs. The tracked layers hold no values. + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ global: { KEPT_SETTING: KEPT_VALUE, SHARED_SETTING: BASE_ONLY_VALUE }, }); - expect(await readLayerValues(repo, "hack.env.staging.yaml")).toEqual({ + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ global: { SHARED_SETTING: STAGING_VALUE }, }); + expect(await readLayerValues(repo, "hack.env.default.yaml")).toEqual({ + global: {}, + }); + expect(await readLayerValues(repo, "hack.env.staging.yaml")).toEqual({ + global: {}, + }); + // The resolved env still carries every value. + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: "staging", + serviceNames: ["api"], + }); + expect(resolved?.globalEnv).toEqual({ + KEPT_SETTING: KEPT_VALUE, + SHARED_SETTING: STAGING_VALUE, + }); // Without a contract nothing names secret-backend values: the project config // and its secret backend store are left alone and never offered for cleanup. expect(migrated.updatedProjectConfig).toBe(false); @@ -556,20 +580,249 @@ test("the migration the refusal names converts a project with only .hack/.env, a expect(run.exitCode).toBe(0); expect(`${run.stdout}${run.stderr}`).not.toContain(NEW_VALUE); const values = await readLayerValues(repo, "hack.env.default.yaml"); - expect(values.global).toEqual({ - KEPT_SETTING: KEPT_VALUE, - SHARED_SETTING: BASE_ONLY_VALUE, - }); + expect(values.global).toEqual({}); expect(Object.keys(values.api ?? {})).toEqual(["NEW_SETTING"]); expect(await lstat(repo.storePath).catch(() => null)).toBeNull(); }, 60_000); +const PASSWORD_VALUE = ["synthetic", "db", "password", "e8a1"].join("-"); +const STRIPE_VALUE = ["sk", "test", "synthetic", "4f0c9b"].join("_"); + +async function git(opts: { + readonly repo: Repo; + readonly args: readonly string[]; +}): Promise { + const proc = Bun.spawn(["git", "-C", opts.repo.projectRoot, ...opts.args], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + HOME: opts.repo.root, + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_AUTHOR_NAME: "Synthetic", + GIT_AUTHOR_EMAIL: "synthetic@example.invalid", + GIT_COMMITTER_NAME: "Synthetic", + GIT_COMMITTER_EMAIL: "synthetic@example.invalid", + }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + proc.exited, + ]); + expect(exitCode).toBe(0); + return stdout; +} + +test("the migration keeps secret-shaped values from the gitignored .hack/.env out of every file git would commit", async () => { + const repo = await createLegacyRepo(); + await git({ repo, args: ["init", "-q"] }); + await writeFile( + resolve(repo.projectDir, ".env"), + `DATABASE_PASSWORD=${PASSWORD_VALUE}\nSTRIPE_SECRET_KEY=${STRIPE_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `STRIPE_SECRET_KEY=${STRIPE_VALUE}-staging\n` + ); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.wroteFiles).toContain( + resolve(repo.projectDir, "hack.env.default.yaml") + ); + // Every file git would add (tracked or untracked, not ignored) is free of + // the values; the machine-local layers that hold them are ignored. + const committable = ( + await git({ + repo, + args: ["ls-files", "--cached", "--others", "--exclude-standard"], + }) + ) + .split("\n") + .filter((line) => line.length > 0); + expect(committable).toContain(".hack/hack.env.default.yaml"); + expect(committable).toContain(".hack/hack.env.staging.yaml"); + expect(committable).not.toContain(".hack/hack.env.local.yaml"); + expect(committable).not.toContain(".hack/hack.env.staging.local.yaml"); + // The legacy overlay input itself is outside the managed ignore block; the + // migration only reads it (and offers it for cleanup). + for (const path of committable.filter( + (entry) => entry !== ".hack/.env.staging" + )) { + const text = await readFile(resolve(repo.projectRoot, path), "utf8"); + expect(text).not.toContain(PASSWORD_VALUE); + expect(text).not.toContain(STRIPE_VALUE); + } + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { + DATABASE_PASSWORD: PASSWORD_VALUE, + STRIPE_SECRET_KEY: STRIPE_VALUE, + }, + }); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { STRIPE_SECRET_KEY: `${STRIPE_VALUE}-staging` }, + }); +}, 60_000); + +/** The legacy files and the .hack entries, to prove a refusal wrote nothing. */ +async function legacySnapshot(repo: Repo): Promise<{ + readonly entries: readonly string[]; + readonly files: Readonly>; +}> { + const entries = await hackDirEntries(repo); + const files: Record = {}; + for (const name of entries.filter((entry) => entry.startsWith(".env"))) { + files[name] = await readFile(resolve(repo.projectDir, name), "utf8"); + } + return { entries, files }; +} + +async function expectMigrationRefusal(opts: { + readonly repo: Repo; + readonly code: LegacyDotenvMigrationRefusalError["code"]; + readonly reason: string; + readonly hidden: readonly string[]; +}): Promise { + const before = await legacySnapshot(opts.repo); + const error = await migrateLegacyRepo(opts.repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + const refusal = error as LegacyDotenvMigrationRefusalError; + expect(refusal.code).toBe(opts.code); + expect(refusal.message).toContain(opts.reason); + expect(refusal.message).toContain( + "Nothing was written, and the legacy files are kept." + ); + expect(refusal.message.split("\n")).toHaveLength(1); + for (const hidden of opts.hidden) { + expect(refusal.message).not.toContain(hidden); + } + expect(await legacySnapshot(opts.repo)).toEqual(before); +} + +test("the migration refuses keys an env layer cannot hold, counting them without showing keys or values", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `GOOD_KEY=${KEPT_VALUE}\nnode_env=${BASE_ONLY_VALUE}\nexport OTHER=${STAGING_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `GOOD_KEY=${NEW_VALUE}\nlower_case=${NEW_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "2 keys in .env, 1 key in .env.staging do not", + hidden: [ + "node_env", + "export OTHER", + "lower_case", + KEPT_VALUE, + BASE_ONLY_VALUE, + STAGING_VALUE, + NEW_VALUE, + ], + }); +}); + +for (const name of [".env.default", ".env.local", ".env.staging.local"]) { + test(`the migration refuses a legacy ${name}, whose layer would not be an overlay`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, name), + `SHARED_SETTING=${STAGING_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: `${name} would become the default or a machine-local env layer`, + hidden: [KEPT_VALUE, STAGING_VALUE], + }); + }); +} + +test("the migration refuses a machine-local target that git tracks", async () => { + const repo = await createLegacyRepo(); + await git({ repo, args: ["init", "-q"] }); + const localPath = resolve(repo.projectDir, "hack.env.local.yaml"); + await writeFile(localPath, LAYER); + await git({ repo, args: ["add", "-f", ".hack/hack.env.local.yaml"] }); + // Still in the index, gone from the working tree. + await rm(localPath); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: `git tracks the machine-local env layer ${localPath}`, + hidden: [KEPT_VALUE], + }); +}, 60_000); + +test("the migration refuses existing layer entries and never writes through or replaces them", async () => { + // A symlinked overlay counts as an existing env layer: refused, and the + // link target keeps its content. + const linked = await createLegacyRepo(); + await writeFile(resolve(linked.projectDir, ".env"), `OWNED=${KEPT_VALUE}\n`); + await writeFile( + resolve(linked.projectDir, ".env.staging"), + `OWNED=${STAGING_VALUE}\n` + ); + const target = resolve(linked.root, "outside-staging.yaml"); + const targetText = LAYER.replace( + "environment: default", + "environment: staging" + ); + await writeFile(target, targetText); + await symlink(target, resolve(linked.projectDir, "hack.env.staging.yaml")); + await expectMigrationRefusal({ + repo: linked, + code: "E_PATH_CONFLICT", + reason: `env layer files already exist in ${linked.projectDir}`, + hidden: [KEPT_VALUE, STAGING_VALUE], + }); + expect(await readFile(target, "utf8")).toBe(targetText); + + // A dangling default layer is a path conflict, and stays a symlink. + const dangling = await createLegacyRepo(); + await writeFile( + resolve(dangling.projectDir, ".env"), + `OWNED=${KEPT_VALUE}\n` + ); + const path = await placeLayerEntry(dangling, { + name: "hack.env.default.yaml", + kind: "dangling-link", + }); + const before = await legacySnapshot(dangling); + const error = await migrateLegacyRepo(dangling, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ProjectEnvLayerPathConflictError); + expect((error as ProjectEnvLayerPathConflictError).path).toBe(path); + expect((await lstat(path)).isSymbolicLink()).toBe(true); + expect(await legacySnapshot(dangling)).toEqual(before); +}); + test("an explicit migration of a project with no legacy env file switches it to an empty default layer", async () => { const repo = await createLegacyRepo(); const migrated = await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true, }); expect(migrated.legacyDetected).toBe(true); + expect(migrated.wroteFiles).toEqual([ + resolve(repo.projectDir, "hack.env.default.yaml"), + ]); expect(migrated.cleanupCandidates).toEqual([]); expect(await readLayerValues(repo, "hack.env.default.yaml")).toEqual({ global: {}, From 23c8e7ea9c812879964e2f56612fd55aff18457d Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:14:42 +0000 Subject: [PATCH 04/55] fix(env): refuse a non-file or unreadable env layer on reads instead of falling back to .hack/.env env list, env explain and runtime env resolution read a selected layer with readTextFile, which returns null for a dangling symlink, a directory or an unreadable file. The read then looked like "no env layers" and fell back to the legacy .hack/.env format with exit 0. Only ENOENT is now an absent layer; a non-file entry refuses with E_PATH_CONFLICT and an unreadable one with a closed one-line error. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 5 ++- src/lib/project-env-config.ts | 33 +++++++++++++++++++- tests/env-legacy-service-scope.test.ts | 28 +++++++++++++++++ tests/project-env-metadata.test.ts | 18 +++++++---- tests/project-env-write-refusal-gaps.test.ts | 3 +- 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/docs/env.md b/docs/env.md index 0b18787d0..b7d66ce82 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1075,7 +1075,10 @@ looking for `.hack/hack.env.*.yaml` entries, `hack.env.default.yaml` included. A file, or a symlink to one, counts as an env layer. Any other entry with a layer name, such as a dangling symlink or a directory, is refused with `E_PATH_CONFLICT` and nothing is written; it never silently selects the legacy `.hack/.env`. Replace or remove the entry, -then retry. +then retry. Reads refuse the same way: `hack env list`, `hack env explain` and runtime +env injection stop with `E_PATH_CONFLICT` on a selected layer entry that is not a regular +file (such as a dangling `hack.env.default.yaml`), and with a closed one-line error on one +that cannot be read, instead of falling back to the legacy file. ## Compatibility notes diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 510a8ce2c..d63602669 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -985,12 +985,43 @@ export async function listProjectEnvOverlayNames(opts: { } /** Strict planning reads distinguish missing paths from failed/non-file layers. */ +/** + * Read a selected layer for the CLI and runtime paths. Only an absent entry + * (ENOENT on lstat) is an absent layer. An entry that exists but is not a + * readable regular file (a dangling symlink, a link to a directory, a + * directory, an unreadable file) refuses closed, so a broken layer never + * makes a read silently fall back to the legacy `.hack/.env` format. + */ +async function readLenientProjectEnvLayerText(opts: { + readonly path: string; +}): Promise { + const text = await readTextFile(opts.path); + if (text !== null) { + return text; + } + try { + await lstat(opts.path); + } catch (error: unknown) { + if (isRecord(error) && error.code === "ENOENT") { + return null; + } + throw new ClosedCliError(`Cannot read the env layer ${opts.path}.`); + } + const target = await stat(opts.path).catch(() => null); + if (!target?.isFile()) { + throw new ProjectEnvLayerPathConflictError({ path: opts.path }); + } + throw new ClosedCliError( + `Cannot read the env layer ${opts.path}. Check its permissions, then retry.` + ); +} + async function readProjectEnvLayerText(opts: { readonly path: string; readonly strictRead?: boolean; }): Promise { if (!opts.strictRead) { - return await readTextFile(opts.path); + return await readLenientProjectEnvLayerText({ path: opts.path }); } try { const selected = await stat(opts.path); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index cfae30831..cf5db1abc 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -348,6 +348,34 @@ for (const entry of UNUSABLE_LAYER_ENTRIES) { }, 60_000); } +for (const entry of UNUSABLE_LAYER_ENTRIES) { + test(`env list and explain refuse a ${entry.kind} ${entry.name} instead of reading .hack/.env`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const path = await placeLayerEntry(repo, entry); + const before = await hackDirEntries(repo); + const refusal = `E_PATH_CONFLICT: Refusing to choose the env format: ${path} is not a regular file`; + // An overlay layer is read only when that overlay is selected. + const envArgs = + entry.name === "hack.env.staging.yaml" ? ["--env", "staging"] : []; + + for (const args of [ + ["env", "list"], + ["env", "explain", "KEPT_SETTING"], + ]) { + const run = await runCli({ + repo, + args: [...args, ...envArgs, "--path", repo.projectRoot], + }); + expectRefusal(run, refusal, { lines: 1 }); + } + expect(await hackDirEntries(repo)).toEqual(before); + }, 60_000); +} + test("a symlinked overlay that resolves to a regular layer selects the env layers", async () => { const repo = await createLegacyRepo(); const target = resolve(repo.root, "shared-staging.yaml"); diff --git a/tests/project-env-metadata.test.ts b/tests/project-env-metadata.test.ts index 4baf8354c..8cf48db48 100644 --- a/tests/project-env-metadata.test.ts +++ b/tests/project-env-metadata.test.ts @@ -13,6 +13,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { PROJECT_ENV_KEY_FILENAME } from "../src/constants.ts"; import { + ProjectEnvLayerPathConflictError, type ProjectEnvResolvedConfig, type ProjectEnvValuesByScope, resolveProjectEnvConfig, @@ -429,8 +430,11 @@ test("metadata refuses unreadable selected layers instead of silently omitting t const path = join(f.projectDir, file); await mkdir(path); await expect(resolveProjectEnvMetadata(f)).rejects.toThrow(message); - // Existing runtime lookup behavior remains unchanged by strict planning reads. - expect(await resolveProjectEnvConfig(f)).toBeNull(); + // Runtime lookup refuses too (HACK-1370): a non-file layer never reads + // as absent, which would fall back to the legacy .hack/.env format. + await expect(resolveProjectEnvConfig(f)).rejects.toBeInstanceOf( + ProjectEnvLayerPathConflictError + ); await rm(path, { recursive: true }); } await writeLayer(f.projectDir, "hack.env.default.yaml", { @@ -438,9 +442,9 @@ test("metadata refuses unreadable selected layers instead of silently omitting t }); await mkdir(join(f.projectDir, "hack.env.qa.yaml")); await expect(resolveProjectEnvMetadata(f)).rejects.toThrow(message); - expect((await resolveProjectEnvConfig(f))?.globalEnv).toEqual({ - BASE: plainSentinel, - }); + await expect(resolveProjectEnvConfig(f)).rejects.toBeInstanceOf( + ProjectEnvLayerPathConflictError + ); expect( (await resolveProjectEnvMetadata({ ...f, envName: null })) ?.effectiveMetadata.global @@ -454,7 +458,9 @@ test("metadata rejects a dangling selected layer and retains readable symlink be await expect(resolveProjectEnvMetadata(f)).rejects.toThrow( "Cannot resolve project env metadata: selected configuration is invalid or unreadable." ); - expect(await resolveProjectEnvConfig(f)).toBeNull(); + await expect(resolveProjectEnvConfig(f)).rejects.toBeInstanceOf( + ProjectEnvLayerPathConflictError + ); await writeLayer(f.projectDir, "missing.yaml", { global: { PRESENT: plainSentinel }, }); diff --git a/tests/project-env-write-refusal-gaps.test.ts b/tests/project-env-write-refusal-gaps.test.ts index 79e3e8a57..90504b431 100644 --- a/tests/project-env-write-refusal-gaps.test.ts +++ b/tests/project-env-write-refusal-gaps.test.ts @@ -875,9 +875,10 @@ test("a directory at the layer path is a closed E_PATH_CONFLICT", async () => { args: setArgs(repo, "NEW_SETTING"), stdin: `${NEW_VALUE}\n`, }); + // The env format check (HACK-1370) refuses before the atomic writer runs. expectOneLineRefusal( run, - `E_PATH_CONFLICT: Refusing to replace ${repo.tracked}: it is not a regular file.` + `E_PATH_CONFLICT: Refusing to choose the env format: ${repo.tracked} is not a regular file` ); }, 60_000); From 69fb8ea0bad9e90bcc10a68a8541eebbfe18ea04 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:34:14 +0000 Subject: [PATCH 05/55] fix(env): refuse unusable layers in the contract migration and doctor checks, and keep tracked or compose-referenced legacy files out of cleanup Review round 1 (HACK-1370): - the contract migration decides the format with the mutation check, so a dangling default layer refuses (E_PATH_CONFLICT) instead of being replaced; its layer writes are create-only - doctor's env checks and project metadata report an unusable layer entry instead of the legacy format - the .env-only migration refuses .env. files whose name is not an overlay name, skips template and backup copies, and never offers a git-tracked legacy file for cleanup - doctor offers compose-referenced legacy files for cleanup only after the compose repair removed those references - a test for an unreadable default layer Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 17 ++- src/commands/doctor.ts | 96 ++++++++++--- src/lib/project-env-config.ts | 122 ++++++++++++++-- src/lib/project-meta.ts | 41 ++++-- tests/doctor-fix-noninteractive.test.ts | 16 +++ tests/env-legacy-service-scope.test.ts | 181 ++++++++++++++++++++++++ 6 files changed, 423 insertions(+), 50 deletions(-) diff --git a/docs/env.md b/docs/env.md index b7d66ce82..3c10f9c72 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1044,6 +1044,12 @@ files, when: - a legacy file is named `.hack/.env.default`, `.hack/.env.local` or `.hack/.env..local`, whose layer would be the default or a machine-local layer rather than an overlay (`E_CONFIG_INVALID`). Rename or remove it; +- a legacy `.hack/.env.` file has a name that is not an overlay name (lowercase + letters, digits and single dashes), such as `.env.staging.old` or `.env.Prod` + (`E_CONFIG_INVALID`). The error counts such files. Rename or remove them. Template and + backup copies (`.env.example`, `.env.sample`, `.env.template`, `.env.dist`, `.env.bak`, + `.env.backup`, `.env.old`, `.env.orig`) are not overlays: they are neither migrated + nor offered for cleanup; - an entry named like an env layer (`.hack/hack.env.*.yaml`) is a dangling symlink, a directory or another non-file (`E_PATH_CONFLICT`). A regular layer file, or a symlink to one, means the repo already uses env layers: `hack doctor` reports that and writes @@ -1057,9 +1063,14 @@ any secret backend it configures, is left unchanged. The migration changes proje only, so it runs even when Docker is not reachable; the other `--migrate-env-config` repairs still need Docker. As with any migration, `hack doctor` then offers to remove the legacy `.hack/.env*` files that Compose no longer references (a non-interactive run -accepts). - -`hack doctor` also warns when a repo still depends on the old format. +accepts). Files Compose still names as `env_file` are offered only when the Compose repair +removed those references. Without `hack.env.json`, a legacy file that git tracks, or one +whose status git cannot report in a git checkout, is never offered, because the migrated +values now live only in this machine's local layers. + +`hack doctor` also warns when a repo still depends on the old format. Its `env mode` +check reports an error, not the legacy format, when a `.hack/hack.env.*.yaml` entry is a +dangling symlink or another non-file. For modern env repos, `hack doctor` also warns when materialized compatibility output in `.hack/.env` or `.hack/.env.state.json` is stale and points to `hack env materialize`. diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 3a4aada93..92fe4d979 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -169,9 +169,9 @@ import { inspectLegacyComposeEnvFileReferences, inspectProjectEnvMaterialization, type LegacyComposeEnvFileReference, + type LegacyProjectEnvMigration, migrateLegacyProjectEnv, ProjectEnvLayerPathConflictError, - projectEnvConfigExists, projectEnvConfigExistsForMutation, removeLegacyProjectEnvArtifacts, repairLegacyComposeEnvFileReferences, @@ -2656,11 +2656,11 @@ async function checkProjectEnvMode({ }; } - if ( - await projectEnvConfigExists({ - projectDir: ctx.projectDir, - }) - ) { + const format = await doctorEnvFormat({ projectDir: ctx.projectDir }); + if (format.kind === "conflict") { + return { name: "env mode", status: "error", message: format.message }; + } + if (format.kind === "layers") { const defaultOverlay = cfg.env?.defaultOverlay ?? cfg.defaultEnvConfig ?? null; return { @@ -2690,6 +2690,31 @@ async function checkProjectEnvMode({ }; } +/** + * The env format the doctor checks report, decided by the same check that + * `hack env set`/`unset` use. A dangling or non-file `.hack/hack.env.*.yaml` + * entry is a conflict to report, never a silent "legacy format" (HACK-1370). + */ +async function doctorEnvFormat(opts: { + readonly projectDir: string; +}): Promise< + | { readonly kind: "layers" | "legacy" } + | { readonly kind: "conflict"; readonly message: string } +> { + try { + return (await projectEnvConfigExistsForMutation({ + projectDir: opts.projectDir, + })) + ? { kind: "layers" } + : { kind: "legacy" }; + } catch (error: unknown) { + if (error instanceof ProjectEnvLayerPathConflictError) { + return { kind: "conflict", message: error.message }; + } + throw error; + } +} + async function checkProjectEnvOverlayWarnings({ startDir, }: { @@ -2713,11 +2738,15 @@ async function checkProjectEnvOverlayWarnings({ }; } - if ( - await projectEnvConfigExists({ - projectDir: ctx.projectDir, - }) - ) { + const format = await doctorEnvFormat({ projectDir: ctx.projectDir }); + if (format.kind === "conflict") { + return { + name: "env overlay warnings", + status: "warn", + message: "Skipped: an env layer entry is unusable (see env mode)", + }; + } + if (format.kind === "layers") { const serviceNames = await discoverComposeServiceNames({ composeFile: ctx.composeFile, }); @@ -2783,11 +2812,15 @@ async function checkProjectEnvMaterialization({ }; } - if ( - !(await projectEnvConfigExists({ - projectDir: ctx.projectDir, - })) - ) { + const format = await doctorEnvFormat({ projectDir: ctx.projectDir }); + if (format.kind === "conflict") { + return { + name: "env materialization", + status: "warn", + message: "Skipped: an env layer entry is unusable (see env mode)", + }; + } + if (format.kind === "legacy") { return { name: "env materialization", status: "ok", @@ -3743,7 +3776,7 @@ async function maybeMigrateProjectEnvConfig(opts: { } note(migrationNotes.join("\n"), "doctor"); - let blockedCleanupCandidates = [...migrated.blockedCleanupCandidates]; + let composeRepaired = false; if (migrated.composeEnvFileReferences.length > 0) { note( buildComposeEnvRepairNote({ @@ -3772,7 +3805,7 @@ async function maybeMigrateProjectEnvConfig(opts: { ].join("\n"), "compose env repair" ); - blockedCleanupCandidates = []; + composeRepaired = true; } else { note( "No compose env_file updates were needed; legacy compatibility files remain blocked from cleanup.", @@ -3787,10 +3820,10 @@ async function maybeMigrateProjectEnvConfig(opts: { } } - const cleanupCandidates = [ - ...migrated.cleanupCandidates, - ...blockedCleanupCandidates, - ].sort((left, right) => left.localeCompare(right)); + const cleanupCandidates = doctorEnvCleanupCandidates({ + migrated, + composeRepaired, + }); if (cleanupCandidates.length === 0) { return; } @@ -3820,6 +3853,25 @@ async function maybeMigrateProjectEnvConfig(opts: { } } +/** + * Legacy env files doctor offers to delete after a migration. Files compose + * still names as `env_file` (`blockedCleanupCandidates`) are offered only when + * the compose repair removed those references; deleting them otherwise would + * break the next `hack up`. + */ +export function doctorEnvCleanupCandidates(opts: { + readonly migrated: Pick< + LegacyProjectEnvMigration, + "cleanupCandidates" | "blockedCleanupCandidates" + >; + readonly composeRepaired: boolean; +}): string[] { + return [ + ...opts.migrated.cleanupCandidates, + ...(opts.composeRepaired ? opts.migrated.blockedCleanupCandidates : []), + ].sort((left, right) => left.localeCompare(right)); +} + function buildComposeEnvRepairNote(opts: { readonly references: readonly LegacyComposeEnvFileReference[]; }): string { diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index d63602669..550b0a2be 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -4937,6 +4937,23 @@ export async function migrateLegacyProjectEnv(opts: { composeEnvFileReferences: [], }; } + // Callers gate on projectEnvConfigExists, which reads a dangling default + // layer or a symlinked overlay as absent. Decide the format here with the + // mutation check: an unusable entry refuses (E_PATH_CONFLICT) instead of + // being replaced, and a linked regular layer means env layers are in use. + if ( + await projectEnvConfigExistsForMutation({ projectDir: opts.projectDir }) + ) { + return { + wroteFiles: [], + migratedOverlays: [], + legacyDetected: false, + updatedProjectConfig: false, + cleanupCandidates: [], + blockedCleanupCandidates: [], + composeEnvFileReferences: [], + }; + } const baseResolved = await resolveHackEnv({ projectDir: opts.projectDir, @@ -4970,6 +4987,8 @@ export async function migrateLegacyProjectEnv(opts: { await writeProjectEnvConfigFile({ path: defaultPath, config: defaultConfig, + // Create only: no layer existed at the format check above. + expectedExisting: null, }) ) { wroteFiles.push(defaultPath); @@ -5006,6 +5025,7 @@ export async function migrateLegacyProjectEnv(opts: { await writeProjectEnvConfigFile({ path: overlayPath, config: overlayConfig, + expectedExisting: null, }) ) { wroteFiles.push(overlayPath); @@ -5074,6 +5094,40 @@ const LEGACY_DOTENV_STATE_OVERLAY = PROJECT_ENV_STATE_FILENAME.slice( `${PROJECT_ENV_FILENAME}.`.length ); +/** `.hack/.env.` copies that are templates or backups, not overlays. */ +const LEGACY_DOTENV_NON_OVERLAY_NAMES: ReadonlySet = new Set([ + "example", + "sample", + "template", + "dist", + "bak", + "backup", + "old", + "orig", +]); + +/** Cleanup candidates git does not track; unknown status inside a checkout is kept out. */ +async function filterUntrackedLegacyCleanupCandidates(opts: { + readonly projectRoot: string; + readonly paths: readonly string[]; +}): Promise { + const kept: string[] = []; + for (const path of opts.paths) { + const tracked = await isGitTrackedProjectEnvPath({ + projectRoot: opts.projectRoot, + path, + }); + const untracked = + tracked === null + ? !(await isInsideGitCheckoutMarker({ startDir: opts.projectRoot })) + : !tracked; + if (untracked) { + kept.push(path); + } + } + return kept; +} + /** * Legacy overlay names whose env layers would be the default layer or a * machine-local layer instead of an overlay (`.env.default`, `.env.local`, @@ -5126,6 +5180,46 @@ async function assertUntrackedMigrationLocalLayer(opts: { } } +/** + * Legacy `.hack/.env.` overlays the `.env`-only migration converts. + * Refuses, before anything is written, names that are not overlay names or + * that would land on the default or a machine-local layer. + */ +async function selectLegacyDotenvOverlays(opts: { + readonly projectDir: string; +}): Promise { + // Template and backup copies (`.env.example`, `.env.bak`, ...) are not + // overlays: they are neither migrated nor offered for cleanup. + const overlayNames = ( + await discoverLegacyOverlayNames({ projectDir: opts.projectDir }) + ).filter( + (name) => + name !== LEGACY_DOTENV_STATE_OVERLAY && + !LEGACY_DOTENV_NON_OVERLAY_NAMES.has(name) + ); + const invalidOverlays = overlayNames.filter( + (name) => + !( + isReservedLegacyOverlayName(name) || + NATIVE_ENV_OVERLAY_PATTERN.test(name) + ) + ); + if (invalidOverlays.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${countLabel({ count: invalidOverlays.length, noun: "legacy .env. file" })} in ${opts.projectDir} ${invalidOverlays.length === 1 ? "has" : "have"} a name that is not an overlay name (lowercase letters, digits and single dashes). Rename or remove ${invalidOverlays.length === 1 ? "it" : "them"}, then retry.`, + }); + } + const reservedOverlays = overlayNames.filter(isReservedLegacyOverlayName); + if (reservedOverlays.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${reservedOverlays.map((name) => `${PROJECT_ENV_FILENAME}.${name}`).join(", ")} would become the default or a machine-local env layer rather than an overlay. Rename it to another overlay name or remove it, then retry.`, + }); + } + return overlayNames; +} + /** * Entries of `.hack/.env` (key `null`) and each legacy overlay file. Refuses, * before anything is written, when a key cannot be an env layer key; the @@ -5213,16 +5307,9 @@ async function migrateUndeclaredLegacyDotenv(opts: { reason: `env layer files already exist in ${opts.projectDir}.`, }); } - const overlayNames = ( - await discoverLegacyOverlayNames({ projectDir: opts.projectDir }) - ).filter((name) => name !== LEGACY_DOTENV_STATE_OVERLAY); - const reservedOverlays = overlayNames.filter(isReservedLegacyOverlayName); - if (reservedOverlays.length > 0) { - throw new LegacyDotenvMigrationRefusalError({ - code: "E_CONFIG_INVALID", - reason: `${reservedOverlays.map((name) => `${PROJECT_ENV_FILENAME}.${name}`).join(", ")} would become the default or a machine-local env layer rather than an overlay. Rename it to another overlay name or remove it, then retry.`, - }); - } + const overlayNames = await selectLegacyDotenvOverlays({ + projectDir: opts.projectDir, + }); const legacyValues = await readValidatedLegacyDotenvFiles({ projectDir: opts.projectDir, @@ -5343,14 +5430,25 @@ async function migrateUndeclaredLegacyDotenv(opts: { configCleanupCandidates: [], composeEnvFileReferences, }); + // A legacy file git tracks may carry values teammates rely on; the values + // now live only in this machine's local layers, so it is never offered + // for deletion. Neither is one whose status git cannot report. + const allowedCleanup = await filterUntrackedLegacyCleanupCandidates({ + projectRoot: opts.projectRoot, + paths: cleanupCandidates.allowed, + }); + const blockedCleanup = await filterUntrackedLegacyCleanupCandidates({ + projectRoot: opts.projectRoot, + paths: cleanupCandidates.blocked, + }); return { wroteFiles, migratedOverlays: overlayNames, legacyDetected: true, updatedProjectConfig: false, - cleanupCandidates: cleanupCandidates.allowed, - blockedCleanupCandidates: cleanupCandidates.blocked, + cleanupCandidates: allowedCleanup, + blockedCleanupCandidates: blockedCleanup, composeEnvFileReferences, }; } diff --git a/src/lib/project-meta.ts b/src/lib/project-meta.ts index bc153cb8a..45bd453f9 100644 --- a/src/lib/project-meta.ts +++ b/src/lib/project-meta.ts @@ -13,7 +13,7 @@ import type { ProjectOwnershipConfig } from "./project.ts"; import { readProjectConfig } from "./project.ts"; import { discoverComposeServiceNames, - projectEnvConfigExists, + projectEnvConfigExistsForMutation, resolveProjectEnvConfig, } from "./project-env-config.ts"; import { assertLegacyProjectDirectory } from "./project-input-selection.ts"; @@ -322,16 +322,38 @@ export async function resolveEnvMeta(opts: { }; } +function invalidModernEnvMeta(opts: { + readonly projectDir: string; + readonly error: unknown; +}): EnvMeta { + return { + contractPath: resolve(opts.projectDir, "hack.env.default.yaml"), + contractExists: true, + contractParseError: + opts.error instanceof Error + ? opts.error.message + : "Invalid modern env config.", + vars: [], + missingRequired: [], + }; +} + async function resolveModernEnvMeta(opts: { readonly projectRoot: string; readonly projectDir: string; readonly composeFile: string; }): Promise { - if ( - !(await projectEnvConfigExists({ + // The mutation format check: a dangling or non-file layer entry is reported + // as an invalid env config instead of reading as "legacy" (HACK-1370). + let layersInUse: boolean; + try { + layersInUse = await projectEnvConfigExistsForMutation({ projectDir: opts.projectDir, - })) - ) { + }); + } catch (error: unknown) { + return invalidModernEnvMeta({ projectDir: opts.projectDir, error }); + } + if (!layersInUse) { return null; } @@ -347,14 +369,7 @@ async function resolveModernEnvMeta(opts: { serviceNames, }); } catch (error: unknown) { - return { - contractPath: resolve(opts.projectDir, "hack.env.default.yaml"), - contractExists: true, - contractParseError: - error instanceof Error ? error.message : "Invalid modern env config.", - vars: [], - missingRequired: [], - }; + return invalidModernEnvMeta({ projectDir: opts.projectDir, error }); } if (!resolved) { return null; diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index b5ff2d611..180a718ea 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -720,3 +720,19 @@ test("doctor --migrate-env-config refuses a dangling default layer with E_PATH_C ).toBeNull(); expect(noteCalls.join("\n")).not.toContain(legacy); }, 30_000); + +test("doctor offers files compose still references for cleanup only after the compose repair removed them (HACK-1370)", async () => { + const { doctorEnvCleanupCandidates } = await import( + "../src/commands/doctor.ts" + ); + const migrated = { + cleanupCandidates: ["/p/.hack/.env.staging"], + blockedCleanupCandidates: ["/p/.hack/.env"], + }; + expect( + doctorEnvCleanupCandidates({ migrated, composeRepaired: false }) + ).toEqual(["/p/.hack/.env.staging"]); + expect( + doctorEnvCleanupCandidates({ migrated, composeRepaired: true }) + ).toEqual(["/p/.hack/.env", "/p/.hack/.env.staging"]); +}); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index cf5db1abc..604d8cb86 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, test } from "bun:test"; import { + chmod, lstat, mkdir, mkdtemp, @@ -29,6 +30,7 @@ import { projectEnvConfigExistsForMutation, resolveProjectEnvConfig, } from "../src/lib/project-env-config.ts"; +import { resolveEnvMeta } from "../src/lib/project-meta.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; /** @@ -873,3 +875,182 @@ test("the implicit migrations other commands run still need the hack.env.json co expect(migrated.wroteFiles).toEqual([]); expect(await hackDirEntries(repo)).toEqual(before); }); + +test("the contract migration refuses a dangling default layer and never replaces it", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.json"), + `${JSON.stringify({ + version: 1, + vars: [{ key: "KEPT_SETTING", source: "plain_env", required: false }], + })}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const path = await placeLayerEntry(repo, { + name: "hack.env.default.yaml", + kind: "dangling-link", + }); + const before = await hackDirEntries(repo); + const error = await migrateLegacyRepo(repo, {}).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(ProjectEnvLayerPathConflictError); + expect((error as ProjectEnvLayerPathConflictError).path).toBe(path); + expect(await readlink(path)).toBe( + resolve(repo.projectDir, "missing-layer.yaml") + ); + expect(await hackDirEntries(repo)).toEqual(before); + + // A linked regular overlay means env layers are in use: nothing is migrated. + await rm(path); + const target = resolve(repo.root, "shared-staging.yaml"); + await writeFile( + target, + LAYER.replace("environment: default", "environment: staging") + ); + await symlink(target, resolve(repo.projectDir, "hack.env.staging.yaml")); + const linked = await hackDirEntries(repo); + const migrated = await migrateLegacyRepo(repo, {}); + expect(migrated.legacyDetected).toBe(false); + expect(migrated.wroteFiles).toEqual([]); + expect(await hackDirEntries(repo)).toEqual(linked); +}); + +test("doctor and project metadata report a dangling default layer instead of the legacy format", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const path = await placeLayerEntry(repo, { + name: "hack.env.default.yaml", + kind: "dangling-link", + }); + const refusal = `Refusing to choose the env format: ${path} is not a regular file`; + + const meta = await resolveEnvMeta({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + composeFile: resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + projectName: "env-legacy-scope", + }); + expect(meta.contractParseError).toContain(refusal); + expect(meta.vars).toEqual([]); + + const run = await runCli({ + repo, + args: ["doctor", "--json", "--path", repo.projectRoot], + }); + expect(`${run.stdout}${run.stderr}`).not.toContain(KEPT_VALUE); + const checks = ( + JSON.parse(run.stdout) as { + readonly data: { + readonly checks: readonly { + readonly id: string; + readonly status: string; + readonly detail: string; + }[]; + }; + } + ).data.checks; + const envMode = checks.find((check) => check.id === "env mode"); + expect(envMode?.status).toBe("error"); + expect(envMode?.detail).toContain(refusal); + expect(await readlink(path)).toBe( + resolve(repo.projectDir, "missing-layer.yaml") + ); +}, 120_000); + +const RUNS_AS_ROOT = (process.geteuid?.() ?? process.getuid?.() ?? -1) === 0; + +test.skipIf(RUNS_AS_ROOT)( + "env list and explain refuse an unreadable default layer with one closed line", + async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const file = resolve(repo.projectDir, "hack.env.default.yaml"); + await writeFile(file, LAYER); + await chmod(file, 0o000); + try { + for (const args of [ + ["env", "list"], + ["env", "explain", "KEPT_SETTING"], + ]) { + const run = await runCli({ + repo, + args: [...args, "--path", repo.projectRoot], + }); + expectRefusal(run, `Cannot read the env layer ${file}.`, { + lines: 1, + }); + } + } finally { + await chmod(file, 0o600); + } + expect(await readFile(file, "utf8")).toBe(LAYER); + }, + 60_000 +); + +test("the migration refuses a legacy .env. whose name is not an overlay name", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + for (const name of [".env.staging.old", ".env.Prod"]) { + await writeFile( + resolve(repo.projectDir, name), + `SHARED_SETTING=${STAGING_VALUE}\n` + ); + } + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "2 legacy .env. files in", + hidden: [KEPT_VALUE, STAGING_VALUE], + }); +}); + +test("the migration leaves template and backup copies alone and never offers a git-tracked legacy file for cleanup", async () => { + const repo = await createLegacyRepo(); + await git({ repo, args: ["init", "-q"] }); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const staging = resolve(repo.projectDir, ".env.staging"); + await writeFile(staging, `KEPT_SETTING=${STAGING_VALUE}\n`); + await git({ repo, args: ["add", "-f", ".hack/.env.staging"] }); + const copies: Record = { + ".env.example": "KEPT_SETTING=placeholder\n", + ".env.bak": `KEPT_SETTING=${BASE_ONLY_VALUE}\n`, + }; + for (const [name, text] of Object.entries(copies)) { + await writeFile(resolve(repo.projectDir, name), text); + } + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + // The tracked overlay is migrated, but only the untracked .env is offered. + expect(migrated.migratedOverlays).toEqual(["staging"]); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { KEPT_SETTING: STAGING_VALUE }, + }); + expect(migrated.cleanupCandidates).toEqual([ + resolve(repo.projectDir, ".env"), + ]); + for (const [name, text] of Object.entries(copies)) { + expect(await readFile(resolve(repo.projectDir, name), "utf8")).toBe(text); + } + const entries = await hackDirEntries(repo); + expect(entries).not.toContain("hack.env.example.yaml"); + expect(entries).not.toContain("hack.env.bak.yaml"); +}, 60_000); From 0e7329283591e1e780d9b6d6dfb681d228811610 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:38:53 +0000 Subject: [PATCH 06/55] fix(env): keep migrated plaintext private, ignored and complete in the .env-only migration Codex review on #436: - machine-local layers are published create-only as 0600 files - in a git checkout, each machine-local layer must be ignored after the .hack/.gitignore step; a read-only .gitignore without the managed block refuses instead of only warning - an unreadable legacy .env or overlay refuses (E_PERMISSION) instead of migrating as empty and then being offered for deletion Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 10 ++- src/lib/project-env-config.ts | 96 ++++++++++++++++++++++++-- tests/env-legacy-service-scope.test.ts | 79 +++++++++++++++++++++ 3 files changed, 176 insertions(+), 9 deletions(-) diff --git a/docs/env.md b/docs/env.md index 3c10f9c72..bed37d2c7 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1028,7 +1028,8 @@ A repo that has no `.hack/hack.env.json` is migrated as well. Its values live on API keys, so the migration keeps them out of tracked files: each `.hack/.env` entry becomes a plaintext global value in the gitignored machine-local layer `hack.env.local.yaml`, and each overlay entry that differs from `.hack/.env` becomes a -global value in `hack.env..local.yaml`. The tracked `hack.env.default.yaml`, and +global value in `hack.env..local.yaml`. These machine-local layers are created +as private (mode 0600) files and never replace an existing entry. The tracked `hack.env.default.yaml`, and `hack.env..yaml` for each migrated overlay, are written with no values, so the repo switches to env layers. Teammates do not receive these values through git; share the ones they need with `hack env set` (plain settings) or `hack env set --secret` @@ -1055,7 +1056,12 @@ files, when: to one, means the repo already uses env layers: `hack doctor` reports that and writes nothing, never through the link; - git tracks a machine-local layer it would write, or, in a git checkout, git cannot - report whether it does (`E_PATH_CONFLICT`). + report whether it does (`E_PATH_CONFLICT`); +- in a git checkout, git does not ignore a machine-local layer it would write, for + example because `.hack/.gitignore` is read-only and lacks Hack's managed block + (`E_PATH_CONFLICT`). Make `.hack/.gitignore` writable, or ignore the path, and retry; +- a legacy `.hack/.env` or overlay file exists but cannot be read, for example mode 000 + or a dangling symlink (`E_PERMISSION`), so its values would be lost. Without `hack.env.json`, nothing lists values kept in a secret backend, so none are migrated: set them again with `hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 550b0a2be..520a4b42f 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -39,6 +39,7 @@ import { mergeManagedGitignoreBlock, pathExists, readAtomicWriteTargetText, + readOptionalTextFile, readTextFile, writeTextFile, writeTextFileAtomicIfChanged, @@ -3552,14 +3553,44 @@ function isProcessAlive(pid: number): boolean { } } +function serializeProjectEnvConfig(config: ProjectEnvConfig): string { + const yaml = YAML.stringify(config, null, 2); + return yaml.endsWith("\n") ? yaml : `${yaml}\n`; +} + +/** + * Creates a machine-local layer of the `.env`-only migration as a private + * (0600) file, published only where no entry exists. An existing entry, + * including a symlink, refuses; nothing is replaced or followed. + */ +async function publishMigrationLocalLayer(opts: { + readonly projectDir: string; + readonly write: PlannedEnvLayerWrite; +}): Promise { + await assertLegacyProjectDirectory({ projectDir: opts.projectDir }); + const refusal = () => + new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `${opts.write.path} already exists or is not a regular file path.`, + }); + const published = await publishPrivateFileExclusive({ + path: opts.write.path, + text: serializeProjectEnvConfig(opts.write.config), + directories: [opts.projectDir], + refusal, + }); + if (!published) { + throw refusal(); + } +} + async function writeProjectEnvConfigFile(opts: { readonly path: string; readonly config: ProjectEnvConfig; /** Compare-and-swap guard; see `writeTextFileAtomicIfChanged`. */ readonly expectedExisting?: string | null; }): Promise { - const yaml = YAML.stringify(opts.config, null, 2); - const text = yaml.endsWith("\n") ? yaml : `${yaml}\n`; + const text = serializeProjectEnvConfig(opts.config); await assertLegacyProjectDirectory({ projectDir: dirname(opts.path) }); // Rename-committed: an interrupted mutation leaves the previous layer intact. return ( @@ -5078,7 +5109,7 @@ export async function migrateLegacyProjectEnv(opts: { */ export class LegacyDotenvMigrationRefusalError extends HackCliError { constructor(opts: { - readonly code: "E_CONFIG_INVALID" | "E_PATH_CONFLICT"; + readonly code: "E_CONFIG_INVALID" | "E_PATH_CONFLICT" | "E_PERMISSION"; readonly reason: string; }) { super({ @@ -5220,6 +5251,31 @@ async function selectLegacyDotenvOverlays(opts: { return overlayNames; } +/** + * In a git checkout, a machine-local layer the migration writes must be + * ignored by git, or its plaintext values could be committed. Unknown status + * inside a checkout refuses too. Outside a checkout nothing can be committed. + */ +async function assertIgnoredMigrationLocalLayer(opts: { + readonly projectRoot: string; + readonly path: string; +}): Promise { + const ignored = await isGitIgnoredProjectEnvPath(opts); + if (ignored === true) { + return; + } + if ( + ignored === null && + !(await isInsideGitCheckoutMarker({ startDir: opts.projectRoot })) + ) { + return; + } + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `git does not ignore the machine-local env layer ${opts.path}, so migrated values could be committed. Make .hack/.gitignore writable (or ignore that path), then retry.`, + }); +} + /** * Entries of `.hack/.env` (key `null`) and each legacy overlay file. Refuses, * before anything is written, when a key cannot be an env layer key; the @@ -5384,12 +5440,28 @@ async function migrateUndeclaredLegacyDotenv(opts: { projectDir: opts.projectDir, }); } + // The managed block may not have been written (a read-only .gitignore), so + // check the outcome, not the attempt. + for (const write of localWrites) { + await assertIgnoredMigrationLocalLayer({ + projectRoot: opts.projectRoot, + path: write.path, + }); + } // Machine-local layers first, so the tracked layers switch the format only // once the values have a home. const wroteFiles: string[] = []; try { - for (const write of [...localWrites, ...trackedWrites]) { + for (const write of localWrites) { + // Created 0600 from the start: the values were in a private .env. + await publishMigrationLocalLayer({ + projectDir: opts.projectDir, + write, + }); + wroteFiles.push(write.path); + } + for (const write of trackedWrites) { if ( await writeProjectEnvConfigFile({ path: write.path, @@ -5453,12 +5525,22 @@ async function migrateUndeclaredLegacyDotenv(opts: { }; } -/** Entries of a legacy dotenv file; an absent file has none. */ +/** + * Entries of a legacy dotenv file; an absent file has none. One that exists + * but cannot be read (for example mode 000, or a dangling symlink) refuses: + * reading it as empty would drop its values and then offer it for deletion. + */ async function readLegacyDotenvFile(opts: { readonly path: string; }): Promise> { - const text = await readTextFile(opts.path); - return text === null ? {} : parseDotEnv(text); + const read = await readOptionalTextFile(opts.path); + if (read.kind === "unreadable") { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PERMISSION", + reason: `${opts.path} exists but cannot be read. Make it a readable regular file, then retry.`, + }); + } + return read.kind === "absent" ? {} : parseDotEnv(read.text); } export async function removeLegacyProjectEnvArtifacts(opts: { diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 604d8cb86..4da303f8b 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1054,3 +1054,82 @@ test("the migration leaves template and backup copies alone and never offers a g expect(entries).not.toContain("hack.env.example.yaml"); expect(entries).not.toContain("hack.env.bak.yaml"); }, 60_000); + +test.skipIf(RUNS_AS_ROOT)( + "the migration refuses an unreadable legacy file instead of migrating it as empty", + async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const staging = resolve(repo.projectDir, ".env.staging"); + await writeFile(staging, `KEPT_SETTING=${STAGING_VALUE}\n`); + await chmod(staging, 0o000); + const before = await hackDirEntries(repo); + try { + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + const refusal = error as LegacyDotenvMigrationRefusalError; + expect(refusal.code).toBe("E_PERMISSION"); + expect(refusal.message).toContain(`${staging} exists but cannot be read`); + expect(refusal.message).not.toContain(KEPT_VALUE); + expect(await hackDirEntries(repo)).toEqual(before); + } finally { + await chmod(staging, 0o600); + } + expect(await readFile(staging, "utf8")).toBe( + `KEPT_SETTING=${STAGING_VALUE}\n` + ); + } +); + +test.skipIf(RUNS_AS_ROOT)( + "the migration refuses when a read-only .hack/.gitignore leaves the machine-local layer unignored", + async () => { + const repo = await createLegacyRepo(); + await git({ repo, args: ["init", "-q"] }); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const gitignore = resolve(repo.projectDir, ".gitignore"); + await writeFile(gitignore, "# no managed block\n"); + await chmod(gitignore, 0o444); + try { + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: `git does not ignore the machine-local env layer ${resolve(repo.projectDir, "hack.env.local.yaml")}`, + hidden: [KEPT_VALUE], + }); + } finally { + await chmod(gitignore, 0o644); + } + }, + 60_000 +); + +test("the migration creates machine-local layers as private 0600 files", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `KEPT_SETTING=${STAGING_VALUE}\n` + ); + const previous = process.umask(0o022); + try { + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + } finally { + process.umask(previous); + } + for (const name of ["hack.env.local.yaml", "hack.env.staging.local.yaml"]) { + const mode = (await lstat(resolve(repo.projectDir, name))).mode & 0o777; + expect({ name, mode }).toEqual({ name, mode: 0o600 }); + } +}); From 69def4f009119fd66a979ce8bc77346a777f571f Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:49:12 -0400 Subject: [PATCH 07/55] fix(env): migrate symlinked legacy overlays instead of dropping them discoverLegacyOverlayNames skipped symlinks, so a symlinked .hack/.env. was left out of the migration and its values were lost once the project switched formats. A link to a readable file now migrates like a regular overlay, and cleanup removes only the link. A dangling link or a link to a directory refuses with E_PERMISSION before anything is written. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 4 ++- src/lib/project-env-config.ts | 4 ++- tests/env-legacy-service-scope.test.ts | 46 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docs/env.md b/docs/env.md index bed37d2c7..7e797cff6 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1061,7 +1061,9 @@ files, when: example because `.hack/.gitignore` is read-only and lacks Hack's managed block (`E_PATH_CONFLICT`). Make `.hack/.gitignore` writable, or ignore the path, and retry; - a legacy `.hack/.env` or overlay file exists but cannot be read, for example mode 000 - or a dangling symlink (`E_PERMISSION`), so its values would be lost. + or a dangling symlink (`E_PERMISSION`), so its values would be lost. A symlinked overlay + that resolves to a readable file is migrated like a regular one; cleanup removes only the + link, never its target. Without `hack.env.json`, nothing lists values kept in a secret backend, so none are migrated: set them again with `hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 520a4b42f..711e6b1e9 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5566,7 +5566,9 @@ async function discoverLegacyOverlayNames(opts: { const names = new Set(); const entries = await readdir(opts.projectDir, { withFileTypes: true }); for (const entry of entries) { - if (!entry.isFile()) { + // A symlinked overlay is an overlay too: skipping it would switch formats + // without its values. What it points at is checked when it is read. + if (!(entry.isFile() || entry.isSymbolicLink())) { continue; } if (!entry.name.startsWith(`${PROJECT_ENV_FILENAME}.`)) { diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 4da303f8b..dd91c9725 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1133,3 +1133,49 @@ test("the migration creates machine-local layers as private 0600 files", async ( expect({ name, mode }).toEqual({ name, mode: 0o600 }); } }); + +test("the migration carries a symlinked legacy overlay's values and refuses one it cannot read", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const target = resolve(repo.root, "shared-staging.env"); + const targetText = `KEPT_SETTING=${KEPT_VALUE}-linked\n`; + await writeFile(target, targetText); + const link = resolve(repo.projectDir, ".env.staging"); + await symlink(target, link); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.migratedOverlays).toContain("staging"); + expect( + await readFile( + resolve(repo.projectDir, "hack.env.staging.local.yaml"), + "utf8" + ) + ).toContain(`${KEPT_VALUE}-linked`); + expect(migrated.cleanupCandidates).toContain(link); + expect(await readFile(target, "utf8")).toBe(targetText); + + for (const place of [ + (root: string) => resolve(root, "missing"), + (root: string) => root, + ]) { + const unusable = await createLegacyRepo(); + await symlink( + place(unusable.root), + resolve(unusable.projectDir, ".env.staging") + ); + const before = await hackDirEntries(unusable); + const error = await migrateLegacyRepo(unusable, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as LegacyDotenvMigrationRefusalError).code).toBe( + "E_PERMISSION" + ); + expect(await hackDirEntries(unusable)).toEqual(before); + } +}); From ede4cb9bf457c5e2c38e5f460e061705c2be9655 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:37:32 +0000 Subject: [PATCH 08/55] fix(env): name the legacy overlay files a migration refuses, quoted on one line Review round 2 (optional P3s): the non-overlay-name refusal lists the files instead of counting them and says to select a renamed overlay with --env. Both name refusals JSON-quote each file name, so a control character cannot break the one-line message. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 3 ++- src/lib/project-env-config.ts | 14 ++++++++++++-- tests/env-legacy-service-scope.test.ts | 7 ++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/env.md b/docs/env.md index 7e797cff6..227959cdb 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1047,7 +1047,8 @@ files, when: rather than an overlay (`E_CONFIG_INVALID`). Rename or remove it; - a legacy `.hack/.env.` file has a name that is not an overlay name (lowercase letters, digits and single dashes), such as `.env.staging.old` or `.env.Prod` - (`E_CONFIG_INVALID`). The error counts such files. Rename or remove them. Template and + (`E_CONFIG_INVALID`). The error names such files. Rename or remove them; a renamed + overlay is then selected by its new name (`--env `). Template and backup copies (`.env.example`, `.env.sample`, `.env.template`, `.env.dist`, `.env.bak`, `.env.backup`, `.env.old`, `.env.orig`) are not overlays: they are neither migrated nor offered for cleanup; diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 711e6b1e9..f3e943443 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5216,6 +5216,16 @@ async function assertUntrackedMigrationLocalLayer(opts: { * Refuses, before anything is written, names that are not overlay names or * that would land on the default or a machine-local layer. */ +/** + * Names legacy overlay files for a refusal. File names are not values, but + * each is JSON-quoted so a control character cannot break the one-line message. + */ +function legacyOverlayFileList(names: readonly string[]): string { + return names + .map((name) => JSON.stringify(`${PROJECT_ENV_FILENAME}.${name}`)) + .join(", "); +} + async function selectLegacyDotenvOverlays(opts: { readonly projectDir: string; }): Promise { @@ -5238,14 +5248,14 @@ async function selectLegacyDotenvOverlays(opts: { if (invalidOverlays.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `${countLabel({ count: invalidOverlays.length, noun: "legacy .env. file" })} in ${opts.projectDir} ${invalidOverlays.length === 1 ? "has" : "have"} a name that is not an overlay name (lowercase letters, digits and single dashes). Rename or remove ${invalidOverlays.length === 1 ? "it" : "them"}, then retry.`, + reason: `${legacyOverlayFileList(invalidOverlays)} in ${opts.projectDir} ${invalidOverlays.length === 1 ? "is not named" : "are not named"} like an overlay (lowercase letters, digits and single dashes). Rename ${invalidOverlays.length === 1 ? "it" : "them"} (and select the overlay by its new name with --env) or remove ${invalidOverlays.length === 1 ? "it" : "them"}, then retry.`, }); } const reservedOverlays = overlayNames.filter(isReservedLegacyOverlayName); if (reservedOverlays.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `${reservedOverlays.map((name) => `${PROJECT_ENV_FILENAME}.${name}`).join(", ")} would become the default or a machine-local env layer rather than an overlay. Rename it to another overlay name or remove it, then retry.`, + reason: `${legacyOverlayFileList(reservedOverlays)} would become the default or a machine-local env layer rather than an overlay. Rename it to another overlay name or remove it, then retry.`, }); } return overlayNames; diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index dd91c9725..41973d83a 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -774,7 +774,7 @@ for (const name of [".env.default", ".env.local", ".env.staging.local"]) { await expectMigrationRefusal({ repo, code: "E_CONFIG_INVALID", - reason: `${name} would become the default or a machine-local env layer`, + reason: `${JSON.stringify(name)} would become the default or a machine-local env layer`, hidden: [KEPT_VALUE, STAGING_VALUE], }); }); @@ -1004,7 +1004,8 @@ test("the migration refuses a legacy .env. whose name is not an overlay na resolve(repo.projectDir, ".env"), `KEPT_SETTING=${KEPT_VALUE}\n` ); - for (const name of [".env.staging.old", ".env.Prod"]) { + // A control character in a file name must not break the one-line refusal. + for (const name of [".env.staging.old", ".env.Prod", ".env.bad\nname"]) { await writeFile( resolve(repo.projectDir, name), `SHARED_SETTING=${STAGING_VALUE}\n` @@ -1013,7 +1014,7 @@ test("the migration refuses a legacy .env. whose name is not an overlay na await expectMigrationRefusal({ repo, code: "E_CONFIG_INVALID", - reason: "2 legacy .env. files in", + reason: '".env.bad\\nname", ".env.Prod", ".env.staging.old" in', hidden: [KEPT_VALUE, STAGING_VALUE], }); }); From 1b1806d15f97ac0fffcf1e62b90b6788c3194ed3 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:47:24 -0400 Subject: [PATCH 09/55] fix(env): refuse a migration that would lose Compose-referenced template files A Compose env_file that uses a template or backup copy such as .env.sample reached no migrated layer, and the Compose repair then removed the reference, so the next launch lost those values. The .env-only migration now refuses first. Docs: overriding a migrated key for one service needs --local, because the migrated global value in hack.env.local.yaml wins over a tracked service value. A test pins both halves of that behaviour. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 9 ++++ src/lib/project-env-config.ts | 47 ++++++++++++++++--- tests/env-legacy-service-scope.test.ts | 62 ++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 5 deletions(-) diff --git a/docs/env.md b/docs/env.md index 227959cdb..430371a9d 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1052,6 +1052,9 @@ files, when: backup copies (`.env.example`, `.env.sample`, `.env.template`, `.env.dist`, `.env.bak`, `.env.backup`, `.env.old`, `.env.orig`) are not overlays: they are neither migrated nor offered for cleanup; +- Compose `env_file` uses one of those template or backup copies (`E_CONFIG_INVALID`), so + its values would be lost. Rename the file to an overlay name or move it out of `.hack`, + and update `env_file` to match; - an entry named like an env layer (`.hack/hack.env.*.yaml`) is a dangling symlink, a directory or another non-file (`E_PATH_CONFLICT`). A regular layer file, or a symlink to one, means the repo already uses env layers: `hack doctor` reports that and writes @@ -1090,6 +1093,12 @@ the same way in both formats). `--service global`, or no `--service`, still work env layers first with `hack doctor --migrate-env-config`, which also converts a project that has only `.hack/.env` (see above), then set or unset the service-scoped value. +Values migrated from `.hack/.env` live in the machine-local layer, and a later layer wins +over a more specific scope in an earlier one. So to override a migrated key for one service, +add `--local`: `hack env set KEY --service api --local`. Without it the new value goes to the +tracked default layer, where the migrated global value in `hack.env.local.yaml` still wins +and the command has no effect on the runtime value. + `hack env set` and `hack env unset` choose between the env layers and the legacy file by looking for `.hack/hack.env.*.yaml` entries, `hack.env.default.yaml` included. A regular file, or a symlink to one, counts as an env layer. Any other entry with a layer name, such diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index f3e943443..6b71ce3a7 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5226,6 +5226,44 @@ function legacyOverlayFileList(names: readonly string[]): string { .join(", "); } +function isExcludedLegacyDotenvName(name: string): boolean { + return ( + name === LEGACY_DOTENV_STATE_OVERLAY || + LEGACY_DOTENV_NON_OVERLAY_NAMES.has(name) + ); +} + +/** + * Refuses when Compose uses a `.hack/.env.` file the migration leaves + * out: its values reach no layer, and the Compose repair that follows removes + * every `.hack/.env*` reference, so the next launch would lose them. + */ +async function assertComposeUsesNoExcludedLegacyDotenv(opts: { + readonly projectDir: string; +}): Promise { + const references = await inspectLegacyComposeEnvFileReferences({ + composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), + projectDir: opts.projectDir, + }); + const excluded = [ + ...new Set( + references + .map((reference) => + basename(reference.resolvedPath).slice( + `${PROJECT_ENV_FILENAME}.`.length + ) + ) + .filter(isExcludedLegacyDotenvName) + ), + ].sort((left, right) => left.localeCompare(right)); + if (excluded.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `Compose env_file uses ${legacyOverlayFileList(excluded)}, which the migration does not carry over, so those values would be lost. Rename the file to an overlay name or move it out of .hack, update env_file to match, then retry.`, + }); + } +} + async function selectLegacyDotenvOverlays(opts: { readonly projectDir: string; }): Promise { @@ -5233,11 +5271,7 @@ async function selectLegacyDotenvOverlays(opts: { // overlays: they are neither migrated nor offered for cleanup. const overlayNames = ( await discoverLegacyOverlayNames({ projectDir: opts.projectDir }) - ).filter( - (name) => - name !== LEGACY_DOTENV_STATE_OVERLAY && - !LEGACY_DOTENV_NON_OVERLAY_NAMES.has(name) - ); + ).filter((name) => !isExcludedLegacyDotenvName(name)); const invalidOverlays = overlayNames.filter( (name) => !( @@ -5376,6 +5410,9 @@ async function migrateUndeclaredLegacyDotenv(opts: { const overlayNames = await selectLegacyDotenvOverlays({ projectDir: opts.projectDir, }); + await assertComposeUsesNoExcludedLegacyDotenv({ + projectDir: opts.projectDir, + }); const legacyValues = await readValidatedLegacyDotenvFiles({ projectDir: opts.projectDir, diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 41973d83a..9838e1896 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -29,6 +29,7 @@ import { ProjectEnvLayerPathConflictError, projectEnvConfigExistsForMutation, resolveProjectEnvConfig, + selectProjectEnvValues, } from "../src/lib/project-env-config.ts"; import { resolveEnvMeta } from "../src/lib/project-meta.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -1180,3 +1181,64 @@ test("the migration carries a symlinked legacy overlay's values and refuses one expect(await hackDirEntries(unusable)).toEqual(before); } }); + +test("the migration refuses when Compose uses a legacy file it would leave out", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: .env.sample\n" + ); + await writeFile( + resolve(repo.projectDir, ".env.sample"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'Compose env_file uses ".env.sample"', + hidden: [KEPT_VALUE], + }); +}); + +test("after the migration, a migrated key's service override takes effect only with --local", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + const apiValue = async (): Promise => { + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + }); + if (!resolved) { + throw new Error("expected env layers"); + } + return selectProjectEnvValues({ resolved, scopeName: "api" }).KEPT_SETTING; + }; + const set = async (extra: readonly string[]): Promise => { + const run = await runCli({ + repo, + args: [ + "env", + "set", + "KEPT_SETTING", + "--service", + "api", + ...extra, + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${NEW_VALUE}\n`, + }); + expect(run.exitCode).toBe(0); + }; + + await set([]); + expect(await apiValue()).toBe(KEPT_VALUE); + await set(["--local"]); + expect(await apiValue()).toBe(NEW_VALUE); +}, 60_000); From 36fab6c79c4e3507efaa595e82d6e2ea7847e7b8 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:59:48 -0400 Subject: [PATCH 10/55] fix(env): refuse a migration that would drop a Compose-selected overlay Compose loads env_file: .env.staging on every launch, but the .env-only migration put those values only in the staging layers and the Compose repair removed the reference. With no default overlay, the next launch without --env lost them although the migration reported success. It now refuses first, unless the overlay is env.defaultOverlay or adds nothing beyond the base. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 5 ++ src/lib/project-env-config.ts | 74 +++++++++++++++++++++++--- tests/env-legacy-service-scope.test.ts | 72 +++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 6 deletions(-) diff --git a/docs/env.md b/docs/env.md index 430371a9d..d7c5ab53e 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1055,6 +1055,11 @@ files, when: - Compose `env_file` uses one of those template or backup copies (`E_CONFIG_INVALID`), so its values would be lost. Rename the file to an overlay name or move it out of `.hack`, and update `env_file` to match; +- Compose `env_file` uses an overlay such as `.env.staging` that holds values of its own + and is not `env.defaultOverlay` (`E_CONFIG_INVALID`). Compose loads that file on every + launch, but after the migration a launch without `--env` would not select the overlay, + so the service would lose those values. Set `env.defaultOverlay` to it, or remove the + `env_file` entry and select the overlay with `--env`; - an entry named like an env layer (`.hack/hack.env.*.yaml`) is a dangling symlink, a directory or another non-file (`E_PATH_CONFLICT`). A regular layer file, or a symlink to one, means the repo already uses env layers: `hack doctor` reports that and writes diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 6b71ce3a7..c5a3af014 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5264,6 +5264,62 @@ async function assertComposeUsesNoExcludedLegacyDotenv(opts: { } } +/** + * Refuses when Compose uses a migrated overlay that a launch without `--env` + * would not select. Compose loads that file unconditionally, but the migration + * puts its values only in the overlay's layers and the Compose repair removes + * the reference, so the service would lose them. An overlay with no entries + * beyond the base loses nothing, because the base layer already holds them. + */ +async function assertComposeOverlaysStaySelected(opts: { + readonly projectDir: string; + readonly overlayNames: readonly string[]; + readonly legacyValues: ReadonlyMap>; +}): Promise { + const references = await inspectLegacyComposeEnvFileReferences({ + composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), + projectDir: opts.projectDir, + }); + const referenced = new Set( + references.map((reference) => + basename(reference.resolvedPath).slice(`${PROJECT_ENV_FILENAME}.`.length) + ) + ); + const defaultOverlay = await readProjectDefaultEnvConfig({ + projectDir: opts.projectDir, + }); + const baseValues = opts.legacyValues.get(null) ?? {}; + const unselected = opts.overlayNames.filter( + (name) => + name !== defaultOverlay && + referenced.has(name) && + Object.keys( + legacyOverlayChangedValues({ + overlayValues: opts.legacyValues.get(name) ?? {}, + baseValues, + }) + ).length > 0 + ); + if (unselected.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `Compose env_file uses ${legacyOverlayFileList(unselected)}, but a launch without --env would not select ${unselected.length === 1 ? "that overlay" : "those overlays"}, so the service would lose its values. Set env.defaultOverlay to the overlay every launch should use, or remove the env_file entry and select the overlay with --env, then retry.`, + }); + } +} + +/** Overlay entries a migrated overlay layer must hold: those that differ from the base. */ +function legacyOverlayChangedValues(opts: { + readonly overlayValues: Record; + readonly baseValues: Record; +}): Record { + return Object.fromEntries( + Object.entries(opts.overlayValues).filter( + ([key, value]) => opts.baseValues[key] !== value + ) + ); +} + async function selectLegacyDotenvOverlays(opts: { readonly projectDir: string; }): Promise { @@ -5379,7 +5435,9 @@ async function readValidatedLegacyDotenvFiles(opts: { * - Refused before anything is written ({@link LegacyDotenvMigrationRefusalError}, * or {@link ProjectEnvLayerPathConflictError} for an unusable layer entry): * any existing env layer entry, a legacy overlay named like the default or a - * machine-local layer, a key that does not match `^[A-Z_][A-Z0-9_]*$` + * machine-local layer, a Compose `env_file` on a file the migration leaves + * out or on an overlay a launch without `--env` would not select, a key that + * does not match `^[A-Z_][A-Z0-9_]*$` * (counted per file, never shown), and a machine-local target that git * tracks or, inside a git checkout, cannot report on. * - Layers are created only where nothing exists (compare-and-swap against an @@ -5418,6 +5476,11 @@ async function migrateUndeclaredLegacyDotenv(opts: { projectDir: opts.projectDir, overlayNames, }); + await assertComposeOverlaysStaySelected({ + projectDir: opts.projectDir, + overlayNames, + legacyValues, + }); const baseValues = legacyValues.get(null) ?? {}; const localWrites: PlannedEnvLayerWrite[] = []; @@ -5446,11 +5509,10 @@ async function migrateUndeclaredLegacyDotenv(opts: { }); } for (const overlayName of overlayNames) { - const changedValues = Object.fromEntries( - Object.entries(legacyValues.get(overlayName) ?? {}).filter( - ([key, value]) => baseValues[key] !== value - ) - ); + const changedValues = legacyOverlayChangedValues({ + overlayValues: legacyValues.get(overlayName) ?? {}, + baseValues, + }); if (Object.keys(changedValues).length === 0) { continue; } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 9838e1896..e1d54eb02 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1200,6 +1200,78 @@ test("the migration refuses when Compose uses a legacy file it would leave out", }); }); +test("the migration refuses when Compose uses an overlay a launch would not select", async () => { + const repo = await createLegacyRepo(); + const compose = resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME); + await writeFile( + compose, + "services:\n api:\n image: alpine:3.20\n env_file: .env.staging\n" + ); + await writeFile(resolve(repo.projectDir, ".env"), "SHARED=base\n"); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `SHARED=base\nKEPT_SETTING=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'Compose env_file uses ".env.staging"', + hidden: [KEPT_VALUE], + }); + + // Selected by default, every launch still gets the values. + await writeFile( + resolve(repo.projectDir, "hack.config.json"), + `${JSON.stringify({ env: { defaultOverlay: "staging" } })}\n` + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.migratedOverlays).toContain("staging"); + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + }); + if (!resolved) { + throw new Error("expected env layers"); + } + expect( + selectProjectEnvValues({ resolved, scopeName: "api" }).KEPT_SETTING + ).toBe(KEPT_VALUE); +}); + +test("the migration accepts a Compose overlay that adds nothing to the base", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: .env.staging\n" + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.migratedOverlays).toContain("staging"); + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + }); + if (!resolved) { + throw new Error("expected env layers"); + } + expect( + selectProjectEnvValues({ resolved, scopeName: "api" }).KEPT_SETTING + ).toBe(KEPT_VALUE); +}); + test("after the migration, a migrated key's service override takes effect only with --local", async () => { const repo = await createLegacyRepo(); await writeFile( From 95ffaa40539ac076a208367cf477ff1dd9e8ba4b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:21:33 +0000 Subject: [PATCH 11/55] fix(env): refuse a .env-only migration that would override an explicit Compose environment value Compose ranks a service's own environment above env_file, but the migration turns every .hack/.env entry into a global layer value, and Hack's generated Compose override applies those after environment. A service with env_file .env and environment PORT "4000" would switch to the .env PORT. Preserving the Compose value is not sound (global values reach every service, and Compose values may be interpolations or passthrough entries), so the migration now refuses, naming the keys and services (JSON-quoted), never values. An environment entry it cannot read refuses too. Codex review on #436 (PRRT_kwDOQwrGd86yWkPh). Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 5 ++ src/lib/project-env-config.ts | 85 ++++++++++++++++++++++++++ tests/env-legacy-service-scope.test.ts | 34 +++++++++++ 3 files changed, 124 insertions(+) diff --git a/docs/env.md b/docs/env.md index d7c5ab53e..07e0943e3 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1069,6 +1069,11 @@ files, when: - in a git checkout, git does not ignore a machine-local layer it would write, for example because `.hack/.gitignore` is read-only and lacks Hack's managed block (`E_PATH_CONFLICT`). Make `.hack/.gitignore` writable, or ignore the path, and retry; +- a key the legacy files set is also set in a Compose service's own `environment` + (`E_CONFIG_INVALID`). Compose ranks `environment` above `env_file`, but migrated values + become global layer values that Hack applies after `environment`, so the service would + switch to the `.env` value. The error names the keys and services, never values. Remove + the key from the `.hack/.env` file or from that service's `environment`, then retry; - a legacy `.hack/.env` or overlay file exists but cannot be read, for example mode 000 or a dangling symlink (`E_PERMISSION`), so its values would be lost. A symlinked overlay that resolves to a readable file is migrated like a regular one; cleanup removes only the diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index c5a3af014..a8cc9e611 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5308,6 +5308,87 @@ async function assertComposeOverlaysStaySelected(opts: { } } +/** + * Keys a Compose service sets in its own `environment` (map keys, or the + * names of `KEY=value` / `KEY` list entries). Null when the entry has a shape + * this check cannot read. + */ +function composeServiceEnvironmentKeys( + environment: unknown +): ReadonlySet | null { + if (environment === undefined || environment === null) { + return new Set(); + } + if (Array.isArray(environment)) { + const keys = new Set(); + for (const entry of environment) { + if (typeof entry !== "string") { + return null; + } + const key = entry.split("=", 1)[0]?.trim() ?? ""; + if (key.length > 0) { + keys.add(key); + } + } + return keys; + } + return isRecord(environment) ? new Set(Object.keys(environment)) : null; +} + +/** + * Refuses when a migrated key is also set in a Compose service's own + * `environment`. Legacy `env_file` values rank below `environment`, but the + * migrated values become global layer values, which Hack's generated Compose + * override applies to every service after `environment`; the service would + * silently switch to the dotenv value. The refusal names services and keys, + * JSON-quoted, never values. + */ +async function assertComposeEnvironmentKeepsPrecedence(opts: { + readonly projectDir: string; + readonly legacyValues: ReadonlyMap>; +}): Promise { + const parsed = await readComposeFile({ + composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), + }); + const services = parsed ? getRecord(parsed, "services") : null; + if (!services) { + return; + } + const migratedKeys = new Set(); + for (const values of opts.legacyValues.values()) { + for (const key of Object.keys(values)) { + migratedKeys.add(key); + } + } + const conflicts: string[] = []; + const unreadable: string[] = []; + for (const [service, rawService] of Object.entries(services)) { + if (!isRecord(rawService)) { + continue; + } + const keys = composeServiceEnvironmentKeys(rawService.environment); + if (keys === null) { + unreadable.push(JSON.stringify(service)); + continue; + } + for (const key of [...keys].filter((name) => migratedKeys.has(name))) { + conflicts.push(`${JSON.stringify(key)} in ${JSON.stringify(service)}`); + } + } + if (unreadable.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `the Compose environment of ${unreadable.sort().join(", ")} cannot be read, so the migration cannot check that it keeps precedence over the migrated values. Use a map or a list of KEY=value entries, then retry.`, + }); + } + if (conflicts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `Compose environment sets ${conflicts.sort().join(", ")}, which the legacy .env files also set; after the migration the .env value would override the Compose value. Remove the key from the .hack/.env file or from that service's environment, then retry.`, + }); + } +} + /** Overlay entries a migrated overlay layer must hold: those that differ from the base. */ function legacyOverlayChangedValues(opts: { readonly overlayValues: Record; @@ -5481,6 +5562,10 @@ async function migrateUndeclaredLegacyDotenv(opts: { overlayNames, legacyValues, }); + await assertComposeEnvironmentKeepsPrecedence({ + projectDir: opts.projectDir, + legacyValues, + }); const baseValues = legacyValues.get(null) ?? {}; const localWrites: PlannedEnvLayerWrite[] = []; diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index e1d54eb02..a30d325c9 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1314,3 +1314,37 @@ test("after the migration, a migrated key's service override takes effect only w await set(["--local"]); expect(await apiValue()).toBe(NEW_VALUE); }, 60_000); + +test("the migration refuses when a Compose environment value would lose precedence to a migrated .env value", async () => { + const repo = await createLegacyRepo(); + const composeLegacyPort = ["3", "000"].join(""); + const composeExplicitPort = ["4", "000"].join(""); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + `services:\n api:\n image: alpine:3.20\n env_file: .env\n environment:\n PORT: "${composeExplicitPort}"\n worker:\n image: alpine:3.20\n environment:\n - OTHER=1\n` + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `PORT=${composeLegacyPort}\nKEPT_SETTING=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: + 'Compose environment sets "PORT" in "api", which the legacy .env files also set', + hidden: [composeLegacyPort, composeExplicitPort, KEPT_VALUE], + }); + + // Without the overlapping key the same shape migrates. + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); +}); From ff06193978fc9fa4ed0d9d3e9fefd3272e9532df Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:30:40 -0400 Subject: [PATCH 12/55] fix(env): keep legacy symlinks and other Compose env files from changing values A repository could link .hack/.env. to a file outside .hack, such as the root .env, and the contract migration would write its plain_env values to a tracked layer. That path now reads regular overlay files only, as it did before this PR, and the .env-only migration refuses a legacy link that leaves .hack. The .env-only migration also refuses when another Compose env_file sets a migrated key: that file stays in Compose, but the migrated value would override it. The refusal names keys, files and services, never values. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 14 ++- src/lib/project-env-config.ts | 126 +++++++++++++++++++++++-- tests/env-legacy-service-scope.test.ts | 79 +++++++++++++++- 3 files changed, 205 insertions(+), 14 deletions(-) diff --git a/docs/env.md b/docs/env.md index 07e0943e3..5867c2bff 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1074,10 +1074,20 @@ files, when: become global layer values that Hack applies after `environment`, so the service would switch to the `.env` value. The error names the keys and services, never values. Remove the key from the `.hack/.env` file or from that service's `environment`, then retry; +- a key the legacy files set is also set in another file a Compose service uses as + `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the + migrated value would override it. The check does not look at the order of the entries, + or expand variables in an `env_file` path. Remove the key from one of the files, then retry; +- `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` + (`E_PATH_CONFLICT`), so its values would be copied from elsewhere into this project's + layers. Replace the link with a regular file in `.hack`, or remove it; - a legacy `.hack/.env` or overlay file exists but cannot be read, for example mode 000 or a dangling symlink (`E_PERMISSION`), so its values would be lost. A symlinked overlay - that resolves to a readable file is migrated like a regular one; cleanup removes only the - link, never its target. + that resolves to a readable file inside `.hack` is migrated like a regular one; cleanup + removes only the link, never its target. + +With `hack.env.json`, the migration reads only regular `.hack/.env.` files, so a +symlinked overlay is not migrated and its values cannot reach a tracked layer. Without `hack.env.json`, nothing lists values kept in a secret backend, so none are migrated: set them again with `hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index a8cc9e611..0f22dc02d 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5,8 +5,16 @@ import { randomBytes, } from "node:crypto"; import { constants } from "node:fs"; -import { access, lstat, readdir, rm, stat, unlink } from "node:fs/promises"; -import { basename, dirname, relative, resolve } from "node:path"; +import { + access, + lstat, + readdir, + realpath, + rm, + stat, + unlink, +} from "node:fs/promises"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; import { YAML } from "bun"; import { HACK_PROJECT_DIR_PRIMARY, @@ -4992,8 +5000,11 @@ export async function migrateLegacyProjectEnv(opts: { envName: null, }); + // Regular files only: this path writes plain_env values to tracked layers, so + // a repository's symlink must not pull an outside file's values into them. const overlayNames = await discoverLegacyOverlayNames({ projectDir: opts.projectDir, + includeSymlinks: false, }); const keyInfo = baseResolved.values.some( (value) => value.source === "keychain" @@ -5335,9 +5346,45 @@ function composeServiceEnvironmentKeys( return isRecord(environment) ? new Set(Object.keys(environment)) : null; } +/** + * Migrated keys a service's other `env_file` entries set, as refusal labels. + * Those files stay in Compose, but Hack's generated override applies migrated + * values after them. An absent file sets nothing; an unreadable one refuses. + */ +async function composeEnvFileKeyConflicts(opts: { + readonly projectDir: string; + readonly service: string; + readonly rawService: Record; + readonly migratedKeys: ReadonlySet; +}): Promise { + const conflicts: string[] = []; + for (const configuredPath of readComposeEnvFilePaths({ + rawService: opts.rawService, + })) { + const path = resolve(opts.projectDir, configuredPath); + if ( + isLegacyProjectEnvFilePath({ + projectDir: opts.projectDir, + candidatePath: path, + }) + ) { + continue; + } + const values = await readLegacyDotenvFile({ path }); + for (const key of Object.keys(values)) { + if (opts.migratedKeys.has(key)) { + conflicts.push( + `${JSON.stringify(key)} (${JSON.stringify(configuredPath)} in ${JSON.stringify(opts.service)})` + ); + } + } + } + return conflicts; +} + /** * Refuses when a migrated key is also set in a Compose service's own - * `environment`. Legacy `env_file` values rank below `environment`, but the + * `environment` or another of its `env_file` entries. Legacy `env_file` values rank below `environment`, but the * migrated values become global layer values, which Hack's generated Compose * override applies to every service after `environment`; the service would * silently switch to the dotenv value. The refusal names services and keys, @@ -5362,10 +5409,19 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { } const conflicts: string[] = []; const unreadable: string[] = []; + const envFileConflicts: string[] = []; for (const [service, rawService] of Object.entries(services)) { if (!isRecord(rawService)) { continue; } + envFileConflicts.push( + ...(await composeEnvFileKeyConflicts({ + projectDir: opts.projectDir, + service, + rawService, + migratedKeys, + })) + ); const keys = composeServiceEnvironmentKeys(rawService.environment); if (keys === null) { unreadable.push(JSON.stringify(service)); @@ -5387,6 +5443,12 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { reason: `Compose environment sets ${conflicts.sort().join(", ")}, which the legacy .env files also set; after the migration the .env value would override the Compose value. Remove the key from the .hack/.env file or from that service's environment, then retry.`, }); } + if (envFileConflicts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `Compose env_file sets ${envFileConflicts.sort().join(", ")}, which the legacy .env files also set; after the migration the .env value would override that file's value. Remove the key from the .hack/.env file or from that env_file, then retry.`, + }); + } } /** Overlay entries a migrated overlay layer must hold: those that differ from the base. */ @@ -5406,9 +5468,15 @@ async function selectLegacyDotenvOverlays(opts: { }): Promise { // Template and backup copies (`.env.example`, `.env.bak`, ...) are not // overlays: they are neither migrated nor offered for cleanup. - const overlayNames = ( - await discoverLegacyOverlayNames({ projectDir: opts.projectDir }) - ).filter((name) => !isExcludedLegacyDotenvName(name)); + const overlayNames = + // A symlinked overlay is an overlay too: skipping it would switch formats + // without its values. Its target is checked by the migration before reading. + ( + await discoverLegacyOverlayNames({ + projectDir: opts.projectDir, + includeSymlinks: true, + }) + ).filter((name) => !isExcludedLegacyDotenvName(name)); const invalidOverlays = overlayNames.filter( (name) => !( @@ -5432,6 +5500,40 @@ async function selectLegacyDotenvOverlays(opts: { return overlayNames; } +/** + * Refuses a legacy `.hack/.env` or overlay that is a symlink resolving outside + * `.hack`, so a repository cannot pull an outside file, such as a root `.env`, + * into its env layers. A dangling link stays for the read to refuse. + */ +async function assertLegacyDotenvLinksStayInProject(opts: { + readonly projectDir: string; + readonly overlayNames: readonly string[]; +}): Promise { + const projectDir = await realpath(opts.projectDir); + const escaping: string[] = []; + for (const envName of [null, ...opts.overlayNames]) { + const path = resolveEnvFilePath({ projectDir: opts.projectDir, envName }); + const entry = await lstat(path).catch(() => null); + if (!entry?.isSymbolicLink()) { + continue; + } + const target = await realpath(path).catch(() => null); + if (target === null) { + continue; + } + const inside = relative(projectDir, target); + if (inside.startsWith("..") || isAbsolute(inside)) { + escaping.push(JSON.stringify(basename(path))); + } + } + if (escaping.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `${escaping.join(", ")} ${escaping.length === 1 ? "is a symlink" : "are symlinks"} to a file outside .hack, so migrating would copy that file's values into this project's env layers. Replace the link with a regular file in .hack, or remove it, then retry.`, + }); + } +} + /** * In a git checkout, a machine-local layer the migration writes must be * ignored by git, or its plaintext values could be committed. Unknown status @@ -5515,7 +5617,8 @@ async function readValidatedLegacyDotenvFiles(opts: { * With no legacy file at all only the empty default layer is written. * - Refused before anything is written ({@link LegacyDotenvMigrationRefusalError}, * or {@link ProjectEnvLayerPathConflictError} for an unusable layer entry): - * any existing env layer entry, a legacy overlay named like the default or a + * any existing env layer entry, a legacy file that is a symlink out of + * `.hack`, a legacy overlay named like the default or a * machine-local layer, a Compose `env_file` on a file the migration leaves * out or on an overlay a launch without `--env` would not select, a key that * does not match `^[A-Z_][A-Z0-9_]*$` @@ -5552,6 +5655,10 @@ async function migrateUndeclaredLegacyDotenv(opts: { await assertComposeUsesNoExcludedLegacyDotenv({ projectDir: opts.projectDir, }); + await assertLegacyDotenvLinksStayInProject({ + projectDir: opts.projectDir, + overlayNames, + }); const legacyValues = await readValidatedLegacyDotenvFiles({ projectDir: opts.projectDir, @@ -5756,13 +5863,12 @@ export async function removeLegacyProjectEnvArtifacts(opts: { async function discoverLegacyOverlayNames(opts: { readonly projectDir: string; + readonly includeSymlinks: boolean; }): Promise { const names = new Set(); const entries = await readdir(opts.projectDir, { withFileTypes: true }); for (const entry of entries) { - // A symlinked overlay is an overlay too: skipping it would switch formats - // without its values. What it points at is checked when it is read. - if (!(entry.isFile() || entry.isSymbolicLink())) { + if (!(entry.isFile() || (opts.includeSymlinks && entry.isSymbolicLink()))) { continue; } if (!entry.name.startsWith(`${PROJECT_ENV_FILENAME}.`)) { diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index a30d325c9..55c7cab07 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1142,7 +1142,7 @@ test("the migration carries a symlinked legacy overlay's values and refuses one resolve(repo.projectDir, ".env"), `KEPT_SETTING=${KEPT_VALUE}\n` ); - const target = resolve(repo.root, "shared-staging.env"); + const target = resolve(repo.projectDir, "shared-staging.env"); const targetText = `KEPT_SETTING=${KEPT_VALUE}-linked\n`; await writeFile(target, targetText); const link = resolve(repo.projectDir, ".env.staging"); @@ -1163,7 +1163,8 @@ test("the migration carries a symlinked legacy overlay's values and refuses one for (const place of [ (root: string) => resolve(root, "missing"), - (root: string) => root, + // A link to a directory inside .hack: reading it fails. + (root: string) => resolve(root, "repo", ".hack"), ]) { const unusable = await createLegacyRepo(); await symlink( @@ -1348,3 +1349,77 @@ test("the migration refuses when a Compose environment value would lose preceden global: { KEPT_SETTING: KEPT_VALUE }, }); }); + +test("the .env-only migration refuses a legacy symlink that leaves .hack", async () => { + for (const link of [".env", ".env.staging"]) { + const repo = await createLegacyRepo(); + const target = resolve(repo.projectRoot, ".env"); + await writeFile(target, `KEPT_SETTING=${KEPT_VALUE}\n`); + await symlink(target, resolve(repo.projectDir, link)); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: `"${link}" is a symlink to a file outside .hack`, + hidden: [KEPT_VALUE], + }); + } +}); + +test("the contract migration does not copy a symlinked overlay's values into a tracked layer", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.json"), + `${JSON.stringify({ + version: 1, + vars: [{ key: "KEPT_SETTING", source: "plain_env", required: false }], + })}\n` + ); + await writeFile(resolve(repo.projectDir, ".env"), "OTHER=1\n"); + const target = resolve(repo.projectRoot, ".env"); + await writeFile(target, `KEPT_SETTING=${KEPT_VALUE}\n`); + await symlink(target, resolve(repo.projectDir, ".env.staging")); + + const migrated = await migrateLegacyRepo(repo, {}); + expect(migrated.migratedOverlays).not.toContain("staging"); + expect( + await lstat(resolve(repo.projectDir, "hack.env.staging.yaml")).catch( + () => null + ) + ).toBeNull(); + for (const name of await readdir(repo.projectDir)) { + if (name.startsWith("hack.env.") && name.endsWith(".yaml")) { + expect( + await readFile(resolve(repo.projectDir, name), "utf8") + ).not.toContain(KEPT_VALUE); + } + } +}); + +test("the migration refuses when another Compose env_file sets a migrated key", async () => { + const repo = await createLegacyRepo(); + const legacyPort = ["3", "000"].join(""); + const servicePort = ["4", "000"].join(""); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file:\n - .env\n - service.env\n - path: absent.env\n required: false\n" + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `PORT=${legacyPort}\nKEPT_SETTING=${KEPT_VALUE}\n` + ); + const serviceEnv = resolve(repo.projectDir, "service.env"); + await writeFile(serviceEnv, `PORT=${servicePort}\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'Compose env_file sets "PORT" ("service.env" in "api")', + hidden: [legacyPort, servicePort, KEPT_VALUE], + }); + + // With no shared key, the same shape migrates. + await writeFile(serviceEnv, "SERVICE_ONLY=1\n"); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); From de517d4d14e1a71be400bef5169d2fcdb72a7b2a Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:56:18 -0400 Subject: [PATCH 13/55] fix(env): refuse a migration that would widen a legacy file to new services Compose gives a legacy file listed in env_file only to the services that list it, but migrated values are global and reach every service. The .env-only migration now refuses when some services list a file with values to migrate and others do not, naming files and services, never values. A file no service lists has no per-service boundary to keep, so it still migrates, and docs/env.md says its values then reach every service. An environment entry that only refers to its own key (${KEY}, $KEY, ${KEY:-default}) is no longer a precedence conflict: Compose resolves it from the same values before and after. Based on the uncommitted draft from the Hetzner session at ff061939. That draft refused unless every service listed every file, which also refused a project where no service lists .env, so the rule here is narrower. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 11 ++- src/lib/project-env-config.ts | 108 ++++++++++++++++++++-- tests/env-legacy-service-scope.test.ts | 121 ++++++++++++++++++++++++- 3 files changed, 230 insertions(+), 10 deletions(-) diff --git a/docs/env.md b/docs/env.md index 5867c2bff..e28689f3f 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1072,8 +1072,15 @@ files, when: - a key the legacy files set is also set in a Compose service's own `environment` (`E_CONFIG_INVALID`). Compose ranks `environment` above `env_file`, but migrated values become global layer values that Hack applies after `environment`, so the service would - switch to the `.env` value. The error names the keys and services, never values. Remove - the key from the `.hack/.env` file or from that service's `environment`, then retry; + switch to the `.env` value. An entry that only refers to the same key (`${KEY}`, `$KEY`, + `${KEY:-default}`) is not a conflict. The error names the keys and services, never values. + Remove the key from the `.hack/.env` file or from that service's `environment`, then retry; +- some Compose services list a legacy file with values to migrate in `env_file` and others + do not (`E_CONFIG_INVALID`). Migrated values are global and reach every service, so the + others would receive values they never had. The error names the file and services, never + values. List the file for those services if they may have its values, or move the values + out of it and set them with `hack env set --service` after migrating. If no service lists + the file, the migration goes ahead and its values then reach every service; - a key the legacy files set is also set in another file a Compose service uses as `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 0f22dc02d..e38d0c89c 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5321,8 +5321,10 @@ async function assertComposeOverlaysStaySelected(opts: { /** * Keys a Compose service sets in its own `environment` (map keys, or the - * names of `KEY=value` / `KEY` list entries). Null when the entry has a shape - * this check cannot read. + * names of `KEY=value` / `KEY` list entries), except a value that only refers + * to the same key (`${KEY}`, `${KEY:-default}`, `$KEY`): Compose resolves that + * from the same values before and after the migration. Null when the entry + * has a shape this check cannot read. */ function composeServiceEnvironmentKeys( environment: unknown @@ -5330,20 +5332,55 @@ function composeServiceEnvironmentKeys( if (environment === undefined || environment === null) { return new Set(); } + const keys = new Set(); + const add = (key: string, value: unknown) => { + if (key.length > 0 && !isComposeSelfReference({ key, value })) { + keys.add(key); + } + }; if (Array.isArray(environment)) { - const keys = new Set(); for (const entry of environment) { if (typeof entry !== "string") { return null; } - const key = entry.split("=", 1)[0]?.trim() ?? ""; - if (key.length > 0) { - keys.add(key); + const separator = entry.indexOf("="); + if (separator === -1) { + add(entry.trim(), undefined); + } else { + add(entry.slice(0, separator).trim(), entry.slice(separator + 1)); } } return keys; } - return isRecord(environment) ? new Set(Object.keys(environment)) : null; + if (!isRecord(environment)) { + return null; + } + for (const [key, value] of Object.entries(environment)) { + add(key, value); + } + return keys; +} + +/** Default forms only (`:-`, `-`); `${KEY:?err}` and nested references stay conflicts. */ +const COMPOSE_DEFAULT_MODIFIER_PATTERN = /^:?-[^${}]*$/; + +function isComposeSelfReference(opts: { + readonly key: string; + readonly value: unknown; +}): boolean { + if (typeof opts.value !== "string") { + return false; + } + const value = opts.value.trim(); + if (value === `$${opts.key}` || value === `\${${opts.key}}`) { + return true; + } + const prefix = `\${${opts.key}`; + if (!(value.startsWith(prefix) && value.endsWith("}"))) { + return false; + } + const modifier = value.slice(prefix.length, -1); + return COMPOSE_DEFAULT_MODIFIER_PATTERN.test(modifier); } /** @@ -5449,6 +5486,63 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { reason: `Compose env_file sets ${envFileConflicts.sort().join(", ")}, which the legacy .env files also set; after the migration the .env value would override that file's value. Remove the key from the .hack/.env file or from that env_file, then retry.`, }); } + const unlisted = composeServicesMissingLegacyDotenv({ + projectDir: opts.projectDir, + services, + legacyValues: opts.legacyValues, + }); + if (unlisted.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${unlisted.join("; ")}, but other services do. Migrated values are global and reach every service, so those services would receive values they never had. List the file in env_file for those services if they may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, + }); + } +} + +/** + * Before the migration a legacy dotenv file listed in `env_file` reached only + * the services that list it; after it, its values are global layer values that + * Hack applies to every service. For each file with values to migrate (the + * base, and each overlay entry that differs from it) that some service lists, + * the services that do not, as refusal labels naming files and services only. + * A file no service lists has no per-service boundary to keep: it is project + * wide already, and docs/env.md says its values then reach every service. + */ +function composeServicesMissingLegacyDotenv(opts: { + readonly projectDir: string; + readonly services: Record; + readonly legacyValues: ReadonlyMap>; +}): string[] { + const baseValues = opts.legacyValues.get(null) ?? {}; + const labels: string[] = []; + for (const [envName, values] of opts.legacyValues) { + const migrated = + envName === null + ? values + : legacyOverlayChangedValues({ overlayValues: values, baseValues }); + if (Object.keys(migrated).length === 0) { + continue; + } + const path = resolveEnvFilePath({ projectDir: opts.projectDir, envName }); + const composeServices = Object.entries(opts.services).filter( + (entry): entry is [string, Record] => isRecord(entry[1]) + ); + const missing = composeServices + .filter( + ([, rawService]) => + !readComposeEnvFilePaths({ rawService }).some( + (configured) => resolve(opts.projectDir, configured) === path + ) + ) + .map(([service]) => JSON.stringify(service)) + .sort(); + if (missing.length > 0 && missing.length < composeServices.length) { + labels.push( + `${missing.join(", ")} ${missing.length === 1 ? "does" : "do"} not list ${JSON.stringify(basename(path))} in env_file` + ); + } + } + return labels.sort(); } /** Overlay entries a migrated overlay layer must hold: those that differ from the base. */ diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 55c7cab07..8f57e74cf 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1322,7 +1322,7 @@ test("the migration refuses when a Compose environment value would lose preceden const composeExplicitPort = ["4", "000"].join(""); await writeFile( resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), - `services:\n api:\n image: alpine:3.20\n env_file: .env\n environment:\n PORT: "${composeExplicitPort}"\n worker:\n image: alpine:3.20\n environment:\n - OTHER=1\n` + `services:\n api:\n image: alpine:3.20\n env_file: .env\n environment:\n PORT: "${composeExplicitPort}"\n worker:\n image: alpine:3.20\n env_file: .env\n environment:\n - OTHER=1\n` ); await writeFile( resolve(repo.projectDir, ".env"), @@ -1423,3 +1423,122 @@ test("the migration refuses when another Compose env_file sets a migrated key", }); expect(migrated.legacyDetected).toBe(true); }); + +test("the migration refuses when only some Compose services list a legacy file", async () => { + const repo = await createLegacyRepo(); + const compose = resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME); + await writeFile( + compose, + "services:\n api:\n image: alpine:3.20\n env_file: .env\n worker:\n image: alpine:3.20\n" + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"worker" does not list ".env" in env_file', + hidden: [KEPT_VALUE], + }); + + // Once every service lists it, none receives anything new. + await writeFile( + compose, + "services:\n api:\n image: alpine:3.20\n env_file: .env\n worker:\n image: alpine:3.20\n env_file: [.env]\n" + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); +}); + +test("the migration refuses a legacy overlay only some services list, and accepts one that adds nothing", async () => { + const repo = await createLegacyRepo(); + const compose = resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME); + await writeFile( + resolve(repo.projectDir, "hack.config.json"), + `${JSON.stringify({ env: { defaultOverlay: "staging" } })}\n` + ); + await writeFile( + compose, + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n worker:\n image: alpine:3.20\n env_file: .env\n" + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const overlay = resolve(repo.projectDir, ".env.staging"); + await writeFile(overlay, `KEPT_SETTING=${KEPT_VALUE}-staging\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"worker" does not list ".env.staging" in env_file', + hidden: [KEPT_VALUE], + }); + + await writeFile(overlay, `KEPT_SETTING=${KEPT_VALUE}\n`); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); + +test("the migration goes ahead when no Compose service lists the legacy file", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n worker:\n image: alpine:3.20\n" + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); +}); + +for (const entry of ["${PORT}", "$PORT", "${PORT:-1}", "${PORT-1}"]) { + test(`a Compose environment entry ${entry} that only refers to its own key is not a conflict`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + `services:\n api:\n image: alpine:3.20\n env_file: .env\n environment:\n - PORT=${entry}\n` + ); + await writeFile(resolve(repo.projectDir, ".env"), "PORT=2\n"); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + }); +} + +for (const entry of [ + "PORT: ${PORT:?required}", + "PORT: ${OTHER}", + "PORT: ${PORT:-${OTHER}}", + "PORT:", +]) { + test(`a Compose environment entry "${entry}" stays a conflict`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + `services:\n api:\n image: alpine:3.20\n env_file: .env\n environment:\n ${entry}\n` + ); + await writeFile(resolve(repo.projectDir, ".env"), "PORT=2\n"); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'Compose environment sets "PORT" in "api"', + hidden: [], + }); + }); +} From b34821d318c4feff57d371840118169a31a08dea Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:02:56 -0400 Subject: [PATCH 14/55] fix(env): refuse legacy lines Compose reads differently from the migration Compose drops an unquoted inline comment and expands $ outside single quotes, but parseDotEnv keeps both literally. A migrated PORT=3000 # dev would reach the service as that whole string through Hack's generated override. The .env-only migration now refuses such lines, counting them per file and never showing them. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 5 +++ src/lib/project-env-config.ts | 50 +++++++++++++++++++++++--- tests/env-legacy-service-scope.test.ts | 43 ++++++++++++++++++++++ 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/docs/env.md b/docs/env.md index e28689f3f..e35c998cb 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1085,6 +1085,11 @@ files, when: `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, or expand variables in an `env_file` path. Remove the key from one of the files, then retry; +- a line in a legacy file has an unquoted inline comment (`PORT=3000 # dev`) or a `$` outside + single quotes (`E_CONFIG_INVALID`). Compose drops the comment and expands the `$`, but the + migration would keep them as literal text, so the value would change. The error counts + lines per file and never shows them. Remove the comment, or write the value Compose uses + in single quotes; - `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` (`E_PATH_CONFLICT`), so its values would be copied from elsewhere into this project's layers. Replace the link with a regular file in `.hack`, or remove it; diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index e38d0c89c..70ffb24d5 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5653,6 +5653,30 @@ async function assertIgnoredMigrationLocalLayer(opts: { }); } +const COMPOSE_UNQUOTED_INLINE_COMMENT_PATTERN = /\s#/; + +/** + * Whether Compose reads this dotenv line differently from {@link parseDotEnv}: + * it drops an unquoted inline comment and expands `$` outside single quotes, + * while the parser keeps both literally. A migrated value goes through Hack's + * generated override, so the service would receive the literal text instead. + */ +function hasComposeOnlyDotenvSyntax(line: string): boolean { + const separator = line.indexOf("="); + if (separator === -1 || line.trimStart().startsWith("#")) { + return false; + } + const value = line.slice(separator + 1).trimStart(); + if (value.startsWith("'")) { + return false; + } + return ( + value.includes("$") || + (!value.startsWith('"') && + COMPOSE_UNQUOTED_INLINE_COMMENT_PATTERN.test(value)) + ); +} + /** * Entries of `.hack/.env` (key `null`) and each legacy overlay file. Refuses, * before anything is written, when a key cannot be an env layer key; the @@ -5671,14 +5695,24 @@ async function readValidatedLegacyDotenvFiles(opts: { ]; const legacyValues = new Map>(); const invalidKeyCounts: string[] = []; + const composeSyntaxCounts: string[] = []; for (const file of legacyFiles) { - const values = await readLegacyDotenvFile({ - path: resolveEnvFilePath({ - projectDir: opts.projectDir, - envName: file.envName, - }), + const path = resolveEnvFilePath({ + projectDir: opts.projectDir, + envName: file.envName, }); + const values = await readLegacyDotenvFile({ path }); legacyValues.set(file.envName, values); + const read = await readOptionalTextFile(path); + const composeSyntax = + read.kind === "absent" || read.kind === "unreadable" + ? 0 + : read.text.split("\n").filter(hasComposeOnlyDotenvSyntax).length; + if (composeSyntax > 0) { + composeSyntaxCounts.push( + `${countLabel({ count: composeSyntax, noun: "line" })} in ${file.label}` + ); + } const invalid = Object.keys(values).filter( (key) => !PROJECT_ENV_KEY_PATTERN.test(key) ).length; @@ -5694,6 +5728,12 @@ async function readValidatedLegacyDotenvFiles(opts: { reason: `env keys must match ^[A-Z_][A-Z0-9_]*$, but ${invalidKeyCounts.join(", ")} do not (for example a lowercase name or an "export " prefix). The keys are not shown because a malformed line may contain a value. Rename or remove them, then retry.`, }); } + if (composeSyntaxCounts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `Compose reads ${composeSyntaxCounts.join(", ")} differently from Hack: an unquoted " #" starts a comment and "$" outside single quotes is expanded, so the migrated value would change. Remove the inline comment, or write the value Compose uses in single quotes, then retry. The lines are not shown because they may contain values.`, + }); + } return legacyValues; } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 8f57e74cf..6f9f4ba02 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1542,3 +1542,46 @@ for (const entry of [ }); }); } + +for (const line of [ + `PORT=3000 # ${KEPT_VALUE}`, + `PORT=${KEPT_VALUE}$OTHER`, + `PORT="${KEPT_VALUE}\${OTHER}"`, +]) { + test(`the migration refuses a line Compose reads differently: ${line.slice(0, 7)}`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: .env\n" + ); + await writeFile(resolve(repo.projectDir, ".env"), `${line}\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "Compose reads 1 line in .env differently from Hack", + hidden: [KEPT_VALUE], + }); + }); +} + +test("the migration keeps values Compose and Hack read the same", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + [ + "# a comment with $ and # in it", + `SINGLE='${KEPT_VALUE}$literal # kept'`, + `QUOTED="${KEPT_VALUE} # kept"`, + `FRAGMENT=${KEPT_VALUE}#kept`, + "", + ].join("\n") + ); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { + SINGLE: `${KEPT_VALUE}$literal # kept`, + QUOTED: `${KEPT_VALUE} # kept`, + FRAGMENT: `${KEPT_VALUE}#kept`, + }, + }); +}); From 296eb23b1ab0c891ce41d3cebb0b46071b19c2d2 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:11:00 -0400 Subject: [PATCH 15/55] fix(env): drop the self-reference exception and close migration exposure gaps Review of the exact head showed the ${KEY} exception changed values: an overlay or the shell can set the key, and the generated override then wins. It is gone, so those entries refuse again; the remedy is to drop the entry from environment. The service check now compares what each service had: a service that would gain a base key from a file only another service lists is refused, including through an overlay identical to the base. Top-level include, service extends and variable env_file paths refuse because the check cannot see what they add. When no service lists a legacy file the migration still runs, since its values may only feed Compose interpolation, and doctor names the keys every service then receives. The contract migration now refuses .hack/.env or a default overlay linked out of .hack before reading it, so a repository cannot pull an outside dotenv into a tracked layer. That closes the route doctor exposed by migrating before the Docker gate. Refs HACK-1370, HACK-1686 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 29 ++- src/commands/doctor.ts | 25 ++- src/lib/project-env-config.ts | 235 +++++++++++++++++------- tests/doctor-fix-noninteractive.test.ts | 20 ++ tests/env-legacy-service-scope.test.ts | 144 +++++++++++++-- 5 files changed, 353 insertions(+), 100 deletions(-) diff --git a/docs/env.md b/docs/env.md index e35c998cb..4b7b5cb75 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1072,15 +1072,17 @@ files, when: - a key the legacy files set is also set in a Compose service's own `environment` (`E_CONFIG_INVALID`). Compose ranks `environment` above `env_file`, but migrated values become global layer values that Hack applies after `environment`, so the service would - switch to the `.env` value. An entry that only refers to the same key (`${KEY}`, `$KEY`, - `${KEY:-default}`) is not a conflict. The error names the keys and services, never values. + switch to the `.env` value. This includes an entry that only refers to its own key + (`PORT: ${PORT}`), whose result can differ when an overlay or the shell sets the key. The error names the keys and services, never values. Remove the key from the `.hack/.env` file or from that service's `environment`, then retry; -- some Compose services list a legacy file with values to migrate in `env_file` and others - do not (`E_CONFIG_INVALID`). Migrated values are global and reach every service, so the - others would receive values they never had. The error names the file and services, never - values. List the file for those services if they may have its values, or move the values - out of it and set them with `hack env set --service` after migrating. If no service lists - the file, the migration goes ahead and its values then reach every service; +- some Compose services list a legacy file and others do not, so a service would receive + values it never had (`E_CONFIG_INVALID`), because migrated values are global and reach every + service. The error names files, services and keys, never values. List the file for those + services if they may have its values, or move the values out of it and set them with + `hack env set --service` after migrating; +- Compose uses a top-level `include`, a service `extends` or a variable in an `env_file` path + (`E_CONFIG_INVALID`), so the migration cannot tell which services receive the values. Write + those `env_file` entries directly in the Compose file; - a key the legacy files set is also set in another file a Compose service uses as `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, @@ -1098,8 +1100,15 @@ files, when: that resolves to a readable file inside `.hack` is migrated like a regular one; cleanup removes only the link, never its target. -With `hack.env.json`, the migration reads only regular `.hack/.env.` files, so a -symlinked overlay is not migrated and its values cannot reach a tracked layer. +If no Compose service lists a legacy file, the values may only be used for Compose +interpolation today. The migration goes ahead, every service then receives them as environment +variables, and doctor names the keys. To limit a key to one service, unset it and set it with +`hack env set --service --local`. + +With `hack.env.json`, the migration discovers only regular `.hack/.env.` files, so a +symlinked overlay is not migrated. It refuses (`E_PATH_CONFLICT`) when `.hack/.env` or a +configured default overlay is a symlink to a file outside `.hack`, so its values cannot reach a +tracked layer. Without `hack.env.json`, nothing lists values kept in a secret backend, so none are migrated: set them again with `hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 92fe4d979..713bfb489 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -3691,6 +3691,23 @@ function renderSkippedDoctorFixSteps(): void { ); } +function buildEnvMigrationNote(opts: { + readonly migrated: LegacyProjectEnvMigration; +}): string { + const { migrated } = opts; + const notes = [`Wrote ${migrated.wroteFiles.join(", ")}`]; + if (migrated.updatedProjectConfig) { + notes.push("Updated .hack/hack.config.json for the modern env format."); + } + const newKeys = migrated.keysNewToEveryService ?? []; + if (newKeys.length > 0) { + notes.push( + `No Compose service listed the legacy .env files, so every service now receives: ${newKeys.join(", ")}. To limit a key to one service, run \`hack env unset KEY --local\` and then \`hack env set KEY --service --local\`.` + ); + } + return notes.join("\n"); +} + async function maybeMigrateProjectEnvConfig(opts: { readonly startDir: string; }): Promise { @@ -3768,13 +3785,7 @@ async function maybeMigrateProjectEnvConfig(opts: { note("Legacy env migration found nothing to write.", "doctor"); return; } - const migrationNotes = [`Wrote ${migrated.wroteFiles.join(", ")}`]; - if (migrated.updatedProjectConfig) { - migrationNotes.push( - "Updated .hack/hack.config.json for the modern env format." - ); - } - note(migrationNotes.join("\n"), "doctor"); + note(buildEnvMigrationNote({ migrated }), "doctor"); let composeRepaired = false; if (migrated.composeEnvFileReferences.length > 0) { diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 70ffb24d5..8efc0b401 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -4930,6 +4930,8 @@ export type LegacyProjectEnvMigration = { readonly cleanupCandidates: readonly string[]; readonly blockedCleanupCandidates: readonly string[]; readonly composeEnvFileReferences: readonly LegacyComposeEnvFileReference[]; + /** Keys every service now receives though none listed a legacy file. Names only. */ + readonly keysNewToEveryService?: readonly string[]; }; /** @@ -4994,18 +4996,25 @@ export async function migrateLegacyProjectEnv(opts: { }; } + // Discovered overlays are regular files only, and any link that remains (the + // base file, or a configured default overlay) must stay in .hack: this path + // writes plain_env values to tracked layers, so a repository's symlink must + // not pull an outside file's values into them. + const overlayNames = await discoverLegacyOverlayNames({ + projectDir: opts.projectDir, + includeSymlinks: false, + }); + await assertLegacyDotenvLinksStayInProject({ + projectDir: opts.projectDir, + overlayNames, + }); + const baseResolved = await resolveHackEnv({ projectDir: opts.projectDir, projectName: opts.projectName, envName: null, }); - // Regular files only: this path writes plain_env values to tracked layers, so - // a repository's symlink must not pull an outside file's values into them. - const overlayNames = await discoverLegacyOverlayNames({ - projectDir: opts.projectDir, - includeSymlinks: false, - }); const keyInfo = baseResolved.values.some( (value) => value.source === "keychain" ) @@ -5321,10 +5330,8 @@ async function assertComposeOverlaysStaySelected(opts: { /** * Keys a Compose service sets in its own `environment` (map keys, or the - * names of `KEY=value` / `KEY` list entries), except a value that only refers - * to the same key (`${KEY}`, `${KEY:-default}`, `$KEY`): Compose resolves that - * from the same values before and after the migration. Null when the entry - * has a shape this check cannot read. + * names of `KEY=value` / `KEY` list entries). Null when the entry has a shape + * this check cannot read. */ function composeServiceEnvironmentKeys( environment: unknown @@ -5332,55 +5339,44 @@ function composeServiceEnvironmentKeys( if (environment === undefined || environment === null) { return new Set(); } - const keys = new Set(); - const add = (key: string, value: unknown) => { - if (key.length > 0 && !isComposeSelfReference({ key, value })) { - keys.add(key); - } - }; if (Array.isArray(environment)) { + const keys = new Set(); for (const entry of environment) { if (typeof entry !== "string") { return null; } - const separator = entry.indexOf("="); - if (separator === -1) { - add(entry.trim(), undefined); - } else { - add(entry.slice(0, separator).trim(), entry.slice(separator + 1)); + const key = entry.split("=", 1)[0]?.trim() ?? ""; + if (key.length > 0) { + keys.add(key); } } return keys; } - if (!isRecord(environment)) { - return null; - } - for (const [key, value] of Object.entries(environment)) { - add(key, value); - } - return keys; + return isRecord(environment) ? new Set(Object.keys(environment)) : null; } -/** Default forms only (`:-`, `-`); `${KEY:?err}` and nested references stay conflicts. */ -const COMPOSE_DEFAULT_MODIFIER_PATTERN = /^:?-[^${}]*$/; - -function isComposeSelfReference(opts: { - readonly key: string; - readonly value: unknown; -}): boolean { - if (typeof opts.value !== "string") { - return false; - } - const value = opts.value.trim(); - if (value === `$${opts.key}` || value === `\${${opts.key}}`) { - return true; - } - const prefix = `\${${opts.key}`; - if (!(value.startsWith(prefix) && value.endsWith("}"))) { - return false; +/** + * Compose inputs that can add or hide an `env_file` this check cannot read: a + * top-level `include`, a service `extends`, or an `env_file` path with a + * variable. Returned as refusal labels naming the service or field only. + */ +function uncheckableComposeEnvInputs(opts: { + readonly parsed: Record; + readonly services: Record; +}): readonly string[] { + const labels: string[] = + "include" in opts.parsed ? ["top-level include"] : []; + for (const [service, rawService] of composeServiceEntries(opts.services)) { + if ("extends" in rawService) { + labels.push(`extends in ${JSON.stringify(service)}`); + } + if ( + readComposeEnvFilePaths({ rawService }).some((path) => path.includes("$")) + ) { + labels.push(`a variable env_file path in ${JSON.stringify(service)}`); + } } - const modifier = value.slice(prefix.length, -1); - return COMPOSE_DEFAULT_MODIFIER_PATTERN.test(modifier); + return labels; } /** @@ -5444,6 +5440,18 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { migratedKeys.add(key); } } + if (migratedKeys.size === 0) { + return; + } + const uncheckable = parsed + ? uncheckableComposeEnvInputs({ parsed, services }) + : []; + if (uncheckable.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `Compose uses ${uncheckable.join(", ")}, so the migration cannot tell which services receive the legacy .env values. Write those services' env_file entries directly in ${PROJECT_COMPOSE_FILENAME}, then retry.`, + }); + } const conflicts: string[] = []; const unreadable: string[] = []; const envFileConflicts: string[] = []; @@ -5494,19 +5502,47 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { if (unlisted.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `${unlisted.join("; ")}, but other services do. Migrated values are global and reach every service, so those services would receive values they never had. List the file in env_file for those services if they may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, + reason: `${unlisted.join("; ")}. Migrated values are global and reach every service, so those services would receive values they never had. List the file in env_file for those services if they may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, }); } } +/** Compose services as records, skipping entries that are not service maps. */ +function composeServiceEntries( + services: Record +): readonly (readonly [string, Record])[] { + return Object.entries(services).filter( + (entry): entry is [string, Record] => isRecord(entry[1]) + ); +} + +/** Legacy files a service lists in `env_file`, as `legacyValues` keys. */ +function composeServiceLegacyEnvNames(opts: { + readonly projectDir: string; + readonly rawService: Record; + readonly legacyValues: ReadonlyMap>; +}): readonly (string | null)[] { + const listed = readComposeEnvFilePaths({ rawService: opts.rawService }).map( + (configured) => resolve(opts.projectDir, configured) + ); + return [...opts.legacyValues.keys()].filter((envName) => + listed.includes( + resolveEnvFilePath({ projectDir: opts.projectDir, envName }) + ) + ); +} + /** * Before the migration a legacy dotenv file listed in `env_file` reached only * the services that list it; after it, its values are global layer values that - * Hack applies to every service. For each file with values to migrate (the - * base, and each overlay entry that differs from it) that some service lists, - * the services that do not, as refusal labels naming files and services only. - * A file no service lists has no per-service boundary to keep: it is project - * wide already, and docs/env.md says its values then reach every service. + * Hack applies to every service. When any service lists a legacy file, returns + * refusal labels (file, service and key names only) for each file with values + * to migrate that only some services list, and for each service that does not + * list a file setting a base key. + * + * A project where no service lists a legacy file has no `env_file` boundary to + * compare, and its values may be needed for interpolation, so it is not refused + * here; {@link legacyKeysNewToEveryService} reports what those services gain. */ function composeServicesMissingLegacyDotenv(opts: { readonly projectDir: string; @@ -5514,6 +5550,19 @@ function composeServicesMissingLegacyDotenv(opts: { readonly legacyValues: ReadonlyMap>; }): string[] { const baseValues = opts.legacyValues.get(null) ?? {}; + const services = composeServiceEntries(opts.services).map( + ([service, rawService]) => ({ + label: JSON.stringify(service), + listed: composeServiceLegacyEnvNames({ + projectDir: opts.projectDir, + rawService, + legacyValues: opts.legacyValues, + }), + }) + ); + if (services.every((service) => service.listed.length === 0)) { + return []; + } const labels: string[] = []; for (const [envName, values] of opts.legacyValues) { const migrated = @@ -5523,28 +5572,70 @@ function composeServicesMissingLegacyDotenv(opts: { if (Object.keys(migrated).length === 0) { continue; } - const path = resolveEnvFilePath({ projectDir: opts.projectDir, envName }); - const composeServices = Object.entries(opts.services).filter( - (entry): entry is [string, Record] => isRecord(entry[1]) - ); - const missing = composeServices - .filter( - ([, rawService]) => - !readComposeEnvFilePaths({ rawService }).some( - (configured) => resolve(opts.projectDir, configured) === path - ) + const missing = services + .filter((service) => !service.listed.includes(envName)) + .map((service) => service.label) + .sort(); + if (missing.length > 0 && missing.length < services.length) { + labels.push( + `${missing.join(", ")} ${missing.length === 1 ? "does" : "do"} not list ${JSON.stringify(basename(resolveEnvFilePath({ projectDir: opts.projectDir, envName })))} in env_file` + ); + } + } + for (const service of services) { + const had = new Set( + service.listed.flatMap((envName) => + Object.keys(opts.legacyValues.get(envName) ?? {}) ) - .map(([service]) => JSON.stringify(service)) + ); + const gained = Object.keys(baseValues) + .filter((key) => !had.has(key)) + .map((key) => JSON.stringify(key)) .sort(); - if (missing.length > 0 && missing.length < composeServices.length) { + if (gained.length > 0 && !service.listed.includes(null)) { labels.push( - `${missing.join(", ")} ${missing.length === 1 ? "does" : "do"} not list ${JSON.stringify(basename(path))} in env_file` + `${service.label} lists no legacy file that sets ${gained.join(", ")}` ); } } return labels.sort(); } +/** + * Migrated keys that reach every service although no service lists a legacy + * file in `env_file`, for the migration to name. Empty when any service lists + * one, because {@link composeServicesMissingLegacyDotenv} covers that case. + */ +async function legacyKeysNewToEveryService(opts: { + readonly projectDir: string; + readonly legacyValues: ReadonlyMap>; +}): Promise { + const parsed = await readComposeFile({ + composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), + }); + const services = parsed ? getRecord(parsed, "services") : null; + if (!services) { + return []; + } + const entries = composeServiceEntries(services); + const anyListed = entries.some( + ([, rawService]) => + composeServiceLegacyEnvNames({ + projectDir: opts.projectDir, + rawService, + legacyValues: opts.legacyValues, + }).length > 0 + ); + if (entries.length === 0 || anyListed) { + return []; + } + return [ + ...new Set( + [...opts.legacyValues.values()].flatMap((values) => Object.keys(values)) + ), + ].sort((left, right) => left.localeCompare(right)); +} + /** Overlay entries a migrated overlay layer must hold: those that differ from the base. */ function legacyOverlayChangedValues(opts: { readonly overlayValues: Record; @@ -5754,8 +5845,11 @@ async function readValidatedLegacyDotenvFiles(opts: { * any existing env layer entry, a legacy file that is a symlink out of * `.hack`, a legacy overlay named like the default or a * machine-local layer, a Compose `env_file` on a file the migration leaves - * out or on an overlay a launch without `--env` would not select, a key that - * does not match `^[A-Z_][A-Z0-9_]*$` + * out or on an overlay a launch without `--env` would not select, a Compose + * `environment` or `env_file` entry the migrated value would override, a + * service that would gain values a legacy file gave only to others, a Compose + * input that hides `env_file` from this check, a line Compose reads differently + * from Hack, a key that does not match `^[A-Z_][A-Z0-9_]*$` * (counted per file, never shown), and a machine-local target that git * tracks or, inside a git checkout, cannot report on. * - Layers are created only where nothing exists (compare-and-swap against an @@ -5925,6 +6019,10 @@ async function migrateUndeclaredLegacyDotenv(opts: { }); } + const keysNewToEveryService = await legacyKeysNewToEveryService({ + projectDir: opts.projectDir, + legacyValues, + }); const composeEnvFileReferences = await inspectLegacyComposeEnvFileReferences({ composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), projectDir: opts.projectDir, @@ -5957,6 +6055,7 @@ async function migrateUndeclaredLegacyDotenv(opts: { cleanupCandidates: allowedCleanup, blockedCleanupCandidates: blockedCleanup, composeEnvFileReferences, + keysNewToEveryService, }; } diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 180a718ea..82bc0150b 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -692,6 +692,26 @@ test("doctor --migrate-env-config treats a symlinked overlay as env layers and n } }, 30_000); +test("doctor --migrate-env-config names the keys every service gains, never their values (HACK-1370)", async () => { + const projectRoot = await createProjectWithStaleDaemonPid(); + const projectDir = resolve(projectRoot, ".hack"); + const legacy = ["synthetic", "legacy", "4c2e"].join("-"); + await writeFile(resolve(projectDir, ".env"), `OWNED=${legacy}\n`); + + const { runCli } = await import("../src/cli/run.ts"); + const code = await runCli([ + "doctor", + "--migrate-env-config", + "--no-interactive", + "--path", + projectRoot, + ]); + expect(code).toBe(0); + const notes = noteCalls.join("\n"); + expect(notes).toContain("so every service now receives: OWNED."); + expect(notes).not.toContain(legacy); +}, 30_000); + test("doctor --migrate-env-config refuses a dangling default layer with E_PATH_CONFLICT and keeps it (HACK-1370)", async () => { const projectRoot = await createProjectWithStaleDaemonPid(); const projectDir = resolve(projectRoot, ".hack"); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 6f9f4ba02..a1de0790d 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1506,22 +1506,11 @@ test("the migration goes ahead when no Compose service lists the legacy file", a }); }); -for (const entry of ["${PORT}", "$PORT", "${PORT:-1}", "${PORT-1}"]) { - test(`a Compose environment entry ${entry} that only refers to its own key is not a conflict`, async () => { - const repo = await createLegacyRepo(); - await writeFile( - resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), - `services:\n api:\n image: alpine:3.20\n env_file: .env\n environment:\n - PORT=${entry}\n` - ); - await writeFile(resolve(repo.projectDir, ".env"), "PORT=2\n"); - const migrated = await migrateLegacyRepo(repo, { - migrateUndeclaredDotenv: true, - }); - expect(migrated.legacyDetected).toBe(true); - }); -} - for (const entry of [ + "PORT: ${PORT}", + "PORT: $PORT", + "PORT: ${PORT:-1}", + "PORT: ${PORT-1}", "PORT: ${PORT:?required}", "PORT: ${OTHER}", "PORT: ${PORT:-${OTHER}}", @@ -1585,3 +1574,128 @@ test("the migration keeps values Compose and Hack read the same", async () => { }, }); }); + +test("the migration refuses a service that gains keys from a file only another service lists as an identical overlay", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: .env.staging\n worker:\n image: alpine:3.20\n" + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"worker" lists no legacy file that sets "KEPT_SETTING"', + hidden: [KEPT_VALUE], + }); +}); + +const UNCHECKABLE_COMPOSE: readonly (readonly [string, string, string])[] = [ + [ + "a service extends", + "services:\n api:\n image: alpine:3.20\n extends: {file: base.yml, service: b}\n worker:\n image: alpine:3.20\n", + 'extends in "api"', + ], + [ + "a top-level include", + "include:\n - other.yml\nservices:\n worker:\n image: alpine:3.20\n", + "top-level include", + ], + [ + "a variable env_file path", + "services:\n api:\n image: alpine:3.20\n env_file: ${D:-.}/.env\n worker:\n image: alpine:3.20\n", + 'a variable env_file path in "api"', + ], +]; + +for (const [name, compose, reason] of UNCHECKABLE_COMPOSE) { + test(`the migration refuses ${name} it cannot check`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + compose + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason, + hidden: [KEPT_VALUE], + }); + }); +} + +test("the migration names the keys every service gains when no service lists the legacy file", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\nOTHER=1\n` + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.keysNewToEveryService).toEqual(["KEPT_SETTING", "OTHER"]); + expect(JSON.stringify(migrated)).not.toContain(KEPT_VALUE); +}); + +test("the migration reports no new keys when every service lists the legacy file", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: ./.env\n worker:\n image: alpine:3.20\n env_file:\n - path: ../.hack/.env\n required: false\n" + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.keysNewToEveryService).toEqual([]); +}); + +for (const [name, link, config] of [ + [".hack/.env", ".env", null], + ["a default overlay", ".env.staging", "staging"], +] as const) { + test(`the contract migration refuses ${name} linked out of .hack before writing`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.json"), + `${JSON.stringify({ + version: 1, + vars: [{ key: "KEPT_SETTING", source: "plain_env", required: false }], + })}\n` + ); + if (config) { + await writeFile( + resolve(repo.projectDir, PROJECT_CONFIG_FILENAME), + `${JSON.stringify({ name: "env-legacy-scope", dev_host: "env-legacy-scope.hack", env: { defaultOverlay: config } })}\n` + ); + } + const target = resolve(repo.projectRoot, ".env"); + await writeFile(target, `KEPT_SETTING=${KEPT_VALUE}\n`); + await symlink(target, resolve(repo.projectDir, link)); + const before = await hackDirEntries(repo); + const error = await migrateLegacyRepo(repo, {}).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as LegacyDotenvMigrationRefusalError).code).toBe( + "E_PATH_CONFLICT" + ); + expect((error as Error).message).toContain(`"${link}" is a symlink`); + expect((error as Error).message).not.toContain(KEPT_VALUE); + expect(await hackDirEntries(repo)).toEqual(before); + }); +} From b41965c209506cfa2dd64bb4e93071914d3edb44 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:25:48 -0400 Subject: [PATCH 16/55] fix(env): refuse more Compose dotenv forms and default-overlay exposure Compose decodes backslash escapes in double quotes, expands $ across quoted lines, and accepts KEY: value and a bare KEY, so a PEM key written with \n would have migrated corrupted and a colon entry would have been lost. The migration now scans entries, tracking quoted values across lines, and refuses those forms. A default overlay reaches every launch, so one a service does not list now refuses when any service lists a legacy file. A key the Compose file interpolates that the default overlay changes refuses too: Compose takes it from .hack/.env today and the overlay would apply afterwards. Keys from an unlisted non-default overlay are named by doctor, whose note now warns against service-scoping a key Compose interpolates. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 24 +-- src/commands/doctor.ts | 2 +- src/lib/project-env-config.ts | 197 +++++++++++++++++++----- tests/doctor-fix-noninteractive.test.ts | 3 +- tests/env-legacy-service-scope.test.ts | 118 ++++++++++++++ 5 files changed, 298 insertions(+), 46 deletions(-) diff --git a/docs/env.md b/docs/env.md index 4b7b5cb75..8bcde3ae3 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1087,11 +1087,14 @@ files, when: `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, or expand variables in an `env_file` path. Remove the key from one of the files, then retry; -- a line in a legacy file has an unquoted inline comment (`PORT=3000 # dev`) or a `$` outside - single quotes (`E_CONFIG_INVALID`). Compose drops the comment and expands the `$`, but the - migration would keep them as literal text, so the value would change. The error counts - lines per file and never shows them. Remove the comment, or write the value Compose uses - in single quotes; +- a legacy entry is one Compose reads differently from Hack (`E_CONFIG_INVALID`): an unquoted + inline comment (`PORT=3000 # dev`), a `$` outside single quotes, a `\` in double quotes (such + as `\n` in a PEM key), or a line without `=` (`A: 1`, or a bare `A`). The value would change + or be lost. The error counts entries per file and never shows them. Write the value Compose + uses in single quotes, with real line breaks and no escapes, or remove the line; +- the Compose file interpolates a key the default overlay sets differently from `.hack/.env` + (`E_CONFIG_INVALID`). Compose takes it from `.hack/.env` today, but the overlay value would + apply after the migration. Make the two files agree or stop interpolating the key; - `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` (`E_PATH_CONFLICT`), so its values would be copied from elsewhere into this project's layers. Replace the link with a regular file in `.hack`, or remove it; @@ -1100,10 +1103,13 @@ files, when: that resolves to a readable file inside `.hack` is migrated like a regular one; cleanup removes only the link, never its target. -If no Compose service lists a legacy file, the values may only be used for Compose -interpolation today. The migration goes ahead, every service then receives them as environment -variables, and doctor names the keys. To limit a key to one service, unset it and set it with -`hack env set --service --local`. +If no Compose service lists a legacy file, its values may only be used for Compose +interpolation today. The migration goes ahead, every service receives them as environment +variables once that env is selected, and doctor names the keys. The same applies to an overlay +no service lists, unless it is the default overlay, which refuses when any service lists a legacy +file. To limit a key to one service, unset it and set it with `hack env set --service --local`. +Do not do this for a key your Compose file interpolates: service values do not reach +interpolation, so `${KEY}` would become empty. With `hack.env.json`, the migration discovers only regular `.hack/.env.` files, so a symlinked overlay is not migrated. It refuses (`E_PATH_CONFLICT`) when `.hack/.env` or a diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 713bfb489..a7e4585c5 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -3702,7 +3702,7 @@ function buildEnvMigrationNote(opts: { const newKeys = migrated.keysNewToEveryService ?? []; if (newKeys.length > 0) { notes.push( - `No Compose service listed the legacy .env files, so every service now receives: ${newKeys.join(", ")}. To limit a key to one service, run \`hack env unset KEY --local\` and then \`hack env set KEY --service --local\`.` + `No Compose service listed the legacy file that sets these, so every service now receives them once that env is selected: ${newKeys.join(", ")}. To limit a key to one service, run \`hack env unset KEY --local\` and then \`hack env set KEY --service --local\`, but not for a key your Compose file interpolates: service values do not reach interpolation.` ); } return notes.join("\n"); diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 8efc0b401..b64b2a73a 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -4930,7 +4930,7 @@ export type LegacyProjectEnvMigration = { readonly cleanupCandidates: readonly string[]; readonly blockedCleanupCandidates: readonly string[]; readonly composeEnvFileReferences: readonly LegacyComposeEnvFileReference[]; - /** Keys every service now receives though none listed a legacy file. Names only. */ + /** Keys every service receives once their env is selected, from files no service listed. Names only. */ readonly keysNewToEveryService?: readonly string[]; }; @@ -5355,6 +5355,34 @@ function composeServiceEnvironmentKeys( return isRecord(environment) ? new Set(Object.keys(environment)) : null; } +/** + * Keys the Compose file interpolates that the default overlay changes. Compose + * interpolates from `.hack/.env` alone before the migration, and from the + * selected layers after it. Key names only, JSON-quoted. + */ +function composeInterpolatedOverlayKeys(opts: { + readonly parsed: Record | null; + readonly legacyValues: ReadonlyMap>; + readonly defaultOverlay: string | null; +}): readonly string[] { + if (opts.parsed === null || opts.defaultOverlay === null) { + return []; + } + const composeText = JSON.stringify(opts.parsed); + const changed = legacyOverlayChangedValues({ + overlayValues: opts.legacyValues.get(opts.defaultOverlay) ?? {}, + baseValues: opts.legacyValues.get(null) ?? {}, + }); + return Object.keys(changed) + .filter( + (key) => + composeText.includes(`\${${key}`) || + new RegExp(`\\$${key}(?![A-Z0-9_])`).test(composeText) + ) + .map((key) => JSON.stringify(key)) + .sort(); +} + /** * Compose inputs that can add or hide an `env_file` this check cannot read: a * top-level `include`, a service `extends`, or an `env_file` path with a @@ -5494,10 +5522,25 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { reason: `Compose env_file sets ${envFileConflicts.sort().join(", ")}, which the legacy .env files also set; after the migration the .env value would override that file's value. Remove the key from the .hack/.env file or from that env_file, then retry.`, }); } + const defaultOverlay = await readProjectDefaultEnvConfig({ + projectDir: opts.projectDir, + }); + const interpolated = composeInterpolatedOverlayKeys({ + parsed, + legacyValues: opts.legacyValues, + defaultOverlay, + }); + if (interpolated.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `the Compose file interpolates ${interpolated.join(", ")}, which the default overlay sets differently from .hack/.env. Compose takes these from .hack/.env today, but after the migration the overlay value would apply. Make the two files agree or stop interpolating the key, then retry.`, + }); + } const unlisted = composeServicesMissingLegacyDotenv({ projectDir: opts.projectDir, services, legacyValues: opts.legacyValues, + defaultOverlay, }); if (unlisted.length > 0) { throw new LegacyDotenvMigrationRefusalError({ @@ -5537,8 +5580,9 @@ function composeServiceLegacyEnvNames(opts: { * the services that list it; after it, its values are global layer values that * Hack applies to every service. When any service lists a legacy file, returns * refusal labels (file, service and key names only) for each file with values - * to migrate that only some services list, and for each service that does not - * list a file setting a base key. + * to migrate that only some services list, for a default overlay some service + * does not list, and for each service that does not list a file setting a base + * key. * * A project where no service lists a legacy file has no `env_file` boundary to * compare, and its values may be needed for interpolation, so it is not refused @@ -5548,6 +5592,7 @@ function composeServicesMissingLegacyDotenv(opts: { readonly projectDir: string; readonly services: Record; readonly legacyValues: ReadonlyMap>; + readonly defaultOverlay: string | null; }): string[] { const baseValues = opts.legacyValues.get(null) ?? {}; const services = composeServiceEntries(opts.services).map( @@ -5576,7 +5621,14 @@ function composeServicesMissingLegacyDotenv(opts: { .filter((service) => !service.listed.includes(envName)) .map((service) => service.label) .sort(); - if (missing.length > 0 && missing.length < services.length) { + // Every launch selects the default overlay, so its values reach every + // service even when no service lists it. + const reachesEveryLaunch = + envName !== null && envName === opts.defaultOverlay; + if ( + missing.length > 0 && + (missing.length < services.length || reachesEveryLaunch) + ) { labels.push( `${missing.join(", ")} ${missing.length === 1 ? "does" : "do"} not list ${JSON.stringify(basename(resolveEnvFilePath({ projectDir: opts.projectDir, envName })))} in env_file` ); @@ -5602,9 +5654,9 @@ function composeServicesMissingLegacyDotenv(opts: { } /** - * Migrated keys that reach every service although no service lists a legacy - * file in `env_file`, for the migration to name. Empty when any service lists - * one, because {@link composeServicesMissingLegacyDotenv} covers that case. + * Migrated keys from legacy files no service lists in `env_file`, for the + * migration to name: they reach every service once their env is selected. + * Files a service lists are covered by {@link composeServicesMissingLegacyDotenv}. */ async function legacyKeysNewToEveryService(opts: { readonly projectDir: string; @@ -5618,22 +5670,33 @@ async function legacyKeysNewToEveryService(opts: { return []; } const entries = composeServiceEntries(services); - const anyListed = entries.some( - ([, rawService]) => + if (entries.length === 0) { + return []; + } + const listed = new Set( + entries.flatMap(([, rawService]) => composeServiceLegacyEnvNames({ projectDir: opts.projectDir, rawService, legacyValues: opts.legacyValues, - }).length > 0 + }) + ) ); - if (entries.length === 0 || anyListed) { - return []; + const baseValues = opts.legacyValues.get(null) ?? {}; + const keys = new Set(); + for (const [envName, values] of opts.legacyValues) { + if (listed.has(envName)) { + continue; + } + const migrated = + envName === null + ? values + : legacyOverlayChangedValues({ overlayValues: values, baseValues }); + for (const key of Object.keys(migrated)) { + keys.add(key); + } } - return [ - ...new Set( - [...opts.legacyValues.values()].flatMap((values) => Object.keys(values)) - ), - ].sort((left, right) => left.localeCompare(right)); + return [...keys].sort((left, right) => left.localeCompare(right)); } /** Overlay entries a migrated overlay layer must hold: those that differ from the base. */ @@ -5746,26 +5809,90 @@ async function assertIgnoredMigrationLocalLayer(opts: { const COMPOSE_UNQUOTED_INLINE_COMMENT_PATTERN = /\s#/; +/** End of the quoted value in `text`, or -1 when it continues on the next line. */ +function closingQuoteIndex(opts: { + readonly text: string; + readonly quote: string; +}): number { + for (let index = 0; index < opts.text.length; index += 1) { + if (opts.text[index] === "\\" && opts.quote === '"') { + index += 1; + } else if (opts.text[index] === opts.quote) { + return index; + } + } + return -1; +} + /** - * Whether Compose reads this dotenv line differently from {@link parseDotEnv}: - * it drops an unquoted inline comment and expands `$` outside single quotes, - * while the parser keeps both literally. A migrated value goes through Hack's - * generated override, so the service would receive the literal text instead. + * Counts dotenv entries Compose reads differently from {@link parseDotEnv}. + * Compose drops an unquoted inline comment, expands `$` outside single quotes, + * decodes backslash escapes in double quotes (across lines too), and accepts + * `KEY: value` and a bare `KEY`, while the parser keeps the text literally or + * skips the line. A migrated value goes through Hack's generated override, so + * the service would receive the parser's reading instead. */ -function hasComposeOnlyDotenvSyntax(line: string): boolean { - const separator = line.indexOf("="); - if (separator === -1 || line.trimStart().startsWith("#")) { - return false; +function countComposeOnlyDotenvEntries(text: string): number { + let count = 0; + let open: { quote: string; flagged: boolean } | null = null; + for (const rawLine of text.split("\n")) { + let quoted = rawLine; + if (open === null) { + const entry = readComposeDotenvEntryStart(rawLine); + count += entry.differs ? 1 : 0; + if (entry.quote === null) { + continue; + } + open = { quote: entry.quote, flagged: false }; + quoted = entry.text; + } + const end = closingQuoteIndex({ text: quoted, quote: open.quote }); + const content = end === -1 ? quoted : quoted.slice(0, end); + if ( + open.quote === '"' && + !open.flagged && + (content.includes("$") || content.includes("\\")) + ) { + open.flagged = true; + count += 1; + } + if (end !== -1) { + open = null; + } } - const value = line.slice(separator + 1).trimStart(); - if (value.startsWith("'")) { - return false; + return count; +} + +/** + * Reads a dotenv line that starts an entry. `quote` is set when the value is + * quoted, with `text` as what follows the opening quote; otherwise `differs` + * says whether Compose reads the whole line differently. + */ +function readComposeDotenvEntryStart(rawLine: string): { + readonly differs: boolean; + readonly quote: string | null; + readonly text: string; +} { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) { + return { differs: false, quote: null, text: "" }; } - return ( - value.includes("$") || - (!value.startsWith('"') && - COMPOSE_UNQUOTED_INLINE_COMMENT_PATTERN.test(value)) - ); + const separator = rawLine.indexOf("="); + if (separator <= 0) { + return { differs: true, quote: null, text: "" }; + } + const value = rawLine.slice(separator + 1).trimStart(); + const first = value[0]; + if (first === '"' || first === "'") { + return { differs: false, quote: first, text: value.slice(1) }; + } + return { + differs: + value.includes("$") || + COMPOSE_UNQUOTED_INLINE_COMMENT_PATTERN.test(value), + quote: null, + text: "", + }; } /** @@ -5798,7 +5925,7 @@ async function readValidatedLegacyDotenvFiles(opts: { const composeSyntax = read.kind === "absent" || read.kind === "unreadable" ? 0 - : read.text.split("\n").filter(hasComposeOnlyDotenvSyntax).length; + : countComposeOnlyDotenvEntries(read.text); if (composeSyntax > 0) { composeSyntaxCounts.push( `${countLabel({ count: composeSyntax, noun: "line" })} in ${file.label}` @@ -5822,7 +5949,7 @@ async function readValidatedLegacyDotenvFiles(opts: { if (composeSyntaxCounts.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `Compose reads ${composeSyntaxCounts.join(", ")} differently from Hack: an unquoted " #" starts a comment and "$" outside single quotes is expanded, so the migrated value would change. Remove the inline comment, or write the value Compose uses in single quotes, then retry. The lines are not shown because they may contain values.`, + reason: `Compose reads ${composeSyntaxCounts.join(", ")} differently from Hack, so the migrated value would change or be lost: an unquoted " #" starts a comment, "$" outside single quotes is expanded, "\\" in double quotes is an escape, and a line without "=" still sets a key. Write each one as KEY='the value Compose uses' with real line breaks and no escapes, or remove it, then retry. The lines are not shown because they may contain values.`, }); } return legacyValues; diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 82bc0150b..e862557c8 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -708,7 +708,8 @@ test("doctor --migrate-env-config names the keys every service gains, never thei ]); expect(code).toBe(0); const notes = noteCalls.join("\n"); - expect(notes).toContain("so every service now receives: OWNED."); + expect(notes).toContain("once that env is selected: OWNED."); + expect(notes).toContain("not for a key your Compose file interpolates"); expect(notes).not.toContain(legacy); }, 30_000); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index a1de0790d..547f82209 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1699,3 +1699,121 @@ for (const [name, link, config] of [ expect(await hackDirEntries(repo)).toEqual(before); }); } + +const COMPOSE_READS_DIFFERENTLY: readonly (readonly [string, string])[] = [ + ["a double-quoted escape", `P="${KEPT_VALUE}\\nabc"`], + ["a double-quoted tab escape", `P="${KEPT_VALUE}\\tz"`], + ["a multi-line $", `P="${KEPT_VALUE}\n$OTHER"`], + ["a multi-line escape", `P="${KEPT_VALUE}\nabc\\n"`], + ["a colon form", `P: ${KEPT_VALUE}`], + ["a bare key", "P"], +]; + +for (const [name, content] of COMPOSE_READS_DIFFERENTLY) { + test(`the migration refuses ${name}`, async () => { + const repo = await createLegacyRepo(); + await writeFile(resolve(repo.projectDir, ".env"), `${content}\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "Compose reads 1 line in .env differently from Hack", + hidden: [KEPT_VALUE], + }); + }); +} + +test("the migration keeps a multi-line value in single quotes and does not read its lines as entries", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `PEM='${KEPT_VALUE}\nnot an entry $ \\n\nend'\nAFTER=1\nQUOTED="a\nb"\n` + ); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { + PEM: `${KEPT_VALUE}\nnot an entry $ \\n\nend`, + AFTER: "1", + QUOTED: "a\nb", + }, + }); +}); + +async function writeDefaultOverlayRepo(opts: { + readonly compose: string; + readonly base: string; + readonly overlay: string; +}): Promise { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_CONFIG_FILENAME), + `${JSON.stringify({ name: "env-legacy-scope", dev_host: "env-legacy-scope.hack", env: { defaultOverlay: "staging" } })}\n` + ); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + opts.compose + ); + await writeFile(resolve(repo.projectDir, ".env"), opts.base); + await writeFile(resolve(repo.projectDir, ".env.staging"), opts.overlay); + return repo; +} + +test("the migration refuses a default overlay no service lists when a service lists the base", async () => { + const repo = await writeDefaultOverlayRepo({ + compose: + "services:\n api:\n image: alpine:3.20\n env_file: .env\n worker:\n image: alpine:3.20\n env_file: .env\n", + base: "A=1\n", + overlay: `NEW=${KEPT_VALUE}\n`, + }); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"api", "worker" do not list ".env.staging" in env_file', + hidden: [KEPT_VALUE], + }); +}); + +test("the migration names an overlay's keys when no service lists that non-default overlay", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: .env\n" + ); + await writeFile(resolve(repo.projectDir, ".env"), "A=1\n"); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `NEW=${KEPT_VALUE}\n` + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.keysNewToEveryService).toEqual(["NEW"]); +}); + +for (const reference of ["${PORT}", "$PORT", "${PORT:-1}"]) { + test(`the migration refuses ${reference} when the default overlay changes that key`, async () => { + const repo = await writeDefaultOverlayRepo({ + compose: `services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n command: run ${reference}\n`, + base: "PORT=2\n", + overlay: "PORT=3\n", + }); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'the Compose file interpolates "PORT"', + hidden: [], + }); + }); +} + +test("the migration allows interpolating a key the default overlay leaves alone or a longer name", async () => { + const repo = await writeDefaultOverlayRepo({ + compose: + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n command: run ${A} $PORT_EXTRA\n", + base: "A=1\nPORT=2\n", + overlay: "A=1\nPORT=3\n", + }); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); From 49cf4881d0af2018c19863eab0de13db3668a8e4 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:34:58 -0400 Subject: [PATCH 17/55] fix(env): refuse text after a closing quote and backslashes in quoted values Compose reads text after a closing quote as another entry, so A='x' B=2 lost B, and it reads backslashes in single quotes differently, so A='a\\b' changed. A backslash before a single quote also desynced the scanner and hid a later line. The migration now refuses both, keeping a trailing comment. An interpolated name that only starts with an overlay key no longer refuses. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 12 ++++--- src/lib/project-env-config.ts | 38 ++++++++++++++------ tests/env-legacy-service-scope.test.ts | 49 ++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/docs/env.md b/docs/env.md index 8bcde3ae3..2fdf98c87 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1088,10 +1088,11 @@ files, when: migrated value would override it. The check does not look at the order of the entries, or expand variables in an `env_file` path. Remove the key from one of the files, then retry; - a legacy entry is one Compose reads differently from Hack (`E_CONFIG_INVALID`): an unquoted - inline comment (`PORT=3000 # dev`), a `$` outside single quotes, a `\` in double quotes (such - as `\n` in a PEM key), or a line without `=` (`A: 1`, or a bare `A`). The value would change - or be lost. The error counts entries per file and never shows them. Write the value Compose - uses in single quotes, with real line breaks and no escapes, or remove the line; + inline comment (`PORT=3000 # dev`), a `$` outside single quotes, a `\` in a quoted value (such + as `\n` in a PEM key), text after a closing quote (`A='x' B=2`), or a line without `=` (`A: 1`, + or a bare `A`). The value would change or be lost. The error counts entries per file and never + shows them. Write each value on its own line, in single quotes, with real line breaks and no + backslashes, or remove the line; - the Compose file interpolates a key the default overlay sets differently from `.hack/.env` (`E_CONFIG_INVALID`). Compose takes it from `.hack/.env` today, but the overlay value would apply after the migration. Make the two files agree or stop interpolating the key; @@ -1109,7 +1110,8 @@ variables once that env is selected, and doctor names the keys. The same applies no service lists, unless it is the default overlay, which refuses when any service lists a legacy file. To limit a key to one service, unset it and set it with `hack env set --service --local`. Do not do this for a key your Compose file interpolates: service values do not reach -interpolation, so `${KEY}` would become empty. +interpolation, so `${KEY}` would become empty. After the migration a migrated value also takes +precedence over the same variable exported in your shell, including for interpolation. With `hack.env.json`, the migration discovers only regular `.hack/.env.` files, so a symlinked overlay is not migrated. It refuses (`E_PATH_CONFLICT`) when `.hack/.env` or a diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index b64b2a73a..a23bf5fe5 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5375,9 +5375,8 @@ function composeInterpolatedOverlayKeys(opts: { }); return Object.keys(changed) .filter( - (key) => - composeText.includes(`\${${key}`) || - new RegExp(`\\$${key}(?![A-Z0-9_])`).test(composeText) + // Keys match PROJECT_ENV_KEY_PATTERN, so they are safe in a pattern. + (key) => new RegExp(`\\$\\{?${key}(?![A-Z0-9_])`).test(composeText) ) .map((key) => JSON.stringify(key)) .sort(); @@ -5815,7 +5814,7 @@ function closingQuoteIndex(opts: { readonly quote: string; }): number { for (let index = 0; index < opts.text.length; index += 1) { - if (opts.text[index] === "\\" && opts.quote === '"') { + if (opts.text[index] === "\\") { index += 1; } else if (opts.text[index] === opts.quote) { return index; @@ -5827,9 +5826,9 @@ function closingQuoteIndex(opts: { /** * Counts dotenv entries Compose reads differently from {@link parseDotEnv}. * Compose drops an unquoted inline comment, expands `$` outside single quotes, - * decodes backslash escapes in double quotes (across lines too), and accepts - * `KEY: value` and a bare `KEY`, while the parser keeps the text literally or - * skips the line. A migrated value goes through Hack's generated override, so + * reads backslashes in quoted values differently (across lines too), reads + * text after a closing quote as another entry, and accepts `KEY: value` and a + * bare `KEY`, while the parser keeps the text literally or skips it. A migrated value goes through Hack's generated override, so * the service would receive the parser's reading instead. */ function countComposeOnlyDotenvEntries(text: string): number { @@ -5847,11 +5846,13 @@ function countComposeOnlyDotenvEntries(text: string): number { quoted = entry.text; } const end = closingQuoteIndex({ text: quoted, quote: open.quote }); - const content = end === -1 ? quoted : quoted.slice(0, end); if ( - open.quote === '"' && !open.flagged && - (content.includes("$") || content.includes("\\")) + composeReadsQuotedTextDifferently({ + text: quoted, + end, + quote: open.quote, + }) ) { open.flagged = true; count += 1; @@ -5863,6 +5864,21 @@ function countComposeOnlyDotenvEntries(text: string): number { return count; } +/** Whether Compose reads this line of a quoted value differently from the parser. */ +function composeReadsQuotedTextDifferently(opts: { + readonly text: string; + readonly end: number; + readonly quote: string; +}): boolean { + const content = opts.end === -1 ? opts.text : opts.text.slice(0, opts.end); + const trailing = opts.end === -1 ? "" : opts.text.slice(opts.end + 1).trim(); + return ( + content.includes("\\") || + (opts.quote === '"' && content.includes("$")) || + !(trailing.length === 0 || trailing.startsWith("#")) + ); +} + /** * Reads a dotenv line that starts an entry. `quote` is set when the value is * quoted, with `text` as what follows the opening quote; otherwise `differs` @@ -5949,7 +5965,7 @@ async function readValidatedLegacyDotenvFiles(opts: { if (composeSyntaxCounts.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `Compose reads ${composeSyntaxCounts.join(", ")} differently from Hack, so the migrated value would change or be lost: an unquoted " #" starts a comment, "$" outside single quotes is expanded, "\\" in double quotes is an escape, and a line without "=" still sets a key. Write each one as KEY='the value Compose uses' with real line breaks and no escapes, or remove it, then retry. The lines are not shown because they may contain values.`, + reason: `Compose reads ${composeSyntaxCounts.join(", ")} differently from Hack, so the migrated value would change or be lost: an unquoted " #" starts a comment, "$" outside single quotes is expanded, "\\" in a quoted value is read differently, text after a closing quote is another entry, and a line without "=" still sets a key. Write each one on its own as KEY='the value Compose uses' with real line breaks and no backslashes, or remove it, then retry. The lines are not shown because they may contain values.`, }); } return legacyValues; diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 547f82209..a8a398570 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1707,6 +1707,12 @@ const COMPOSE_READS_DIFFERENTLY: readonly (readonly [string, string])[] = [ ["a multi-line escape", `P="${KEPT_VALUE}\nabc\\n"`], ["a colon form", `P: ${KEPT_VALUE}`], ["a bare key", "P"], + ["an entry after a single-quoted value", `P='${KEPT_VALUE}' B=2`], + ["an entry after a double-quoted value", `P="${KEPT_VALUE}" B=2`], + ["an entry touching a closing quote", `P='${KEPT_VALUE}'B=2`], + ["a colon entry after a closing quote", `P='${KEPT_VALUE}' B: 2`], + ["a double backslash in single quotes", `P='${KEPT_VALUE}\\\\b'`], + ["an escaped quote in single quotes", `P='${KEPT_VALUE}\\'`], ]; for (const [name, content] of COMPOSE_READS_DIFFERENTLY) { @@ -1726,12 +1732,12 @@ test("the migration keeps a multi-line value in single quotes and does not read const repo = await createLegacyRepo(); await writeFile( resolve(repo.projectDir, ".env"), - `PEM='${KEPT_VALUE}\nnot an entry $ \\n\nend'\nAFTER=1\nQUOTED="a\nb"\n` + `PEM='${KEPT_VALUE}\nnot an entry $ =\nend'\nAFTER=1\nQUOTED="a\nb"\n` ); await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ global: { - PEM: `${KEPT_VALUE}\nnot an entry $ \\n\nend`, + PEM: `${KEPT_VALUE}\nnot an entry $ =\nend`, AFTER: "1", QUOTED: "a\nb", }, @@ -1817,3 +1823,42 @@ test("the migration allows interpolating a key the default overlay leaves alone }); expect(migrated.legacyDetected).toBe(true); }); + +test("the migration keeps a comment after a closing quote", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `SPACED='${KEPT_VALUE}' # note\nTOUCHING="${KEPT_VALUE}"#note\n` + ); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { SPACED: KEPT_VALUE, TOUCHING: KEPT_VALUE }, + }); +}); + +test("a backslash before a single quote cannot hide a later line from the migration", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `Z=1\nA='q\\'\nB='r\nC=$Z\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "differently from Hack", + hidden: [], + }); +}); + +test("the migration does not read a longer interpolated name as the overlay's key", async () => { + const repo = await writeDefaultOverlayRepo({ + compose: + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n command: run ${PORT_X}\n", + base: "PORT=2\n", + overlay: "PORT=3\n", + }); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); From 581da6499d68465a53f96af676974fb48adef094 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:54:21 -0400 Subject: [PATCH 18/55] fix(env): refuse an env_file order that changes a value on migration Compose lets the last listed env_file win, but env layers always let the default overlay win over the base. env_file: [.env.staging, .env] with differing values would change the service's value and the Compose repair would remove the listing. The .env-only migration now compares each service's ordered legacy values with the selected layers and refuses a difference, naming keys and services only. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 4 ++ src/lib/project-env-config.ts | 79 +++++++++++++++++++++++--- tests/env-legacy-service-scope.test.ts | 39 +++++++++++++ 3 files changed, 114 insertions(+), 8 deletions(-) diff --git a/docs/env.md b/docs/env.md index 2fdf98c87..a7fed31a1 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1093,6 +1093,10 @@ files, when: or a bare `A`). The value would change or be lost. The error counts entries per file and never shows them. Write each value on its own line, in single quotes, with real line breaks and no backslashes, or remove the line; +- a service lists legacy files in an order that gives a key a different value from the migrated + layers (`E_CONFIG_INVALID`), such as `env_file: [.env.staging, .env]` with `.env.staging` as the + default overlay. Compose lets the last file win, but the default overlay always wins over + `.hack/.env` in env layers. List `.env` first, or make the files agree; - the Compose file interpolates a key the default overlay sets differently from `.hack/.env` (`E_CONFIG_INVALID`). Compose takes it from `.hack/.env` today, but the overlay value would apply after the migration. Make the two files agree or stop interpolating the key; diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index a23bf5fe5..d919ef125 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5535,18 +5535,12 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { reason: `the Compose file interpolates ${interpolated.join(", ")}, which the default overlay sets differently from .hack/.env. Compose takes these from .hack/.env today, but after the migration the overlay value would apply. Make the two files agree or stop interpolating the key, then retry.`, }); } - const unlisted = composeServicesMissingLegacyDotenv({ + assertComposeServicesKeepLegacyValues({ projectDir: opts.projectDir, services, legacyValues: opts.legacyValues, defaultOverlay, }); - if (unlisted.length > 0) { - throw new LegacyDotenvMigrationRefusalError({ - code: "E_CONFIG_INVALID", - reason: `${unlisted.join("; ")}. Migrated values are global and reach every service, so those services would receive values they never had. List the file in env_file for those services if they may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, - }); - } } /** Compose services as records, skipping entries that are not service maps. */ @@ -5652,6 +5646,74 @@ function composeServicesMissingLegacyDotenv(opts: { return labels.sort(); } +/** + * Refuses when a service would gain a value it never had, or a listed key + * would change because the migration ignores `env_file` order. + */ +function assertComposeServicesKeepLegacyValues(opts: { + readonly projectDir: string; + readonly services: Record; + readonly legacyValues: ReadonlyMap>; + readonly defaultOverlay: string | null; +}): void { + const unlisted = composeServicesMissingLegacyDotenv(opts); + if (unlisted.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${unlisted.join("; ")}. Migrated values are global and reach every service, so those services would receive values they never had. List the file in env_file for those services if they may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, + }); + } + const orderConflicts = composeLegacyEnvFileOrderConflicts(opts); + if (orderConflicts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `the order of Compose env_file gives ${orderConflicts.join(", ")} a different value from the migrated layers, where the default overlay wins over .hack/.env. List .env before the overlay, or make the files agree, then retry.`, + }); + } +} + +/** + * Keys whose value a service would change because the migration ignores + * `env_file` order. Compose lets the last listed file win; the selected layers + * always let the default overlay win over the base. Refusal labels naming keys + * and services, never values. + */ +function composeLegacyEnvFileOrderConflicts(opts: { + readonly projectDir: string; + readonly services: Record; + readonly legacyValues: ReadonlyMap>; + readonly defaultOverlay: string | null; +}): readonly string[] { + const selected = { + ...(opts.legacyValues.get(null) ?? {}), + ...(opts.defaultOverlay === null + ? {} + : (opts.legacyValues.get(opts.defaultOverlay) ?? {})), + }; + const envNameByPath = new Map( + [...opts.legacyValues.keys()].map((envName) => [ + resolveEnvFilePath({ projectDir: opts.projectDir, envName }), + envName, + ]) + ); + const conflicts: string[] = []; + for (const [service, rawService] of composeServiceEntries(opts.services)) { + const effective: Record = {}; + for (const configured of readComposeEnvFilePaths({ rawService })) { + const envName = envNameByPath.get(resolve(opts.projectDir, configured)); + if (envName !== undefined) { + Object.assign(effective, opts.legacyValues.get(envName) ?? {}); + } + } + for (const [key, value] of Object.entries(effective)) { + if (key in selected && selected[key] !== value) { + conflicts.push(`${JSON.stringify(key)} in ${JSON.stringify(service)}`); + } + } + } + return conflicts.sort(); +} + /** * Migrated keys from legacy files no service lists in `env_file`, for the * migration to name: they reach every service once their env is selected. @@ -5990,7 +6052,8 @@ async function readValidatedLegacyDotenvFiles(opts: { * machine-local layer, a Compose `env_file` on a file the migration leaves * out or on an overlay a launch without `--env` would not select, a Compose * `environment` or `env_file` entry the migrated value would override, a - * service that would gain values a legacy file gave only to others, a Compose + * service that would gain values a legacy file gave only to others, an + * `env_file` order that gives a key a different value from the layers, a Compose * input that hides `env_file` from this check, a line Compose reads differently * from Hack, a key that does not match `^[A-Z_][A-Z0-9_]*$` * (counted per file, never shown), and a machine-local target that git diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index a8a398570..32ad512e5 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1862,3 +1862,42 @@ test("the migration does not read a longer interpolated name as the overlay's ke }); expect(migrated.legacyDetected).toBe(true); }); + +test("the migration refuses an env_file order that gives a key a different value from the layers", async () => { + const compose = (order: string) => + `services:\n api:\n image: alpine:3.20\n env_file: ${order}\n`; + const repo = await writeDefaultOverlayRepo({ + compose: compose("[.env.staging, .env]"), + base: `PORT=${KEPT_VALUE}-base\nSAME=1\n`, + overlay: `PORT=${KEPT_VALUE}-staging\nSAME=1\n`, + }); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'the order of Compose env_file gives "PORT" in "api"', + hidden: [KEPT_VALUE], + }); + + // In the order the layers apply, the value is the same before and after. + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + compose("[.env, .env.staging]") + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); + +test("the migration accepts any env_file order when the files agree", async () => { + const repo = await writeDefaultOverlayRepo({ + compose: + "services:\n api:\n image: alpine:3.20\n env_file: [.env.staging, .env]\n", + base: "PORT=1\n", + overlay: "PORT=1\nONLY=2\n", + }); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); From 93c5702335f12fef3ea5abd5ccbb938b302921b7 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:56:43 -0400 Subject: [PATCH 19/55] fix(env): refuse any symlinked legacy source in the contract migration The contract migration writes plain_env values to tracked layers. A committed .hack/.env -> .env.local stayed inside .hack, passed the containment check, and would have published a machine-local credential. This path now refuses a symlinked .hack/.env or default overlay wherever it points. The .env-only migration keeps allowing links inside .hack, because it writes only to gitignored machine-local layers. Refs HACK-1370, HACK-1686 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 5 ++-- src/lib/project-env-config.ts | 35 ++++++++++++++++++---- tests/env-legacy-service-scope.test.ts | 40 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/docs/env.md b/docs/env.md index a7fed31a1..3215296fd 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1119,8 +1119,9 @@ precedence over the same variable exported in your shell, including for interpol With `hack.env.json`, the migration discovers only regular `.hack/.env.` files, so a symlinked overlay is not migrated. It refuses (`E_PATH_CONFLICT`) when `.hack/.env` or a -configured default overlay is a symlink to a file outside `.hack`, so its values cannot reach a -tracked layer. +configured default overlay is a symlink, wherever it points, because this migration writes +`plain_env` values to tracked layers and another file's values, such as a machine-local +`.env.local`, could then be committed. Without `hack.env.json`, nothing lists values kept in a secret backend, so none are migrated: set them again with `hack env set --secret` after the migration. In that case `.hack/hack.config.json`, and diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index d919ef125..74dc0667d 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -4996,15 +4996,16 @@ export async function migrateLegacyProjectEnv(opts: { }; } - // Discovered overlays are regular files only, and any link that remains (the - // base file, or a configured default overlay) must stay in .hack: this path - // writes plain_env values to tracked layers, so a repository's symlink must - // not pull an outside file's values into them. + // Discovered overlays are regular files only, and a link that remains (the + // base file, or a configured default overlay) refuses wherever it points: this + // path writes plain_env values to tracked layers, so a repository's symlink + // must not pull another file's values into them, even one in .hack such as a + // machine-local .env.local. const overlayNames = await discoverLegacyOverlayNames({ projectDir: opts.projectDir, includeSymlinks: false, }); - await assertLegacyDotenvLinksStayInProject({ + await assertNoLegacyDotenvLinks({ projectDir: opts.projectDir, overlayNames, }); @@ -5809,6 +5810,30 @@ async function selectLegacyDotenvOverlays(opts: { return overlayNames; } +/** + * Refuses a legacy `.hack/.env` or overlay that is a symlink at all, dangling or + * not, for a migration that publishes values to tracked layers. + */ +async function assertNoLegacyDotenvLinks(opts: { + readonly projectDir: string; + readonly overlayNames: readonly string[]; +}): Promise { + const linked: string[] = []; + for (const envName of [null, ...opts.overlayNames]) { + const path = resolveEnvFilePath({ projectDir: opts.projectDir, envName }); + const entry = await lstat(path).catch(() => null); + if (entry?.isSymbolicLink()) { + linked.push(JSON.stringify(basename(path))); + } + } + if (linked.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `${linked.join(", ")} ${linked.length === 1 ? "is a symlink" : "are symlinks"}, and this migration writes plain_env values to tracked env layers, so another file's values could be committed. Replace the link with a regular file, or remove it, then retry.`, + }); + } +} + /** * Refuses a legacy `.hack/.env` or overlay that is a symlink resolving outside * `.hack`, so a repository cannot pull an outside file, such as a root `.env`, diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 32ad512e5..a9d90b7e6 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1901,3 +1901,43 @@ test("the migration accepts any env_file order when the files agree", async () = }); expect(migrated.legacyDetected).toBe(true); }); + +test("the contract migration refuses a link that stays in .hack, so a local credential stays out of tracked layers", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.json"), + `${JSON.stringify({ + version: 1, + vars: [{ key: "KEPT_SETTING", source: "plain_env", required: false }], + })}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.local"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await symlink(".env.local", resolve(repo.projectDir, ".env")); + const before = await hackDirEntries(repo); + const error = await migrateLegacyRepo(repo, {}).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as LegacyDotenvMigrationRefusalError).code).toBe( + "E_PATH_CONFLICT" + ); + expect((error as Error).message).toContain('".env" is a symlink'); + expect((error as Error).message).not.toContain(KEPT_VALUE); + expect(await hackDirEntries(repo)).toEqual(before); +}); + +test("the contract migration refuses a dangling legacy link", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.json"), + `${JSON.stringify({ version: 1, vars: [] })}\n` + ); + await symlink("missing.env", resolve(repo.projectDir, ".env")); + const error = await migrateLegacyRepo(repo, {}).catch( + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); +}); From bb7f7227bcd400b9dbee223b18f5ac682289d663 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:02:42 -0400 Subject: [PATCH 20/55] fix(env): refuse repeated env_file entries and env_file format on migration Compose v5.5.0 drops an exact repeat in env_file, so [.env.staging, .env, .env.staging] passed the order check while the value changed. format: raw makes Compose keep text as written, but the migration parses it as dotenv, so quotes and trailing spaces changed. The .env-only migration now refuses both as inputs it cannot check. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 7 +++--- src/lib/project-env-config.ts | 30 ++++++++++++++++++++------ tests/env-legacy-service-scope.test.ts | 15 +++++++++++++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/env.md b/docs/env.md index 3215296fd..51b26ba20 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1080,9 +1080,10 @@ files, when: service. The error names files, services and keys, never values. List the file for those services if they may have its values, or move the values out of it and set them with `hack env set --service` after migrating; -- Compose uses a top-level `include`, a service `extends` or a variable in an `env_file` path - (`E_CONFIG_INVALID`), so the migration cannot tell which services receive the values. Write - those `env_file` entries directly in the Compose file; +- Compose uses a top-level `include`, a service `extends`, a variable in an `env_file` path, a + file listed twice in one `env_file`, or a `format` on an `env_file` entry (`E_CONFIG_INVALID`), + so the migration cannot tell which values services receive. Write those `env_file` entries + directly in the Compose file, each file once and with no `format`; - a key the legacy files set is also set in another file a Compose service uses as `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 74dc0667d..71b9a5ce6 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5383,10 +5383,20 @@ function composeInterpolatedOverlayKeys(opts: { .sort(); } +/** Whether a map-form `env_file` entry sets a `format`, which changes how Compose reads the file. */ +function composeEnvFileHasFormat(rawEnvFile: unknown): boolean { + return ( + Array.isArray(rawEnvFile) && + rawEnvFile.some((entry) => isRecord(entry) && "format" in entry) + ); +} + /** - * Compose inputs that can add or hide an `env_file` this check cannot read: a - * top-level `include`, a service `extends`, or an `env_file` path with a - * variable. Returned as refusal labels naming the service or field only. + * Compose inputs that can add, hide or reinterpret an `env_file` this check + * cannot read: a top-level `include`, a service `extends`, an `env_file` path + * with a variable, a repeated entry (Compose versions may drop a repeat, which + * changes the order), or a `format`. Returned as refusal labels naming the + * service or field only. */ function uncheckableComposeEnvInputs(opts: { readonly parsed: Record; @@ -5398,11 +5408,17 @@ function uncheckableComposeEnvInputs(opts: { if ("extends" in rawService) { labels.push(`extends in ${JSON.stringify(service)}`); } - if ( - readComposeEnvFilePaths({ rawService }).some((path) => path.includes("$")) - ) { + const paths = readComposeEnvFilePaths({ rawService }); + if (paths.some((path) => path.includes("$"))) { labels.push(`a variable env_file path in ${JSON.stringify(service)}`); } + const resolved = paths.map((path) => resolve("/", path)); + if (new Set(resolved).size < resolved.length) { + labels.push(`a repeated env_file entry in ${JSON.stringify(service)}`); + } + if (composeEnvFileHasFormat(rawService.env_file)) { + labels.push(`an env_file format in ${JSON.stringify(service)}`); + } } return labels; } @@ -5477,7 +5493,7 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { if (uncheckable.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `Compose uses ${uncheckable.join(", ")}, so the migration cannot tell which services receive the legacy .env values. Write those services' env_file entries directly in ${PROJECT_COMPOSE_FILENAME}, then retry.`, + reason: `Compose uses ${uncheckable.join(", ")}, so the migration cannot tell which values services receive from the legacy .env files. Write those services' env_file entries directly in ${PROJECT_COMPOSE_FILENAME}, each file once and with no format, then retry.`, }); } const conflicts: string[] = []; diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index a9d90b7e6..bbcf9cb6b 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1613,6 +1613,21 @@ const UNCHECKABLE_COMPOSE: readonly (readonly [string, string, string])[] = [ "services:\n api:\n image: alpine:3.20\n env_file: ${D:-.}/.env\n worker:\n image: alpine:3.20\n", 'a variable env_file path in "api"', ], + [ + "a repeated env_file entry", + "services:\n api:\n image: alpine:3.20\n env_file: [.env.staging, .env, .env.staging]\n", + 'a repeated env_file entry in "api"', + ], + [ + "the same file spelled twice in env_file", + "services:\n api:\n image: alpine:3.20\n env_file: [.env, ./.env]\n", + 'a repeated env_file entry in "api"', + ], + [ + "an env_file format", + "services:\n api:\n image: alpine:3.20\n env_file:\n - path: .env\n format: raw\n", + 'an env_file format in "api"', + ], ]; for (const [name, compose, reason] of UNCHECKABLE_COMPOSE) { From ed6ebe92559e732679a76705c5987a16f1ff20eb Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:07:48 -0400 Subject: [PATCH 21/55] fix(env): refuse home-relative env_file paths and Compose merge tags on migration Compose expands ~ in env_file, but the check resolved it inside .hack, found no file, and missed a conflict with a migrated key. The .env-only migration now refuses a ~ path and a !reset or !override tag, and treats a top-level secret or config reading a key the default overlay changes like interpolation. A parent directory env_file no longer counts as a repeat of .env. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 12 ++++--- src/lib/project-env-config.ts | 40 +++++++++++++++++++---- tests/env-legacy-service-scope.test.ts | 45 ++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/docs/env.md b/docs/env.md index 51b26ba20..931b8ce80 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1080,10 +1080,11 @@ files, when: service. The error names files, services and keys, never values. List the file for those services if they may have its values, or move the values out of it and set them with `hack env set --service` after migrating; -- Compose uses a top-level `include`, a service `extends`, a variable in an `env_file` path, a - file listed twice in one `env_file`, or a `format` on an `env_file` entry (`E_CONFIG_INVALID`), - so the migration cannot tell which values services receive. Write those `env_file` entries - directly in the Compose file, each file once and with no `format`; +- Compose uses a top-level `include`, a service `extends`, a `!reset` or `!override` tag, a + variable or a `~` in an `env_file` path, a file listed twice in one `env_file`, or a `format` + on an `env_file` entry (`E_CONFIG_INVALID`), so the migration cannot tell which values services + receive. Write those `env_file` entries directly in the Compose file, each file once, with a + plain path and no `format`; - a key the legacy files set is also set in another file a Compose service uses as `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, @@ -1098,7 +1099,8 @@ files, when: layers (`E_CONFIG_INVALID`), such as `env_file: [.env.staging, .env]` with `.env.staging` as the default overlay. Compose lets the last file win, but the default overlay always wins over `.hack/.env` in env layers. List `.env` first, or make the files agree; -- the Compose file interpolates a key the default overlay sets differently from `.hack/.env` +- the Compose file interpolates a key, or a top-level secret or config reads it with + `environment:`, and the default overlay sets it differently from `.hack/.env` (`E_CONFIG_INVALID`). Compose takes it from `.hack/.env` today, but the overlay value would apply after the migration. Make the two files agree or stop interpolating the key; - `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 71b9a5ce6..c35b413b9 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5357,9 +5357,10 @@ function composeServiceEnvironmentKeys( } /** - * Keys the Compose file interpolates that the default overlay changes. Compose - * interpolates from `.hack/.env` alone before the migration, and from the - * selected layers after it. Key names only, JSON-quoted. + * Keys the Compose file interpolates, or a top-level secret or config reads with + * `environment:`, that the default overlay changes. Compose reads these from + * `.hack/.env` alone before the migration, and from the selected layers after + * it. Key names only, JSON-quoted. */ function composeInterpolatedOverlayKeys(opts: { readonly parsed: Record | null; @@ -5370,6 +5371,11 @@ function composeInterpolatedOverlayKeys(opts: { return []; } const composeText = JSON.stringify(opts.parsed); + const environmentSources = ["secrets", "configs"].flatMap((section) => + Object.values(getRecord(opts.parsed ?? {}, section) ?? {}).map((entry) => + isRecord(entry) ? entry.environment : undefined + ) + ); const changed = legacyOverlayChangedValues({ overlayValues: opts.legacyValues.get(opts.defaultOverlay) ?? {}, baseValues: opts.legacyValues.get(null) ?? {}, @@ -5377,12 +5383,16 @@ function composeInterpolatedOverlayKeys(opts: { return Object.keys(changed) .filter( // Keys match PROJECT_ENV_KEY_PATTERN, so they are safe in a pattern. - (key) => new RegExp(`\\$\\{?${key}(?![A-Z0-9_])`).test(composeText) + (key) => + environmentSources.includes(key) || + new RegExp(`\\$\\{?${key}(?![A-Z0-9_])`).test(composeText) ) .map((key) => JSON.stringify(key)) .sort(); } +const COMPOSE_MERGE_TAG_PATTERN = /!(?:reset|override)\b/; + /** Whether a map-form `env_file` entry sets a `format`, which changes how Compose reads the file. */ function composeEnvFileHasFormat(rawEnvFile: unknown): boolean { return ( @@ -5394,16 +5404,21 @@ function composeEnvFileHasFormat(rawEnvFile: unknown): boolean { /** * Compose inputs that can add, hide or reinterpret an `env_file` this check * cannot read: a top-level `include`, a service `extends`, an `env_file` path - * with a variable, a repeated entry (Compose versions may drop a repeat, which + * with a variable or a `~`, a `!reset` or `!override` tag, a repeated entry (Compose versions may drop a repeat, which * changes the order), or a `format`. Returned as refusal labels naming the * service or field only. */ function uncheckableComposeEnvInputs(opts: { + readonly projectDir: string; + readonly composeText: string; readonly parsed: Record; readonly services: Record; }): readonly string[] { const labels: string[] = "include" in opts.parsed ? ["top-level include"] : []; + if (COMPOSE_MERGE_TAG_PATTERN.test(opts.composeText)) { + labels.push("a !reset or !override tag"); + } for (const [service, rawService] of composeServiceEntries(opts.services)) { if ("extends" in rawService) { labels.push(`extends in ${JSON.stringify(service)}`); @@ -5412,7 +5427,12 @@ function uncheckableComposeEnvInputs(opts: { if (paths.some((path) => path.includes("$"))) { labels.push(`a variable env_file path in ${JSON.stringify(service)}`); } - const resolved = paths.map((path) => resolve("/", path)); + if (paths.some((path) => path.startsWith("~"))) { + labels.push( + `a home-relative env_file path in ${JSON.stringify(service)}` + ); + } + const resolved = paths.map((path) => resolve(opts.projectDir, path)); if (new Set(resolved).size < resolved.length) { labels.push(`a repeated env_file entry in ${JSON.stringify(service)}`); } @@ -5487,8 +5507,14 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { if (migratedKeys.size === 0) { return; } + const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); const uncheckable = parsed - ? uncheckableComposeEnvInputs({ parsed, services }) + ? uncheckableComposeEnvInputs({ + projectDir: opts.projectDir, + composeText: (await readTextFile(composeFile)) ?? "", + parsed, + services, + }) : []; if (uncheckable.length > 0) { throw new LegacyDotenvMigrationRefusalError({ diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index bbcf9cb6b..88930c854 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1623,6 +1623,21 @@ const UNCHECKABLE_COMPOSE: readonly (readonly [string, string, string])[] = [ "services:\n api:\n image: alpine:3.20\n env_file: [.env, ./.env]\n", 'a repeated env_file entry in "api"', ], + [ + "a home-relative env_file path", + "services:\n api:\n image: alpine:3.20\n env_file: [.env, ~/x.env]\n", + 'a home-relative env_file path in "api"', + ], + [ + "a !reset tag", + "services:\n api:\n image: alpine:3.20\n env_file: !reset [.env]\n", + "a !reset or !override tag", + ], + [ + "an !override tag", + "services:\n api:\n image: alpine:3.20\n env_file: !override [.env]\n", + "a !reset or !override tag", + ], [ "an env_file format", "services:\n api:\n image: alpine:3.20\n env_file:\n - path: .env\n format: raw\n", @@ -1956,3 +1971,33 @@ test("the contract migration refuses a dangling legacy link", async () => { ); expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); }); + +test("the migration does not count a parent-directory env_file as a repeat", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: [.env, ../.env]\n" + ); + await writeFile(resolve(repo.projectDir, ".env"), "A=1\n"); + await writeFile(resolve(repo.projectRoot, ".env"), "OTHER=2\n"); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); + +for (const section of ["secrets", "configs"]) { + test(`the migration refuses top-level ${section} reading a key the default overlay changes`, async () => { + const repo = await writeDefaultOverlayRepo({ + compose: `services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n${section}:\n item:\n environment: PORT\n`, + base: "PORT=2\n", + overlay: "PORT=3\n", + }); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'the Compose file interpolates "PORT"', + hidden: [], + }); + }); +} From ae03d0372bd04d215bf381eb323bc19b8a2f1181 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:28:58 -0400 Subject: [PATCH 22/55] fix(env): refuse any Compose use of a key the default overlay changes Compose reads a key without a $ in places like a bare build.args entry, so matching interpolation syntax missed it and images would build with a different value. The .env-only migration now refuses when the key's name appears anywhere in the Compose file, or it is a COMPOSE_* or DOCKER_* setting, such as COMPOSE_PROFILES turning on a gated service. This replaces the per-form checks. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 9 ++-- src/lib/project-env-config.ts | 32 +++++++-------- tests/env-legacy-service-scope.test.ts | 57 +++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/docs/env.md b/docs/env.md index 931b8ce80..642598ae5 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1099,10 +1099,11 @@ files, when: layers (`E_CONFIG_INVALID`), such as `env_file: [.env.staging, .env]` with `.env.staging` as the default overlay. Compose lets the last file win, but the default overlay always wins over `.hack/.env` in env layers. List `.env` first, or make the files agree; -- the Compose file interpolates a key, or a top-level secret or config reads it with - `environment:`, and the default overlay sets it differently from `.hack/.env` - (`E_CONFIG_INVALID`). Compose takes it from `.hack/.env` today, but the overlay value would - apply after the migration. Make the two files agree or stop interpolating the key; +- the default overlay sets a key differently from `.hack/.env`, and Compose reads that key: its + name appears anywhere in the Compose file (`${KEY}`, a bare `build.args` entry, a secret's + `environment:`), or it is a `COMPOSE_*` or `DOCKER_*` setting (`E_CONFIG_INVALID`). Compose takes + it from `.hack/.env` today, but the overlay value would apply after the migration. Make the two + files agree or stop using the key in Compose; - `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` (`E_PATH_CONFLICT`), so its values would be copied from elsewhere into this project's layers. Replace the link with a regular file in `.hack`, or remove it; diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index c35b413b9..53c718344 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5356,13 +5356,18 @@ function composeServiceEnvironmentKeys( return isRecord(environment) ? new Set(Object.keys(environment)) : null; } +/** Environment Compose and Docker read for themselves, whatever the Compose file says. */ +const COMPOSE_OWN_KEY_PATTERN = /^(?:COMPOSE|DOCKER)_/; + /** - * Keys the Compose file interpolates, or a top-level secret or config reads with - * `environment:`, that the default overlay changes. Compose reads these from - * `.hack/.env` alone before the migration, and from the selected layers after - * it. Key names only, JSON-quoted. + * Keys the default overlay changes that Compose reads from its own environment: + * any whose name appears in the Compose file, however it is used (`${KEY}`, a + * bare `build.args` entry, a secret's `environment:`), and Compose's and Docker's + * own settings. Compose reads these from `.hack/.env` alone before the migration, + * and from the selected layers after it. Matching the name anywhere can refuse + * more than it must, which is the safe direction. Key names only, JSON-quoted. */ -function composeInterpolatedOverlayKeys(opts: { +function composeReferencedOverlayKeys(opts: { readonly parsed: Record | null; readonly legacyValues: ReadonlyMap>; readonly defaultOverlay: string | null; @@ -5371,11 +5376,6 @@ function composeInterpolatedOverlayKeys(opts: { return []; } const composeText = JSON.stringify(opts.parsed); - const environmentSources = ["secrets", "configs"].flatMap((section) => - Object.values(getRecord(opts.parsed ?? {}, section) ?? {}).map((entry) => - isRecord(entry) ? entry.environment : undefined - ) - ); const changed = legacyOverlayChangedValues({ overlayValues: opts.legacyValues.get(opts.defaultOverlay) ?? {}, baseValues: opts.legacyValues.get(null) ?? {}, @@ -5384,8 +5384,8 @@ function composeInterpolatedOverlayKeys(opts: { .filter( // Keys match PROJECT_ENV_KEY_PATTERN, so they are safe in a pattern. (key) => - environmentSources.includes(key) || - new RegExp(`\\$\\{?${key}(?![A-Z0-9_])`).test(composeText) + COMPOSE_OWN_KEY_PATTERN.test(key) || + new RegExp(`(? JSON.stringify(key)) .sort(); @@ -5519,7 +5519,7 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { if (uncheckable.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `Compose uses ${uncheckable.join(", ")}, so the migration cannot tell which values services receive from the legacy .env files. Write those services' env_file entries directly in ${PROJECT_COMPOSE_FILENAME}, each file once and with no format, then retry.`, + reason: `Compose uses ${uncheckable.join(", ")}, so the migration cannot tell which values services receive from the legacy .env files. Write those services' env_file entries directly in ${PROJECT_COMPOSE_FILENAME}, each file once, with a plain path and no format, and remove any !reset or !override text (a comment or value containing it also matches), then retry.`, }); } const conflicts: string[] = []; @@ -5567,15 +5567,15 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { const defaultOverlay = await readProjectDefaultEnvConfig({ projectDir: opts.projectDir, }); - const interpolated = composeInterpolatedOverlayKeys({ + const referenced = composeReferencedOverlayKeys({ parsed, legacyValues: opts.legacyValues, defaultOverlay, }); - if (interpolated.length > 0) { + if (referenced.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `the Compose file interpolates ${interpolated.join(", ")}, which the default overlay sets differently from .hack/.env. Compose takes these from .hack/.env today, but after the migration the overlay value would apply. Make the two files agree or stop interpolating the key, then retry.`, + reason: `Compose reads ${referenced.join(", ")}, which the default overlay sets differently from .hack/.env. Compose takes these from .hack/.env today, but after the migration the overlay value would apply. Make the two files agree or stop using the key in Compose, then retry.`, }); } assertComposeServicesKeepLegacyValues({ diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 88930c854..1bc0c3ca7 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1835,7 +1835,7 @@ for (const reference of ["${PORT}", "$PORT", "${PORT:-1}"]) { await expectMigrationRefusal({ repo, code: "E_CONFIG_INVALID", - reason: 'the Compose file interpolates "PORT"', + reason: 'Compose reads "PORT"', hidden: [], }); }); @@ -1996,8 +1996,61 @@ for (const section of ["secrets", "configs"]) { await expectMigrationRefusal({ repo, code: "E_CONFIG_INVALID", - reason: 'the Compose file interpolates "PORT"', + reason: 'Compose reads "PORT"', hidden: [], }); }); } + +for (const [name, block] of [ + [ + "a bare build.args list entry", + " build:\n context: .\n args: [PORT]\n", + ], + [ + "a bare build.args map entry", + " build:\n context: .\n args:\n PORT:\n", + ], +] as const) { + test(`the migration refuses ${name} the default overlay changes`, async () => { + const repo = await writeDefaultOverlayRepo({ + compose: `services:\n api:\n image: alpine:3.20\n${block} env_file: [.env, .env.staging]\n`, + base: "PORT=2\n", + overlay: "PORT=3\n", + }); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'Compose reads "PORT"', + hidden: [], + }); + }); +} + +test("the migration refuses a Compose setting the default overlay changes, even if the file never names it", async () => { + const repo = await writeDefaultOverlayRepo({ + compose: + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n", + base: "A=1\n", + overlay: "A=1\nCOMPOSE_PROFILES=dev\n", + }); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'Compose reads "COMPOSE_PROFILES"', + hidden: [], + }); +}); + +test("the migration allows a default overlay key the Compose file never uses, including as part of a longer name", async () => { + const repo = await writeDefaultOverlayRepo({ + compose: + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n environment:\n MY_PORT_X: fixed\n", + base: "PORT=2\n", + overlay: "PORT=3\n", + }); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); From 1e1db2486065f8b1782097fb49abb9e68f2052d3 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:35:21 -0400 Subject: [PATCH 23/55] fix(env): refuse multi-document Compose files and tool settings on migration Compose merges several YAML documents, but the parser returned no map and every check was skipped, so explicit values changed. The .env-only migration now refuses a Compose file it cannot read as one document. It also searches other env_file text for keys the default overlay changes, since Compose expands variables there, and refuses DOCKER_*, BUILDKIT_*, BUILDX_*, proxy settings, SOURCE_DATE_EPOCH, HOME and PATH in any legacy file. Migrated values enter Compose's process environment, so an unchanged DOCKER_HOST could select another daemon. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 15 +++- src/lib/project-env-config.ts | 114 +++++++++++++++++++------ tests/env-legacy-service-scope.test.ts | 70 +++++++++++++++ 3 files changed, 170 insertions(+), 29 deletions(-) diff --git a/docs/env.md b/docs/env.md index 642598ae5..9fbd19c28 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1100,10 +1100,17 @@ files, when: default overlay. Compose lets the last file win, but the default overlay always wins over `.hack/.env` in env layers. List `.env` first, or make the files agree; - the default overlay sets a key differently from `.hack/.env`, and Compose reads that key: its - name appears anywhere in the Compose file (`${KEY}`, a bare `build.args` entry, a secret's - `environment:`), or it is a `COMPOSE_*` or `DOCKER_*` setting (`E_CONFIG_INVALID`). Compose takes - it from `.hack/.env` today, but the overlay value would apply after the migration. Make the two - files agree or stop using the key in Compose; + name appears anywhere in the Compose file or another `env_file` (`${KEY}`, a bare `build.args` + entry, a secret's `environment:`), or it is a `COMPOSE_*` setting (`E_CONFIG_INVALID`). Compose + takes it from `.hack/.env` today, but the overlay value would apply after the migration. Make + the two files agree or stop using the key in Compose; +- a legacy file sets a key that Docker, its build tools or the process read from their + environment (`E_CONFIG_INVALID`): `DOCKER_*`, `BUILDKIT_*`, `BUILDX_*`, a proxy setting, + `SOURCE_DATE_EPOCH`, `HOME` or `PATH`. After the migration it would reach the Docker client too. + Remove it from the legacy file, and set it in your shell or for one service with + `hack env set --service --local` after migrating; +- the Compose file is not a single YAML document this check can read, such as several `---` + documents (`E_CONFIG_INVALID`). Merge it into one document; - `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` (`E_PATH_CONFLICT`), so its values would be copied from elsewhere into this project's layers. Replace the link with a regular file in `.hack`, or remove it; diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 53c718344..83bcb5869 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5356,26 +5356,81 @@ function composeServiceEnvironmentKeys( return isRecord(environment) ? new Set(Object.keys(environment)) : null; } -/** Environment Compose and Docker read for themselves, whatever the Compose file says. */ -const COMPOSE_OWN_KEY_PATTERN = /^(?:COMPOSE|DOCKER)_/; +/** Compose's own settings: it reads these from `.hack/.env`, so only a changed value matters. */ +const COMPOSE_OWN_KEY_PATTERN = /^COMPOSE_/; + +/** + * Settings the Docker client, build tools or the process itself read from their + * environment. Migrated values enter Compose's process environment, where before + * they only reached Compose's project environment, so even an unchanged value + * could select another daemon or proxy. + */ +const COMPOSE_TOOL_KEY_PATTERN = + /^(?:DOCKER_|BUILDKIT_|BUILDX_|HTTP_PROXY$|HTTPS_PROXY$|NO_PROXY$|ALL_PROXY$|SOURCE_DATE_EPOCH$|HOME$|PATH$)/; + +function assertNoComposeToolKeys(opts: { + readonly migratedKeys: ReadonlySet; +}): void { + const toolKeys = [...opts.migratedKeys] + .filter((key) => COMPOSE_TOOL_KEY_PATTERN.test(key)) + .map((key) => JSON.stringify(key)) + .sort(); + if (toolKeys.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `the legacy .env files set ${toolKeys.join(", ")}, which Docker, its build tools or the process read from their environment. After the migration these would reach the Docker client as well as the services. Set them for one service with \`hack env set --service --local\` after migrating, or in your shell, and remove them from the legacy file, then retry.`, + }); + } +} + +/** + * Text of each non-legacy `env_file`, because Compose expands variables inside + * those files too. An absent file adds nothing; an unreadable one was refused + * by {@link composeEnvFileKeyConflicts}. + */ +async function composeOtherEnvFileText(opts: { + readonly projectDir: string; + readonly services: Record; +}): Promise { + const paths = new Set( + composeServiceEntries(opts.services).flatMap(([, rawService]) => + readComposeEnvFilePaths({ rawService }) + .map((configured) => resolve(opts.projectDir, configured)) + .filter( + (candidatePath) => + !isLegacyProjectEnvFilePath({ + projectDir: opts.projectDir, + candidatePath, + }) + ) + ) + ); + const texts: string[] = []; + for (const path of paths) { + const read = await readOptionalTextFile(path); + if (read.kind !== "absent" && read.kind !== "unreadable") { + texts.push(read.text); + } + } + return texts.join("\n"); +} /** * Keys the default overlay changes that Compose reads from its own environment: - * any whose name appears in the Compose file, however it is used (`${KEY}`, a - * bare `build.args` entry, a secret's `environment:`), and Compose's and Docker's - * own settings. Compose reads these from `.hack/.env` alone before the migration, + * any whose name appears in `searchText` (the Compose file and other `env_file` + * text), however it is used (`${KEY}`, a bare `build.args` entry, a secret's + * `environment:`), and Compose's own settings. Compose reads these from `.hack/.env` alone before the migration, * and from the selected layers after it. Matching the name anywhere can refuse * more than it must, which is the safe direction. Key names only, JSON-quoted. */ function composeReferencedOverlayKeys(opts: { - readonly parsed: Record | null; + readonly searchText: string; readonly legacyValues: ReadonlyMap>; readonly defaultOverlay: string | null; }): readonly string[] { - if (opts.parsed === null || opts.defaultOverlay === null) { + if (opts.defaultOverlay === null) { return []; } - const composeText = JSON.stringify(opts.parsed); const changed = legacyOverlayChangedValues({ overlayValues: opts.legacyValues.get(opts.defaultOverlay) ?? {}, baseValues: opts.legacyValues.get(null) ?? {}, @@ -5385,7 +5440,9 @@ function composeReferencedOverlayKeys(opts: { // Keys match PROJECT_ENV_KEY_PATTERN, so they are safe in a pattern. (key) => COMPOSE_OWN_KEY_PATTERN.test(key) || - new RegExp(`(? JSON.stringify(key)) .sort(); @@ -5491,13 +5548,6 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { readonly projectDir: string; readonly legacyValues: ReadonlyMap>; }): Promise { - const parsed = await readComposeFile({ - composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), - }); - const services = parsed ? getRecord(parsed, "services") : null; - if (!services) { - return; - } const migratedKeys = new Set(); for (const values of opts.legacyValues.values()) { for (const key of Object.keys(values)) { @@ -5507,15 +5557,26 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { if (migratedKeys.size === 0) { return; } + assertNoComposeToolKeys({ migratedKeys }); const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); - const uncheckable = parsed - ? uncheckableComposeEnvInputs({ - projectDir: opts.projectDir, - composeText: (await readTextFile(composeFile)) ?? "", - parsed, - services, - }) - : []; + const composeText = (await readTextFile(composeFile)) ?? ""; + const parsed = await readComposeFile({ composeFile }); + if (parsed === null && composeText.trim().length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${PROJECT_COMPOSE_FILENAME} is not a single YAML document this check can read, such as a file with several "---" documents, so the migration cannot tell which values services receive. Merge it into one document, then retry.`, + }); + } + const services = parsed ? getRecord(parsed, "services") : null; + if (!(parsed && services)) { + return; + } + const uncheckable = uncheckableComposeEnvInputs({ + projectDir: opts.projectDir, + composeText, + parsed, + services, + }); if (uncheckable.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", @@ -5568,7 +5629,10 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { projectDir: opts.projectDir, }); const referenced = composeReferencedOverlayKeys({ - parsed, + searchText: `${JSON.stringify(parsed)}\n${await composeOtherEnvFileText({ + projectDir: opts.projectDir, + services, + })}`, legacyValues: opts.legacyValues, defaultOverlay, }); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 1bc0c3ca7..967e2a295 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -2054,3 +2054,73 @@ test("the migration allows a default overlay key the Compose file never uses, in }); expect(migrated.legacyDetected).toBe(true); }); + +for (const reference of ["Z=${PORT}", "Z=$PORT"]) { + test(`the migration refuses another env_file with ${reference} when the default overlay changes that key`, async () => { + const repo = await writeDefaultOverlayRepo({ + compose: + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging, other.env]\n", + base: "PORT=2\n", + overlay: "PORT=3\n", + }); + await writeFile(resolve(repo.projectDir, "other.env"), `${reference}\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'Compose reads "PORT"', + hidden: [], + }); + }); +} + +test("the migration refuses a Compose file with several documents", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file: .env\n---\nservices:\n api:\n environment:\n A: fixed\n" + ); + await writeFile(resolve(repo.projectDir, ".env"), `A=${KEPT_VALUE}\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "is not a single YAML document", + hidden: [KEPT_VALUE], + }); +}); + +for (const key of [ + "DOCKER_HOST", + "BUILDKIT_PROGRESS", + "BUILDX_BUILDER", + "HTTP_PROXY", + "NO_PROXY", + "SOURCE_DATE_EPOCH", + "HOME", + "PATH", +]) { + test(`the migration refuses ${key} in a legacy file, changed or not`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `${key}=${KEPT_VALUE}\nA=1\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: `the legacy .env files set "${key}"`, + hidden: [KEPT_VALUE], + }); + }); +} + +test("the migration allows a name that only starts like a tool setting", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + "HOME_PAGE=/x\nPATH_PREFIX=/y\nMY_DOCKER_HOST=z\n" + ); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); +}); From ef9147f12859655c724a9ad904025c08c61a605a Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:46:17 -0400 Subject: [PATCH 24/55] fix(env): keep a teammate's .env values after a committed migration Once one developer commits the .env-only migration, another checkout's .hack/.env stopped reaching services with no warning, doctor said the project already used env layers, and hack env materialize then erased the file, its only copy. doctor --migrate-env-config now moves such a checkout's values to its own local layers, leaving the tracked layers alone. hack up warns and names the keys, doctor reports them, and hack env materialize refuses to replace the file. This applies when the checkout has regular layer files, no machine-local layer, no contract and no materialization state. A Compose file with include and no services now reaches the checks too, and doctor's note says host lifecycle hooks receive migrated values. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 10 +- src/commands/doctor.ts | 11 +- src/commands/env.ts | 20 ++ src/commands/project.ts | 10 + src/lib/project-env-config.ts | 260 ++++++++++++++++++------ tests/doctor-fix-noninteractive.test.ts | 25 +++ tests/env-legacy-service-scope.test.ts | 129 ++++++++++++ 7 files changed, 402 insertions(+), 63 deletions(-) diff --git a/docs/env.md b/docs/env.md index 9fbd19c28..bf13939f1 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1033,7 +1033,15 @@ as private (mode 0600) files and never replace an existing entry. The tracked `h `hack.env..yaml` for each migrated overlay, are written with no values, so the repo switches to env layers. Teammates do not receive these values through git; share the ones they need with `hack env set` (plain settings) or `hack env set --secret` -(encrypted) after the migration. A repo with no legacy env file gets only an empty +(encrypted) after the migration. + +Once this is committed, a teammate's own `.hack/.env` stops reaching services, because +their checkout now uses env layers. `hack up` warns and names the keys, doctor reports it, +and `hack env materialize` refuses to replace that file. They run +`hack doctor --migrate-env-config` to move their values to their own local layers; this +applies while the checkout has regular layer files, no machine-local layer, no contract and +no materialization state. Tell teammates when you commit the migration. + A repo with no legacy env file gets only an empty `hack.env.default.yaml`. This migration refuses, with a one-line `E_*` error, writes nothing and keeps the legacy diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index a7e4585c5..c7e9c79bd 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -161,6 +161,7 @@ import { } from "../lib/project.ts"; import { runProjectDomainCommand } from "../lib/project-domain-command.ts"; import { + checkoutOnlyLegacyDotenvKeys, discoverComposeServiceNames, ensureHackDirGitignore, HACK_DIR_GITIGNORE_READ_ONLY_WARNING, @@ -3702,7 +3703,7 @@ function buildEnvMigrationNote(opts: { const newKeys = migrated.keysNewToEveryService ?? []; if (newKeys.length > 0) { notes.push( - `No Compose service listed the legacy file that sets these, so every service now receives them once that env is selected: ${newKeys.join(", ")}. To limit a key to one service, run \`hack env unset KEY --local\` and then \`hack env set KEY --service --local\`, but not for a key your Compose file interpolates: service values do not reach interpolation.` + `No Compose service listed the legacy file that sets these, so every service now receives them once that env is selected: ${newKeys.join(", ")}. To limit a key to one service, run \`hack env unset KEY --local\` and then \`hack env set KEY --service --local\`, but not for a key your Compose file interpolates: service values do not reach interpolation. Host lifecycle hooks receive these values too.` ); } return notes.join("\n"); @@ -3721,9 +3722,13 @@ async function maybeMigrateProjectEnvConfig(opts: { // through), and a dangling or non-file layer entry refuses closed with // E_PATH_CONFLICT instead of being replaced. if ( - await projectEnvConfigExistsForMutation({ + (await projectEnvConfigExistsForMutation({ projectDir: project.projectDir, - }) + })) && + // A checkout whose legacy file is its only copy still needs migrating, for + // example after a teammate's migration was committed. + (await checkoutOnlyLegacyDotenvKeys({ projectDir: project.projectDir })) + .length === 0 ) { note("Project already uses the new env config files.", "doctor"); await maybeRepairModernProjectComposeEnvReferences({ diff --git a/src/commands/env.ts b/src/commands/env.ts index a2abf4e1f..044a883ad 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -21,6 +21,7 @@ import { resolveRequestedEnvName } from "../cli/env-selection.ts"; import { optEnv, optJson, optPath, optProject } from "../cli/options.ts"; import { resolveRegisteredProjectOption } from "../cli/project-selector.ts"; import { readControlPlaneConfig } from "../control-plane/sdk/config.ts"; +import { HackCliError } from "../lib/cli-result.ts"; import { updateGlobalConfig } from "../lib/config.ts"; import { resolveGlobalConfigPath } from "../lib/config-paths.ts"; import { @@ -76,6 +77,7 @@ import { assertValidProjectEnvScopeName, discoverComposeServiceNames, ENV_KEY_INVALID_MESSAGE, + findUnmigratedLegacyDotenvKeys, hydrateProjectEnvLocalLayer, materializeProjectEnv, migrateLegacyProjectEnv, @@ -2080,6 +2082,24 @@ const handleEnvMaterialize: CommandHandlerFor = async ({ const serviceNames = await discoverComposeServiceNames({ composeFile: project.composeFile, }); + const unmigrated = await findUnmigratedLegacyDotenvKeys({ + projectDir: project.projectDir, + layerValues: + ( + await resolveProjectEnvConfig({ + projectRoot: project.projectRoot, + projectDir: project.projectDir, + envName, + serviceNames, + }) + )?.globalEnv ?? {}, + }); + if (unmigrated.length > 0) { + throw new HackCliError({ + code: "E_CONFIG_INVALID", + message: `Refusing to replace ${resolve(project.projectDir, ".env")}: it sets ${unmigrated.join(", ")}, which the env layers don't deliver, and it is this checkout's only copy. Run \`hack doctor --migrate-env-config\` first, or remove the file if you don't need it.`, + }); + } const result = await materializeProjectEnv({ projectRoot: project.projectRoot, projectDir: project.projectDir, diff --git a/src/commands/project.ts b/src/commands/project.ts index 8c37f4a73..7333727d2 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -261,6 +261,7 @@ import { import { discoverComposeServiceNames, ensureHackDirGitignore, + findUnmigratedLegacyDotenvKeys, HACK_DIR_GITIGNORE_READ_ONLY_WARNING, migrateLegacyProjectEnv, projectEnvConfigExists, @@ -1320,6 +1321,15 @@ export async function resolveModernComposeEnvOverrides(opts: { if (!modern) { return null; } + const unmigrated = await findUnmigratedLegacyDotenvKeys({ + projectDir: opts.project.projectDir, + layerValues: modern.globalEnv, + }); + if (unmigrated.length > 0) { + logger.warn({ + message: `.hack/.env sets ${unmigrated.join(", ")}, which the env layers don't deliver, so services won't receive them. Run \`hack doctor --migrate-env-config\` to move them to this checkout's local layer.`, + }); + } const lifecycleEnv = selectProjectEnvValuesForExecutionTarget({ resolved: modern, scopeName: "global", diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 83bcb5869..77f948ed3 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -3709,11 +3709,24 @@ export async function inspectProjectEnvMaterialization(opts: { } if (envExists && !stateExists) { + const resolved = await resolveProjectEnvConfig({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + envName: opts.envName, + serviceNames: opts.serviceNames, + }); + const unmigrated = await findUnmigratedLegacyDotenvKeys({ + projectDir: opts.projectDir, + layerValues: resolved?.globalEnv ?? {}, + }); return { envPath, statePath, status: "warn", - message: `Materialized ${envPath} is missing ${statePath} (run: hack env materialize)`, + message: + unmigrated.length > 0 + ? `${envPath} sets ${unmigrated.join(", ")}, which the env layers don't deliver, and it is this checkout's only copy (run: hack doctor --migrate-env-config)` + : `Materialized ${envPath} is missing ${statePath} (run: hack env materialize)`, issues: [`missing ${statePath}`], }; } @@ -5536,6 +5549,43 @@ async function composeEnvFileKeyConflicts(opts: { return conflicts; } +/** Per-service `environment` and other `env_file` entries that set a migrated key. */ +async function collectComposeServiceConflicts(opts: { + readonly projectDir: string; + readonly services: Record; + readonly migratedKeys: ReadonlySet; +}): Promise<{ + readonly conflicts: string[]; + readonly unreadable: string[]; + readonly envFileConflicts: string[]; +}> { + const conflicts: string[] = []; + const unreadable: string[] = []; + const envFileConflicts: string[] = []; + for (const [service, rawService] of Object.entries(opts.services)) { + if (!isRecord(rawService)) { + continue; + } + envFileConflicts.push( + ...(await composeEnvFileKeyConflicts({ + projectDir: opts.projectDir, + service, + rawService, + migratedKeys: opts.migratedKeys, + })) + ); + const keys = composeServiceEnvironmentKeys(rawService.environment); + if (keys === null) { + unreadable.push(JSON.stringify(service)); + continue; + } + for (const key of [...keys].filter((name) => opts.migratedKeys.has(name))) { + conflicts.push(`${JSON.stringify(key)} in ${JSON.stringify(service)}`); + } + } + return { conflicts, unreadable, envFileConflicts }; +} + /** * Refuses when a migrated key is also set in a Compose service's own * `environment` or another of its `env_file` entries. Legacy `env_file` values rank below `environment`, but the @@ -5567,10 +5617,11 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { reason: `${PROJECT_COMPOSE_FILENAME} is not a single YAML document this check can read, such as a file with several "---" documents, so the migration cannot tell which values services receive. Merge it into one document, then retry.`, }); } - const services = parsed ? getRecord(parsed, "services") : null; - if (!(parsed && services)) { + if (!parsed) { return; } + // No `services` still runs the checks: `include` can supply every service. + const services = getRecord(parsed, "services") ?? {}; const uncheckable = uncheckableComposeEnvInputs({ projectDir: opts.projectDir, composeText, @@ -5583,30 +5634,12 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { reason: `Compose uses ${uncheckable.join(", ")}, so the migration cannot tell which values services receive from the legacy .env files. Write those services' env_file entries directly in ${PROJECT_COMPOSE_FILENAME}, each file once, with a plain path and no format, and remove any !reset or !override text (a comment or value containing it also matches), then retry.`, }); } - const conflicts: string[] = []; - const unreadable: string[] = []; - const envFileConflicts: string[] = []; - for (const [service, rawService] of Object.entries(services)) { - if (!isRecord(rawService)) { - continue; - } - envFileConflicts.push( - ...(await composeEnvFileKeyConflicts({ - projectDir: opts.projectDir, - service, - rawService, - migratedKeys, - })) - ); - const keys = composeServiceEnvironmentKeys(rawService.environment); - if (keys === null) { - unreadable.push(JSON.stringify(service)); - continue; - } - for (const key of [...keys].filter((name) => migratedKeys.has(name))) { - conflicts.push(`${JSON.stringify(key)} in ${JSON.stringify(service)}`); - } - } + const { conflicts, unreadable, envFileConflicts } = + await collectComposeServiceConflicts({ + projectDir: opts.projectDir, + services, + migratedKeys, + }); if (unreadable.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", @@ -5916,6 +5949,90 @@ async function selectLegacyDotenvOverlays(opts: { return overlayNames; } +/** + * Keys in this checkout's legacy `.hack/.env*` files when they are its only copy + * of the values: the repo already has tracked env layers, for example because a + * teammate's migration was committed, but this checkout never migrated. That + * means regular layer files only, none of them machine-local, no contract and no + * materialization state. Empty otherwise. + */ +export async function checkoutOnlyLegacyDotenvKeys(opts: { + readonly projectDir: string; + readonly baseOnly?: boolean; +}): Promise { + const entries = await readdir(opts.projectDir, { withFileTypes: true }); + const layers = entries.filter( + (entry) => + entry.name.startsWith(PROJECT_ENV_CONFIG_FILENAME_PREFIX) && + entry.name.endsWith(PROJECT_ENV_CONFIG_FILENAME_SUFFIX) + ); + const notApplicable = + layers.length === 0 || + layers.some( + (entry) => + !entry.isFile() || + entry.name.endsWith( + `.${PROJECT_ENV_LOCAL_SEGMENT}${PROJECT_ENV_CONFIG_FILENAME_SUFFIX}` + ) + ) || + (await pathExists( + resolve(opts.projectDir, PROJECT_ENV_CONTRACT_FILENAME) + )) || + (await pathExists( + resolveProjectEnvStatePath({ projectDir: opts.projectDir }) + )); + if (notApplicable) { + return []; + } + const overlayNames = + opts.baseOnly === true + ? [] + : ( + await discoverLegacyOverlayNames({ + projectDir: opts.projectDir, + includeSymlinks: true, + }) + ).filter((name) => !isExcludedLegacyDotenvName(name)); + const keys = new Set(); + for (const envName of [null, ...overlayNames]) { + const read = await readOptionalTextFile( + resolveEnvFilePath({ projectDir: opts.projectDir, envName }) + ); + if (read.kind !== "absent" && read.kind !== "unreadable") { + for (const key of Object.keys(parseDotEnv(read.text))) { + keys.add(key); + } + } + } + return [...keys].sort((left, right) => left.localeCompare(right)); +} + +/** + * Keys this checkout's `.hack/.env` sets that its env layers don't deliver, when + * that file is its only copy of them. Commands that run from layers would lose + * these, and `hack env materialize` would erase them. Names only. + */ +export async function findUnmigratedLegacyDotenvKeys(opts: { + readonly projectDir: string; + readonly layerValues: Readonly>; +}): Promise { + const keys = await checkoutOnlyLegacyDotenvKeys({ + projectDir: opts.projectDir, + baseOnly: true, + }); + if (keys.length === 0) { + return []; + } + const read = await readOptionalTextFile( + resolveEnvFilePath({ projectDir: opts.projectDir, envName: null }) + ); + const values = + read.kind === "absent" || read.kind === "unreadable" + ? {} + : parseDotEnv(read.text); + return keys.filter((key) => values[key] !== opts.layerValues[key]); +} + /** * Refuses a legacy `.hack/.env` or overlay that is a symlink at all, dangling or * not, for a migration that publishes values to tracked layers. @@ -6164,6 +6281,53 @@ async function readValidatedLegacyDotenvFiles(opts: { return legacyValues; } +/** + * Publishes machine-local layers, then tracked ones. If a write fails, removes + * what this call created, so the checkout keeps using the legacy files. + */ +async function publishMigratedLegacyLayers(opts: { + readonly projectDir: string; + readonly localWrites: readonly PlannedEnvLayerWrite[]; + readonly trackedWrites: readonly PlannedEnvLayerWrite[]; +}): Promise { + // Machine-local layers first, so the tracked layers switch the format only + // once the values have a home. + const wroteFiles: string[] = []; + try { + for (const write of opts.localWrites) { + // Created 0600 from the start: the values were in a private .env. + await publishMigrationLocalLayer({ + projectDir: opts.projectDir, + write, + }); + wroteFiles.push(write.path); + } + for (const write of opts.trackedWrites) { + // Another checkout's migration may already have committed this layer. + if (await pathExists(write.path)) { + continue; + } + if ( + await writeProjectEnvConfigFile({ + path: write.path, + config: write.config, + expectedExisting: null, + }) + ) { + wroteFiles.push(write.path); + } + } + } catch (error: unknown) { + // Every target was absent before this call, so removing what it created + // leaves the project on the legacy files. + for (const path of [...wroteFiles].reverse()) { + await unlink(path).catch(() => undefined); + } + throw error; + } + return wroteFiles; +} + /** * Migrates a legacy project that has no `.hack/hack.env.json` contract, so * its values live only in `.hack/.env` and `.hack/.env.`. @@ -6207,7 +6371,11 @@ async function migrateUndeclaredLegacyDotenv(opts: { // Also refuses a dangling or non-file layer entry (E_PATH_CONFLICT), which // an atomic write would otherwise replace or write through. if ( - await projectEnvConfigExistsForMutation({ projectDir: opts.projectDir }) + (await projectEnvConfigExistsForMutation({ + projectDir: opts.projectDir, + })) && + (await checkoutOnlyLegacyDotenvKeys({ projectDir: opts.projectDir })) + .length === 0 ) { throw new LegacyDotenvMigrationRefusalError({ code: "E_PATH_CONFLICT", @@ -6315,37 +6483,11 @@ async function migrateUndeclaredLegacyDotenv(opts: { }); } - // Machine-local layers first, so the tracked layers switch the format only - // once the values have a home. - const wroteFiles: string[] = []; - try { - for (const write of localWrites) { - // Created 0600 from the start: the values were in a private .env. - await publishMigrationLocalLayer({ - projectDir: opts.projectDir, - write, - }); - wroteFiles.push(write.path); - } - for (const write of trackedWrites) { - if ( - await writeProjectEnvConfigFile({ - path: write.path, - config: write.config, - expectedExisting: null, - }) - ) { - wroteFiles.push(write.path); - } - } - } catch (error: unknown) { - // Every target was absent before this call, so removing what it created - // leaves the project on the legacy files. - for (const path of [...wroteFiles].reverse()) { - await unlink(path).catch(() => undefined); - } - throw error; - } + const wroteFiles = await publishMigratedLegacyLayers({ + projectDir: opts.projectDir, + localWrites, + trackedWrites, + }); if (opts.materialize) { await materializeProjectEnv({ diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index e862557c8..3cbf44c61 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -713,6 +713,31 @@ test("doctor --migrate-env-config names the keys every service gains, never thei expect(notes).not.toContain(legacy); }, 30_000); +test("doctor --migrate-env-config moves a checkout's own .env after a teammate's migration landed (HACK-1370)", async () => { + const projectRoot = await createProjectWithStaleDaemonPid(); + const projectDir = resolve(projectRoot, ".hack"); + const legacy = ["synthetic", "legacy", "71ab"].join("-"); + await writeFile( + resolve(projectDir, "hack.env.default.yaml"), + "version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global: {}\n" + ); + await writeFile(resolve(projectDir, ".env"), `OWNED=${legacy}\n`); + + const { runCli } = await import("../src/cli/run.ts"); + const code = await runCli([ + "doctor", + "--migrate-env-config", + "--no-interactive", + "--path", + projectRoot, + ]); + expect(code).toBe(0); + expect(noteCalls.join("\n")).not.toContain(legacy); + expect( + await readFile(resolve(projectDir, "hack.env.local.yaml"), "utf8") + ).toContain(legacy); +}, 30_000); + test("doctor --migrate-env-config refuses a dangling default layer with E_PATH_CONFLICT and keeps it (HACK-1370)", async () => { const projectRoot = await createProjectWithStaleDaemonPid(); const projectDir = resolve(projectRoot, ".hack"); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 967e2a295..5507bbd04 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -22,7 +22,10 @@ import { } from "../src/constants.ts"; import { ClosedCliError, HackCliError } from "../src/lib/cli-result.ts"; import { + checkoutOnlyLegacyDotenvKeys, ENV_SCOPE_INVALID_MESSAGE, + findUnmigratedLegacyDotenvKeys, + inspectProjectEnvMaterialization, LegacyDotenvMigrationRefusalError, type LegacyProjectEnvMigration, migrateLegacyProjectEnv, @@ -2124,3 +2127,129 @@ test("the migration allows a name that only starts like a tool setting", async ( }); expect(migrated.legacyDetected).toBe(true); }); + +test("the migration refuses a Compose file with include and no services", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "include:\n - inc.yml\n" + ); + await writeFile(resolve(repo.projectDir, ".env"), `A=${KEPT_VALUE}\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "top-level include", + hidden: [KEPT_VALUE], + }); +}); + +async function writeTeammateCheckout(): Promise { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + return repo; +} + +const LAYER_EMPTY = + "version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global: {}\n"; + +test("a checkout with only its own .env migrates to its local layer after a teammate's migration landed", async () => { + const repo = await writeTeammateCheckout(); + const tracked = await readFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + "utf8" + ); + expect( + await findUnmigratedLegacyDotenvKeys({ + projectDir: repo.projectDir, + layerValues: {}, + }) + ).toEqual(["KEPT_SETTING"]); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); + expect( + await readFile(resolve(repo.projectDir, "hack.env.default.yaml"), "utf8") + ).toBe(tracked); + // Now migrated: nothing is unmigrated, and a second run refuses. + expect( + await checkoutOnlyLegacyDotenvKeys({ projectDir: repo.projectDir }) + ).toEqual([]); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: "env layer files already exist", + hidden: [KEPT_VALUE], + }); +}); + +test("a checkout's .env is not its only copy with a state file, a local layer, a contract or a linked layer", async () => { + for (const place of [ + (repo: Repo) => + writeFile(resolve(repo.projectDir, ".env.state.json"), "{}\n"), + (repo: Repo) => + writeFile(resolve(repo.projectDir, "hack.env.local.yaml"), LAYER_EMPTY), + (repo: Repo) => + writeFile( + resolve(repo.projectDir, "hack.env.json"), + `${JSON.stringify({ version: 1, vars: [] })}\n` + ), + (repo: Repo) => + symlink( + resolve(repo.root, "elsewhere.yaml"), + resolve(repo.projectDir, "hack.env.staging.yaml") + ), + ]) { + const repo = await writeTeammateCheckout(); + await place(repo); + expect( + await checkoutOnlyLegacyDotenvKeys({ projectDir: repo.projectDir }) + ).toEqual([]); + } +}); + +test("hack env materialize refuses to replace a checkout's only copy, and keeps the file", async () => { + const repo = await writeTeammateCheckout(); + const envPath = resolve(repo.projectDir, ".env"); + const run = await runCli({ + repo, + args: ["env", "materialize", "--path", repo.projectRoot], + }); + expect(run.exitCode).not.toBe(0); + const output = `${run.stdout}${run.stderr}`; + expect(output).toContain("E_CONFIG_INVALID"); + expect(output).toContain("KEPT_SETTING"); + expect(output).not.toContain(KEPT_VALUE); + expect(await readFile(envPath, "utf8")).toContain(KEPT_VALUE); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + const after = await runCli({ + repo, + args: ["env", "materialize", "--path", repo.projectRoot], + }); + expect(after.exitCode).toBe(0); + expect(await readFile(envPath, "utf8")).toContain(KEPT_VALUE); +}, 60_000); + +test("doctor names a checkout's unmigrated keys, not their values", async () => { + const repo = await writeTeammateCheckout(); + const inspected = await inspectProjectEnvMaterialization({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + }); + expect(inspected.message).toContain("KEPT_SETTING"); + expect(inspected.message).toContain("hack doctor --migrate-env-config"); + expect(inspected.message).not.toContain(KEPT_VALUE); +}); From 190c66b291bf0448f53975d7757251ca72ad4d3d Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:01:06 -0400 Subject: [PATCH 25/55] fix(env): decide unmigrated legacy env by what the layers deliver Protection depended on which files existed, so an unrelated local layer, a stray state file or an ordinary env set --local turned it off and hack env materialize could erase a checkout's only copy. It now compares each legacy file's values with what the layers deliver, including overlays, and keeps protecting them in those states. Migrating adds to an existing local layer, refuses a key it sets differently, and no longer copies values the layers already deliver. Diagnostics printed raw parsed names, so a malformed line could echo a secret. They now name only valid keys, count the rest and name unreadable files, and malformed or unreadable entries still block materialize. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 20 +- src/commands/doctor.ts | 15 +- src/commands/env.ts | 21 +- src/commands/project.ts | 13 +- src/lib/project-env-config.ts | 315 ++++++++++++++++++------- tests/env-legacy-service-scope.test.ts | 215 ++++++++++++++--- 6 files changed, 450 insertions(+), 149 deletions(-) diff --git a/docs/env.md b/docs/env.md index bf13939f1..fd134ab44 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1035,14 +1035,18 @@ repo switches to env layers. Teammates do not receive these values through git; the ones they need with `hack env set` (plain settings) or `hack env set --secret` (encrypted) after the migration. -Once this is committed, a teammate's own `.hack/.env` stops reaching services, because -their checkout now uses env layers. `hack up` warns and names the keys, doctor reports it, -and `hack env materialize` refuses to replace that file. They run -`hack doctor --migrate-env-config` to move their values to their own local layers; this -applies while the checkout has regular layer files, no machine-local layer, no contract and -no materialization state. Tell teammates when you commit the migration. - A repo with no legacy env file gets only an empty -`hack.env.default.yaml`. +A repo with no legacy env file gets only an empty `hack.env.default.yaml`. + +Once this is committed, a teammate's own `.hack/.env` and overlays may stop reaching services, +because their checkout now uses env layers. Hack compares those files with what the layers +deliver. For any value the layers don't deliver, `hack up` warns, doctor reports it, and +`hack env materialize` refuses, so the file isn't replaced. They run +`hack doctor --migrate-env-config` to move those values to their own local layers, adding to a +local layer that already exists. A value equal to what the layers deliver isn't copied, so later +team changes still reach them. A key the local layer sets differently refuses +(`E_PATH_CONFLICT`). Messages name valid keys, count malformed entries and name unreadable files, +and never show a value. A `.hack/.env` with a valid materialization state is generated output, and +this comparison skips it. Tell teammates when you commit the migration. This migration refuses, with a one-line `E_*` error, writes nothing and keeps the legacy files, when: diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index c7e9c79bd..5989e574c 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -161,7 +161,7 @@ import { } from "../lib/project.ts"; import { runProjectDomainCommand } from "../lib/project-domain-command.ts"; import { - checkoutOnlyLegacyDotenvKeys, + canMigrateCheckoutLegacyDotenv, discoverComposeServiceNames, ensureHackDirGitignore, HACK_DIR_GITIGNORE_READ_ONLY_WARNING, @@ -3725,10 +3725,15 @@ async function maybeMigrateProjectEnvConfig(opts: { (await projectEnvConfigExistsForMutation({ projectDir: project.projectDir, })) && - // A checkout whose legacy file is its only copy still needs migrating, for - // example after a teammate's migration was committed. - (await checkoutOnlyLegacyDotenvKeys({ projectDir: project.projectDir })) - .length === 0 + // A checkout whose legacy files hold values its layers don't deliver still + // needs migrating, for example after a teammate's migration was committed. + !(await canMigrateCheckoutLegacyDotenv({ + projectRoot: project.projectRoot, + projectDir: project.projectDir, + serviceNames: await discoverComposeServiceNames({ + composeFile: project.composeFile, + }), + })) ) { note("Project already uses the new env config files.", "doctor"); await maybeRepairModernProjectComposeEnvReferences({ diff --git a/src/commands/env.ts b/src/commands/env.ts index 044a883ad..5f3fac223 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -75,9 +75,11 @@ import { } from "../lib/project.ts"; import { assertValidProjectEnvScopeName, + describeUnmigratedLegacyDotenv, discoverComposeServiceNames, ENV_KEY_INVALID_MESSAGE, - findUnmigratedLegacyDotenvKeys, + findUnmigratedLegacyDotenv, + hasUnmigratedLegacyDotenv, hydrateProjectEnvLocalLayer, materializeProjectEnv, migrateLegacyProjectEnv, @@ -2082,22 +2084,15 @@ const handleEnvMaterialize: CommandHandlerFor = async ({ const serviceNames = await discoverComposeServiceNames({ composeFile: project.composeFile, }); - const unmigrated = await findUnmigratedLegacyDotenvKeys({ + const unmigrated = await findUnmigratedLegacyDotenv({ + projectRoot: project.projectRoot, projectDir: project.projectDir, - layerValues: - ( - await resolveProjectEnvConfig({ - projectRoot: project.projectRoot, - projectDir: project.projectDir, - envName, - serviceNames, - }) - )?.globalEnv ?? {}, + serviceNames, }); - if (unmigrated.length > 0) { + if (hasUnmigratedLegacyDotenv(unmigrated)) { throw new HackCliError({ code: "E_CONFIG_INVALID", - message: `Refusing to replace ${resolve(project.projectDir, ".env")}: it sets ${unmigrated.join(", ")}, which the env layers don't deliver, and it is this checkout's only copy. Run \`hack doctor --migrate-env-config\` first, or remove the file if you don't need it.`, + message: `Refusing to materialize: this checkout ${describeUnmigratedLegacyDotenv(unmigrated)} in its legacy .hack/.env files, and the env layers don't deliver those values, so they could be lost. Run \`hack doctor --migrate-env-config\` first, or remove what you don't need.`, }); } const result = await materializeProjectEnv({ diff --git a/src/commands/project.ts b/src/commands/project.ts index 7333727d2..d39a93892 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -259,10 +259,12 @@ import { sanitizeProjectSlug, } from "../lib/project.ts"; import { + describeUnmigratedLegacyDotenv, discoverComposeServiceNames, ensureHackDirGitignore, - findUnmigratedLegacyDotenvKeys, + findUnmigratedLegacyDotenv, HACK_DIR_GITIGNORE_READ_ONLY_WARNING, + hasUnmigratedLegacyDotenv, migrateLegacyProjectEnv, projectEnvConfigExists, resolveProjectEnvConfig, @@ -1321,13 +1323,14 @@ export async function resolveModernComposeEnvOverrides(opts: { if (!modern) { return null; } - const unmigrated = await findUnmigratedLegacyDotenvKeys({ + const unmigrated = await findUnmigratedLegacyDotenv({ + projectRoot: opts.project.projectRoot, projectDir: opts.project.projectDir, - layerValues: modern.globalEnv, + serviceNames: opts.allServiceNames, }); - if (unmigrated.length > 0) { + if (hasUnmigratedLegacyDotenv(unmigrated)) { logger.warn({ - message: `.hack/.env sets ${unmigrated.join(", ")}, which the env layers don't deliver, so services won't receive them. Run \`hack doctor --migrate-env-config\` to move them to this checkout's local layer.`, + message: `This checkout ${describeUnmigratedLegacyDotenv(unmigrated)} in its legacy .hack/.env files, and the env layers don't deliver those values, so services that rely on them may not get them. Run \`hack doctor --migrate-env-config\` to move them to this checkout's local layers.`, }); } const lifecycleEnv = selectProjectEnvValuesForExecutionTarget({ diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 77f948ed3..3bef3db1b 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -3698,6 +3698,17 @@ export async function inspectProjectEnvMaterialization(opts: { pathExists(statePath), ]); + const unmigrated = await findUnmigratedLegacyDotenv(opts); + if (hasUnmigratedLegacyDotenv(unmigrated)) { + return { + envPath, + statePath, + status: "warn", + message: `The legacy .hack/.env files hold values the env layers don't deliver: this checkout ${describeUnmigratedLegacyDotenv(unmigrated)} (run: hack doctor --migrate-env-config)`, + issues: ["unmigrated legacy env values"], + }; + } + if (!(envExists || stateExists)) { return { envPath, @@ -3709,24 +3720,11 @@ export async function inspectProjectEnvMaterialization(opts: { } if (envExists && !stateExists) { - const resolved = await resolveProjectEnvConfig({ - projectRoot: opts.projectRoot, - projectDir: opts.projectDir, - envName: opts.envName, - serviceNames: opts.serviceNames, - }); - const unmigrated = await findUnmigratedLegacyDotenvKeys({ - projectDir: opts.projectDir, - layerValues: resolved?.globalEnv ?? {}, - }); return { envPath, statePath, status: "warn", - message: - unmigrated.length > 0 - ? `${envPath} sets ${unmigrated.join(", ")}, which the env layers don't deliver, and it is this checkout's only copy (run: hack doctor --migrate-env-config)` - : `Materialized ${envPath} is missing ${statePath} (run: hack env materialize)`, + message: `Materialized ${envPath} is missing ${statePath} (run: hack env materialize)`, issues: [`missing ${statePath}`], }; } @@ -5900,6 +5898,22 @@ async function legacyKeysNewToEveryService(opts: { return [...keys].sort((left, right) => left.localeCompare(right)); } +/** + * Entries the current layers don't deliver with the same value. With no layers + * that is all of them. Skipping the rest keeps a teammate from pinning a team + * value in their local layer, where a later team change wouldn't reach them. + */ +function legacyValuesNotDelivered(opts: { + readonly values: Record; + readonly delivered: Readonly>; +}): Record { + return Object.fromEntries( + Object.entries(opts.values).filter( + ([key, value]) => opts.delivered[key] !== value + ) + ); +} + /** Overlay entries a migrated overlay layer must hold: those that differ from the base. */ function legacyOverlayChangedValues(opts: { readonly overlayValues: Record; @@ -5949,88 +5963,147 @@ async function selectLegacyDotenvOverlays(opts: { return overlayNames; } +export type UnmigratedLegacyDotenv = { + /** Valid key names the layers don't deliver with the same value. */ + readonly keys: readonly string[]; + /** Entries whose name isn't a valid key; a name could hold a value, so never shown. */ + readonly malformedCount: number; + /** Legacy file names that exist but can't be read. */ + readonly unreadableFiles: readonly string[]; +}; + +/** The value each key has from the layers selected for `envName`. Empty when none can be read. */ +async function deliveredLayerValues(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly envName: string | null; +}): Promise>> { + try { + return (await resolveProjectEnvConfig(opts))?.globalEnv ?? {}; + } catch { + // An unusable layer delivers nothing, and its own commands report it. + return {}; + } +} + /** - * Keys in this checkout's legacy `.hack/.env*` files when they are its only copy - * of the values: the repo already has tracked env layers, for example because a - * teammate's migration was committed, but this checkout never migrated. That - * means regular layer files only, none of them machine-local, no contract and no - * materialization state. Empty otherwise. + * What this checkout's legacy `.hack/.env*` files hold that its env layers don't + * deliver, in a project that already has layers, for example after a teammate's + * migration was committed. The answer comes from comparing values, not from which + * files exist: an unrelated local layer doesn't make a value safe to overwrite. + * + * A contract project is out of scope, because its own migration owns those files. + * `.hack/.env` with a valid state file is generated output, so it is skipped; + * overlays are not generated by materialize, so they are always compared. */ -export async function checkoutOnlyLegacyDotenvKeys(opts: { +export async function findUnmigratedLegacyDotenv(opts: { + readonly projectRoot: string; readonly projectDir: string; - readonly baseOnly?: boolean; -}): Promise { + readonly serviceNames: readonly string[]; +}): Promise { + const none = { keys: [], malformedCount: 0, unreadableFiles: [] }; const entries = await readdir(opts.projectDir, { withFileTypes: true }); - const layers = entries.filter( + const hasLayers = entries.some( (entry) => entry.name.startsWith(PROJECT_ENV_CONFIG_FILENAME_PREFIX) && entry.name.endsWith(PROJECT_ENV_CONFIG_FILENAME_SUFFIX) ); - const notApplicable = - layers.length === 0 || - layers.some( - (entry) => - !entry.isFile() || - entry.name.endsWith( - `.${PROJECT_ENV_LOCAL_SEGMENT}${PROJECT_ENV_CONFIG_FILENAME_SUFFIX}` - ) - ) || - (await pathExists( - resolve(opts.projectDir, PROJECT_ENV_CONTRACT_FILENAME) - )) || - (await pathExists( - resolveProjectEnvStatePath({ projectDir: opts.projectDir }) - )); - if (notApplicable) { - return []; + if ( + !hasLayers || + (await pathExists(resolve(opts.projectDir, PROJECT_ENV_CONTRACT_FILENAME))) + ) { + return none; } - const overlayNames = - opts.baseOnly === true - ? [] - : ( - await discoverLegacyOverlayNames({ - projectDir: opts.projectDir, - includeSymlinks: true, - }) - ).filter((name) => !isExcludedLegacyDotenvName(name)); + const state = await readProjectEnvStateFile({ + statePath: resolveProjectEnvStatePath({ projectDir: opts.projectDir }), + }); + const overlayNames = ( + await discoverLegacyOverlayNames({ + projectDir: opts.projectDir, + includeSymlinks: true, + }) + ).filter( + (name) => + !( + isExcludedLegacyDotenvName(name) || isReservedLegacyOverlayName(name) + ) && NATIVE_ENV_OVERLAY_PATTERN.test(name) + ); const keys = new Set(); - for (const envName of [null, ...overlayNames]) { - const read = await readOptionalTextFile( - resolveEnvFilePath({ projectDir: opts.projectDir, envName }) - ); - if (read.kind !== "absent" && read.kind !== "unreadable") { - for (const key of Object.keys(parseDotEnv(read.text))) { + const unreadableFiles: string[] = []; + let malformedCount = 0; + for (const envName of [...(state.ok ? [] : [null]), ...overlayNames]) { + const path = resolveEnvFilePath({ projectDir: opts.projectDir, envName }); + const read = await readOptionalTextFile(path); + if (read.kind === "unreadable") { + unreadableFiles.push(basename(path)); + } + if (read.kind !== "text") { + continue; + } + const delivered = await deliveredLayerValues({ ...opts, envName }); + for (const [key, value] of Object.entries(parseDotEnv(read.text))) { + if (!PROJECT_ENV_KEY_PATTERN.test(key)) { + malformedCount += 1; + } else if (delivered[key] !== value) { keys.add(key); } } } - return [...keys].sort((left, right) => left.localeCompare(right)); + return { + keys: [...keys].sort((left, right) => left.localeCompare(right)), + malformedCount, + unreadableFiles, + }; } -/** - * Keys this checkout's `.hack/.env` sets that its env layers don't deliver, when - * that file is its only copy of them. Commands that run from layers would lose - * these, and `hack env materialize` would erase them. Names only. - */ -export async function findUnmigratedLegacyDotenvKeys(opts: { - readonly projectDir: string; - readonly layerValues: Readonly>; -}): Promise { - const keys = await checkoutOnlyLegacyDotenvKeys({ - projectDir: opts.projectDir, - baseOnly: true, - }); - if (keys.length === 0) { - return []; +export function hasUnmigratedLegacyDotenv( + unmigrated: UnmigratedLegacyDotenv +): boolean { + return ( + unmigrated.keys.length > 0 || + unmigrated.malformedCount > 0 || + unmigrated.unreadableFiles.length > 0 + ); +} + +/** Value-free description: valid key names, a count of malformed names, and file names. */ +export function describeUnmigratedLegacyDotenv( + unmigrated: UnmigratedLegacyDotenv +): string { + const parts: string[] = []; + if (unmigrated.keys.length > 0) { + parts.push(`sets ${unmigrated.keys.join(", ")}`); + } + if (unmigrated.malformedCount > 0) { + parts.push( + `has ${countLabel({ count: unmigrated.malformedCount, noun: "malformed entry" })} (not shown, because a malformed line may contain a value)` + ); + } + if (unmigrated.unreadableFiles.length > 0) { + parts.push(`can't read ${unmigrated.unreadableFiles.join(", ")}`); } - const read = await readOptionalTextFile( - resolveEnvFilePath({ projectDir: opts.projectDir, envName: null }) + return parts.join("; "); +} + +/** Whether `doctor --migrate-env-config` can move these values: regular layer files only. */ +export async function canMigrateCheckoutLegacyDotenv(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; +}): Promise { + const entries = await readdir(opts.projectDir, { withFileTypes: true }); + const allRegular = entries + .filter( + (entry) => + entry.name.startsWith(PROJECT_ENV_CONFIG_FILENAME_PREFIX) && + entry.name.endsWith(PROJECT_ENV_CONFIG_FILENAME_SUFFIX) + ) + .every((entry) => entry.isFile()); + return ( + allRegular && + hasUnmigratedLegacyDotenv(await findUnmigratedLegacyDotenv(opts)) ); - const values = - read.kind === "absent" || read.kind === "unreadable" - ? {} - : parseDotEnv(read.text); - return keys.filter((key) => values[key] !== opts.layerValues[key]); } /** @@ -6281,6 +6354,61 @@ async function readValidatedLegacyDotenvFiles(opts: { return legacyValues; } +/** + * Plans adding migrated keys to machine-local layers that already exist. Refuses + * a key the layer sets to something else, because either value could be the one + * to keep. The refusal names keys and files, never values. + */ +async function planLocalLayerMerges(opts: { + readonly localWrites: readonly PlannedEnvLayerWrite[]; +}): Promise< + readonly { + readonly path: string; + readonly config: ProjectEnvConfig; + readonly expectedExisting: string; + }[] +> { + const merges: { + path: string; + config: ProjectEnvConfig; + expectedExisting: string; + }[] = []; + const conflicts: string[] = []; + for (const write of opts.localWrites) { + const text = await readTextFile(write.path); + if (text === null) { + continue; + } + const existing = parseProjectEnvConfigText({ + path: write.path, + environment: write.config.environment, + text, + }).config; + const global = existing.values.global ?? {}; + const added = write.config.values.global ?? {}; + conflicts.push( + ...Object.keys(added) + .filter((key) => key in global) + .map((key) => `${JSON.stringify(key)} in ${basename(write.path)}`) + ); + merges.push({ + path: write.path, + config: { + ...existing, + values: { ...existing.values, global: { ...global, ...added } }, + }, + expectedExisting: text, + }); + } + if (conflicts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `the legacy .env files and the local layer set ${conflicts.sort().join(", ")} to different values. Keep the one you want, by removing the key from the other, then retry.`, + }); + } + return merges; +} + /** * Publishes machine-local layers, then tracked ones. If a write fails, removes * what this call created, so the checkout keeps using the legacy files. @@ -6295,6 +6423,9 @@ async function publishMigratedLegacyLayers(opts: { const wroteFiles: string[] = []; try { for (const write of opts.localWrites) { + if (await pathExists(write.path)) { + continue; + } // Created 0600 from the start: the values were in a private .env. await publishMigrationLocalLayer({ projectDir: opts.projectDir, @@ -6374,8 +6505,7 @@ async function migrateUndeclaredLegacyDotenv(opts: { (await projectEnvConfigExistsForMutation({ projectDir: opts.projectDir, })) && - (await checkoutOnlyLegacyDotenvKeys({ projectDir: opts.projectDir })) - .length === 0 + !(await canMigrateCheckoutLegacyDotenv(opts)) ) { throw new LegacyDotenvMigrationRefusalError({ code: "E_PATH_CONFLICT", @@ -6408,6 +6538,10 @@ async function migrateUndeclaredLegacyDotenv(opts: { }); const baseValues = legacyValues.get(null) ?? {}; + const baseToMigrate = legacyValuesNotDelivered({ + values: baseValues, + delivered: await deliveredLayerValues({ ...opts, envName: null }), + }); const localWrites: PlannedEnvLayerWrite[] = []; const trackedWrites: PlannedEnvLayerWrite[] = [ { @@ -6421,7 +6555,7 @@ async function migrateUndeclaredLegacyDotenv(opts: { }), }, ]; - if (Object.keys(baseValues).length > 0) { + if (Object.keys(baseToMigrate).length > 0) { localWrites.push({ path: resolveProjectEnvLocalConfigPath({ projectDir: opts.projectDir, @@ -6429,14 +6563,17 @@ async function migrateUndeclaredLegacyDotenv(opts: { }), config: buildMigratedProjectEnvConfig({ environment: "default", - values: { global: { ...baseValues } }, + values: { global: { ...baseToMigrate } }, }), }); } for (const overlayName of overlayNames) { - const changedValues = legacyOverlayChangedValues({ - overlayValues: legacyValues.get(overlayName) ?? {}, - baseValues, + const changedValues = legacyValuesNotDelivered({ + values: legacyOverlayChangedValues({ + overlayValues: legacyValues.get(overlayName) ?? {}, + baseValues, + }), + delivered: await deliveredLayerValues({ ...opts, envName: overlayName }), }); if (Object.keys(changedValues).length === 0) { continue; @@ -6463,6 +6600,8 @@ async function migrateUndeclaredLegacyDotenv(opts: { }); } + // Before anything is written, so this refusal leaves the checkout unchanged. + const merges = await planLocalLayerMerges({ localWrites }); for (const write of localWrites) { await assertUntrackedMigrationLocalLayer({ projectRoot: opts.projectRoot, @@ -6488,6 +6627,12 @@ async function migrateUndeclaredLegacyDotenv(opts: { localWrites, trackedWrites, }); + // Last, after everything that can roll back: each only adds keys, so a failure + // here leaves the rest for the next run. + for (const merge of merges) { + await writeProjectEnvConfigFile(merge); + wroteFiles.push(merge.path); + } if (opts.materialize) { await materializeProjectEnv({ diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 5507bbd04..14e9da6ed 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -22,9 +22,8 @@ import { } from "../src/constants.ts"; import { ClosedCliError, HackCliError } from "../src/lib/cli-result.ts"; import { - checkoutOnlyLegacyDotenvKeys, ENV_SCOPE_INVALID_MESSAGE, - findUnmigratedLegacyDotenvKeys, + findUnmigratedLegacyDotenv, inspectProjectEnvMaterialization, LegacyDotenvMigrationRefusalError, type LegacyProjectEnvMigration, @@ -2159,18 +2158,21 @@ async function writeTeammateCheckout(): Promise { const LAYER_EMPTY = "version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global: {}\n"; +function unmigratedIn(repo: Repo) { + return findUnmigratedLegacyDotenv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + }); +} + test("a checkout with only its own .env migrates to its local layer after a teammate's migration landed", async () => { const repo = await writeTeammateCheckout(); const tracked = await readFile( resolve(repo.projectDir, "hack.env.default.yaml"), "utf8" ); - expect( - await findUnmigratedLegacyDotenvKeys({ - projectDir: repo.projectDir, - layerValues: {}, - }) - ).toEqual(["KEPT_SETTING"]); + expect((await unmigratedIn(repo)).keys).toEqual(["KEPT_SETTING"]); const migrated = await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true, @@ -2182,10 +2184,8 @@ test("a checkout with only its own .env migrates to its local layer after a team expect( await readFile(resolve(repo.projectDir, "hack.env.default.yaml"), "utf8") ).toBe(tracked); - // Now migrated: nothing is unmigrated, and a second run refuses. - expect( - await checkoutOnlyLegacyDotenvKeys({ projectDir: repo.projectDir }) - ).toEqual([]); + // Now delivered: nothing is unmigrated, and a second run refuses. + expect((await unmigratedIn(repo)).keys).toEqual([]); await expectMigrationRefusal({ repo, code: "E_PATH_CONFLICT", @@ -2194,28 +2194,177 @@ test("a checkout with only its own .env migrates to its local layer after a team }); }); -test("a checkout's .env is not its only copy with a state file, a local layer, a contract or a linked layer", async () => { - for (const place of [ - (repo: Repo) => - writeFile(resolve(repo.projectDir, ".env.state.json"), "{}\n"), - (repo: Repo) => - writeFile(resolve(repo.projectDir, "hack.env.local.yaml"), LAYER_EMPTY), - (repo: Repo) => - writeFile( - resolve(repo.projectDir, "hack.env.json"), - `${JSON.stringify({ version: 1, vars: [] })}\n` - ), - (repo: Repo) => - symlink( - resolve(repo.root, "elsewhere.yaml"), - resolve(repo.projectDir, "hack.env.staging.yaml") - ), +test("an unrelated local layer doesn't hide a checkout's undelivered values, and migrating adds to its layer", async () => { + const repo = await writeTeammateCheckout(); + await writeFile( + resolve(repo.projectDir, "hack.env.x.local.yaml"), + LAYER_EMPTY.replace("environment: default", "environment: x") + ); + await writeFile( + resolve(repo.projectDir, "hack.env.local.yaml"), + `${LAYER_EMPTY.replace("global: {}", "global:\n OTHER: kept")}` + ); + expect((await unmigratedIn(repo)).keys).toEqual(["KEPT_SETTING"]); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { OTHER: "kept", KEPT_SETTING: KEPT_VALUE }, + }); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}); + +test("a stray state file doesn't hide undelivered values", async () => { + const repo = await writeTeammateCheckout(); + await writeFile(resolve(repo.projectDir, ".env.state.json"), "{}\n"); + expect((await unmigratedIn(repo)).keys).toEqual(["KEPT_SETTING"]); +}); + +test("the migration refuses a key the local layer sets differently, and keeps both", async () => { + const repo = await writeTeammateCheckout(); + const layer = LAYER_EMPTY.replace( + "global: {}", + "global:\n KEPT_SETTING: other" + ); + await writeFile(resolve(repo.projectDir, "hack.env.local.yaml"), layer); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: '"KEPT_SETTING" in hack.env.local.yaml', + hidden: [KEPT_VALUE], + }); + expect( + await readFile(resolve(repo.projectDir, "hack.env.local.yaml"), "utf8") + ).toBe(layer); +}); + +test("the migration doesn't copy a value the tracked layers already deliver", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY.replace("global: {}", "global:\n SAME: team") + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `SAME=team\nMINE=${KEPT_VALUE}\n` + ); + expect((await unmigratedIn(repo)).keys).toEqual(["MINE"]); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { MINE: KEPT_VALUE }, + }); +}); + +test("an overlay-only checkout is reported, and migrates", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_CONFIG_FILENAME), + `${JSON.stringify({ name: "env-legacy-scope", dev_host: "env-legacy-scope.hack", env: { defaultOverlay: "staging" } })}\n` + ); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + LAYER_EMPTY.replace("environment: default", "environment: staging") + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `ONLY_HERE=${KEPT_VALUE}\n` + ); + expect((await unmigratedIn(repo)).keys).toEqual(["ONLY_HERE"]); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { ONLY_HERE: KEPT_VALUE }, + }); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}); + +test("a contract project or a linked layer keeps the older behaviour", async () => { + const contract = await writeTeammateCheckout(); + await writeFile( + resolve(contract.projectDir, "hack.env.json"), + `${JSON.stringify({ version: 1, vars: [] })}\n` + ); + expect((await unmigratedIn(contract)).keys).toEqual([]); + + const linked = await writeTeammateCheckout(); + const target = resolve(linked.root, "elsewhere.yaml"); + await writeFile( + target, + LAYER_EMPTY.replace("environment: default", "environment: staging") + ); + await symlink(target, resolve(linked.projectDir, "hack.env.staging.yaml")); + await expectMigrationRefusal({ + repo: linked, + code: "E_PATH_CONFLICT", + reason: "env layer files already exist", + hidden: [KEPT_VALUE], + }); +}); + +test("diagnostics never echo a malformed entry, and it still blocks materialize", async () => { + const secret = ["hunter", "2-synthetic"].join(""); + for (const line of [ + `sk-${secret}=x`, + `postgres://u:${secret}@db/app?x=1`, + `SECRET ${secret}=1`, + `\u001b[31m${secret}=1`, ]) { - const repo = await writeTeammateCheckout(); - await place(repo); - expect( - await checkoutOnlyLegacyDotenvKeys({ projectDir: repo.projectDir }) - ).toEqual([]); + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile(resolve(repo.projectDir, ".env"), `${line}\n`); + const unmigrated = await unmigratedIn(repo); + expect(unmigrated.keys).toEqual([]); + expect(unmigrated.malformedCount).toBe(1); + const inspected = await inspectProjectEnvMaterialization({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + }); + expect(inspected.message).toContain("1 malformed entry"); + expect(inspected.message).not.toContain(secret); + const run = await runCli({ + repo, + args: ["env", "materialize", "--path", repo.projectRoot], + }); + expect(run.exitCode).not.toBe(0); + expect(`${run.stdout}${run.stderr}`).not.toContain(secret); + expect(await readFile(resolve(repo.projectDir, ".env"), "utf8")).toContain( + secret + ); + } +}, 120_000); + +test("an unreadable legacy file is reported and goes to the migration, which refuses it", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + const envPath = resolve(repo.projectDir, ".env"); + await writeFile(envPath, `KEPT_SETTING=${KEPT_VALUE}\n`); + await chmod(envPath, 0o000); + try { + expect((await unmigratedIn(repo)).unreadableFiles).toEqual([".env"]); + const inspected = await inspectProjectEnvMaterialization({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + }); + expect(inspected.message).toContain("can't read .env"); + expect(inspected.message).toContain("hack doctor --migrate-env-config"); + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as LegacyDotenvMigrationRefusalError).code).toBe( + "E_PERMISSION" + ); + } finally { + await chmod(envPath, 0o600); } }); From 6361682761d5c5daa495dd1fd5352ba03ec048c3 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:19:26 -0400 Subject: [PATCH 26/55] fix(env): refuse unreadable local layers and overlay disagreements on migration A local layer that failed to parse gave an empty default, and adding migrated keys then rewrote the file, losing its entries. It now refuses and leaves the file alone. A local overlay that disagreed with its legacy overlay could pass unnoticed when the entry equalled the base, leaving a key reported as undelivered that no further run could clear. It now refuses too. Adding to an existing local layer now makes it private, because the values were in a private .env and a layer from hack env set --local may be readable by others. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 7 +-- src/lib/project-env-config.ts | 65 +++++++++++++++++++++++-- tests/env-legacy-service-scope.test.ts | 66 ++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 6 deletions(-) diff --git a/docs/env.md b/docs/env.md index fd134ab44..3bfa57e67 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1042,9 +1042,10 @@ because their checkout now uses env layers. Hack compares those files with what deliver. For any value the layers don't deliver, `hack up` warns, doctor reports it, and `hack env materialize` refuses, so the file isn't replaced. They run `hack doctor --migrate-env-config` to move those values to their own local layers, adding to a -local layer that already exists. A value equal to what the layers deliver isn't copied, so later -team changes still reach them. A key the local layer sets differently refuses -(`E_PATH_CONFLICT`). Messages name valid keys, count malformed entries and name unreadable files, +local layer that already exists and making it private (mode 0600). A value equal to what the +layers deliver isn't copied, so later team changes still reach them. A key the local layer sets +differently refuses (`E_PATH_CONFLICT`), and so does a local layer that can't be read as an env +layer (`E_CONFIG_INVALID`), which is left as it is. Adding to a layer drops its YAML comments. Messages name valid keys, count malformed entries and name unreadable files, and never show a value. A `.hack/.env` with a valid materialization state is generated output, and this comparison skips it. Tell teammates when you commit the migration. diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 3bef3db1b..8181975d8 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -7,6 +7,7 @@ import { import { constants } from "node:fs"; import { access, + chmod, lstat, readdir, realpath, @@ -6354,6 +6355,46 @@ async function readValidatedLegacyDotenvFiles(opts: { return legacyValues; } +/** + * Refuses when an existing local overlay sets a key its legacy overlay file sets + * differently. Planning drops an overlay entry equal to the base, and this layer + * would keep winning over it, so the disagreement would pass unnoticed and leave + * a key reported as undelivered that no further migration could clear. + */ +async function assertLocalOverlaysAgreeWithLegacy(opts: { + readonly projectDir: string; + readonly overlayNames: readonly string[]; + readonly legacyValues: ReadonlyMap>; +}): Promise { + const conflicts: string[] = []; + for (const overlayName of opts.overlayNames) { + const path = resolveProjectEnvLocalConfigPath({ + projectDir: opts.projectDir, + envName: overlayName, + }); + const text = await readTextFile(path); + if (text === null) { + continue; + } + const global = + parseProjectEnvConfigText({ path, environment: overlayName, text }).config + .values.global ?? {}; + for (const [key, value] of Object.entries( + opts.legacyValues.get(overlayName) ?? {} + )) { + if (key in global && global[key] !== value) { + conflicts.push(`${JSON.stringify(key)} in ${basename(path)}`); + } + } + } + if (conflicts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `the legacy .env files and the local layer set ${conflicts.sort().join(", ")} to different values. Keep the one you want, by removing the key from the other, then retry.`, + }); + } +} + /** * Plans adding migrated keys to machine-local layers that already exist. Refuses * a key the layer sets to something else, because either value could be the one @@ -6379,11 +6420,21 @@ async function planLocalLayerMerges(opts: { if (text === null) { continue; } - const existing = parseProjectEnvConfigText({ + const read = parseProjectEnvConfigText({ path: write.path, environment: write.config.environment, text, - }).config; + }); + const entry = await lstat(write.path); + // A failed parse gives an empty default, and rewriting that would lose the + // layer's real entries. The refusal names the file, never its content. + if (read.parseError !== undefined || !entry.isFile()) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${basename(write.path)} can't be read as an env layer, so migrated values can't be added to it safely. Fix or move that file, then retry.`, + }); + } + const existing = read.config; const global = existing.values.global ?? {}; const added = write.config.values.global ?? {}; conflicts.push( @@ -6600,7 +6651,12 @@ async function migrateUndeclaredLegacyDotenv(opts: { }); } - // Before anything is written, so this refusal leaves the checkout unchanged. + // Before anything is written, so these refusals leave the checkout unchanged. + await assertLocalOverlaysAgreeWithLegacy({ + projectDir: opts.projectDir, + overlayNames, + legacyValues, + }); const merges = await planLocalLayerMerges({ localWrites }); for (const write of localWrites) { await assertUntrackedMigrationLocalLayer({ @@ -6630,6 +6686,9 @@ async function migrateUndeclaredLegacyDotenv(opts: { // Last, after everything that can roll back: each only adds keys, so a failure // here leaves the rest for the next run. for (const merge of merges) { + // The values were in a private .env. The atomic write keeps the file's mode, + // and a layer from `hack env set --local` may be readable by others. + await chmod(merge.path, 0o600); await writeProjectEnvConfigFile(merge); wroteFiles.push(merge.path); } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 14e9da6ed..23be36a0c 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -2402,3 +2402,69 @@ test("doctor names a checkout's unmigrated keys, not their values", async () => expect(inspected.message).toContain("hack doctor --migrate-env-config"); expect(inspected.message).not.toContain(KEPT_VALUE); }); + +for (const [name, layer] of [ + ["invalid YAML", "version: ["], + [ + "a newer version", + "version: 2\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global:\n KEEP: x\n", + ], +] as const) { + test(`the migration leaves a local layer with ${name} alone and refuses`, async () => { + const repo = await writeTeammateCheckout(); + const path = resolve(repo.projectDir, "hack.env.local.yaml"); + await writeFile(path, layer); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "hack.env.local.yaml can't be read as an env layer", + hidden: [KEPT_VALUE], + }); + expect(await readFile(path, "utf8")).toBe(layer); + }); +} + +test("the migration refuses a local overlay that disagrees with its legacy overlay, even when that entry equals the base", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + const local = LAYER_EMPTY.replace( + "environment: default", + "environment: staging" + ).replace("global: {}", "global:\n A: stage"); + const path = resolve(repo.projectDir, "hack.env.staging.local.yaml"); + await writeFile(path, local); + await writeFile(resolve(repo.projectDir, ".env"), `A=${KEPT_VALUE}\n`); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `A=${KEPT_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: '"A" in hack.env.staging.local.yaml', + hidden: [KEPT_VALUE], + }); + expect(await readFile(path, "utf8")).toBe(local); +}); + +test("adding migrated values to a readable local layer makes it private, and keeps its other entries", async () => { + const repo = await writeTeammateCheckout(); + const path = resolve(repo.projectDir, "hack.env.local.yaml"); + await writeFile( + path, + LAYER_EMPTY.replace( + "global: {}", + "global:\n OTHER: kept\n api:\n SCOPED: yes" + ) + ); + await chmod(path, 0o644); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect((await lstat(path)).mode & 0o777).toBe(0o600); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { OTHER: "kept", KEPT_SETTING: KEPT_VALUE }, + api: { SCOPED: "yes" }, + }); +}); From 1f23d033891fa448dc05816d4688c9190b8133fb Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:30:47 -0400 Subject: [PATCH 27/55] fix(env): refuse unreadable local layers and merge without following links readTextFile returns null on read errors, so an unreadable local layer looked absent: the publish step then skipped it, the value was written nowhere, and cleanup deleted .env. The migration now refuses a local layer it can't read or parse, and skips only layers it actually planned to merge into. Merging now uses the private-file helpers instead of chmod and the shared atomic write. They open no-follow, stage privately from creation and rename over the path itself, so a link swapped in can't redirect the write or the mode change. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 115 +++++++++++++++++++------ tests/env-legacy-service-scope.test.ts | 44 ++++++++++ 2 files changed, 132 insertions(+), 27 deletions(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 8181975d8..b57d55085 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -7,7 +7,6 @@ import { import { constants } from "node:fs"; import { access, - chmod, lstat, readdir, realpath, @@ -6355,6 +6354,42 @@ async function readValidatedLegacyDotenvFiles(opts: { return legacyValues; } +/** + * An existing machine-local layer the migration may add to, or null when absent. + * Refuses one it can't read or parse, or that isn't a regular file: `.config` + * would be an empty default, and treating it as absent or empty loses values. + * The refusal names the file, never its content. + */ +async function readExistingLocalLayer(opts: { + readonly path: string; + readonly environment: string; +}): Promise<{ + readonly text: string; + readonly config: ProjectEnvConfig; +} | null> { + const read = await readOptionalTextFile(opts.path); + if (read.kind === "absent") { + return null; + } + const parsed = + read.kind === "text" + ? parseProjectEnvConfigText({ ...opts, text: read.text }) + : null; + const entry = await lstat(opts.path).catch(() => null); + if ( + read.kind !== "text" || + parsed?.parseError !== undefined || + !entry?.isFile() || + !parsed + ) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `${basename(opts.path)} can't be read as an env layer, so migrated values can't be added to it safely. Fix or move that file, then retry.`, + }); + } + return { text: read.text, config: parsed.config }; +} + /** * Refuses when an existing local overlay sets a key its legacy overlay file sets * differently. Planning drops an overlay entry equal to the base, and this layer @@ -6372,13 +6407,14 @@ async function assertLocalOverlaysAgreeWithLegacy(opts: { projectDir: opts.projectDir, envName: overlayName, }); - const text = await readTextFile(path); - if (text === null) { + const layer = await readExistingLocalLayer({ + path, + environment: overlayName, + }); + if (layer === null) { continue; } - const global = - parseProjectEnvConfigText({ path, environment: overlayName, text }).config - .values.global ?? {}; + const global = layer.config.values.global ?? {}; for (const [key, value] of Object.entries( opts.legacyValues.get(overlayName) ?? {} )) { @@ -6395,6 +6431,37 @@ async function assertLocalOverlaysAgreeWithLegacy(opts: { } } +/** + * Replaces a local layer with mode 0600, because the values were in a private + * `.env` and a layer from `hack env set --local` may be readable by others. It + * doesn't follow a link at the path: the reader opens no-follow, staging is + * private from creation, and rename replaces the path itself. + */ +async function replaceLocalLayerPrivately(opts: { + readonly projectDir: string; + readonly path: string; + readonly config: ProjectEnvConfig; + readonly expectedExisting: string; +}): Promise { + const options = { + path: opts.path, + directories: [opts.projectDir], + refusal: () => + new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `${basename(opts.path)} changed during the migration or is not a regular file. Check it, then retry.`, + }), + }; + const current = await readReplaceablePrivateFile(options); + if (current?.text !== opts.expectedExisting) { + throw options.refusal(); + } + await replacePrivateFileAtomic({ + ...options, + text: serializeProjectEnvConfig(opts.config), + }); +} + /** * Plans adding migrated keys to machine-local layers that already exist. Refuses * a key the layer sets to something else, because either value could be the one @@ -6416,25 +6483,14 @@ async function planLocalLayerMerges(opts: { }[] = []; const conflicts: string[] = []; for (const write of opts.localWrites) { - const text = await readTextFile(write.path); - if (text === null) { - continue; - } - const read = parseProjectEnvConfigText({ + const layer = await readExistingLocalLayer({ path: write.path, environment: write.config.environment, - text, }); - const entry = await lstat(write.path); - // A failed parse gives an empty default, and rewriting that would lose the - // layer's real entries. The refusal names the file, never its content. - if (read.parseError !== undefined || !entry.isFile()) { - throw new LegacyDotenvMigrationRefusalError({ - code: "E_CONFIG_INVALID", - reason: `${basename(write.path)} can't be read as an env layer, so migrated values can't be added to it safely. Fix or move that file, then retry.`, - }); + if (layer === null) { + continue; } - const existing = read.config; + const existing = layer.config; const global = existing.values.global ?? {}; const added = write.config.values.global ?? {}; conflicts.push( @@ -6448,7 +6504,7 @@ async function planLocalLayerMerges(opts: { ...existing, values: { ...existing.values, global: { ...global, ...added } }, }, - expectedExisting: text, + expectedExisting: layer.text, }); } if (conflicts.length > 0) { @@ -6468,13 +6524,15 @@ async function publishMigratedLegacyLayers(opts: { readonly projectDir: string; readonly localWrites: readonly PlannedEnvLayerWrite[]; readonly trackedWrites: readonly PlannedEnvLayerWrite[]; + /** Local layers that exist and receive these values by merging instead. */ + readonly mergedPaths: ReadonlySet; }): Promise { // Machine-local layers first, so the tracked layers switch the format only // once the values have a home. const wroteFiles: string[] = []; try { for (const write of opts.localWrites) { - if (await pathExists(write.path)) { + if (opts.mergedPaths.has(write.path)) { continue; } // Created 0600 from the start: the values were in a private .env. @@ -6682,14 +6740,17 @@ async function migrateUndeclaredLegacyDotenv(opts: { projectDir: opts.projectDir, localWrites, trackedWrites, + mergedPaths: new Set(merges.map((merge) => merge.path)), }); // Last, after everything that can roll back: each only adds keys, so a failure // here leaves the rest for the next run. for (const merge of merges) { - // The values were in a private .env. The atomic write keeps the file's mode, - // and a layer from `hack env set --local` may be readable by others. - await chmod(merge.path, 0o600); - await writeProjectEnvConfigFile(merge); + await replaceLocalLayerPrivately({ + projectDir: opts.projectDir, + path: merge.path, + config: merge.config, + expectedExisting: merge.expectedExisting, + }); wroteFiles.push(merge.path); } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 23be36a0c..3ebf74e54 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -2468,3 +2468,47 @@ test("adding migrated values to a readable local layer makes it private, and kee api: { SCOPED: "yes" }, }); }); + +test("the migration refuses an unreadable local layer, so no value is lost and .env survives", async () => { + const repo = await writeTeammateCheckout(); + await writeFile(resolve(repo.projectDir, ".env.staging"), "OVERLAY_ONLY=2\n"); + const path = resolve(repo.projectDir, "hack.env.local.yaml"); + await writeFile(path, LAYER_EMPTY); + await chmod(path, 0o000); + try { + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as Error).message).toContain( + "hack.env.local.yaml can't be read as an env layer" + ); + expect(await readFile(resolve(repo.projectDir, ".env"), "utf8")).toContain( + KEPT_VALUE + ); + expect( + await lstat( + resolve(repo.projectDir, "hack.env.staging.local.yaml") + ).catch(() => null) + ).toBeNull(); + } finally { + await chmod(path, 0o600); + } +}); + +test("the migration refuses an unreadable local overlay that its legacy overlay could disagree with", async () => { + const repo = await writeTeammateCheckout(); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const path = resolve(repo.projectDir, "hack.env.staging.local.yaml"); + await writeFile(path, "version: ["); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "hack.env.staging.local.yaml can't be read as an env layer", + hidden: [KEPT_VALUE], + }); + expect(await readFile(path, "utf8")).toBe("version: ["); +}); From b3f8f0021989c2b52de92d2339f10940699d4027 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:35:34 -0400 Subject: [PATCH 28/55] fix(env): recheck a local layer's content just before the private rename Moving merges to the private-file writer dropped the shared writer's second compare, made after staging and fsync, so a change after the first read could be overwritten. replacePrivateFileAtomic now takes a beforeRename hook, and the merge rechecks the expected content there. Tests change the layer after the first read, and plant a link at the path, and confirm neither is overwritten or followed. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- src/lib/private-file.ts | 7 ++ src/lib/project-env-config.ts | 24 +++++-- tests/env-legacy-service-scope.test.ts | 97 ++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 5 deletions(-) diff --git a/src/lib/private-file.ts b/src/lib/private-file.ts index 307034a52..3014a1843 100644 --- a/src/lib/private-file.ts +++ b/src/lib/private-file.ts @@ -382,6 +382,12 @@ export async function replacePrivateFileAtomic( readonly text: string; /** Permission bits for the new file; defaults to 0600. */ readonly mode?: number; + /** + * Runs after staging and fsync, immediately before the rename. A throw + * abandons the replacement and removes the staging file, so a caller can + * make its last compare-and-swap check here. + */ + readonly beforeRename?: () => Promise; } ): Promise { const mode = (opts.mode ?? 0o600) & 0o777; @@ -400,6 +406,7 @@ export async function replacePrivateFileAtomic( await file.sync(); await recheckRealDirectories({ directories, refusal: opts.refusal }); await observeReplaceableTarget(opts); + await opts.beforeRename?.(); await rename(staging, opts.path); committed = true; } catch (error: unknown) { diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index b57d55085..3f7e88914 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6436,12 +6436,19 @@ async function assertLocalOverlaysAgreeWithLegacy(opts: { * `.env` and a layer from `hack env set --local` may be readable by others. It * doesn't follow a link at the path: the reader opens no-follow, staging is * private from creation, and rename replaces the path itself. + * + * The content is compared with `expectedExisting` before staging and again after + * the staging file is fsynced, immediately before the rename. As with the shared + * atomic writer, the second check narrows, but cannot close, the window in which + * a writer that does not use this check can still be overwritten. */ -async function replaceLocalLayerPrivately(opts: { +export async function replaceLocalLayerPrivately(opts: { readonly projectDir: string; readonly path: string; readonly config: ProjectEnvConfig; readonly expectedExisting: string; + /** Test seam: runs just before the last comparison. */ + readonly beforeFinalCheck?: () => Promise; }): Promise { const options = { path: opts.path, @@ -6452,13 +6459,20 @@ async function replaceLocalLayerPrivately(opts: { reason: `${basename(opts.path)} changed during the migration or is not a regular file. Check it, then retry.`, }), }; - const current = await readReplaceablePrivateFile(options); - if (current?.text !== opts.expectedExisting) { - throw options.refusal(); - } + const assertUnchanged = async (): Promise => { + const current = await readReplaceablePrivateFile(options); + if (current?.text !== opts.expectedExisting) { + throw options.refusal(); + } + }; + await assertUnchanged(); await replacePrivateFileAtomic({ ...options, text: serializeProjectEnvConfig(opts.config), + beforeRename: async () => { + await opts.beforeFinalCheck?.(); + await assertUnchanged(); + }, }); } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 3ebf74e54..eea6d1d5e 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -30,6 +30,7 @@ import { migrateLegacyProjectEnv, ProjectEnvLayerPathConflictError, projectEnvConfigExistsForMutation, + replaceLocalLayerPrivately, resolveProjectEnvConfig, selectProjectEnvValues, } from "../src/lib/project-env-config.ts"; @@ -2512,3 +2513,99 @@ test("the migration refuses an unreadable local overlay that its legacy overlay }); expect(await readFile(path, "utf8")).toBe("version: ["); }); + +async function localLayerFixture(): Promise<{ + readonly repo: Repo; + readonly path: string; + readonly config: { + readonly version: 1; + readonly environment: string; + readonly secretsprovider: "project_key"; + readonly values: { readonly global: { readonly NEW_KEY: string } }; + }; +}> { + const repo = await createLegacyRepo(); + const path = resolve(repo.projectDir, "hack.env.local.yaml"); + await writeFile(path, LAYER_EMPTY); + return { + repo, + path, + config: { + version: 1, + environment: "default", + secretsprovider: "project_key", + values: { global: { NEW_KEY: KEPT_VALUE } }, + }, + }; +} + +async function stagingFiles(repo: Repo): Promise { + return (await readdir(repo.projectDir)).filter((name) => + name.endsWith(".tmp") + ); +} + +test("a local layer that changed before the first read is refused and left alone", async () => { + const { repo, path, config } = await localLayerFixture(); + await writeFile(path, `${LAYER_EMPTY}# changed\n`); + const error = await replaceLocalLayerPrivately({ + projectDir: repo.projectDir, + path, + config, + expectedExisting: LAYER_EMPTY, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as Error).message).toContain("changed during the migration"); + expect(await readFile(path, "utf8")).toBe(`${LAYER_EMPTY}# changed\n`); +}); + +test("a local layer that changed after staging is refused at the last check, with nothing lost or left behind", async () => { + const { repo, path, config } = await localLayerFixture(); + const changed = `${LAYER_EMPTY}# changed after the first read\n`; + const error = await replaceLocalLayerPrivately({ + projectDir: repo.projectDir, + path, + config, + expectedExisting: LAYER_EMPTY, + beforeFinalCheck: () => writeFile(path, changed), + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as Error).message).toContain("changed during the migration"); + expect(await readFile(path, "utf8")).toBe(changed); + expect(await stagingFiles(repo)).toEqual([]); +}); + +test("a link planted at the local layer path after staging is refused, and its target is untouched", async () => { + const { repo, path, config } = await localLayerFixture(); + const outside = resolve(repo.root, "outside.yaml"); + await writeFile(outside, LAYER_EMPTY); + await chmod(outside, 0o644); + const error = await replaceLocalLayerPrivately({ + projectDir: repo.projectDir, + path, + config, + expectedExisting: LAYER_EMPTY, + beforeFinalCheck: async () => { + await rm(path); + await symlink(outside, path); + }, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect(await readFile(outside, "utf8")).toBe(LAYER_EMPTY); + expect((await lstat(outside)).mode & 0o777).toBe(0o644); + expect(await stagingFiles(repo)).toEqual([]); +}); + +test("an unchanged local layer is replaced privately, with the new content", async () => { + const { repo, path, config } = await localLayerFixture(); + await chmod(path, 0o644); + await replaceLocalLayerPrivately({ + projectDir: repo.projectDir, + path, + config, + expectedExisting: LAYER_EMPTY, + }); + expect((await lstat(path)).mode & 0o777).toBe(0o600); + expect(await readFile(path, "utf8")).toContain("NEW_KEY"); + expect(await stagingFiles(repo)).toEqual([]); +}); From 9b41047c3e516c1a6b2a1ad16b7ed17a48d3d1cb Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:06:31 -0500 Subject: [PATCH 29/55] fix(env): compare overlay values with the layers as they will be after migrating The planner skipped an overlay entry the tracked overlay delivers today, but the local base about to be written outranks a tracked overlay, so staging changed to the base value and doctor could then offer the legacy files for deletion. Overlay entries are now compared with the layers including the planned base write. A local overlay already holding the same value no longer counts as a conflict. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 9 +++- tests/env-legacy-service-scope.test.ts | 63 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 3f7e88914..cdb8fbb8a 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6509,7 +6509,7 @@ async function planLocalLayerMerges(opts: { const added = write.config.values.global ?? {}; conflicts.push( ...Object.keys(added) - .filter((key) => key in global) + .filter((key) => key in global && global[key] !== added[key]) .map((key) => `${JSON.stringify(key)} in ${basename(write.path)}`) ); merges.push({ @@ -6696,7 +6696,12 @@ async function migrateUndeclaredLegacyDotenv(opts: { overlayValues: legacyValues.get(overlayName) ?? {}, baseValues, }), - delivered: await deliveredLayerValues({ ...opts, envName: overlayName }), + // The local base about to be written outranks a tracked overlay, so compare + // with the layers as they will be, not as they are. + delivered: { + ...(await deliveredLayerValues({ ...opts, envName: overlayName })), + ...baseToMigrate, + }, }); if (Object.keys(changedValues).length === 0) { continue; diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index eea6d1d5e..ef20ba55f 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -2609,3 +2609,66 @@ test("an unchanged local layer is replaced privately, with the new content", asy expect(await readFile(path, "utf8")).toContain("NEW_KEY"); expect(await stagingFiles(repo)).toEqual([]); }); + +async function stagingValue(repo: Repo): Promise { + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: "staging", + serviceNames: ["api"], + }); + return resolved?.globalEnv.PORT; +} + +const TRACKED_STAGING = LAYER_EMPTY.replace( + "environment: default", + "environment: staging" +).replace("global: {}", "global:\n PORT: stage"); + +test("the migration keeps an overlay value the tracked overlay delivers today, when the new local base would outrank it", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + TRACKED_STAGING + ); + await writeFile(resolve(repo.projectDir, ".env"), "PORT=legacy\n"); + await writeFile(resolve(repo.projectDir, ".env.staging"), "PORT=stage\n"); + expect(await stagingValue(repo)).toBe("stage"); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await stagingValue(repo)).toBe("stage"); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { PORT: "legacy" }, + }); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { PORT: "stage" }, + }); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}); + +test("the migration accepts a local overlay already holding the value the overlay file sets", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.local.yaml"), + TRACKED_STAGING + ); + await writeFile(resolve(repo.projectDir, ".env"), "PORT=legacy\n"); + await writeFile(resolve(repo.projectDir, ".env.staging"), "PORT=stage\n"); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await stagingValue(repo)).toBe("stage"); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { PORT: "legacy" }, + }); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}); From d88ff02f9a5ed025ba2ea6c4641502d6c4539e20 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:21:27 -0500 Subject: [PATCH 30/55] fix(env): refuse a migration that would widen values to services that lack them Migrated values are global and reach every service, including third-party images. A .env no service lists feeds only Compose interpolation today, so migrating it put credentials in containers that never had them, and a note afterwards didn't prevent that. Each service must now list each file with values to migrate, which is also how a developer accepts global delivery. This includes non-default overlays. A migrated base value that would replace a tracked overlay's value its legacy overlay doesn't set now refuses too. An overlay entry equal to the base is compared with what the layers will deliver, so it no longer stays reported with nothing left to migrate. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 25 ++-- src/commands/doctor.ts | 9 +- src/lib/project-env-config.ts | 135 +++++++++------------ tests/doctor-fix-noninteractive.test.ts | 18 +-- tests/env-legacy-service-scope.test.ts | 154 +++++++++++++++++------- 5 files changed, 194 insertions(+), 147 deletions(-) diff --git a/docs/env.md b/docs/env.md index 3bfa57e67..0b8084ce5 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1088,11 +1088,14 @@ files, when: switch to the `.env` value. This includes an entry that only refers to its own key (`PORT: ${PORT}`), whose result can differ when an overlay or the shell sets the key. The error names the keys and services, never values. Remove the key from the `.hack/.env` file or from that service's `environment`, then retry; -- some Compose services list a legacy file and others do not, so a service would receive - values it never had (`E_CONFIG_INVALID`), because migrated values are global and reach every - service. The error names files, services and keys, never values. List the file for those - services if they may have its values, or move the values out of it and set them with - `hack env set --service` after migrating; +- a service does not list a legacy file with values to migrate in `env_file` (`E_CONFIG_INVALID`). + Migrated values are global and reach every service, including third-party images, so that + service would receive values it never had. This includes a file no service lists, whose values + today reach only Compose interpolation. Listing the file for each service is how you accept + that. Otherwise, move those values out of the file and set them with + `hack env set --service` after migrating, or keep them in your shell. Values Compose interpolates + need the shell or a listed file, because service values do not reach interpolation. The error + names files, services and keys, never values; - Compose uses a top-level `include`, a service `extends`, a `!reset` or `!override` tag, a variable or a `~` in an `env_file` path, a file listed twice in one `env_file`, or a `format` on an `env_file` entry (`E_CONFIG_INVALID`), so the migration cannot tell which values services @@ -1117,6 +1120,9 @@ files, when: entry, a secret's `environment:`), or it is a `COMPOSE_*` setting (`E_CONFIG_INVALID`). Compose takes it from `.hack/.env` today, but the overlay value would apply after the migration. Make the two files agree or stop using the key in Compose; +- a migrated `.hack/.env` value would replace what a tracked overlay delivers for a key its legacy + overlay file doesn't set (`E_CONFIG_INVALID`), because a local base outranks tracked overlays. Set + the value you want in that legacy overlay file, or remove the key from `.hack/.env`; - a legacy file sets a key that Docker, its build tools or the process read from their environment (`E_CONFIG_INVALID`): `DOCKER_*`, `BUILDKIT_*`, `BUILDX_*`, a proxy setting, `SOURCE_DATE_EPOCH`, `HOME` or `PATH`. After the migration it would reach the Docker client too. @@ -1132,15 +1138,6 @@ files, when: that resolves to a readable file inside `.hack` is migrated like a regular one; cleanup removes only the link, never its target. -If no Compose service lists a legacy file, its values may only be used for Compose -interpolation today. The migration goes ahead, every service receives them as environment -variables once that env is selected, and doctor names the keys. The same applies to an overlay -no service lists, unless it is the default overlay, which refuses when any service lists a legacy -file. To limit a key to one service, unset it and set it with `hack env set --service --local`. -Do not do this for a key your Compose file interpolates: service values do not reach -interpolation, so `${KEY}` would become empty. After the migration a migrated value also takes -precedence over the same variable exported in your shell, including for interpolation. - With `hack.env.json`, the migration discovers only regular `.hack/.env.` files, so a symlinked overlay is not migrated. It refuses (`E_PATH_CONFLICT`) when `.hack/.env` or a configured default overlay is a symlink, wherever it points, because this migration writes diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 5989e574c..1ce393ac0 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -3700,12 +3700,9 @@ function buildEnvMigrationNote(opts: { if (migrated.updatedProjectConfig) { notes.push("Updated .hack/hack.config.json for the modern env format."); } - const newKeys = migrated.keysNewToEveryService ?? []; - if (newKeys.length > 0) { - notes.push( - `No Compose service listed the legacy file that sets these, so every service now receives them once that env is selected: ${newKeys.join(", ")}. To limit a key to one service, run \`hack env unset KEY --local\` and then \`hack env set KEY --service --local\`, but not for a key your Compose file interpolates: service values do not reach interpolation. Host lifecycle hooks receive these values too.` - ); - } + notes.push( + "Migrated values are global: every service and host lifecycle hook now receives them." + ); return notes.join("\n"); } diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index cdb8fbb8a..eb8e059d2 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -4941,8 +4941,6 @@ export type LegacyProjectEnvMigration = { readonly cleanupCandidates: readonly string[]; readonly blockedCleanupCandidates: readonly string[]; readonly composeEnvFileReferences: readonly LegacyComposeEnvFileReference[]; - /** Keys every service receives once their env is selected, from files no service listed. Names only. */ - readonly keysNewToEveryService?: readonly string[]; }; /** @@ -5709,21 +5707,19 @@ function composeServiceLegacyEnvNames(opts: { /** * Before the migration a legacy dotenv file listed in `env_file` reached only * the services that list it; after it, its values are global layer values that - * Hack applies to every service. When any service lists a legacy file, returns - * refusal labels (file, service and key names only) for each file with values - * to migrate that only some services list, for a default overlay some service - * does not list, and for each service that does not list a file setting a base - * key. + * Hack applies to every service, including third-party images. So each service + * must list each file with values to migrate, which is also how a developer + * consents to that exposure. Returns refusal labels (file, service and key names + * only) for the services that don't. * - * A project where no service lists a legacy file has no `env_file` boundary to - * compare, and its values may be needed for interpolation, so it is not refused - * here; {@link legacyKeysNewToEveryService} reports what those services gain. + * This includes a file no service lists, whose values today reach only Compose + * interpolation: migrating it would put credentials in containers that never had + * them, and a warning afterwards wouldn't prevent that. */ function composeServicesMissingLegacyDotenv(opts: { readonly projectDir: string; readonly services: Record; readonly legacyValues: ReadonlyMap>; - readonly defaultOverlay: string | null; }): string[] { const baseValues = opts.legacyValues.get(null) ?? {}; const services = composeServiceEntries(opts.services).map( @@ -5736,9 +5732,6 @@ function composeServicesMissingLegacyDotenv(opts: { }), }) ); - if (services.every((service) => service.listed.length === 0)) { - return []; - } const labels: string[] = []; for (const [envName, values] of opts.legacyValues) { const migrated = @@ -5752,14 +5745,7 @@ function composeServicesMissingLegacyDotenv(opts: { .filter((service) => !service.listed.includes(envName)) .map((service) => service.label) .sort(); - // Every launch selects the default overlay, so its values reach every - // service even when no service lists it. - const reachesEveryLaunch = - envName !== null && envName === opts.defaultOverlay; - if ( - missing.length > 0 && - (missing.length < services.length || reachesEveryLaunch) - ) { + if (missing.length > 0) { labels.push( `${missing.join(", ")} ${missing.length === 1 ? "does" : "do"} not list ${JSON.stringify(basename(resolveEnvFilePath({ projectDir: opts.projectDir, envName })))} in env_file` ); @@ -5798,7 +5784,7 @@ function assertComposeServicesKeepLegacyValues(opts: { if (unlisted.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `${unlisted.join("; ")}. Migrated values are global and reach every service, so those services would receive values they never had. List the file in env_file for those services if they may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, + reason: `${unlisted.join("; ")}. Migrated values are global and reach every service, so those services would receive values they never had, including a value that today only feeds Compose interpolation. List the file in env_file for each service that may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, }); } const orderConflicts = composeLegacyEnvFileOrderConflicts(opts); @@ -5852,52 +5838,6 @@ function composeLegacyEnvFileOrderConflicts(opts: { return conflicts.sort(); } -/** - * Migrated keys from legacy files no service lists in `env_file`, for the - * migration to name: they reach every service once their env is selected. - * Files a service lists are covered by {@link composeServicesMissingLegacyDotenv}. - */ -async function legacyKeysNewToEveryService(opts: { - readonly projectDir: string; - readonly legacyValues: ReadonlyMap>; -}): Promise { - const parsed = await readComposeFile({ - composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), - }); - const services = parsed ? getRecord(parsed, "services") : null; - if (!services) { - return []; - } - const entries = composeServiceEntries(services); - if (entries.length === 0) { - return []; - } - const listed = new Set( - entries.flatMap(([, rawService]) => - composeServiceLegacyEnvNames({ - projectDir: opts.projectDir, - rawService, - legacyValues: opts.legacyValues, - }) - ) - ); - const baseValues = opts.legacyValues.get(null) ?? {}; - const keys = new Set(); - for (const [envName, values] of opts.legacyValues) { - if (listed.has(envName)) { - continue; - } - const migrated = - envName === null - ? values - : legacyOverlayChangedValues({ overlayValues: values, baseValues }); - for (const key of Object.keys(migrated)) { - keys.add(key); - } - } - return [...keys].sort((left, right) => left.localeCompare(right)); -} - /** * Entries the current layers don't deliver with the same value. With no layers * that is all of them. Skipping the rest keeps a teammate from pinning a team @@ -6354,6 +6294,48 @@ async function readValidatedLegacyDotenvFiles(opts: { return legacyValues; } +/** + * Refuses when a migrated base value would replace what a tracked overlay + * delivers for a key its legacy overlay file doesn't set. The local base outranks + * tracked overlays, so that env would silently change value and stop receiving + * later team changes to the key. Names keys and overlays, never values. + */ +async function assertBaseDoesNotOutrankTrackedOverlays(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly baseToMigrate: Record; + readonly legacyValues: ReadonlyMap>; +}): Promise { + const conflicts: string[] = []; + for (const overlayName of await listProjectEnvOverlayNames({ + projectDir: opts.projectDir, + })) { + const delivered = await deliveredLayerValues({ + ...opts, + envName: overlayName, + }); + const legacyOverlay = opts.legacyValues.get(overlayName) ?? {}; + for (const [key, value] of Object.entries(opts.baseToMigrate)) { + if ( + key in delivered && + delivered[key] !== value && + !(key in legacyOverlay) + ) { + conflicts.push( + `${JSON.stringify(key)} for ${JSON.stringify(overlayName)}` + ); + } + } + } + if (conflicts.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `.hack/.env sets ${conflicts.sort().join(", ")}, and its migrated value would replace what that overlay's env layers deliver. Set the value you want in that legacy overlay file, or remove the key from .hack/.env, then retry.`, + }); + } +} + /** * An existing machine-local layer the migration may add to, or null when absent. * Refuses one it can't read or parse, or that isn't a regular file: `.config` @@ -6692,10 +6674,7 @@ async function migrateUndeclaredLegacyDotenv(opts: { } for (const overlayName of overlayNames) { const changedValues = legacyValuesNotDelivered({ - values: legacyOverlayChangedValues({ - overlayValues: legacyValues.get(overlayName) ?? {}, - baseValues, - }), + values: legacyValues.get(overlayName) ?? {}, // The local base about to be written outranks a tracked overlay, so compare // with the layers as they will be, not as they are. delivered: { @@ -6729,6 +6708,11 @@ async function migrateUndeclaredLegacyDotenv(opts: { } // Before anything is written, so these refusals leave the checkout unchanged. + await assertBaseDoesNotOutrankTrackedOverlays({ + ...opts, + baseToMigrate, + legacyValues, + }); await assertLocalOverlaysAgreeWithLegacy({ projectDir: opts.projectDir, overlayNames, @@ -6782,10 +6766,6 @@ async function migrateUndeclaredLegacyDotenv(opts: { }); } - const keysNewToEveryService = await legacyKeysNewToEveryService({ - projectDir: opts.projectDir, - legacyValues, - }); const composeEnvFileReferences = await inspectLegacyComposeEnvFileReferences({ composeFile: resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME), projectDir: opts.projectDir, @@ -6818,7 +6798,6 @@ async function migrateUndeclaredLegacyDotenv(opts: { cleanupCandidates: allowedCleanup, blockedCleanupCandidates: blockedCleanup, composeEnvFileReferences, - keysNewToEveryService, }; } diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 3cbf44c61..26d8b9bf7 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -214,7 +214,7 @@ async function createProjectWithStaleDaemonPid(): Promise { await mkdir(resolve(projectRoot, ".hack"), { recursive: true }); await writeFile( resolve(projectRoot, ".hack", "docker-compose.yml"), - "services:\n api:\n image: alpine:3.19\n" + "services:\n api:\n image: alpine:3.19\n env_file:\n - path: .env\n required: false\n" ); // Stale daemon pid: a pid that cannot be running (very large, unlikely to @@ -692,10 +692,14 @@ test("doctor --migrate-env-config treats a symlinked overlay as env layers and n } }, 30_000); -test("doctor --migrate-env-config names the keys every service gains, never their values (HACK-1370)", async () => { +test("doctor --migrate-env-config refuses a .env no service lists, and keeps it (HACK-1370)", async () => { const projectRoot = await createProjectWithStaleDaemonPid(); const projectDir = resolve(projectRoot, ".hack"); const legacy = ["synthetic", "legacy", "4c2e"].join("-"); + await writeFile( + resolve(projectDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.19\n" + ); await writeFile(resolve(projectDir, ".env"), `OWNED=${legacy}\n`); const { runCli } = await import("../src/cli/run.ts"); @@ -706,11 +710,11 @@ test("doctor --migrate-env-config names the keys every service gains, never thei "--path", projectRoot, ]); - expect(code).toBe(0); - const notes = noteCalls.join("\n"); - expect(notes).toContain("once that env is selected: OWNED."); - expect(notes).toContain("not for a key your Compose file interpolates"); - expect(notes).not.toContain(legacy); + expect(code).not.toBe(0); + expect(await readFile(resolve(projectDir, ".env"), "utf8")).toContain(legacy); + expect( + await lstat(resolve(projectDir, "hack.env.local.yaml")).catch(() => null) + ).toBeNull(); }, 30_000); test("doctor --migrate-env-config moves a checkout's own .env after a teammate's migration landed (HACK-1370)", async () => { diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index ef20ba55f..137eb321c 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -76,6 +76,27 @@ type Repo = { readonly storePath: string; }; +/** Every service lists the base file, which is how a project accepts global delivery. */ +const LISTED_COMPOSE = + "services:\n api:\n image: alpine:3.20\n env_file:\n - path: .env\n required: false\n"; + +/** Selects staging on every launch and lists it, so its values can migrate. */ +async function useStagingAsDefaultOverlay(repo: Repo): Promise { + const configPath = resolve(repo.projectDir, PROJECT_CONFIG_FILENAME); + const config = JSON.parse(await readFile(configPath, "utf8")) as Record< + string, + unknown + >; + await writeFile( + configPath, + `${JSON.stringify({ ...config, env: { defaultOverlay: "staging" } })}\n` + ); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n env_file:\n - path: .env\n required: false\n - path: .env.staging\n required: false\n" + ); +} + async function createLegacyRepo(): Promise { const root = await mkdtemp(join(tmpdir(), "hack-env-legacy-scope-")); tempDirs.add(root); @@ -85,7 +106,7 @@ async function createLegacyRepo(): Promise { await mkdir(projectDir, { recursive: true }); await writeFile( resolve(projectDir, PROJECT_COMPOSE_FILENAME), - "services:\n api:\n image: alpine:3.20\n" + LISTED_COMPOSE ); await writeFile( resolve(projectDir, PROJECT_CONFIG_FILENAME), @@ -547,6 +568,7 @@ async function serviceScopedSet(repo: Repo): Promise { test("the migration the refusal names converts a project with only .hack/.env, and a service-scoped set then works", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, ".env"), `KEPT_SETTING=${KEPT_VALUE}\nSHARED_SETTING=${BASE_ONLY_VALUE}\n` @@ -603,11 +625,12 @@ test("the migration the refusal names converts a project with only .hack/.env, a // and its secret backend store are left alone and never offered for cleanup. expect(migrated.updatedProjectConfig).toBe(false); expect(await readFile(configPath, "utf8")).toBe(configBefore); - expect(migrated.cleanupCandidates).toEqual([ + // Compose still lists these, so they wait for the Compose repair before cleanup. + expect(migrated.blockedCleanupCandidates).toEqual([ resolve(repo.projectDir, ".env"), resolve(repo.projectDir, ".env.staging"), ]); - expect(migrated.blockedCleanupCandidates).toEqual([]); + expect(migrated.cleanupCandidates).toEqual([]); // The refusal no longer applies: the set lands in the api scope. const run = await serviceScopedSet(repo); @@ -651,6 +674,7 @@ async function git(opts: { test("the migration keeps secret-shaped values from the gitignored .hack/.env out of every file git would commit", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await git({ repo, args: ["init", "-q"] }); await writeFile( resolve(repo.projectDir, ".env"), @@ -1025,6 +1049,7 @@ test("the migration refuses a legacy .env. whose name is not an overlay na test("the migration leaves template and backup copies alone and never offers a git-tracked legacy file for cleanup", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await git({ repo, args: ["init", "-q"] }); await writeFile( resolve(repo.projectDir, ".env"), @@ -1049,7 +1074,8 @@ test("the migration leaves template and backup copies alone and never offers a g expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ global: { KEPT_SETTING: STAGING_VALUE }, }); - expect(migrated.cleanupCandidates).toEqual([ + // Compose still lists these, so they wait for the Compose repair before cleanup. + expect(migrated.blockedCleanupCandidates).toEqual([ resolve(repo.projectDir, ".env"), ]); for (const [name, text] of Object.entries(copies)) { @@ -1119,6 +1145,7 @@ test.skipIf(RUNS_AS_ROOT)( test("the migration creates machine-local layers as private 0600 files", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, ".env"), `KEPT_SETTING=${KEPT_VALUE}\n` @@ -1141,6 +1168,7 @@ test("the migration creates machine-local layers as private 0600 files", async ( test("the migration carries a symlinked legacy overlay's values and refuses one it cannot read", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, ".env"), `KEPT_SETTING=${KEPT_VALUE}\n` @@ -1161,7 +1189,8 @@ test("the migration carries a symlinked legacy overlay's values and refuses one "utf8" ) ).toContain(`${KEPT_VALUE}-linked`); - expect(migrated.cleanupCandidates).toContain(link); + // Compose still lists it, so it waits for the Compose repair before cleanup. + expect(migrated.blockedCleanupCandidates).toContain(link); expect(await readFile(target, "utf8")).toBe(targetText); for (const place of [ @@ -1209,7 +1238,7 @@ test("the migration refuses when Compose uses an overlay a launch would not sele const compose = resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME); await writeFile( compose, - "services:\n api:\n image: alpine:3.20\n env_file: .env.staging\n" + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n" ); await writeFile(resolve(repo.projectDir, ".env"), "SHARED=base\n"); await writeFile( @@ -1249,7 +1278,7 @@ test("the migration accepts a Compose overlay that adds nothing to the base", as const repo = await createLegacyRepo(); await writeFile( resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), - "services:\n api:\n image: alpine:3.20\n env_file: .env.staging\n" + "services:\n api:\n image: alpine:3.20\n env_file: [.env, .env.staging]\n" ); await writeFile( resolve(repo.projectDir, ".env"), @@ -1490,23 +1519,25 @@ test("the migration refuses a legacy overlay only some services list, and accept expect(migrated.legacyDetected).toBe(true); }); -test("the migration goes ahead when no Compose service lists the legacy file", async () => { +test("the migration refuses a legacy file no service lists, whose values only feed Compose interpolation", async () => { const repo = await createLegacyRepo(); await writeFile( resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), - "services:\n api:\n image: alpine:3.20\n worker:\n image: alpine:3.20\n" + "services:\n db:\n image: alpine:3.20\n command: serve --password ${KEPT_SETTING}\n third_party:\n image: alpine:3.20\n" ); await writeFile( resolve(repo.projectDir, ".env"), `KEPT_SETTING=${KEPT_VALUE}\n` ); - const migrated = await migrateLegacyRepo(repo, { - migrateUndeclaredDotenv: true, - }); - expect(migrated.legacyDetected).toBe(true); - expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ - global: { KEPT_SETTING: KEPT_VALUE }, + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"db", "third_party" do not list ".env" in env_file', + hidden: [KEPT_VALUE], }); + expect(await readFile(resolve(repo.projectDir, ".env"), "utf8")).toContain( + KEPT_VALUE + ); }); for (const entry of [ @@ -1668,20 +1699,7 @@ for (const [name, compose, reason] of UNCHECKABLE_COMPOSE) { }); } -test("the migration names the keys every service gains when no service lists the legacy file", async () => { - const repo = await createLegacyRepo(); - await writeFile( - resolve(repo.projectDir, ".env"), - `KEPT_SETTING=${KEPT_VALUE}\nOTHER=1\n` - ); - const migrated = await migrateLegacyRepo(repo, { - migrateUndeclaredDotenv: true, - }); - expect(migrated.keysNewToEveryService).toEqual(["KEPT_SETTING", "OTHER"]); - expect(JSON.stringify(migrated)).not.toContain(KEPT_VALUE); -}); - -test("the migration reports no new keys when every service lists the legacy file", async () => { +test("the migration accepts every path spelling of a listed legacy file", async () => { const repo = await createLegacyRepo(); await writeFile( resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), @@ -1694,7 +1712,7 @@ test("the migration reports no new keys when every service lists the legacy file const migrated = await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true, }); - expect(migrated.keysNewToEveryService).toEqual([]); + expect(migrated.legacyDetected).toBe(true); }); for (const [name, link, config] of [ @@ -1811,21 +1829,34 @@ test("the migration refuses a default overlay no service lists when a service li }); }); -test("the migration names an overlay's keys when no service lists that non-default overlay", async () => { +test("the migration refuses a non-default overlay with its own values that no service lists", async () => { const repo = await createLegacyRepo(); - await writeFile( - resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), - "services:\n api:\n image: alpine:3.20\n env_file: .env\n" - ); await writeFile(resolve(repo.projectDir, ".env"), "A=1\n"); await writeFile( resolve(repo.projectDir, ".env.staging"), `NEW=${KEPT_VALUE}\n` ); - const migrated = await migrateLegacyRepo(repo, { - migrateUndeclaredDotenv: true, + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"api" does not list ".env.staging" in env_file', + hidden: [KEPT_VALUE], + }); +}); + +test("a checkout whose Compose was already repaired refuses rather than widening its values", async () => { + const repo = await writeTeammateCheckout(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n third_party:\n image: alpine:3.20\n" + ); + expect((await unmigratedIn(repo)).keys).toEqual(["KEPT_SETTING"]); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: 'do not list ".env" in env_file', + hidden: [KEPT_VALUE], }); - expect(migrated.keysNewToEveryService).toEqual(["NEW"]); }); for (const reference of ["${PORT}", "$PORT", "${PORT:-1}"]) { @@ -2256,10 +2287,7 @@ test("the migration doesn't copy a value the tracked layers already deliver", as test("an overlay-only checkout is reported, and migrates", async () => { const repo = await createLegacyRepo(); - await writeFile( - resolve(repo.projectDir, PROJECT_CONFIG_FILENAME), - `${JSON.stringify({ name: "env-legacy-scope", dev_host: "env-legacy-scope.hack", env: { defaultOverlay: "staging" } })}\n` - ); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, "hack.env.default.yaml"), LAYER_EMPTY @@ -2427,6 +2455,7 @@ for (const [name, layer] of [ test("the migration refuses a local overlay that disagrees with its legacy overlay, even when that entry equals the base", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, "hack.env.default.yaml"), LAYER_EMPTY @@ -2472,6 +2501,7 @@ test("adding migrated values to a readable local layer makes it private, and kee test("the migration refuses an unreadable local layer, so no value is lost and .env survives", async () => { const repo = await writeTeammateCheckout(); + await useStagingAsDefaultOverlay(repo); await writeFile(resolve(repo.projectDir, ".env.staging"), "OVERLAY_ONLY=2\n"); const path = resolve(repo.projectDir, "hack.env.local.yaml"); await writeFile(path, LAYER_EMPTY); @@ -2499,6 +2529,7 @@ test("the migration refuses an unreadable local layer, so no value is lost and . test("the migration refuses an unreadable local overlay that its legacy overlay could disagree with", async () => { const repo = await writeTeammateCheckout(); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, ".env.staging"), `KEPT_SETTING=${KEPT_VALUE}\n` @@ -2627,6 +2658,7 @@ const TRACKED_STAGING = LAYER_EMPTY.replace( test("the migration keeps an overlay value the tracked overlay delivers today, when the new local base would outrank it", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, "hack.env.default.yaml"), LAYER_EMPTY @@ -2653,6 +2685,7 @@ test("the migration keeps an overlay value the tracked overlay delivers today, w test("the migration accepts a local overlay already holding the value the overlay file sets", async () => { const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); await writeFile( resolve(repo.projectDir, "hack.env.default.yaml"), LAYER_EMPTY @@ -2672,3 +2705,40 @@ test("the migration accepts a local overlay already holding the value the overla }); expect((await unmigratedIn(repo)).keys).toEqual([]); }); + +test("the migration refuses a base value that would replace a tracked overlay's value its legacy overlay doesn't set", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + TRACKED_STAGING + ); + await writeFile(resolve(repo.projectDir, ".env"), `PORT=${KEPT_VALUE}\n`); + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '.hack/.env sets "PORT" for "staging"', + hidden: [KEPT_VALUE], + }); + expect(await stagingValue(repo)).toBe("stage"); +}); + +test("an overlay entry equal to a base the tracked default delivers is migrated, so the key doesn't stay reported", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY.replace("global: {}", "global:\n PORT: legacy") + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + TRACKED_STAGING + ); + await writeFile(resolve(repo.projectDir, ".env"), "PORT=legacy\n"); + await writeFile(resolve(repo.projectDir, ".env.staging"), "PORT=legacy\n"); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await stagingValue(repo)).toBe("legacy"); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}); From ca5ef35fc2e4a3e7b7f76edf954001ac118d882c Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:32:39 -0500 Subject: [PATCH 31/55] fix(env): correct migration refusal guidance and pin the overlay regressions The refusal offered hack env set --service for a value that only feeds Compose interpolation, which silently drops it. It now says to keep that in the shell or list the file, and names --env for an overlay. A listed non-default overlay's message no longer sends people back to the refusal they came from. The tracked-overlay refusal now fires only when that overlay's layers actually override the key, so an empty tracked overlay no longer refuses. doctor's global delivery note is limited to the migration without a contract. Adds regression tests for an equal-to-base overlay entry a tracked overlay overrides, for an unlisted overlay identical to the base, and for a git-tracked legacy file staying out of cleanup. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 4 + src/commands/doctor.ts | 15 +++- src/commands/project.ts | 2 +- src/lib/project-env-config.ts | 7 +- tests/doctor-fix-noninteractive.test.ts | 2 +- tests/env-legacy-service-scope.test.ts | 103 ++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 8 deletions(-) diff --git a/docs/env.md b/docs/env.md index 0b8084ce5..9b9d2b197 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1049,6 +1049,10 @@ layer (`E_CONFIG_INVALID`), which is left as it is. Adding to a layer drops its and never show a value. A `.hack/.env` with a valid materialization state is generated output, and this comparison skips it. Tell teammates when you commit the migration. +Migrated values are global, so a service added later receives them too, and so do host lifecycle +hooks. A migrated value also takes precedence over the same variable exported in your shell, +including for Compose interpolation. + This migration refuses, with a one-line `E_*` error, writes nothing and keeps the legacy files, when: diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 1ce393ac0..c28c5097b 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -3694,15 +3694,19 @@ function renderSkippedDoctorFixSteps(): void { function buildEnvMigrationNote(opts: { readonly migrated: LegacyProjectEnvMigration; + /** Only the migration without a contract makes every value global. */ + readonly globalDelivery: boolean; }): string { const { migrated } = opts; const notes = [`Wrote ${migrated.wroteFiles.join(", ")}`]; if (migrated.updatedProjectConfig) { notes.push("Updated .hack/hack.config.json for the modern env format."); } - notes.push( - "Migrated values are global: every service and host lifecycle hook now receives them." - ); + if (opts.globalDelivery) { + notes.push( + "Migrated values are global: every service, including one added later, and every host lifecycle hook now receives them." + ); + } return notes.join("\n"); } @@ -3792,7 +3796,10 @@ async function maybeMigrateProjectEnvConfig(opts: { note("Legacy env migration found nothing to write.", "doctor"); return; } - note(buildEnvMigrationNote({ migrated }), "doctor"); + note( + buildEnvMigrationNote({ migrated, globalDelivery: !hasContract }), + "doctor" + ); let composeRepaired = false; if (migrated.composeEnvFileReferences.length > 0) { diff --git a/src/commands/project.ts b/src/commands/project.ts index d39a93892..1bb92ebf6 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -1330,7 +1330,7 @@ export async function resolveModernComposeEnvOverrides(opts: { }); if (hasUnmigratedLegacyDotenv(unmigrated)) { logger.warn({ - message: `This checkout ${describeUnmigratedLegacyDotenv(unmigrated)} in its legacy .hack/.env files, and the env layers don't deliver those values, so services that rely on them may not get them. Run \`hack doctor --migrate-env-config\` to move them to this checkout's local layers.`, + message: `This checkout ${describeUnmigratedLegacyDotenv(unmigrated)} in its legacy .hack/.env files, and the env layers don't deliver those values, so services that rely on them may not get them. Run \`hack doctor --migrate-env-config\`, which moves them to this checkout's local layers or says what to change first.`, }); } const lifecycleEnv = selectProjectEnvValuesForExecutionTarget({ diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index eb8e059d2..c39422579 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5333,7 +5333,7 @@ async function assertComposeOverlaysStaySelected(opts: { if (unselected.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `Compose env_file uses ${legacyOverlayFileList(unselected)}, but a launch without --env would not select ${unselected.length === 1 ? "that overlay" : "those overlays"}, so the service would lose its values. Set env.defaultOverlay to the overlay every launch should use, or remove the env_file entry and select the overlay with --env, then retry.`, + reason: `Compose env_file uses ${legacyOverlayFileList(unselected)}, but a launch without --env would not select ${unselected.length === 1 ? "that overlay" : "those overlays"}, so the service would lose its values. Set env.defaultOverlay to the overlay every launch should use, or move its values out of the file and set them with \`hack env set --env --service\` after migrating, then retry.`, }); } } @@ -5784,7 +5784,7 @@ function assertComposeServicesKeepLegacyValues(opts: { if (unlisted.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `${unlisted.join("; ")}. Migrated values are global and reach every service, so those services would receive values they never had, including a value that today only feeds Compose interpolation. List the file in env_file for each service that may have its values, or move the values out of it and set them with \`hack env set --service\` after migrating, then retry.`, + reason: `${unlisted.join("; ")}. Migrated values are global and reach every service, so those services would receive values they never had, including a value that today only feeds Compose interpolation. List the file in env_file for each service that may have its values (an overlay must also be env.defaultOverlay), or move the values out of it and set them with \`hack env set --service\` after migrating, adding \`--env \` for an overlay. A value Compose interpolates can't be set per service, because service values don't reach interpolation: keep it in your shell, or list the file. Then retry.`, }); } const orderConflicts = composeLegacyEnvFileOrderConflicts(opts); @@ -6308,6 +6308,7 @@ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { readonly legacyValues: ReadonlyMap>; }): Promise { const conflicts: string[] = []; + const deliveredBase = await deliveredLayerValues({ ...opts, envName: null }); for (const overlayName of await listProjectEnvOverlayNames({ projectDir: opts.projectDir, })) { @@ -6317,9 +6318,11 @@ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { }); const legacyOverlay = opts.legacyValues.get(overlayName) ?? {}; for (const [key, value] of Object.entries(opts.baseToMigrate)) { + // Equal to the base means the overlay's layers set nothing different. if ( key in delivered && delivered[key] !== value && + delivered[key] !== deliveredBase[key] && !(key in legacyOverlay) ) { conflicts.push( diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 26d8b9bf7..5093b9783 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -717,7 +717,7 @@ test("doctor --migrate-env-config refuses a .env no service lists, and keeps it ).toBeNull(); }, 30_000); -test("doctor --migrate-env-config moves a checkout's own .env after a teammate's migration landed (HACK-1370)", async () => { +test("doctor --migrate-env-config moves a checkout's own .env that Compose still lists, after a teammate's layers landed (HACK-1370)", async () => { const projectRoot = await createProjectWithStaleDaemonPid(); const projectDir = resolve(projectRoot, ".hack"); const legacy = ["synthetic", "legacy", "71ab"].join("-"); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 137eb321c..ebd5cf50b 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -2742,3 +2742,106 @@ test("an overlay entry equal to a base the tracked default delivers is migrated, expect(await stagingValue(repo)).toBe("legacy"); expect((await unmigratedIn(repo)).keys).toEqual([]); }); + +test("an overlay entry equal to the base is kept when a tracked overlay overrides it, alongside a new base key", async () => { + const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY.replace("global: {}", "global:\n PORT: legacy") + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + TRACKED_STAGING + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `PORT=legacy\nNEW_KEY=${KEPT_VALUE}\n` + ); + await writeFile(resolve(repo.projectDir, ".env.staging"), "PORT=legacy\n"); + expect(await stagingValue(repo)).toBe("stage"); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await stagingValue(repo)).toBe("legacy"); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { NEW_KEY: KEPT_VALUE }, + }); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { PORT: "legacy" }, + }); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}); + +test("the migration doesn't refuse a base value over an empty tracked overlay", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY.replace("global: {}", "global:\n PORT: team") + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + LAYER_EMPTY.replace("environment: default", "environment: staging") + ); + await writeFile(resolve(repo.projectDir, ".env"), "PORT=mine\n"); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { PORT: "mine" }, + }); +}); + +test("the refusal warns that a value Compose interpolates can't be set per service, and names --env for an overlay", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + 'services:\n web:\n image: alpine:3.20\n ports:\n - "${SYNTH_PORT}:3000"\n' + ); + await writeFile(resolve(repo.projectDir, ".env"), "SYNTH_PORT=8080\n"); + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + const { message } = error as Error; + expect(message).toContain( + "A value Compose interpolates can't be set per service" + ); + expect(message).toContain("keep it in your shell"); + expect(message).toContain("--env "); + expect(message).not.toContain("8080"); +}); + +test("an unlisted non-default overlay identical to the base migrates without an overlay layer", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect( + await lstat(resolve(repo.projectDir, "hack.env.staging.local.yaml")).catch( + () => null + ) + ).toBeNull(); +}); + +test("a git-tracked legacy file isn't offered for cleanup when nothing in Compose holds it back", async () => { + const repo = await createLegacyRepo(); + await git({ repo, args: ["init", "-q"] }); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services: {}\n" + ); + const tracked = resolve(repo.projectDir, ".env"); + await writeFile(tracked, `KEPT_SETTING=${KEPT_VALUE}\n`); + await git({ repo, args: ["add", "-f", ".hack/.env"] }); + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(migrated.legacyDetected).toBe(true); + expect(migrated.cleanupCandidates).not.toContain(tracked); + expect(await readFile(tracked, "utf8")).toContain(KEPT_VALUE); +}); From dadc9d2548ca86b1f28f09530cae9bd86b7ef6d8 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:48:19 -0500 Subject: [PATCH 32/55] fix(env): don't migrate generated .env output as legacy base values The detector skips .hack/.env when a valid state file says it was materialized, but the planner still read it, so migrating an overlay key could pin another env's materialized values into the local base and change the plain launch. The planner now applies the same exclusion. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 13 ++++++- tests/env-legacy-service-scope.test.ts | 53 ++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index c39422579..96e5ec2a7 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6242,9 +6242,11 @@ function readComposeDotenvEntryStart(rawLine: string): { async function readValidatedLegacyDotenvFiles(opts: { readonly projectDir: string; readonly overlayNames: readonly string[]; + /** Leaves out `.hack/.env` when it is generated output, not a legacy source. */ + readonly skipBase: boolean; }): Promise>> { const legacyFiles = [ - { envName: null, label: PROJECT_ENV_FILENAME }, + ...(opts.skipBase ? [] : [{ envName: null, label: PROJECT_ENV_FILENAME }]), ...opts.overlayNames.map((name) => ({ envName: name, label: `${PROJECT_ENV_FILENAME}.${name}`, @@ -6631,9 +6633,18 @@ async function migrateUndeclaredLegacyDotenv(opts: { overlayNames, }); + // A valid state file means `.hack/.env` was generated from the layers, possibly + // for another env. As in findUnmigratedLegacyDotenv, it is not a legacy source, + // and migrating it would pin that env's values into the local base. + const generatedBase = ( + await readProjectEnvStateFile({ + statePath: resolveProjectEnvStatePath({ projectDir: opts.projectDir }), + }) + ).ok; const legacyValues = await readValidatedLegacyDotenvFiles({ projectDir: opts.projectDir, overlayNames, + skipBase: generatedBase, }); await assertComposeOverlaysStaySelected({ projectDir: opts.projectDir, diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index ebd5cf50b..4a0f221f7 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -27,6 +27,7 @@ import { inspectProjectEnvMaterialization, LegacyDotenvMigrationRefusalError, type LegacyProjectEnvMigration, + materializeProjectEnv, migrateLegacyProjectEnv, ProjectEnvLayerPathConflictError, projectEnvConfigExistsForMutation, @@ -2845,3 +2846,55 @@ test("a git-tracked legacy file isn't offered for cleanup when nothing in Compos expect(migrated.cleanupCandidates).not.toContain(tracked); expect(await readFile(tracked, "utf8")).toContain(KEPT_VALUE); }); + +test("the migration doesn't treat generated .env output as legacy base values", async () => { + const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY.replace("global: {}", "global:\n PORT: team") + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + TRACKED_STAGING + ); + const serviceNames = ["api"]; + await materializeProjectEnv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: "staging", + serviceNames, + }); + expect(await readFile(resolve(repo.projectDir, ".env"), "utf8")).toContain( + "PORT=stage" + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `ONLY_HERE=${KEPT_VALUE}\n` + ); + const baseValue = async () => + ( + await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: null, + serviceNames, + }) + )?.globalEnv.PORT; + expect(await baseValue()).toBe("team"); + expect((await unmigratedIn(repo)).keys).toEqual(["ONLY_HERE"]); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await baseValue()).toBe("team"); + expect(await stagingValue(repo)).toBe("stage"); + expect( + await lstat(resolve(repo.projectDir, "hack.env.local.yaml")).catch( + () => null + ) + ).toBeNull(); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { ONLY_HERE: KEPT_VALUE }, + }); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}); From e78c870b922caedca1b6228b362bef29e3ea282c Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:06:41 -0500 Subject: [PATCH 33/55] fix(env): treat .env as generated only when its text proves it A parsed state file isn't proof. It and .env are gitignored, so both survive a switch to a branch without layers, where the migration then deleted a real .env without reading it. A value added by hand to a generated .env was deleted too. .env now counts as generated only when its text is exactly what materialize writes for the selection its state file records. Without layers it is a legacy file. With layers and other text, the migration refuses and names both ways out. Generated output is no longer offered for cleanup. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 6 + src/lib/project-env-config.ts | 70 +++++++++-- tests/env-legacy-service-scope.test.ts | 163 +++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 11 deletions(-) diff --git a/docs/env.md b/docs/env.md index 9b9d2b197..5e9421c0e 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1127,6 +1127,12 @@ files, when: - a migrated `.hack/.env` value would replace what a tracked overlay delivers for a key its legacy overlay file doesn't set (`E_CONFIG_INVALID`), because a local base outranks tracked overlays. Set the value you want in that legacy overlay file, or remove the key from `.hack/.env`; +- `.hack/.env` has a state file, env layers exist, and its text isn't what `hack env materialize` + would write (`E_PATH_CONFLICT`). The file is out of date or was edited by hand, and Hack can't + tell which. If it's generated, move any value of your own with `hack env set`, then run + `hack env materialize`, which replaces it. If it's your own file, delete `.hack/.env.state.json` + so it's read as a legacy file. When the text does match, the migration doesn't read the file or + offer it for cleanup; - a legacy file sets a key that Docker, its build tools or the process read from their environment (`E_CONFIG_INVALID`): `DOCKER_*`, `BUILDKIT_*`, `BUILDX_*`, a proxy setting, `SOURCE_DATE_EPOCH`, `HOME` or `PATH`. After the migration it would reach the Docker client too. diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 96e5ec2a7..72596419e 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5934,8 +5934,10 @@ async function deliveredLayerValues(opts: { * files exist: an unrelated local layer doesn't make a value safe to overwrite. * * A contract project is out of scope, because its own migration owns those files. - * `.hack/.env` with a valid state file is generated output, so it is skipped; - * overlays are not generated by materialize, so they are always compared. + * `.hack/.env` with a valid state file is treated as generated output, so it is + * skipped; the migration checks its text before relying on that + * ({@link legacyBaseDotenvIsGenerated}). Overlays are not generated by materialize, + * so they are always compared. */ export async function findUnmigratedLegacyDotenv(opts: { readonly projectRoot: string; @@ -6633,14 +6635,7 @@ async function migrateUndeclaredLegacyDotenv(opts: { overlayNames, }); - // A valid state file means `.hack/.env` was generated from the layers, possibly - // for another env. As in findUnmigratedLegacyDotenv, it is not a legacy source, - // and migrating it would pin that env's values into the local base. - const generatedBase = ( - await readProjectEnvStateFile({ - statePath: resolveProjectEnvStatePath({ projectDir: opts.projectDir }), - }) - ).ok; + const generatedBase = await legacyBaseDotenvIsGenerated(opts); const legacyValues = await readValidatedLegacyDotenvFiles({ projectDir: opts.projectDir, overlayNames, @@ -6788,7 +6783,8 @@ async function migrateUndeclaredLegacyDotenv(opts: { projectRoot: opts.projectRoot, projectDir: opts.projectDir, overlayNames, - materialize: opts.materialize, + // Generated output wasn't read, so its values aren't in any layer this run wrote. + materialize: opts.materialize || generatedBase, configCleanupCandidates: [], composeEnvFileReferences, }); @@ -6815,6 +6811,58 @@ async function migrateUndeclaredLegacyDotenv(opts: { }; } +/** + * Whether `.hack/.env` is generated output, not a legacy source: its text must be + * exactly what `hack env materialize` writes for the selection its state file + * records. Migrating generated output would pin that selection's values into the + * local base, possibly another env's. + * + * A valid state file alone proves nothing. It and `.env` are gitignored, so both + * survive a switch to a branch without layers, and `.env` may hold real values by + * then. With layers and different text, the file is either out of date or edited by + * hand, which this can't tell apart, so it refuses instead of dropping or pinning. + */ +async function legacyBaseDotenvIsGenerated(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; +}): Promise { + const stateRead = await readProjectEnvStateFile({ + statePath: resolveProjectEnvStatePath({ projectDir: opts.projectDir }), + }); + if (!stateRead.ok) { + return false; + } + const read = await readOptionalTextFile( + resolve(opts.projectDir, PROJECT_ENV_FILENAME) + ); + if (read.kind !== "text") { + return false; + } + const resolved = await resolveProjectEnvConfig({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + envName: stateRead.state.selectedOverlay, + serviceNames: opts.serviceNames, + }); + if (!resolved) { + return false; + } + const generated = serializeDotEnv( + selectProjectEnvValues({ + resolved, + scopeName: stateRead.state.selectedService, + }) + ); + if (generated !== read.text) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: `.hack/.env has a state file, but its text isn't what \`hack env materialize\` would write, so it may hold values of your own. If it's generated output, move any value of your own with \`hack env set\`, then run \`hack env materialize\`, which replaces the file. If it's your own file, delete .hack/.env.state.json so it's read as a legacy file.`, + }); + } + return true; +} + /** * Entries of a legacy dotenv file; an absent file has none. One that exists * but cannot be read (for example mode 000, or a dangling symlink) refuses: diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 4a0f221f7..afde565be 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, test } from "bun:test"; import { + appendFile, chmod, lstat, mkdir, @@ -2898,3 +2899,165 @@ test("the migration doesn't treat generated .env output as legacy base values", }); expect((await unmigratedIn(repo)).keys).toEqual([]); }); + +/** A checkout whose `.env` was generated from tracked layers, with a new overlay key to migrate. */ +async function writeGeneratedCheckout(opts: { + readonly envName: string | null; + readonly serviceName?: string; + readonly defaultLayerValues?: string; +}): Promise { + const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY.replace( + " global: {}\n", + opts.defaultLayerValues ?? " global:\n PORT: team\n" + ) + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + TRACKED_STAGING + ); + await materializeProjectEnv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: opts.envName, + serviceName: opts.serviceName, + serviceNames: ["api"], + }); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `ONLY_HERE=${KEPT_VALUE}\n` + ); + return repo; +} + +const GENERATED_MISMATCH = "isn't what `hack env materialize` would write"; + +test("a state file left from another branch doesn't make a real .env generated output", async () => { + const repo = await createLegacyRepo(); + const defaultLayer = resolve(repo.projectDir, "hack.env.default.yaml"); + await writeFile(defaultLayer, LAYER_EMPTY); + await materializeProjectEnv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: null, + serviceNames: ["api"], + }); + // Both files are gitignored, so switching to a branch without layers leaves them. + await rm(defaultLayer); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); +}); + +test("the migration refuses a generated .env with a value added by hand, and keeps it", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + await appendFile( + resolve(repo.projectDir, ".env"), + `EXTRA=${BASE_ONLY_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: GENERATED_MISMATCH, + hidden: [BASE_ONLY_VALUE, KEPT_VALUE], + }); +}); + +test("after that refusal, deleting the state file migrates .env as the legacy file it now is", async () => { + const repo = await writeGeneratedCheckout({ envName: null }); + await appendFile( + resolve(repo.projectDir, ".env"), + `EXTRA=${BASE_ONLY_VALUE}\n` + ); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: GENERATED_MISMATCH, + hidden: [BASE_ONLY_VALUE, KEPT_VALUE], + }); + + await rm(resolve(repo.projectDir, ".env.state.json")); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { EXTRA: BASE_ONLY_VALUE }, + }); +}); + +test("the migration refuses a generated .env its layers have moved past, until materialize runs again", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + TRACKED_STAGING.replace("PORT: stage", "PORT: moved") + ); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: GENERATED_MISMATCH, + hidden: [KEPT_VALUE], + }); + + await materializeProjectEnv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: "staging", + serviceNames: ["api"], + }); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect( + await lstat(resolve(repo.projectDir, "hack.env.local.yaml")).catch( + () => null + ) + ).toBeNull(); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { ONLY_HERE: KEPT_VALUE }, + }); +}); + +test("a generated .env is not offered for cleanup, because the migration didn't read it", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const envPath = resolve(repo.projectDir, ".env"); + const before = await readFile(envPath, "utf8"); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + + expect(migrated.cleanupCandidates).not.toContain(envPath); + expect(migrated.blockedCleanupCandidates).not.toContain(envPath); + expect(await readFile(envPath, "utf8")).toBe(before); +}); + +test("a .env generated for one service matches using the scope its state file records", async () => { + const repo = await writeGeneratedCheckout({ + envName: null, + serviceName: "api", + defaultLayerValues: + " global:\n PORT: team\n api:\n SCOPED: only-api\n", + }); + expect(await readFile(resolve(repo.projectDir, ".env"), "utf8")).toContain( + "SCOPED=only-api" + ); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect( + await lstat(resolve(repo.projectDir, "hack.env.local.yaml")).catch( + () => null + ) + ).toBeNull(); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { ONLY_HERE: KEPT_VALUE }, + }); +}); From 2cafd2a789314acfabc63307a5411b058bfaca4e Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:11:20 -0500 Subject: [PATCH 34/55] test(env): pin the stale state file through a real git branch switch Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- tests/env-legacy-service-scope.test.ts | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index afde565be..6f50f5073 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -3061,3 +3061,43 @@ test("a .env generated for one service matches using the scope its state file re global: { ONLY_HERE: KEPT_VALUE }, }); }); + +test("after a real branch switch, a stale state file doesn't cost a real .env its values", async () => { + const repo = await createLegacyRepo(); + await git({ repo, args: ["init", "--quiet", "--initial-branch=main"] }); + await git({ repo, args: ["add", "-A"] }); + await git({ repo, args: ["commit", "--quiet", "-m", "legacy"] }); + + await git({ repo, args: ["checkout", "--quiet", "-b", "layers"] }); + const defaultLayer = resolve(repo.projectDir, "hack.env.default.yaml"); + await writeFile(defaultLayer, LAYER_EMPTY); + await git({ repo, args: ["add", "-A"] }); + await git({ repo, args: ["commit", "--quiet", "-m", "layers"] }); + await materializeProjectEnv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: null, + serviceNames: ["api"], + }); + + await git({ repo, args: ["checkout", "--quiet", "main"] }); + expect(await lstat(defaultLayer).catch(() => null)).toBeNull(); + const statePath = resolve(repo.projectDir, ".env.state.json"); + expect(await lstat(statePath).catch(() => null)).not.toBeNull(); + const envPath = resolve(repo.projectDir, ".env"); + await appendFile(envPath, `KEPT_SETTING=${KEPT_VALUE}\n`); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); + expect(await readFile(envPath, "utf8")).toContain(KEPT_VALUE); + // Offered only now that the layer holds what the file held. + expect([ + ...migrated.cleanupCandidates, + ...migrated.blockedCleanupCandidates, + ]).toContain(envPath); +}); From ce5c770c097f05f8ae825170b820c5029a118721 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:28:58 -0500 Subject: [PATCH 35/55] fix(env): let materialize run with overlay keys waiting, and refuse a corrupt state The migration's refusal names `hack env materialize` as the recovery for a stale or edited generated .env, but materialize also refused on overlay keys, which is the only work that reaches that refusal. It replaces only .hack/.env, so it now checks that file alone, and still refuses a real .env with no state file. A state file that can't be parsed no longer lets .env read as legacy while layers exist, which could pin generated values. Without layers it is still a legacy file. Refs HACK-1370 Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 3 + src/commands/env.ts | 2 + src/lib/project-env-config.ts | 46 +++++++----- tests/env-legacy-service-scope.test.ts | 98 ++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 19 deletions(-) diff --git a/docs/env.md b/docs/env.md index 5e9421c0e..07bc21405 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1133,6 +1133,9 @@ files, when: `hack env materialize`, which replaces it. If it's your own file, delete `.hack/.env.state.json` so it's read as a legacy file. When the text does match, the migration doesn't read the file or offer it for cleanup; +- `.hack/.env` has a state file that can't be parsed, and env layers exist (`E_PATH_CONFLICT`). If + `.env` is your own file, delete `.hack/.env.state.json`. If it's generated output you don't need, + delete both, then run `hack env materialize`; - a legacy file sets a key that Docker, its build tools or the process read from their environment (`E_CONFIG_INVALID`): `DOCKER_*`, `BUILDKIT_*`, `BUILDX_*`, a proxy setting, `SOURCE_DATE_EPOCH`, `HOME` or `PATH`. After the migration it would reach the Docker client too. diff --git a/src/commands/env.ts b/src/commands/env.ts index 5f3fac223..f31050ff9 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -2084,10 +2084,12 @@ const handleEnvMaterialize: CommandHandlerFor = async ({ const serviceNames = await discoverComposeServiceNames({ composeFile: project.composeFile, }); + // Materialize replaces only `.hack/.env`, so overlay values stay where they are. const unmigrated = await findUnmigratedLegacyDotenv({ projectRoot: project.projectRoot, projectDir: project.projectDir, serviceNames, + baseOnly: true, }); if (hasUnmigratedLegacyDotenv(unmigrated)) { throw new HackCliError({ diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 72596419e..c4c2a959a 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5943,6 +5943,8 @@ export async function findUnmigratedLegacyDotenv(opts: { readonly projectRoot: string; readonly projectDir: string; readonly serviceNames: readonly string[]; + /** Only `.hack/.env`, for a caller that replaces that file and leaves overlays alone. */ + readonly baseOnly?: boolean; }): Promise { const none = { keys: [], malformedCount: 0, unreadableFiles: [] }; const entries = await readdir(opts.projectDir, { withFileTypes: true }); @@ -5960,17 +5962,20 @@ export async function findUnmigratedLegacyDotenv(opts: { const state = await readProjectEnvStateFile({ statePath: resolveProjectEnvStatePath({ projectDir: opts.projectDir }), }); - const overlayNames = ( - await discoverLegacyOverlayNames({ - projectDir: opts.projectDir, - includeSymlinks: true, - }) - ).filter( - (name) => - !( - isExcludedLegacyDotenvName(name) || isReservedLegacyOverlayName(name) - ) && NATIVE_ENV_OVERLAY_PATTERN.test(name) - ); + const overlayNames = opts.baseOnly + ? [] + : ( + await discoverLegacyOverlayNames({ + projectDir: opts.projectDir, + includeSymlinks: true, + }) + ).filter( + (name) => + !( + isExcludedLegacyDotenvName(name) || + isReservedLegacyOverlayName(name) + ) && NATIVE_ENV_OVERLAY_PATTERN.test(name) + ); const keys = new Set(); const unreadableFiles: string[] = []; let malformedCount = 0; @@ -6827,27 +6832,30 @@ async function legacyBaseDotenvIsGenerated(opts: { readonly projectDir: string; readonly serviceNames: readonly string[]; }): Promise { - const stateRead = await readProjectEnvStateFile({ - statePath: resolveProjectEnvStatePath({ projectDir: opts.projectDir }), - }); - if (!stateRead.ok) { - return false; - } + const statePath = resolveProjectEnvStatePath({ projectDir: opts.projectDir }); const read = await readOptionalTextFile( resolve(opts.projectDir, PROJECT_ENV_FILENAME) ); - if (read.kind !== "text") { + if (read.kind !== "text" || !(await pathExists(statePath))) { return false; } + const stateRead = await readProjectEnvStateFile({ statePath }); const resolved = await resolveProjectEnvConfig({ projectRoot: opts.projectRoot, projectDir: opts.projectDir, - envName: stateRead.state.selectedOverlay, + envName: stateRead.ok ? stateRead.state.selectedOverlay : null, serviceNames: opts.serviceNames, }); if (!resolved) { return false; } + if (!stateRead.ok) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PATH_CONFLICT", + reason: + ".hack/.env's state file can't be read, so Hack can't tell whether .env is generated output or holds values of your own. If it's your own file, delete .hack/.env.state.json so it's read as a legacy file. If it's generated output you don't need, delete both files, then run `hack env materialize`.", + }); + } const generated = serializeDotEnv( selectProjectEnvValues({ resolved, diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 6f50f5073..e3b649654 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -3101,3 +3101,101 @@ test("after a real branch switch, a stale state file doesn't cost a real .env it ...migrated.blockedCleanupCandidates, ]).toContain(envPath); }); + +test("the refusal's materialize recovery runs with an overlay key waiting, and keeps every value", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const envPath = resolve(repo.projectDir, ".env"); + await appendFile(envPath, `EXTRA=${BASE_ONLY_VALUE}\n`); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: GENERATED_MISMATCH, + hidden: [BASE_ONLY_VALUE, KEPT_VALUE], + }); + + const moved = await runCli({ + repo, + args: [ + "env", + "set", + "EXTRA", + "--local", + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${BASE_ONLY_VALUE}\n`, + }); + expect(moved.exitCode).toBe(0); + const materialized = await runCli({ + repo, + args: [ + "env", + "materialize", + "--env", + "staging", + "--path", + repo.projectRoot, + ], + }); + expect(materialized.exitCode).toBe(0); + // Materialize writes only .env, so the overlay's value is still there to migrate. + expect( + await readFile(resolve(repo.projectDir, ".env.staging"), "utf8") + ).toContain(KEPT_VALUE); + expect((await unmigratedIn(repo)).keys).toEqual(["ONLY_HERE"]); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { EXTRA: BASE_ONLY_VALUE }, + }); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { ONLY_HERE: KEPT_VALUE }, + }); + expect((await unmigratedIn(repo)).keys).toEqual([]); +}, 60_000); + +test("materialize still refuses a real .env with no state file when an overlay key is waiting too", async () => { + const repo = await writeTeammateCheckout(); + const envPath = resolve(repo.projectDir, ".env"); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `ONLY_HERE=${NEW_VALUE}\n` + ); + const run = await runCli({ + repo, + args: ["env", "materialize", "--path", repo.projectRoot], + }); + expect(run.exitCode).not.toBe(0); + const output = `${run.stdout}${run.stderr}`; + expect(output).toContain("KEPT_SETTING"); + expect(output).not.toContain(KEPT_VALUE); + expect(await readFile(envPath, "utf8")).toContain(KEPT_VALUE); +}, 60_000); + +test("the migration refuses a .env whose state file can't be parsed, while layers exist", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + await writeFile(resolve(repo.projectDir, ".env.state.json"), "{not json\n"); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: "state file can't be read", + hidden: [KEPT_VALUE], + }); +}); + +test("without layers, an unparseable state file doesn't stop a real .env migrating", async () => { + const repo = await createLegacyRepo(); + await writeFile(resolve(repo.projectDir, ".env.state.json"), "{not json\n"); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); +}); From 64dc985d3f3c04fc16ef4339342bd5f0028a0849 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:32:50 -0500 Subject: [PATCH 36/55] test: deny implicit daemon starts in private CLI fixtures HACK-1696. `hack doctor --fix`, `--migrate-env-config` and `global up` can run `hack daemon start` from PATH with no TTY. That is a plausible way for a fixture run to leave a daemon behind; it is not proven to be how the daemons in that ticket started. privateCliEnv and the E2E harness now set HACK_DAEMON_AUTOSTART_DISABLE=1 by default. A test that asserts daemon starts against stubs opts in explicitly (launchd-isolation, agent-surface-freeze). The new guard test runs the source CLI with a recording stub `hack` first on PATH, with a positive control for each doctor route, and stops its children before removing their fixture. The retired HACK_DAEMON_DISABLE name, which nothing reads, is removed from six test files. Co-Authored-By: Claude Opus 5.5 --- .ai/skills/hack-repo-verify/SKILL.md | 5 + tests/cli-spawn-env-isolation.test.ts | 15 +- tests/daemon-autostart-fixture-guard.test.ts | 171 ++++++++++++++++++ tests/doctor-adopted-generated-files.test.ts | 1 - tests/doctor-adopted-project.test.ts | 1 - tests/doctor-command.test.ts | 1 - tests/e2e-launchd-guard.test.ts | 16 ++ tests/e2e/README.md | 19 ++ tests/e2e/harness.ts | 4 + tests/e2e/scenarios/agent-surface-freeze.ts | 3 + tests/explicit-agent-integrations.test.ts | 1 - tests/helpers/private-cli-env.ts | 17 +- tests/helpers/project-name-fixture.ts | 1 - tests/launchd-isolation.test.ts | 3 + tests/mcp-repeat-install.test.ts | 2 +- ...tive-compose-adoption-search-depth.test.ts | 1 - 16 files changed, 251 insertions(+), 10 deletions(-) create mode 100644 tests/daemon-autostart-fixture-guard.test.ts diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index d3fa9a6d5..72ad8a959 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -146,6 +146,11 @@ needs source mapping and implementation regression evidence; use Do not overlap stateful runs against one checkout, daemon, port range or runtime. Record owned resources before a live run, use bounded fixtures and verify cleanup. +A harness outside `privateCliEnv` and `buildCliEnv` sets +`HACK_DAEMON_AUTOSTART_DISABLE=1` for every CLI run: `doctor --fix`, +`doctor --migrate-env-config` and `global up` start a daemon that outlives the run, +even with `HACK_NO_INTERACTIVE=1`. `gateway enable` and `gateway setup` start one +regardless of it. Never hand-edit managed internal state or prune unowned resources to repair a test. Report exact command, source/executable, host/container context, result, skips/cache and remaining gate. Do not upgrade a mocked/unit result into live runtime evidence. diff --git a/tests/cli-spawn-env-isolation.test.ts b/tests/cli-spawn-env-isolation.test.ts index 37ef5df61..298fb76cd 100644 --- a/tests/cli-spawn-env-isolation.test.ts +++ b/tests/cli-spawn-env-isolation.test.ts @@ -9,6 +9,7 @@ import { } from "./helpers/cli-spawn-env-scan.ts"; import { restoreEnv } from "./helpers/env.ts"; import { + PRIVATE_CLI_DAEMON_AUTOSTART_ENV, PRIVATE_CLI_INHERITED_ENV, privateCliEnv, } from "./helpers/private-cli-env.ts"; @@ -426,11 +427,21 @@ test("privateCliEnv keeps only the allow-list, test values and a private HOME", expect(env.PATH).toBe("/fixture/bin"); expect(env.DOCKER_CONFIG).toBe(join(root, "docker-client")); expect(Object.keys(env).sort()).toEqual( - ["DOCKER_CONFIG", "HACK_HOME", "HOME", "PATH"].sort() + [ + "DOCKER_CONFIG", + "HACK_HOME", + "HOME", + "PATH", + PRIVATE_CLI_DAEMON_AUTOSTART_ENV, + ].sort() ); const inherited = privateCliEnv({ root, isolateDockerClient: false }); for (const key of Object.keys(inherited)) { - expect([...PRIVATE_CLI_INHERITED_ENV, "HOME"]).toContain(key); + expect([ + ...PRIVATE_CLI_INHERITED_ENV, + "HOME", + PRIVATE_CLI_DAEMON_AUTOSTART_ENV, + ]).toContain(key); } // A HOME the test created first is still made private to the owner. const existing = join(root, "existing-home"); diff --git a/tests/daemon-autostart-fixture-guard.test.ts b/tests/daemon-autostart-fixture-guard.test.ts new file mode 100644 index 000000000..dc88b52e2 --- /dev/null +++ b/tests/daemon-autostart-fixture-guard.test.ts @@ -0,0 +1,171 @@ +import { afterEach, expect, test } from "bun:test"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, relative, resolve } from "node:path"; +import { + DAEMON_AUTOSTART_DISABLE_ENV, + resolveDaemonAutostartAccess, +} from "../src/daemon/client.ts"; +import { fixtureChildren } from "./helpers/fixture-children.ts"; +import { + PRIVATE_CLI_DAEMON_AUTOSTART_ENV, + privateCliEnv, + privateCliRunEnv, +} from "./helpers/private-cli-env.ts"; + +/* + * HACK-1696: `hack doctor --fix`, `--migrate-env-config` and `global up` run + * `hack daemon start` through `resolveHackInvocation`, with no TTY needed. + * That is a plausible way for a fixture run to leave a daemon behind. It is + * not proven to be how the daemons in that ticket started. + * The doctor cases run the source CLI with a recording stub first on PATH, so + * a regression records a call and cannot start a real daemon. + */ + +const ENTRYPOINT = resolve(import.meta.dir, "..", "index.ts"); +const TESTS_ROOT = resolve(import.meta.dir); +const DOCTOR_REPAIR_TIMEOUT_MS = 30_000; + +const children = fixtureChildren("daemon-autostart-fixture-guard"); +let root: string | null = null; + +afterEach(async () => { + // A test timeout does not end the child or its pipe readers, so stop and + // await any child that is still running before removing its fixture. + await children.stopAll(); + if (root) { + await rm(root, { recursive: true, force: true }); + root = null; + } +}); + +async function fixtureRoot(): Promise { + root = await realpath(await mkdtemp(join(tmpdir(), "hack-daemon-guard-"))); + return root; +} + +async function writeStub(path: string, body: string): Promise { + await writeFile(path, `#!/bin/sh\n${body}\n`, { mode: 0o700 }); +} + +async function runDoctorRepair(opts: { + readonly flag: string; + readonly env?: Readonly>; +}): Promise<{ readonly hackCalls: string[]; readonly output: string }> { + const fixture = await fixtureRoot(); + const tools = join(fixture, "tools"); + const project = join(fixture, "project"); + const hackLog = join(fixture, "hack-calls.log"); + await mkdir(tools); + await mkdir(join(project, ".hack"), { recursive: true }); + await writeFile( + join(project, ".hack", "docker-compose.yml"), + "services:\n api:\n image: alpine:3.19\n" + ); + await writeStub(join(tools, "hack"), `printf '%s\\n' "$*" >> "${hackLog}"`); + // Docker is unavailable, so doctor stops after the hackd repair step. + await writeStub(join(tools, "docker"), "exit 1"); + await writeStub(join(tools, "mutagen"), "exit 0"); + + const { stdout, stderr } = await children.run({ + cmd: [ + process.execPath, + ENTRYPOINT, + "doctor", + opts.flag, + "--no-interactive", + "--path", + project, + ], + cwd: project, + env: privateCliEnv({ + root: fixture, + env: { + HACK_HOME: join(fixture, "hack-home"), + HACK_GLOBAL_CONFIG_PATH: join(fixture, "hack-config.json"), + HACK_LOGGER: "console", + HACK_MUTAGEN_PATH: join(tools, "mutagen"), + HACK_NO_INTERACTIVE: "1", + NO_COLOR: "1", + PATH: `${tools}:/usr/bin:/bin`, + ...opts.env, + }, + }), + }); + const log = await readFile(hackLog, "utf8").catch(() => ""); + return { + hackCalls: log.split("\n").filter((line) => line.length > 0), + output: stdout + stderr, + }; +} + +test("the private env names the variable the client reads", () => { + expect(PRIVATE_CLI_DAEMON_AUTOSTART_ENV).toBe(DAEMON_AUTOSTART_DISABLE_ENV); +}); + +test("privateCliEnv denies implicit daemon starts and a test can opt in", async () => { + const fixture = await fixtureRoot(); + const denied = privateCliEnv({ root: fixture }); + expect(denied[DAEMON_AUTOSTART_DISABLE_ENV]).toBe("1"); + expect(resolveDaemonAutostartAccess({ env: denied })).toMatchObject({ + allowed: false, + reason: "disabled", + }); + expect(privateCliRunEnv({ root: fixture }).env).toMatchObject({ + [DAEMON_AUTOSTART_DISABLE_ENV]: "1", + }); + + const optedIn = privateCliEnv({ + root: fixture, + env: { [DAEMON_AUTOSTART_DISABLE_ENV]: undefined }, + }); + expect(Object.hasOwn(optedIn, DAEMON_AUTOSTART_DISABLE_ENV)).toBe(false); + expect(resolveDaemonAutostartAccess({ env: optedIn }).allowed).toBe(true); +}); + +for (const flag of ["--fix", "--migrate-env-config"]) { + test( + `doctor ${flag} with the private env never runs hack daemon`, + async () => { + const { hackCalls, output } = await runDoctorRepair({ flag }); + expect(hackCalls).toEqual([]); + expect(output).toContain("Skipped hackd repair"); + }, + DOCTOR_REPAIR_TIMEOUT_MS + ); + + // Positive control: the same run reaches the stub without the variable, even + // with HACK_NO_INTERACTIVE=1 and no TTY. + test( + `doctor ${flag} that opts in runs hack daemon start from PATH`, + async () => { + const { hackCalls } = await runDoctorRepair({ + flag, + env: { [DAEMON_AUTOSTART_DISABLE_ENV]: undefined }, + }); + expect(hackCalls).toContain("daemon start"); + }, + DOCTOR_REPAIR_TIMEOUT_MS + ); +} + +test("no test sets the old variable name that nothing reads", async () => { + const retired = new RegExp(`${["HACK", "DAEMON", "DISABLE"].join("_")}(?!_)`); + const offenders: string[] = []; + for await (const path of new Bun.Glob("**/*.{ts,tsx,mts}").scan({ + cwd: TESTS_ROOT, + absolute: true, + })) { + if (retired.test(await readFile(path, "utf8"))) { + offenders.push(relative(TESTS_ROOT, path)); + } + } + expect(offenders).toEqual([]); +}); diff --git a/tests/doctor-adopted-generated-files.test.ts b/tests/doctor-adopted-generated-files.test.ts index cc6face52..61f662850 100644 --- a/tests/doctor-adopted-generated-files.test.ts +++ b/tests/doctor-adopted-generated-files.test.ts @@ -61,7 +61,6 @@ async function doctorJson(): Promise { env: { HACK_HOME: state, HACK_GLOBAL_CONFIG_PATH: join(state, "hack.config.json"), - HACK_DAEMON_DISABLE: "1", HACK_NO_INTERACTIVE: "1", HACK_LOGGER: "console", HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", diff --git a/tests/doctor-adopted-project.test.ts b/tests/doctor-adopted-project.test.ts index 3df9be0a9..b0760f6ba 100644 --- a/tests/doctor-adopted-project.test.ts +++ b/tests/doctor-adopted-project.test.ts @@ -67,7 +67,6 @@ async function doctorJson(): Promise<{ env: { HACK_HOME: state, HACK_GLOBAL_CONFIG_PATH: join(state, "hack.config.json"), - HACK_DAEMON_DISABLE: "1", HACK_NO_INTERACTIVE: "1", HACK_LOGGER: "console", XDG_CACHE_HOME: join(h.outer, "cache"), diff --git a/tests/doctor-command.test.ts b/tests/doctor-command.test.ts index 9234aa63f..9a3ebe414 100644 --- a/tests/doctor-command.test.ts +++ b/tests/doctor-command.test.ts @@ -81,7 +81,6 @@ test("doctor refuses native input before probing or repairing global runtime", a env: { HACK_HOME: join(root, "state"), HACK_GLOBAL_CONFIG_PATH: join(root, "state", "hack.config.json"), - HACK_DAEMON_DISABLE: "1", HACK_NO_INTERACTIVE: "1", HACK_LOGGER: "console", XDG_CACHE_HOME: cache, diff --git a/tests/e2e-launchd-guard.test.ts b/tests/e2e-launchd-guard.test.ts index 6e2f6ca7c..beba261f9 100644 --- a/tests/e2e-launchd-guard.test.ts +++ b/tests/e2e-launchd-guard.test.ts @@ -3,6 +3,10 @@ import { mkdtemp, rm, stat, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { DAEMON_LAUNCHD_LABEL } from "../src/constants.ts"; +import { + DAEMON_AUTOSTART_DISABLE_ENV, + resolveDaemonAutostartAccess, +} from "../src/daemon/client.ts"; import { LAUNCHD_ALLOW_ISOLATED_ENV, LAUNCHD_DISABLE_ENV, @@ -50,6 +54,18 @@ test("buildCliEnv forces the launchd hard deny over scenario env", () => { expect(rest.join(":")).toBe("/scenario/tools:/usr/bin:/bin"); }); +test("buildCliEnv denies implicit daemon starts unless a scenario opts in (HACK-1696)", () => { + const denied = buildCliEnv({ hackHome: "/tmp/hack-e2e-home-guard" }); + expect(denied[DAEMON_AUTOSTART_DISABLE_ENV]).toBe("1"); + expect(resolveDaemonAutostartAccess({ env: denied }).allowed).toBe(false); + + const optedIn = buildCliEnv({ + hackHome: "/tmp/hack-e2e-home-guard", + extra: { [DAEMON_AUTOSTART_DISABLE_ENV]: "0" }, + }); + expect(resolveDaemonAutostartAccess({ env: optedIn }).allowed).toBe(true); +}); + test("the launchctl guard shim refuses and records hackd service calls", async () => { const env = buildCliEnv({ hackHome: "/tmp/hack-e2e-home-guard" }); const guardDir = (env.PATH ?? "").split(":")[0] ?? ""; diff --git a/tests/e2e/README.md b/tests/e2e/README.md index bc3030b78..456d8a31f 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -651,6 +651,25 @@ stop that now: code 2. A `hack daemon install` you run outside the suite during the run also trips this check. +### Daemon guard (HACK-1696) + +`hack doctor --fix` and `hack doctor --migrate-env-config` run +`hack daemon start` with or without a TTY, so `HACK_NO_INTERACTIVE=1` does not +stop them. `hack global up` does the same after its Docker check when +`daemon.autoStart` is on. The command comes from `resolveHackInvocation`: +`HACK_MCP_COMMAND`, then the first `hack` on `PATH`, then the running CLI, so +taking `hack` off `PATH` does not prevent a start. That daemon does not exit +when its scenario does. `buildCliEnv` and `privateCliEnv` therefore set +`HACK_DAEMON_AUTOSTART_DISABLE=1`. An explicit `hack daemon start` still works. +A test that needs the implicit start opts in and stops what it starts: +`agent-surface-freeze` passes `0` and runs `hack daemon stop` when it finishes, +and `privateCliEnv` callers pass `undefined`. A harness outside these helpers +sets the variable itself. + +The variable does not cover `hack gateway enable` or `hack gateway setup`, which +start a daemon as part of the command. `tests/gateway-e2e.test.ts` needs +`HACK_GATEWAY_E2E=1` to run. + ## Tiers - **Tier 1 (`local`)** — runs everywhere; no docker needed: diff --git a/tests/e2e/harness.ts b/tests/e2e/harness.ts index e64fed106..8b034ba96 100644 --- a/tests/e2e/harness.ts +++ b/tests/e2e/harness.ts @@ -240,6 +240,10 @@ export function buildCliEnv(opts: { } env.HACK_HOME = opts.hackHome; env.HACK_NO_INTERACTIVE = "1"; + // `doctor --fix` runs `hack daemon start` without a TTY, and nothing stops + // that daemon with the scenario (HACK-1696). Unlike the launchd guard below, + // a scenario that stops its own daemon may opt in through `extra`. + env.HACK_DAEMON_AUTOSTART_DISABLE = "1"; env.NO_COLOR = "1"; env.CLICOLOR = "0"; env.TERM = "dumb"; diff --git a/tests/e2e/scenarios/agent-surface-freeze.ts b/tests/e2e/scenarios/agent-surface-freeze.ts index 7246547ad..c3239ff82 100644 --- a/tests/e2e/scenarios/agent-surface-freeze.ts +++ b/tests/e2e/scenarios/agent-surface-freeze.ts @@ -235,6 +235,9 @@ function freezeEnv(opts: { HACK_HOME: opts.ctx.hackHome, DOCKER_HOST: `unix://${join(opts.ctx.tempRoot, "no-docker.sock")}`, HACK_MUTAGEN_PATH: join(opts.ctx.tempRoot, "no-mutagen"), + // Opts in to the `doctor --fix` hackd start: the freeze covers that step, + // and this scenario stops the daemon itself when it finishes. + HACK_DAEMON_AUTOSTART_DISABLE: "0", }; } diff --git a/tests/explicit-agent-integrations.test.ts b/tests/explicit-agent-integrations.test.ts index b63299ab1..22cfd9e4d 100644 --- a/tests/explicit-agent-integrations.test.ts +++ b/tests/explicit-agent-integrations.test.ts @@ -36,7 +36,6 @@ test("CLI retirement preserves project/global originals; selected skill and MCP CODEX_HOME: join(root, "home/.codex"), HACK_GLOBAL_CONFIG_PATH: join(root, "state/hack.config.json"), HACK_NO_INTERACTIVE: "1", - HACK_DAEMON_DISABLE: "1", HACK_LOGGER: "console", }, }), diff --git a/tests/helpers/private-cli-env.ts b/tests/helpers/private-cli-env.ts index 981b05e0f..59cb4306f 100644 --- a/tests/helpers/private-cli-env.ts +++ b/tests/helpers/private-cli-env.ts @@ -12,6 +12,19 @@ import { requireFixtureRoot } from "./fixture-root.ts"; */ export const PRIVATE_CLI_INHERITED_ENV = ["PATH", "TMPDIR"] as const; +/** + * Set to `1` on every private child unless the test passes its own value in + * `env`. `hack doctor --fix`, `hack doctor --migrate-env-config` and + * `hack global up` run `hack daemon start` through `resolveHackInvocation`, + * with or without a TTY, and nothing stops that daemon with the fixture + * (HACK-1696). Taking `hack` off PATH does not prevent it: the CLI then runs + * itself. `hack gateway enable` and `setup` do not read this variable. A test + * that asserts a daemon start opts in with `undefined`. + * Same name as `DAEMON_AUTOSTART_DISABLE_ENV` in src/daemon/client.ts; it is + * not imported so that this helper does not load CLI modules a test may mock. + */ +export const PRIVATE_CLI_DAEMON_AUTOSTART_ENV = "HACK_DAEMON_AUTOSTART_DISABLE"; + /** The default private HOME of a fixture root. */ export function privateCliHome(root: string): string { return join(root, "user-home"); @@ -33,7 +46,8 @@ export type PrivateCliEnvOptions = { readonly home?: string; /** * Values the test needs, applied after the inherited allow-list. An - * `undefined` value removes an inherited key. `HOME` is not accepted here. + * `undefined` value removes an inherited key or a default, such as + * {@link PRIVATE_CLI_DAEMON_AUTOSTART_ENV}. `HOME` is not accepted here. */ readonly env?: Readonly>; /** @@ -77,6 +91,7 @@ export function privateCliEnv( env[key] = value; } } + env[PRIVATE_CLI_DAEMON_AUTOSTART_ENV] = "1"; if (opts.isolateDockerClient !== false) { Object.assign(env, isolatedDockerClientEnv(root)); } diff --git a/tests/helpers/project-name-fixture.ts b/tests/helpers/project-name-fixture.ts index 0f43449c5..d4e750b84 100644 --- a/tests/helpers/project-name-fixture.ts +++ b/tests/helpers/project-name-fixture.ts @@ -19,7 +19,6 @@ export async function projectNameFixture() { env: { HACK_HOME: state, HACK_GLOBAL_CONFIG_PATH: join(state, "hack.config.json"), - HACK_DAEMON_DISABLE: "1", NO_COLOR: "1", }, }); diff --git a/tests/launchd-isolation.test.ts b/tests/launchd-isolation.test.ts index f22b4e8c0..9eed35825 100644 --- a/tests/launchd-isolation.test.ts +++ b/tests/launchd-isolation.test.ts @@ -135,6 +135,9 @@ async function makeFixture(): Promise { HACK_NO_INTERACTIVE: "1", NO_COLOR: "1", HACK_TEST_STUB_LAUNCHCTL: join(tools, "launchctl"), + // This file asserts daemon starts against stubs, so it opts in. A + // case that tests the opt-out sets it again through `extra`. + [DAEMON_AUTOSTART_DISABLE_ENV]: undefined, ...extra, }, }), diff --git a/tests/mcp-repeat-install.test.ts b/tests/mcp-repeat-install.test.ts index 147774736..26ad500d1 100644 --- a/tests/mcp-repeat-install.test.ts +++ b/tests/mcp-repeat-install.test.ts @@ -87,7 +87,7 @@ async function fixture(kind: "managed" | "custom") { CODEX_HOME: join(home, ".codex"), PATH: join(root, "unavailable-bin"), HACK_NO_INTERACTIVE: "1", - HACK_DAEMON_DISABLE: "1", + HACK_DAEMON_AUTOSTART_DISABLE: "1", DOCKER_HOST: `unix://${root}/unavailable.sock`, NO_COLOR: "1", }; diff --git a/tests/native-compose-adoption-search-depth.test.ts b/tests/native-compose-adoption-search-depth.test.ts index 02773155b..2d29bfda2 100644 --- a/tests/native-compose-adoption-search-depth.test.ts +++ b/tests/native-compose-adoption-search-depth.test.ts @@ -69,7 +69,6 @@ async function cli(opts: { env: { HACK_HOME: state, HACK_GLOBAL_CONFIG_PATH: join(state, "hack.config.json"), - HACK_DAEMON_DISABLE: "1", HACK_NO_INTERACTIVE: "1", HACK_LOGGER: "console", HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", From af73f1d2e3c3ce85216a23c924db8f3e0e2c40df Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:36:50 -0500 Subject: [PATCH 37/55] fix(env): refuse a state path that exists but can't be read The legacy `.env` migration treated a dangling link or an unreadable entry at the state path as absent, so generated output could be migrated as real values. Only an empty path now counts as absent; anything else refuses, writing nothing. Docs and comments now say that an edited `.hack/.env` with a valid state file is only checked when a migration runs for another reason. That existing behavior is HACK-1694's open decision and does not change here. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 22 ++-- src/lib/project-env-config.ts | 18 ++- tests/env-legacy-service-scope.test.ts | 167 +++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 15 deletions(-) diff --git a/docs/env.md b/docs/env.md index 07bc21405..c2b16f08a 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1039,15 +1039,19 @@ A repo with no legacy env file gets only an empty `hack.env.default.yaml`. Once this is committed, a teammate's own `.hack/.env` and overlays may stop reaching services, because their checkout now uses env layers. Hack compares those files with what the layers -deliver. For any value the layers don't deliver, `hack up` warns, doctor reports it, and -`hack env materialize` refuses, so the file isn't replaced. They run +deliver. For any value the layers don't deliver, `hack up` warns and doctor reports it. For a +value in `.hack/.env`, `hack env materialize` also refuses, so the file isn't replaced; it doesn't +write overlay files, so an overlay value doesn't stop it. They run `hack doctor --migrate-env-config` to move those values to their own local layers, adding to a local layer that already exists and making it private (mode 0600). A value equal to what the layers deliver isn't copied, so later team changes still reach them. A key the local layer sets differently refuses (`E_PATH_CONFLICT`), and so does a local layer that can't be read as an env layer (`E_CONFIG_INVALID`), which is left as it is. Adding to a layer drops its YAML comments. Messages name valid keys, count malformed entries and name unreadable files, -and never show a value. A `.hack/.env` with a valid materialization state is generated output, and -this comparison skips it. Tell teammates when you commit the migration. +and never show a value. This comparison skips a `.hack/.env` with a valid materialization state. +A migration that runs for another reason, such as an overlay value, also checks that file's text, +as described below. With nothing else to migrate, doctor reports that the project already uses env +layers without checking the text, and a later `hack env materialize` replaces the file. Tell +teammates when you commit the migration. Migrated values are global, so a service added later receives them too, and so do host lifecycle hooks. A migrated value also takes precedence over the same variable exported in your shell, @@ -1131,11 +1135,11 @@ files, when: would write (`E_PATH_CONFLICT`). The file is out of date or was edited by hand, and Hack can't tell which. If it's generated, move any value of your own with `hack env set`, then run `hack env materialize`, which replaces it. If it's your own file, delete `.hack/.env.state.json` - so it's read as a legacy file. When the text does match, the migration doesn't read the file or - offer it for cleanup; -- `.hack/.env` has a state file that can't be parsed, and env layers exist (`E_PATH_CONFLICT`). If - `.env` is your own file, delete `.hack/.env.state.json`. If it's generated output you don't need, - delete both, then run `hack env materialize`; + so it's read as a legacy file, but not for generated output, whose values would then migrate as + your own. When the text does match, the migration doesn't read the file or offer it for cleanup; +- `.hack/.env` has a state entry that can't be read or parsed, such as a dangling link, and env + layers exist (`E_PATH_CONFLICT`). If `.env` is your own file, delete `.hack/.env.state.json`. If + it's generated output you don't need, delete both, then run `hack env materialize`; - a legacy file sets a key that Docker, its build tools or the process read from their environment (`E_CONFIG_INVALID`): `DOCKER_*`, `BUILDKIT_*`, `BUILDX_*`, a proxy setting, `SOURCE_DATE_EPOCH`, `HOME` or `PATH`. After the migration it would reach the Docker client too. diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index c4c2a959a..9284b3e94 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5935,9 +5935,9 @@ async function deliveredLayerValues(opts: { * * A contract project is out of scope, because its own migration owns those files. * `.hack/.env` with a valid state file is treated as generated output, so it is - * skipped; the migration checks its text before relying on that - * ({@link legacyBaseDotenvIsGenerated}). Overlays are not generated by materialize, - * so they are always compared. + * skipped. A migration that runs for another reason checks its text + * ({@link legacyBaseDotenvIsGenerated}); with nothing else to migrate, nothing does. + * Overlays are not generated by materialize, so they are always compared. */ export async function findUnmigratedLegacyDotenv(opts: { readonly projectRoot: string; @@ -6826,6 +6826,9 @@ async function migrateUndeclaredLegacyDotenv(opts: { * survive a switch to a branch without layers, and `.env` may hold real values by * then. With layers and different text, the file is either out of date or edited by * hand, which this can't tell apart, so it refuses instead of dropping or pinning. + * + * A state entry counts as absent only when nothing is at that path: a dangling link + * or an unreadable entry is a state file this can't read, so it refuses too. */ async function legacyBaseDotenvIsGenerated(opts: { readonly projectRoot: string; @@ -6836,7 +6839,10 @@ async function legacyBaseDotenvIsGenerated(opts: { const read = await readOptionalTextFile( resolve(opts.projectDir, PROJECT_ENV_FILENAME) ); - if (read.kind !== "text" || !(await pathExists(statePath))) { + if ( + read.kind !== "text" || + (await readOptionalTextFile(statePath)).kind === "absent" + ) { return false; } const stateRead = await readProjectEnvStateFile({ statePath }); @@ -6853,7 +6859,7 @@ async function legacyBaseDotenvIsGenerated(opts: { throw new LegacyDotenvMigrationRefusalError({ code: "E_PATH_CONFLICT", reason: - ".hack/.env's state file can't be read, so Hack can't tell whether .env is generated output or holds values of your own. If it's your own file, delete .hack/.env.state.json so it's read as a legacy file. If it's generated output you don't need, delete both files, then run `hack env materialize`.", + ".hack/.env's state file can't be read, so Hack can't tell whether .env is generated output or holds values of your own. If it's your own file, delete .hack/.env.state.json so it's read as a legacy file. If it's generated output you don't need, delete both files, then run `hack env materialize`. Don't delete only the state file of generated output, because its values would then migrate as your own.", }); } const generated = serializeDotEnv( @@ -6865,7 +6871,7 @@ async function legacyBaseDotenvIsGenerated(opts: { if (generated !== read.text) { throw new LegacyDotenvMigrationRefusalError({ code: "E_PATH_CONFLICT", - reason: `.hack/.env has a state file, but its text isn't what \`hack env materialize\` would write, so it may hold values of your own. If it's generated output, move any value of your own with \`hack env set\`, then run \`hack env materialize\`, which replaces the file. If it's your own file, delete .hack/.env.state.json so it's read as a legacy file.`, + reason: `.hack/.env has a state file, but its text isn't what \`hack env materialize\` would write, so it may hold values of your own. If it's generated output, move any value of your own with \`hack env set\`, then run \`hack env materialize\`, which replaces the file. If it's your own file, delete .hack/.env.state.json so it's read as a legacy file. Don't do that for generated output, because its values would then migrate as your own.`, }); } return true; diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index e3b649654..210aec376 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -150,6 +150,7 @@ async function runCli(opts: { HACK_GLOBAL_CONFIG_PATH: join(opts.repo.root, "hack-config.json"), HACK_HOME: join(opts.repo.root, "hack-home"), HACK_LOGGER: opts.logger ?? "console", + HACK_DAEMON_AUTOSTART_DISABLE: "1", HACK_NO_INTERACTIVE: "1", HACK_SECRETS_FILE_KEY: ["legacy", "scope", "test", "key"].join("-"), HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", @@ -3199,3 +3200,169 @@ test("without layers, an unparseable state file doesn't stop a real .env migrati global: { KEPT_SETTING: KEPT_VALUE }, }); }); + +/** For a state entry `legacySnapshot` can't read: refuses, and `.env` and the entries stay. */ +async function expectUnreadableStateRefused(repo: Repo): Promise { + const envPath = resolve(repo.projectDir, ".env"); + const envBefore = await readFile(envPath, "utf8"); + const entriesBefore = await hackDirEntries(repo); + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + const refusal = error as LegacyDotenvMigrationRefusalError; + expect(refusal.code).toBe("E_PATH_CONFLICT"); + expect(refusal.message).toContain("state file can't be read"); + expect(refusal.message).not.toContain(KEPT_VALUE); + expect(await readFile(envPath, "utf8")).toBe(envBefore); + expect(await hackDirEntries(repo)).toEqual(entriesBefore); +} + +test("a dangling state link isn't absent: the migration refuses, and generated values aren't pinned", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const statePath = resolve(repo.projectDir, ".env.state.json"); + await rm(statePath); + await symlink(resolve(repo.root, "state-file-that-is-gone"), statePath); + await expectUnreadableStateRefused(repo); + expect(await readlink(statePath)).toBe( + resolve(repo.root, "state-file-that-is-gone") + ); +}); + +test.skipIf(RUNS_AS_ROOT)( + "a state entry that can't be read refuses the same way", + async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const statePath = resolve(repo.projectDir, ".env.state.json"); + await chmod(statePath, 0o000); + try { + await expectUnreadableStateRefused(repo); + } finally { + await chmod(statePath, 0o600); + } + } +); + +test("a directory at the state path refuses the same way", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const statePath = resolve(repo.projectDir, ".env.state.json"); + await rm(statePath); + await mkdir(statePath); + await expectUnreadableStateRefused(repo); +}); + +test("without layers, a dangling state link doesn't stop a real .env migrating", async () => { + const repo = await createLegacyRepo(); + await symlink( + resolve(repo.root, "state-file-that-is-gone"), + resolve(repo.projectDir, ".env.state.json") + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); +}); + +test("following the printed recovery through doctor keeps the overlay's and the manual values", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const envPath = resolve(repo.projectDir, ".env"); + const stagingPath = resolve(repo.projectDir, ".env.staging"); + const stagingBytes = await readFile(stagingPath); + const manualLine = `EXTRA=${BASE_ONLY_VALUE}\n`; + await appendFile(envPath, manualLine); + const editedBytes = await readFile(envPath, "utf8"); + const doctor = () => + runCli({ + repo, + args: ["doctor", "--migrate-env-config", "--path", repo.projectRoot], + }); + + const refused = await doctor(); + const refusedOutput = `${refused.stdout}${refused.stderr}`; + expect(refusedOutput).toContain("hack env materialize"); + expect(refusedOutput).not.toContain(BASE_ONLY_VALUE); + expect(refusedOutput).not.toContain(KEPT_VALUE); + expect(await readFile(envPath, "utf8")).toBe(editedBytes); + expect((await readFile(stagingPath)).equals(stagingBytes)).toBe(true); + + const moved = await runCli({ + repo, + args: [ + "env", + "set", + "EXTRA", + "--local", + "--value-stdin", + "--path", + repo.projectRoot, + ], + stdin: `${BASE_ONLY_VALUE}\n`, + }); + expect(moved.exitCode).toBe(0); + const materialized = await runCli({ + repo, + args: [ + "env", + "materialize", + "--env", + "staging", + "--path", + repo.projectRoot, + ], + }); + expect(materialized.exitCode).toBe(0); + expect(await readFile(envPath, "utf8")).toContain(manualLine); + expect((await readFile(stagingPath)).equals(stagingBytes)).toBe(true); + + const recovered = await doctor(); + const recoveredOutput = `${recovered.stdout}${recovered.stderr}`; + expect(recoveredOutput).not.toContain(BASE_ONLY_VALUE); + expect(recoveredOutput).not.toContain(KEPT_VALUE); + // Doctor's exit code also reflects Docker, so assert on what it migrated. + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { EXTRA: BASE_ONLY_VALUE }, + }); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { ONLY_HERE: KEPT_VALUE }, + }); + expect(await readFile(envPath, "utf8")).toContain(manualLine); +}, 60_000); + +test("with a corrupt state and a .env generated for staging, deleting both files breaks the loop", async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const envPath = resolve(repo.projectDir, ".env"); + const statePath = resolve(repo.projectDir, ".env.state.json"); + await writeFile(statePath, "{not json\n"); + const materialize = () => + runCli({ + repo, + args: ["env", "materialize", "--path", repo.projectRoot], + }); + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: "state file can't be read", + hidden: [KEPT_VALUE], + }); + expect((await materialize()).exitCode).not.toBe(0); + + await rm(envPath); + await rm(statePath); + expect((await materialize()).exitCode).toBe(0); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + + expect( + await lstat(resolve(repo.projectDir, "hack.env.local.yaml")).catch( + () => null + ) + ).toBeNull(); + expect(await readLayerValues(repo, "hack.env.staging.local.yaml")).toEqual({ + global: { ONLY_HERE: KEPT_VALUE }, + }); +}, 60_000); From ddfa55741093246abbcd6ee35044e2d24944b695 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:52:17 -0500 Subject: [PATCH 38/55] fix(env): refuse a migration when a layer or Compose file can't be read Three inputs let the legacy `.env` migration write when it should refuse: - An env layer that can't be read counted as delivering nothing, so legacy values were pinned locally over what that layer sets once it is readable again. - Another `env_file` that sets a migrated key as `export KEY=value`, `KEY: value` or a bare `KEY` wasn't seen, so the migrated value replaced the one Compose used. - A Compose file that can't be read skipped every Compose check, so migrated values reached services that never listed `.env`. Each now refuses and writes nothing. An absent Compose file still migrates, and `hack doctor` reporting still treats an unusable layer as delivering nothing. Adds regression tests for these inputs and for a state link into a directory without search permission. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 6 +- src/lib/project-env-config.ts | 81 +++++++++- tests/env-legacy-service-scope.test.ts | 200 +++++++++++++++++++++++++ 3 files changed, 278 insertions(+), 9 deletions(-) diff --git a/docs/env.md b/docs/env.md index c2b16f08a..605f1e178 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1110,7 +1110,8 @@ files, when: receive. Write those `env_file` entries directly in the Compose file, each file once, with a plain path and no `format`; - a key the legacy files set is also set in another file a Compose service uses as - `env_file` (`E_CONFIG_INVALID`), for the same reason: that file stays in Compose, but the + `env_file` (`E_CONFIG_INVALID`), in any form Compose reads, such as `export KEY=value`, + `KEY: value` or a bare `KEY`, for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, or expand variables in an `env_file` path. Remove the key from one of the files, then retry; - a legacy entry is one Compose reads differently from Hack (`E_CONFIG_INVALID`): an unquoted @@ -1150,6 +1151,9 @@ files, when: - `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` (`E_PATH_CONFLICT`), so its values would be copied from elsewhere into this project's layers. Replace the link with a regular file in `.hack`, or remove it; +- the Compose file exists but cannot be read (`E_PERMISSION`), or an env layer can't be read + (`E_CONFIG_INVALID`), so the migration can't tell which values services receive or which + values the layers already deliver. Make the file readable or fix the layer, then retry; - a legacy `.hack/.env` or overlay file exists but cannot be read, for example mode 000 or a dangling symlink (`E_PERMISSION`), so its values would be lost. A symlinked overlay that resolves to a readable file inside `.hack` is migrated like a regular one; cleanup diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 9284b3e94..d01922b4c 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5533,8 +5533,8 @@ async function composeEnvFileKeyConflicts(opts: { ) { continue; } - const values = await readLegacyDotenvFile({ path }); - for (const key of Object.keys(values)) { + const text = await readLegacyDotenvText({ path }); + for (const key of composeEnvFileEntryNames(text ?? "")) { if (opts.migratedKeys.has(key)) { conflicts.push( `${JSON.stringify(key)} (${JSON.stringify(configuredPath)} in ${JSON.stringify(opts.service)})` @@ -5545,6 +5545,25 @@ async function composeEnvFileKeyConflicts(opts: { return conflicts; } +const COMPOSE_ENV_FILE_ENTRY_NAME = /^?[ \t]*(?:export[ \t]+)?([^\s=:#]+)/; +const ENV_FILE_LINE_BREAK = /\r?\n/; + +/** + * Key names a Compose `env_file` can set, read line by line instead of with + * `parseDotEnv`, which doesn't see `export KEY=value`, `KEY: value` or a bare `KEY`. + * A line inside a multi-line value can match too, which refuses more than it must. + */ +function composeEnvFileEntryNames(text: string): ReadonlySet { + const names = new Set(Object.keys(parseDotEnv(text))); + for (const line of text.split(ENV_FILE_LINE_BREAK)) { + const name = COMPOSE_ENV_FILE_ENTRY_NAME.exec(line)?.[1]; + if (name !== undefined) { + names.add(name); + } + } + return names; +} + /** Per-service `environment` and other `env_file` entries that set a migrated key. */ async function collectComposeServiceConflicts(opts: { readonly projectDir: string; @@ -5605,7 +5624,14 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { } assertNoComposeToolKeys({ migratedKeys }); const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); - const composeText = (await readTextFile(composeFile)) ?? ""; + const composeRead = await readOptionalTextFile(composeFile); + if (composeRead.kind === "unreadable") { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_PERMISSION", + reason: `${PROJECT_COMPOSE_FILENAME} exists but cannot be read, so the migration cannot tell which values services receive. Make it a readable regular file, then retry.`, + }); + } + const composeText = composeRead.kind === "text" ? composeRead.text : ""; const parsed = await readComposeFile({ composeFile }); if (parsed === null && composeText.trim().length > 0) { throw new LegacyDotenvMigrationRefusalError({ @@ -5927,6 +5953,28 @@ async function deliveredLayerValues(opts: { } } +/** + * {@link deliveredLayerValues} for a caller that plans writes from the answer. A + * layer it can't read might set any key, so it refuses instead of reporting nothing + * delivered, which would pin legacy values over that layer once it's readable again. + */ +async function deliveredLayerValuesForMigration(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly envName: string | null; +}): Promise>> { + try { + return (await resolveProjectEnvConfig(opts))?.globalEnv ?? {}; + } catch { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: + "an env layer can't be read, so the migration can't tell which values the layers already deliver. Run `hack env list` to see the error, fix that layer, then retry.", + }); + } +} + /** * What this checkout's legacy `.hack/.env*` files hold that its env layers don't * deliver, in a project that already has layers, for example after a teammate's @@ -6317,11 +6365,14 @@ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { readonly legacyValues: ReadonlyMap>; }): Promise { const conflicts: string[] = []; - const deliveredBase = await deliveredLayerValues({ ...opts, envName: null }); + const deliveredBase = await deliveredLayerValuesForMigration({ + ...opts, + envName: null, + }); for (const overlayName of await listProjectEnvOverlayNames({ projectDir: opts.projectDir, })) { - const delivered = await deliveredLayerValues({ + const delivered = await deliveredLayerValuesForMigration({ ...opts, envName: overlayName, }); @@ -6659,7 +6710,10 @@ async function migrateUndeclaredLegacyDotenv(opts: { const baseValues = legacyValues.get(null) ?? {}; const baseToMigrate = legacyValuesNotDelivered({ values: baseValues, - delivered: await deliveredLayerValues({ ...opts, envName: null }), + delivered: await deliveredLayerValuesForMigration({ + ...opts, + envName: null, + }), }); const localWrites: PlannedEnvLayerWrite[] = []; const trackedWrites: PlannedEnvLayerWrite[] = [ @@ -6692,7 +6746,10 @@ async function migrateUndeclaredLegacyDotenv(opts: { // The local base about to be written outranks a tracked overlay, so compare // with the layers as they will be, not as they are. delivered: { - ...(await deliveredLayerValues({ ...opts, envName: overlayName })), + ...(await deliveredLayerValuesForMigration({ + ...opts, + envName: overlayName, + })), ...baseToMigrate, }, }); @@ -6885,6 +6942,14 @@ async function legacyBaseDotenvIsGenerated(opts: { async function readLegacyDotenvFile(opts: { readonly path: string; }): Promise> { + const text = await readLegacyDotenvText(opts); + return text === null ? {} : parseDotEnv(text); +} + +/** Text of a legacy dotenv file, null when absent; refuses like {@link readLegacyDotenvFile}. */ +async function readLegacyDotenvText(opts: { + readonly path: string; +}): Promise { const read = await readOptionalTextFile(opts.path); if (read.kind === "unreadable") { throw new LegacyDotenvMigrationRefusalError({ @@ -6892,7 +6957,7 @@ async function readLegacyDotenvFile(opts: { reason: `${opts.path} exists but cannot be read. Make it a readable regular file, then retry.`, }); } - return read.kind === "absent" ? {} : parseDotEnv(read.text); + return read.kind === "absent" ? null : read.text; } export async function removeLegacyProjectEnvArtifacts(opts: { diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 210aec376..1bbab35fa 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -3366,3 +3366,203 @@ test("with a corrupt state and a .env generated for staging, deleting both files global: { ONLY_HERE: KEPT_VALUE }, }); }, 60_000); + +const TWO_ENV_FILES = + "services:\n api:\n image: alpine:3.20\n env_file: [.env, other.env]\n"; + +/** Refuses with `code`, and writes no layer and changes nothing already there. */ +async function expectMigrationRefused( + repo: Repo, + code: string, + reason: string +): Promise { + const entriesBefore = await hackDirEntries(repo); + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + const refusal = error as LegacyDotenvMigrationRefusalError; + expect(refusal.code).toBe(code as typeof refusal.code); + expect(refusal.message).toContain(reason); + expect(refusal.message).not.toContain(KEPT_VALUE); + expect(await hackDirEntries(repo)).toEqual(entriesBefore); +} + +const LAYER_UNREADABLE = "an env layer can't be read"; + +test("an unparseable tracked overlay layer refuses instead of pinning the base over it", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + "<<<<<<< HEAD\nvalues: [\n" + ); + await expectMigrationRefused(repo, "E_CONFIG_INVALID", LAYER_UNREADABLE); +}); + +test("an unparseable default layer refuses instead of pinning every value locally", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + "<<<<<<< HEAD\nvalues: [\n" + ); + await expectMigrationRefused(repo, "E_CONFIG_INVALID", LAYER_UNREADABLE); +}); + +test.skipIf(RUNS_AS_ROOT)( + "a default layer that can't be read refuses the same way", + async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const layer = resolve(repo.projectDir, "hack.env.default.yaml"); + await writeFile(layer, LAYER_EMPTY); + await chmod(layer, 0o000); + try { + await expectMigrationRefused(repo, "E_CONFIG_INVALID", LAYER_UNREADABLE); + } finally { + await chmod(layer, 0o600); + } + } +); + +for (const [form, line] of [ + ["export", "export KEPT_SETTING=other"], + ["colon", "KEPT_SETTING: other"], + ["bare", "KEPT_SETTING"], +] as const) { + test(`another env_file that sets a migrated key in ${form} form refuses`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + TWO_ENV_FILES + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile(resolve(repo.projectDir, "other.env"), `${line}\n`); + await expectMigrationRefused( + repo, + "E_CONFIG_INVALID", + '"KEPT_SETTING" ("other.env" in "api")' + ); + }); +} + +test("another env_file that only mentions a migrated key in a comment or a value still migrates", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + TWO_ENV_FILES + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, "other.env"), + "# KEPT_SETTING is set elsewhere\nOTHER=KEPT_SETTING\n" + ); + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); +}); + +test.skipIf(RUNS_AS_ROOT)( + "a Compose file that can't be read refuses instead of skipping the Compose checks", + async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const composeFile = resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME); + await chmod(composeFile, 0o000); + try { + await expectMigrationRefused( + repo, + "E_PERMISSION", + "exists but cannot be read" + ); + } finally { + await chmod(composeFile, 0o644); + } + } +); + +test("a dangling Compose link refuses the same way", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const composeFile = resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME); + await rm(composeFile); + await symlink(resolve(repo.root, "compose-that-is-gone"), composeFile); + await expectMigrationRefused( + repo, + "E_PERMISSION", + "exists but cannot be read" + ); +}); + +test.skipIf(RUNS_AS_ROOT)( + "a state link into a directory without search permission refuses, and generated values aren't pinned", + async () => { + const repo = await writeGeneratedCheckout({ envName: "staging" }); + const locked = resolve(repo.root, "locked"); + await mkdir(locked); + const statePath = resolve(repo.projectDir, ".env.state.json"); + const target = resolve(locked, "state.json"); + await writeFile(target, await readFile(statePath)); + await rm(statePath); + await symlink(target, statePath); + await chmod(locked, 0o600); + try { + await expectUnreadableStateRefused(repo); + } finally { + await chmod(locked, 0o700); + } + } +); + +test.skipIf(RUNS_AS_ROOT)( + "without layers, a state link into a directory without search permission doesn't stop a real .env migrating", + async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const locked = resolve(repo.root, "locked"); + await mkdir(locked); + await symlink( + resolve(locked, "state.json"), + resolve(repo.projectDir, ".env.state.json") + ); + await chmod(locked, 0o600); + try { + await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); + } finally { + await chmod(locked, 0o700); + } + expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ + global: { KEPT_SETTING: KEPT_VALUE }, + }); + } +); From 17d946f01c3dd3f124096f6b441f3c84689d2df3 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:53:43 -0500 Subject: [PATCH 39/55] fix(env): keep a local layer's own refusal ahead of the delivered-values read A local layer that can't be read refused with a line naming the file. The strict read added in ddfa5574 fired first with a generic line, so check the local layers before it. Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index d01922b4c..50cc9f9b3 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6707,6 +6707,17 @@ async function migrateUndeclaredLegacyDotenv(opts: { legacyValues, }); + // Before the delivered values, so a local layer's refusal can name that file. + for (const envName of [null, ...overlayNames]) { + await readExistingLocalLayer({ + path: resolveProjectEnvLocalConfigPath({ + projectDir: opts.projectDir, + envName, + }), + environment: envName ?? "default", + }); + } + const baseValues = legacyValues.get(null) ?? {}; const baseToMigrate = legacyValuesNotDelivered({ values: baseValues, From c7a3f3122cc9cabf7b183ac607dda93896f1c183 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:03:17 -0500 Subject: [PATCH 40/55] fix(env): name the overlay in the layer refusal and read quoted env_file tails The layer refusal's `hack env list` showed nothing for an overlay that isn't selected, so it now gives `hack env list --env `, and it says a project env key may be missing. Another env_file's `A='x' KEY=value` sets KEY in Compose, so that form now refuses too. An unreadable Compose file refuses even with nothing to migrate. The refusal test helper now compares file bytes as well as names. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 13 ++-- src/lib/project-env-config.ts | 36 +++++++---- tests/env-legacy-service-scope.test.ts | 88 ++++++++++++++++++++++++-- 3 files changed, 116 insertions(+), 21 deletions(-) diff --git a/docs/env.md b/docs/env.md index 605f1e178..6f29e6eb7 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1110,8 +1110,8 @@ files, when: receive. Write those `env_file` entries directly in the Compose file, each file once, with a plain path and no `format`; - a key the legacy files set is also set in another file a Compose service uses as - `env_file` (`E_CONFIG_INVALID`), in any form Compose reads, such as `export KEY=value`, - `KEY: value` or a bare `KEY`, for the same reason: that file stays in Compose, but the + `env_file` (`E_CONFIG_INVALID`), including as `export KEY=value`, `KEY: value`, a bare `KEY` + or after a closing quote (`A='x' KEY=value`), for the same reason: that file stays in Compose, but the migrated value would override it. The check does not look at the order of the entries, or expand variables in an `env_file` path. Remove the key from one of the files, then retry; - a legacy entry is one Compose reads differently from Hack (`E_CONFIG_INVALID`): an unquoted @@ -1151,9 +1151,12 @@ files, when: - `.hack/.env` or a legacy overlay is a symlink to a file outside `.hack` (`E_PATH_CONFLICT`), so its values would be copied from elsewhere into this project's layers. Replace the link with a regular file in `.hack`, or remove it; -- the Compose file exists but cannot be read (`E_PERMISSION`), or an env layer can't be read - (`E_CONFIG_INVALID`), so the migration can't tell which values services receive or which - values the layers already deliver. Make the file readable or fix the layer, then retry; +- the Compose file exists but cannot be read (`E_PERMISSION`), so the migration can't tell which + values services receive. Make it a readable regular file, then retry; +- an env layer can't be read or resolved (`E_CONFIG_INVALID`), for example it doesn't parse or its + project env key is missing, including a layer for an overlay you don't use, so the migration + can't tell which values the layers already deliver. The error gives the `hack env list` command + that shows the problem. Fix it, then retry; - a legacy `.hack/.env` or overlay file exists but cannot be read, for example mode 000 or a dangling symlink (`E_PERMISSION`), so its values would be lost. A symlinked overlay that resolves to a readable file inside `.hack` is migrated like a regular one; cleanup diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 50cc9f9b3..ddca01494 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5545,20 +5545,28 @@ async function composeEnvFileKeyConflicts(opts: { return conflicts; } -const COMPOSE_ENV_FILE_ENTRY_NAME = /^?[ \t]*(?:export[ \t]+)?([^\s=:#]+)/; +// `\s` includes a BOM. The quotes are there for text after a closing quote. +const COMPOSE_ENV_FILE_ENTRY_NAME = + /(?:^|[\s'"])(?:export[ \t]+)?([^\s=:#'"]+)[ \t]*(?:[=:#]|$)/g; const ENV_FILE_LINE_BREAK = /\r?\n/; /** * Key names a Compose `env_file` can set, read line by line instead of with - * `parseDotEnv`, which doesn't see `export KEY=value`, `KEY: value` or a bare `KEY`. - * A line inside a multi-line value can match too, which refuses more than it must. + * `parseDotEnv`, which doesn't see `export KEY=value`, `KEY: value`, a bare `KEY`, or + * `A='x' KEY=value`, where Compose reads the text after the closing quote as another + * entry. A value that holds such text, or a line inside a multi-line value, can match + * too, which refuses more than it must. */ function composeEnvFileEntryNames(text: string): ReadonlySet { const names = new Set(Object.keys(parseDotEnv(text))); for (const line of text.split(ENV_FILE_LINE_BREAK)) { - const name = COMPOSE_ENV_FILE_ENTRY_NAME.exec(line)?.[1]; - if (name !== undefined) { - names.add(name); + if (line.trimStart().startsWith("#")) { + continue; + } + for (const match of line.matchAll(COMPOSE_ENV_FILE_ENTRY_NAME)) { + if (match[1] !== undefined) { + names.add(match[1]); + } } } return names; @@ -5619,10 +5627,6 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { migratedKeys.add(key); } } - if (migratedKeys.size === 0) { - return; - } - assertNoComposeToolKeys({ migratedKeys }); const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); const composeRead = await readOptionalTextFile(composeFile); if (composeRead.kind === "unreadable") { @@ -5631,6 +5635,10 @@ async function assertComposeEnvironmentKeepsPrecedence(opts: { reason: `${PROJECT_COMPOSE_FILENAME} exists but cannot be read, so the migration cannot tell which values services receive. Make it a readable regular file, then retry.`, }); } + if (migratedKeys.size === 0) { + return; + } + assertNoComposeToolKeys({ migratedKeys }); const composeText = composeRead.kind === "text" ? composeRead.text : ""; const parsed = await readComposeFile({ composeFile }); if (parsed === null && composeText.trim().length > 0) { @@ -5957,6 +5965,7 @@ async function deliveredLayerValues(opts: { * {@link deliveredLayerValues} for a caller that plans writes from the answer. A * layer it can't read might set any key, so it refuses instead of reporting nothing * delivered, which would pin legacy values over that layer once it's readable again. + * The caught message isn't repeated, because a parser's message might quote the file. */ async function deliveredLayerValuesForMigration(opts: { readonly projectRoot: string; @@ -5967,10 +5976,13 @@ async function deliveredLayerValuesForMigration(opts: { try { return (await resolveProjectEnvConfig(opts))?.globalEnv ?? {}; } catch { + const listCommand = + opts.envName === null + ? "hack env list" + : `hack env list --env ${opts.envName}`; throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: - "an env layer can't be read, so the migration can't tell which values the layers already deliver. Run `hack env list` to see the error, fix that layer, then retry.", + reason: `an env layer can't be read or resolved, for example it doesn't parse or its project env key is missing, so the migration can't tell which values the layers already deliver. Run \`${listCommand}\` to see the error, fix it, then retry.`, }); } } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 1bbab35fa..11e0fd123 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -3370,13 +3370,27 @@ test("with a corrupt state and a .env generated for staging, deleting both files const TWO_ENV_FILES = "services:\n api:\n image: alpine:3.20\n env_file: [.env, other.env]\n"; +/** Every entry's bytes, null for one that can't be read. */ +async function hackDirContents( + repo: Repo +): Promise> { + const contents: Record = {}; + for (const name of await hackDirEntries(repo)) { + contents[name] = await readFile( + resolve(repo.projectDir, name), + "base64" + ).catch(() => null); + } + return contents; +} + /** Refuses with `code`, and writes no layer and changes nothing already there. */ async function expectMigrationRefused( repo: Repo, code: string, reason: string ): Promise { - const entriesBefore = await hackDirEntries(repo); + const contentsBefore = await hackDirContents(repo); const error = await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true, }).catch((caught: unknown) => caught); @@ -3385,10 +3399,10 @@ async function expectMigrationRefused( expect(refusal.code).toBe(code as typeof refusal.code); expect(refusal.message).toContain(reason); expect(refusal.message).not.toContain(KEPT_VALUE); - expect(await hackDirEntries(repo)).toEqual(entriesBefore); + expect(await hackDirContents(repo)).toEqual(contentsBefore); } -const LAYER_UNREADABLE = "an env layer can't be read"; +const LAYER_UNREADABLE = "an env layer can't be read or resolved"; test("an unparseable tracked overlay layer refuses instead of pinning the base over it", async () => { const repo = await createLegacyRepo(); @@ -3404,7 +3418,50 @@ test("an unparseable tracked overlay layer refuses instead of pinning the base o resolve(repo.projectDir, "hack.env.staging.yaml"), "<<<<<<< HEAD\nvalues: [\n" ); - await expectMigrationRefused(repo, "E_CONFIG_INVALID", LAYER_UNREADABLE); + await expectMigrationRefused( + repo, + "E_CONFIG_INVALID", + "Run `hack env list --env staging` to see the error" + ); + // The command the refusal names shows the problem, which a plain list doesn't. + const named = await runCli({ + repo, + args: ["env", "list", "--env", "staging", "--path", repo.projectRoot], + }); + expect(named.exitCode).not.toBe(0); + expect(`${named.stdout}\n${named.stderr}`).toContain("hack.env.staging.yaml"); +}); + +test("an overlay layer whose project env key is missing refuses and names that overlay", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + LAYER_EMPTY.replace( + " global: {}\n", + " global:\n STAGING_SECRET:\n secure: v1:synthetic-ciphertext-not-valid-for-decryption\n" + ) + ); + const inherited = process.env.HACK_ENV_SECRET_KEY; + Reflect.deleteProperty(process.env, "HACK_ENV_SECRET_KEY"); + try { + await expectMigrationRefused( + repo, + "E_CONFIG_INVALID", + "Run `hack env list --env staging` to see the error" + ); + } finally { + if (inherited !== undefined) { + process.env.HACK_ENV_SECRET_KEY = inherited; + } + } }); test("an unparseable default layer refuses instead of pinning every value locally", async () => { @@ -3443,6 +3500,11 @@ for (const [form, line] of [ ["export", "export KEPT_SETTING=other"], ["colon", "KEPT_SETTING: other"], ["bare", "KEPT_SETTING"], + ["after a closing quote", "A='x' KEPT_SETTING=other"], + ["right after a closing quote", "A='x'KEPT_SETTING=other"], + ["CRLF", "export KEPT_SETTING=other\r"], + ["indented", " KEPT_SETTING=other"], + ["BOM", "\uFEFFKEPT_SETTING=other"], ] as const) { test(`another env_file that sets a migrated key in ${form} form refuses`, async () => { const repo = await createLegacyRepo(); @@ -3505,6 +3567,24 @@ test.skipIf(RUNS_AS_ROOT)( } ); +test.skipIf(RUNS_AS_ROOT)( + "a Compose file that can't be read refuses with nothing to migrate too", + async () => { + const repo = await createLegacyRepo(); + const composeFile = resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME); + await chmod(composeFile, 0o000); + try { + await expectMigrationRefused( + repo, + "E_PERMISSION", + "exists but cannot be read" + ); + } finally { + await chmod(composeFile, 0o644); + } + } +); + test("a dangling Compose link refuses the same way", async () => { const repo = await createLegacyRepo(); await writeFile( From be1b652a8f73f085d3a5a10766fbfe5d0dd936e0 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:13:42 -0500 Subject: [PATCH 41/55] fix(env): pass the project root to the migration's layer writes HACK-1685's layer writer enforces the project boundary, so every migration write was refused without the project root. Also from review: only a name `--env` accepts goes into the refusal's command, and any other is named as a quoted file; a comment at the end of an unquoted env_file line no longer refuses. Two symlink fixtures now point inside the project, which HACK-1685 requires, with their assertions unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 4 +-- src/lib/project-env-config.ts | 46 ++++++++++++++++++++------ tests/env-legacy-service-scope.test.ts | 37 ++++++++++++++++++--- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/docs/env.md b/docs/env.md index 34047ae70..ab5a7c2fe 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1169,8 +1169,8 @@ files, when: values services receive. Make it a readable regular file, then retry; - an env layer can't be read or resolved (`E_CONFIG_INVALID`), for example it doesn't parse or its project env key is missing, including a layer for an overlay you don't use, so the migration - can't tell which values the layers already deliver. The error gives the `hack env list` command - that shows the problem. Fix it, then retry; + can't tell which values the layers already deliver. The error names the file or gives the + `hack env list` command that shows the problem. Fix it, then retry; - a legacy `.hack/.env` or overlay file exists but cannot be read, for example mode 000 or a dangling symlink (`E_PERMISSION`), so its values would be lost. A symlinked overlay that resolves to a readable file inside `.hack` is migrated like a regular one; cleanup diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 95541e5da..f9c37b635 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -87,7 +87,10 @@ import { readReplaceablePrivateFile, replacePrivateFileAtomic, } from "./private-file.ts"; -import { readProjectDefaultEnvConfig } from "./project.ts"; +import { + parseEnvConfigSelection, + readProjectDefaultEnvConfig, +} from "./project.ts"; import { assertProjectEnvLayerPath } from "./project-env-layer-path.ts"; import { assertLegacyProjectDirectory, @@ -5635,17 +5638,21 @@ async function composeEnvFileKeyConflicts(opts: { return conflicts; } -// `\s` includes a BOM. The quotes are there for text after a closing quote. +// `\s` includes a BOM. The quotes are there for text after a closing quote, and the +// space is what finds the key in `export KEY=value`. const COMPOSE_ENV_FILE_ENTRY_NAME = - /(?:^|[\s'"])(?:export[ \t]+)?([^\s=:#'"]+)[ \t]*(?:[=:#]|$)/g; + /(?:^|[\s'"])([^\s=:#'"]+)[ \t]*(?:[=:#]|$)/g; const ENV_FILE_LINE_BREAK = /\r?\n/; +const ENV_FILE_QUOTE = /['"]/; +const ENV_FILE_TRAILING_COMMENT = /\s#.*$/; /** * Key names a Compose `env_file` can set, read line by line instead of with * `parseDotEnv`, which doesn't see `export KEY=value`, `KEY: value`, a bare `KEY`, or * `A='x' KEY=value`, where Compose reads the text after the closing quote as another - * entry. A value that holds such text, or a line inside a multi-line value, can match - * too, which refuses more than it must. + * entry. A value that holds such text, a comment after a quoted value, or a line inside + * a multi-line value can match too, which refuses more than it must. A trailing comment + * is dropped only from a line without quotes, because `A='x # y' KEY=value` sets KEY. */ function composeEnvFileEntryNames(text: string): ReadonlySet { const names = new Set(Object.keys(parseDotEnv(text))); @@ -5653,7 +5660,10 @@ function composeEnvFileEntryNames(text: string): ReadonlySet { if (line.trimStart().startsWith("#")) { continue; } - for (const match of line.matchAll(COMPOSE_ENV_FILE_ENTRY_NAME)) { + const entries = ENV_FILE_QUOTE.test(line) + ? line + : line.replace(ENV_FILE_TRAILING_COMMENT, ""); + for (const match of entries.matchAll(COMPOSE_ENV_FILE_ENTRY_NAME)) { if (match[1] !== undefined) { names.add(match[1]); } @@ -6051,6 +6061,21 @@ async function deliveredLayerValues(opts: { } } +/** + * How to see a layer's error. A file name comes from the repository and this text + * gets pasted, so only a name `--env` accepts unchanged goes into a command; `base`, + * `default` and `none` select no overlay, so they get the file name too. + */ +function unusableLayerRemedy(envName: string | null): string { + if (envName === null) { + return "Run `hack env list` to see the error, fix it, then retry."; + } + if (parseEnvConfigSelection(envName) === envName) { + return `Run \`hack env list --env ${envName}\` to see the error, fix it, then retry.`; + } + return `Check ${JSON.stringify(`hack.env.${envName}.yaml`)}, fix it, then retry.`; +} + /** * {@link deliveredLayerValues} for a caller that plans writes from the answer. A * layer it can't read might set any key, so it refuses instead of reporting nothing @@ -6066,13 +6091,9 @@ async function deliveredLayerValuesForMigration(opts: { try { return (await resolveProjectEnvConfig(opts))?.globalEnv ?? {}; } catch { - const listCommand = - opts.envName === null - ? "hack env list" - : `hack env list --env ${opts.envName}`; throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `an env layer can't be read or resolved, for example it doesn't parse or its project env key is missing, so the migration can't tell which values the layers already deliver. Run \`${listCommand}\` to see the error, fix it, then retry.`, + reason: `an env layer can't be read or resolved, for example it doesn't parse or its project env key is missing, so the migration can't tell which values the layers already deliver. ${unusableLayerRemedy(opts.envName)}`, }); } } @@ -6682,6 +6703,7 @@ async function planLocalLayerMerges(opts: { * what this call created, so the checkout keeps using the legacy files. */ async function publishMigratedLegacyLayers(opts: { + readonly projectRoot: string; readonly projectDir: string; readonly localWrites: readonly PlannedEnvLayerWrite[]; readonly trackedWrites: readonly PlannedEnvLayerWrite[]; @@ -6710,6 +6732,7 @@ async function publishMigratedLegacyLayers(opts: { } if ( await writeProjectEnvConfigFile({ + projectRoot: opts.projectRoot, path: write.path, config: write.config, expectedExisting: null, @@ -6924,6 +6947,7 @@ async function migrateUndeclaredLegacyDotenv(opts: { } const wroteFiles = await publishMigratedLegacyLayers({ + projectRoot: opts.projectRoot, projectDir: opts.projectDir, localWrites, trackedWrites, diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 11e0fd123..29c654c60 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -333,8 +333,9 @@ async function placeLayerEntry(repo: Repo, entry: LayerEntry): Promise { if (entry.kind === "directory") { await mkdir(path); } else if (entry.kind === "link-to-directory") { - await mkdir(resolve(repo.root, "elsewhere"), { recursive: true }); - await symlink(resolve(repo.root, "elsewhere"), path); + // Inside the project: a link leaving it is refused earlier (HACK-1685). + await mkdir(resolve(repo.projectRoot, "elsewhere"), { recursive: true }); + await symlink(resolve(repo.projectRoot, "elsewhere"), path); } else { await symlink(resolve(repo.projectDir, "missing-layer.yaml"), path); } @@ -408,7 +409,7 @@ for (const entry of UNUSABLE_LAYER_ENTRIES) { test("a symlinked overlay that resolves to a regular layer selects the env layers", async () => { const repo = await createLegacyRepo(); - const target = resolve(repo.root, "shared-staging.yaml"); + const target = resolve(repo.projectRoot, "shared-staging.yaml"); await writeFile( target, LAYER.replace("environment: default", "environment: staging") @@ -3464,6 +3465,34 @@ test("an overlay layer whose project env key is missing refuses and names that o } }); +for (const [fileName, shown] of [ + ["hack.env.$(touch x).yaml", '"hack.env.$(touch x).yaml"'], + ["hack.env.My_Overlay.yaml", '"hack.env.My_Overlay.yaml"'], + ["hack.env.base.yaml", '"hack.env.base.yaml"'], +] as const) { + test(`the layer refusal names ${fileName} instead of building a command from it`, async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, fileName), + "<<<<<<< HEAD\nvalues: [\n" + ); + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((error as Error).message).toContain(`Check ${shown}`); + expect((error as Error).message).not.toContain("hack env list"); + }); +} + test("an unparseable default layer refuses instead of pinning every value locally", async () => { const repo = await createLegacyRepo(); await writeFile( @@ -3537,7 +3566,7 @@ test("another env_file that only mentions a migrated key in a comment or a value ); await writeFile( resolve(repo.projectDir, "other.env"), - "# KEPT_SETTING is set elsewhere\nOTHER=KEPT_SETTING\n" + "# KEPT_SETTING is set elsewhere\n# KEPT_SETTING=old\nOTHER=KEPT_SETTING\nFOO=bar # used by KEPT_SETTING\n" ); await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true }); expect(await readLayerValues(repo, "hack.env.local.yaml")).toEqual({ From 55a1136b36552305dc972ed34fea6cab5e896204 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:26:32 -0500 Subject: [PATCH 42/55] fix(env): keep the project boundary check when a layer is a symlink The mutation gate skipped HACK-1685's boundary check once it saw a linked layer, so `hack doctor --fix --migrate-env-config` rewrote a Compose file outside the project through a linked `.hack`, and rewrote the project's own while a local layer linked outside. The check now always runs, so both refuse and change nothing, as on `next`. Fixtures that linked outside the project on the migration route now link inside it, with their assertions unchanged, and new cases cover the outside links. Docs now say `hack env set` and `unset` refuse an in-project dangling link. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 13 +- docs/security/env-secrets-threat-model.md | 2 +- src/commands/doctor.ts | 3 +- src/lib/project-env-config.ts | 18 ++- tests/env-legacy-service-scope.test.ts | 155 +++++++++++++++++++++- 5 files changed, 173 insertions(+), 18 deletions(-) diff --git a/docs/env.md b/docs/env.md index ab5a7c2fe..ca3ed0d47 100644 --- a/docs/env.md +++ b/docs/env.md @@ -620,9 +620,9 @@ rename has these effects: YAML layers, including case-variant names on case-insensitive filesystems. `hack doctor --fix` reports an error for a `.hack` linked outside the project and leaves the link and outside files in place -- an in-project symlink whose target does not exist keeps the existing behavior: the - link is replaced by a regular file. Strict planning and hydrate reads still refuse - dangling links +- `hack env set` and `hack env unset` refuse an in-project symlink whose target does not + exist with `E_PATH_CONFLICT`, and strict planning and hydrate reads refuse it too. Replace + or remove the link, then retry - the writer rechecks both the layer and its resolved destination before staging and commit. These path checks narrow concurrent-swap races; they are not an atomic filesystem fence against a directory changed after the last check @@ -1097,8 +1097,9 @@ files, when: `env_file` entry and select the overlay with `--env`; - an entry named like an env layer (`.hack/hack.env.*.yaml`) is a dangling symlink, a directory or another non-file (`E_PATH_CONFLICT`). A regular layer file, or a symlink - to one, means the repo already uses env layers: `hack doctor` reports that and writes - nothing, never through the link; + inside the project to one, means the repo already uses env layers: `hack doctor` reports + that and writes nothing, never through the link. A `.hack` or a layer linked outside the + project is refused with a closed error, and the link and outside files stay as they are; - git tracks a machine-local layer it would write, or, in a git checkout, git cannot report whether it does (`E_PATH_CONFLICT`); - in a git checkout, git does not ignore a machine-local layer it would write, for @@ -1214,7 +1215,7 @@ and the command has no effect on the runtime value. `hack env set` and `hack env unset` choose between the env layers and the legacy file by looking for `.hack/hack.env.*.yaml` entries, `hack.env.default.yaml` included. A regular -file, or a symlink to one, counts as an env layer. Any other entry with a layer name, such +file, or a symlink inside the project to one, counts as an env layer. Any other entry with a layer name, such as a dangling symlink or a directory, is refused with `E_PATH_CONFLICT` and nothing is written; it never silently selects the legacy `.hack/.env`. Replace or remove the entry, then retry. Reads refuse the same way: `hack env list`, `hack env explain` and runtime diff --git a/docs/security/env-secrets-threat-model.md b/docs/security/env-secrets-threat-model.md index 93e9ff539..3d8563788 100644 --- a/docs/security/env-secrets-threat-model.md +++ b/docs/security/env-secrets-threat-model.md @@ -169,7 +169,7 @@ These gaps are known and not closed on `next`. Each one needs its own change. path-based, not descriptor-relative `openat`/`renameat`: an ancestor directory swapped after the last check can still redirect an operation. This residual race remains open; ordinary outside-link redirection is refused. In-project dangling - links retain set/unset's prior replacement behavior; strict reads refuse them. + links are refused by `hack env set`, `hack env unset` and strict reads. 6. **`encrypted_file` key file reads.** Key writes now use the `src/lib/private-file.ts` helpers: explicit provisioning replaces the key atomically, and first-use generation publishes it exclusively. `readEncryptedFileKey` diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index c28c5097b..2f33696ea 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4041,7 +4041,8 @@ async function buildEnvMigrationPlanStep(opts: { return "Skip env migration because no project was detected from this path."; } - // The same format decision as the migration itself; a plan never throws. + // The same format decision as the migration itself. A layer entry it refuses is + // reported as a plan; a path leaving the project still stops the run (HACK-1685). let modernExists: boolean; try { modernExists = await projectEnvConfigExistsForMutation({ diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index f9c37b635..84ada53e7 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -973,9 +973,9 @@ export class ProjectEnvLayerPathConflictError extends HackCliError { * {@link projectEnvConfigExists} reads a dangling default layer, and any * symlinked overlay, as absent, so such a project would silently fall back to * the legacy file. Here every entry named like an env layer must be a regular - * file or a symlink that resolves to one, and either counts as an env layer; - * any other entry is refused with {@link ProjectEnvLayerPathConflictError} - * before anything is read from stdin or written. + * file or a symlink inside the project that resolves to one, and either counts + * as an env layer; any other entry is refused before anything is read from stdin + * or written. */ export async function projectEnvConfigExistsForMutation(opts: { readonly projectDir: string; @@ -997,7 +997,10 @@ export async function projectEnvConfigExistsForMutation(opts: { } linkedLayer = true; } - return linkedLayer || (await projectEnvConfigExists(opts)); + // Always, not only without a linked layer: this is what checks that `.hack` and + // every layer stay inside the project (HACK-1685). + const regularLayerExists = await projectEnvConfigExists(opts); + return linkedLayer || regularLayerExists; } export async function listProjectEnvOverlayNames(opts: { @@ -5652,7 +5655,8 @@ const ENV_FILE_TRAILING_COMMENT = /\s#.*$/; * `A='x' KEY=value`, where Compose reads the text after the closing quote as another * entry. A value that holds such text, a comment after a quoted value, or a line inside * a multi-line value can match too, which refuses more than it must. A trailing comment - * is dropped only from a line without quotes, because `A='x # y' KEY=value` sets KEY. + * is dropped only from a line without quotes, because `A='x # y' KEY=value` sets KEY, so + * an apostrophe in a comment keeps that comment. */ function composeEnvFileEntryNames(text: string): ReadonlySet { const names = new Set(Object.keys(parseDotEnv(text))); @@ -6073,7 +6077,7 @@ function unusableLayerRemedy(envName: string | null): string { if (parseEnvConfigSelection(envName) === envName) { return `Run \`hack env list --env ${envName}\` to see the error, fix it, then retry.`; } - return `Check ${JSON.stringify(`hack.env.${envName}.yaml`)}, fix it, then retry.`; + return `Check ${JSON.stringify(`hack.env.${envName}.yaml`)} and its local layer, fix the problem, then retry.`; } /** @@ -6093,7 +6097,7 @@ async function deliveredLayerValuesForMigration(opts: { } catch { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `an env layer can't be read or resolved, for example it doesn't parse or its project env key is missing, so the migration can't tell which values the layers already deliver. ${unusableLayerRemedy(opts.envName)}`, + reason: `an env layer can't be read or resolved, for example it doesn't parse, its project env key is missing or its path leaves the project, so the migration can't tell which values the layers already deliver. ${unusableLayerRemedy(opts.envName)}`, }); } } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 29c654c60..2b852527a 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -841,7 +841,7 @@ test("the migration refuses existing layer entries and never writes through or r resolve(linked.projectDir, ".env.staging"), `OWNED=${STAGING_VALUE}\n` ); - const target = resolve(linked.root, "outside-staging.yaml"); + const target = resolve(linked.projectRoot, "linked-staging.yaml"); const targetText = LAYER.replace( "environment: default", "environment: staging" @@ -938,7 +938,7 @@ test("the contract migration refuses a dangling default layer and never replaces // A linked regular overlay means env layers are in use: nothing is migrated. await rm(path); - const target = resolve(repo.root, "shared-staging.yaml"); + const target = resolve(repo.projectRoot, "shared-staging.yaml"); await writeFile( target, LAYER.replace("environment: default", "environment: staging") @@ -2321,7 +2321,7 @@ test("a contract project or a linked layer keeps the older behaviour", async () expect((await unmigratedIn(contract)).keys).toEqual([]); const linked = await writeTeammateCheckout(); - const target = resolve(linked.root, "elsewhere.yaml"); + const target = resolve(linked.projectRoot, "elsewhere.yaml"); await writeFile( target, LAYER_EMPTY.replace("environment: default", "environment: staging") @@ -3675,3 +3675,152 @@ test.skipIf(RUNS_AS_ROOT)( }); } ); + +const LEAVES_PROJECT = "its path must resolve inside the project"; + +test("a layer linked outside the project refuses the migration and every mutation", async () => { + for (const kind of ["file", "directory"] as const) { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const target = resolve(repo.root, "outside-target"); + if (kind === "file") { + await writeFile(target, LAYER); + } else { + await mkdir(target); + } + await symlink(target, resolve(repo.projectDir, "hack.env.staging.yaml")); + const before = await hackDirContents(repo); + + const error = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect( + kind === "file" + ? message.includes(LEAVES_PROJECT) + : message.includes("is not a regular file") + ).toBe(true); + expect(message).not.toContain(KEPT_VALUE); + expect(await hackDirContents(repo)).toEqual(before); + if (kind === "file") { + expect(await readFile(target, "utf8")).toBe(LAYER); + } + } +}); + +/** `hack doctor` with stub `hack` and `docker`, so neither a daemon nor Docker is reached. */ +async function runStubbedDoctor(opts: { + readonly root: string; + readonly projectRoot: string; +}): Promise { + const tools = resolve(opts.root, "tools"); + await mkdir(tools); + for (const [name, body] of [ + ["hack", "exit 0"], + ["docker", "exit 1"], + ["mutagen", "exit 0"], + ] as const) { + await writeFile(join(tools, name), `#!/bin/sh\n${body}\n`, { mode: 0o700 }); + } + const proc = Bun.spawn( + [ + process.execPath, + ENTRYPOINT, + "doctor", + "--fix", + "--migrate-env-config", + "--no-interactive", + "--path", + opts.projectRoot, + ], + { + cwd: opts.projectRoot, + env: privateCliEnv({ + root: opts.root, + env: { + HACK_GLOBAL_CONFIG_PATH: join(opts.root, "hack-config.json"), + HACK_HOME: join(opts.root, "hack-home"), + HACK_LOGGER: "console", + HACK_MUTAGEN_PATH: join(tools, "mutagen"), + HACK_NO_INTERACTIVE: "1", + HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", + NO_COLOR: "1", + PATH: `${tools}:/usr/bin:/bin`, + }, + }), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + } + ); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { exitCode, stdout, stderr }; +} + +test("doctor doesn't rewrite a Compose file outside the project through a linked .hack", async () => { + const root = await mkdtemp(join(tmpdir(), "hack-env-legacy-scope-")); + tempDirs.add(root); + const projectRoot = resolve(root, "repo"); + const outside = resolve(root, "outside"); + await mkdir(projectRoot); + await mkdir(outside); + const layerTarget = resolve(outside, "layer-target.yaml"); + await writeFile(layerTarget, LAYER); + await writeFile(resolve(outside, PROJECT_COMPOSE_FILENAME), LISTED_COMPOSE); + await writeFile( + resolve(outside, PROJECT_CONFIG_FILENAME), + `${JSON.stringify({ name: "env-legacy-scope", dev_host: "env-legacy-scope.hack" })}\n` + ); + await writeFile(resolve(outside, ".env"), `KEPT_SETTING=${KEPT_VALUE}\n`); + await symlink(layerTarget, resolve(outside, "hack.env.x.yaml")); + await symlink(outside, resolve(projectRoot, ".hack")); + const contents = async () => + Object.fromEntries( + await Promise.all( + (await readdir(outside)) + .sort() + .map( + async (name) => + [name, await readFile(resolve(outside, name), "base64")] as const + ) + ) + ); + const before = await contents(); + + const run = await runStubbedDoctor({ root, projectRoot }); + + expect(run.exitCode).not.toBe(0); + expect(`${run.stdout}${run.stderr}`).toContain(LEAVES_PROJECT); + expect(`${run.stdout}${run.stderr}`).not.toContain(KEPT_VALUE); + expect(await contents()).toEqual(before); +}, 60_000); + +test("doctor doesn't rewrite Compose while a local layer links outside the project", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const target = resolve(repo.root, "outside-layer.yaml"); + await writeFile(target, LAYER); + await symlink(target, resolve(repo.projectDir, "hack.env.local.yaml")); + const before = await hackDirContents(repo); + + const run = await runStubbedDoctor({ + root: repo.root, + projectRoot: repo.projectRoot, + }); + + expect(run.exitCode).not.toBe(0); + expect(`${run.stdout}${run.stderr}`).toContain(LEAVES_PROJECT); + expect(await hackDirContents(repo)).toEqual(before); + expect(await readFile(target, "utf8")).toBe(LAYER); +}, 60_000); From d03761730440041305da03e6c93e3dafe686ea0e Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:28:18 -0500 Subject: [PATCH 43/55] test(env): point the doctor symlink fixture inside the project HACK-1685 refuses a layer linked outside the project, so this fixture now links inside it. Its assertions are unchanged, and the outside case is covered in env-legacy-service-scope. Co-Authored-By: Claude Opus 5.5 --- tests/doctor-fix-noninteractive.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 3d3ac991a..1a09f45c6 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -672,7 +672,7 @@ test("doctor --migrate-env-config treats a symlinked overlay as env layers and n const legacy = ["synthetic", "legacy", "b9e0"].join("-"); await writeFile(resolve(projectDir, ".env"), `OWNED=${legacy}\n`); await writeFile(resolve(projectDir, ".env.staging"), `OWNED=${legacy}-s\n`); - const target = resolve(projectRoot, "..", "outside-staging.yaml"); + const target = resolve(projectRoot, "linked-staging.yaml"); const targetText = `version: 1\nenvironment: staging\nsecretsprovider: project_key\nvalues:\n global:\n OWNED: ${owned}\n`; await writeFile(target, targetText); const link = resolve(projectDir, "hack.env.staging.yaml"); From f980705d421d74773cc99e9de58270d4b57c6bc0 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:39:40 -0500 Subject: [PATCH 44/55] docs(env): correct what doctor and env add do with linked layers Doctor can still remove `env_file: .env` when a layer is a symlink, and `hack env add` still replaces an in-project dangling link. Adds the contract-route case for a layer linked outside the project. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 13 ++++++---- docs/security/env-secrets-threat-model.md | 3 ++- tests/env-legacy-service-scope.test.ts | 30 ++++++++++++++++++++++- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/docs/env.md b/docs/env.md index ca3ed0d47..41ef64f8d 100644 --- a/docs/env.md +++ b/docs/env.md @@ -622,7 +622,7 @@ rename has these effects: and leaves the link and outside files in place - `hack env set` and `hack env unset` refuse an in-project symlink whose target does not exist with `E_PATH_CONFLICT`, and strict planning and hydrate reads refuse it too. Replace - or remove the link, then retry + or remove the link, then retry. `hack env add` replaces that link with a regular file - the writer rechecks both the layer and its resolved destination before staging and commit. These path checks narrow concurrent-swap races; they are not an atomic filesystem fence against a directory changed after the last check @@ -1098,7 +1098,9 @@ files, when: - an entry named like an env layer (`.hack/hack.env.*.yaml`) is a dangling symlink, a directory or another non-file (`E_PATH_CONFLICT`). A regular layer file, or a symlink inside the project to one, means the repo already uses env layers: `hack doctor` reports - that and writes nothing, never through the link. A `.hack` or a layer linked outside the + that and migrates nothing, and never writes through the link. It can still remove + `env_file: .env` from the Compose file, and a service then stops receiving a value only that + file set, so move such values with `hack env set` first. A `.hack` or a layer linked outside the project is refused with a closed error, and the link and outside files stay as they are; - git tracks a machine-local layer it would write, or, in a git checkout, git cannot report whether it does (`E_PATH_CONFLICT`); @@ -1215,9 +1217,10 @@ and the command has no effect on the runtime value. `hack env set` and `hack env unset` choose between the env layers and the legacy file by looking for `.hack/hack.env.*.yaml` entries, `hack.env.default.yaml` included. A regular -file, or a symlink inside the project to one, counts as an env layer. Any other entry with a layer name, such -as a dangling symlink or a directory, is refused with `E_PATH_CONFLICT` and nothing is -written; it never silently selects the legacy `.hack/.env`. Replace or remove the entry, +file, or a symlink inside the project to one, counts as an env layer. An entry linked outside +the project is refused with a closed error. Any other entry with a layer name, such +as a dangling symlink or a directory, is refused with `E_PATH_CONFLICT`. Nothing is +written either way, and neither silently selects the legacy `.hack/.env`. Replace or remove the entry, then retry. Reads refuse the same way: `hack env list`, `hack env explain` and runtime env injection stop with `E_PATH_CONFLICT` on a selected layer entry that is not a regular file (such as a dangling `hack.env.default.yaml`), and with a closed one-line error on one diff --git a/docs/security/env-secrets-threat-model.md b/docs/security/env-secrets-threat-model.md index 3d8563788..418f43b6a 100644 --- a/docs/security/env-secrets-threat-model.md +++ b/docs/security/env-secrets-threat-model.md @@ -169,7 +169,8 @@ These gaps are known and not closed on `next`. Each one needs its own change. path-based, not descriptor-relative `openat`/`renameat`: an ancestor directory swapped after the last check can still redirect an operation. This residual race remains open; ordinary outside-link redirection is refused. In-project dangling - links are refused by `hack env set`, `hack env unset` and strict reads. + links are refused by `hack env set`, `hack env unset` and strict reads; `hack env add` + replaces them. 6. **`encrypted_file` key file reads.** Key writes now use the `src/lib/private-file.ts` helpers: explicit provisioning replaces the key atomically, and first-use generation publishes it exclusively. `readEncryptedFileKey` diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 2b852527a..7713fcc20 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -3678,7 +3678,7 @@ test.skipIf(RUNS_AS_ROOT)( const LEAVES_PROJECT = "its path must resolve inside the project"; -test("a layer linked outside the project refuses the migration and every mutation", async () => { +test("a layer linked outside the project refuses the migration", async () => { for (const kind of ["file", "directory"] as const) { const repo = await createLegacyRepo(); await writeFile( @@ -3824,3 +3824,31 @@ test("doctor doesn't rewrite Compose while a local layer links outside the proje expect(await hackDirContents(repo)).toEqual(before); expect(await readFile(target, "utf8")).toBe(LAYER); }, 60_000); + +test("a contract project with a layer linked outside the project refuses the migration", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.json"), + `${JSON.stringify({ + version: 1, + vars: [{ key: "KEPT_SETTING", source: "plain_env", required: false }], + })}\n` + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + const target = resolve(repo.root, "outside-layer.yaml"); + await writeFile(target, LAYER); + await symlink(target, resolve(repo.projectDir, "hack.env.staging.yaml")); + const before = await hackDirContents(repo); + + const error = await migrateLegacyRepo(repo, {}).catch( + (caught: unknown) => caught + ); + + expect((error as Error).message).toContain(LEAVES_PROJECT); + expect((error as Error).message).not.toContain(KEPT_VALUE); + expect(await hackDirContents(repo)).toEqual(before); + expect(await readFile(target, "utf8")).toBe(LAYER); +}); From 11d5e8ffc03e72d30342effa13b61a16d4e3f7cc Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:52:15 -0500 Subject: [PATCH 45/55] docs(env): say when doctor migrates nothing, and move values with --local The "already uses env layers" sentence was false for regular layers with values left to move. Moving a value with plain `hack env set` could put it in a tracked layer. Also notes that `set` and `add` differ on a dangling layer link. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/env.md b/docs/env.md index 41ef64f8d..08010599f 100644 --- a/docs/env.md +++ b/docs/env.md @@ -430,7 +430,7 @@ into a host command, and `--target` on env and host commands chooses the `host` | `hack env explain` | `--env`, `--service`, `--target` | Per invocation; reports source, precedence and delivery without the value. | Not needed. | Reads files; no runtime required. | | `hack env apply` | `--env`, `--service` | Runs `hack restart ` with the same `--env`, recreating one service. | This is the refresh path for one service. | `--service` is required. | | `hack env add` | `--env`, `--service` | Selects the file written (`hack.env..yaml`, or the base file for `base`/`default`/`none`; `--local` writes the local variant). | Running workloads keep old values until `hack env apply` or `hack restart`. | Mutates only env files. | -| `hack env set` | `--env`, `--service` | Alias for `hack env add`. | Same as `hack env add`. | Mutates only env files. | +| `hack env set` | `--env`, `--service` | Alias for `hack env add`, except that it refuses a dangling layer link. | Same as `hack env add`. | Mutates only env files. | | `hack env hydrate` | `--env`, `--service` | Selects the local file written, as for `hack env add --local`; imports every entry in one commit. | Same as `hack env add`. | Mutates only the machine-local env file; refuses a git-tracked one. | | `hack env unset` | `--env`, `--service` | Selects the file the key is removed from, as for `hack env add`. | Same as `hack env add`. | Mutates only env files. | | `hack env materialize` | `--env`, `--service` | Writes `.hack/.env` once, at invocation. | Rerun `hack env materialize`. | Compatibility output for external tools; runtime injection does not depend on it. | @@ -562,6 +562,7 @@ hack env add API_BASE_URL https://api.example.com --local ``` `hack env set` still works as a compatibility alias for `hack env add`, but `add` is the primary UX now. +The two differ on an in-project dangling layer link: `set` refuses it and `add` replaces it. ### Entering secret values @@ -1096,12 +1097,13 @@ files, when: so the service would lose those values. Set `env.defaultOverlay` to it, or remove the `env_file` entry and select the overlay with `--env`; - an entry named like an env layer (`.hack/hack.env.*.yaml`) is a dangling symlink, a - directory or another non-file (`E_PATH_CONFLICT`). A regular layer file, or a symlink - inside the project to one, means the repo already uses env layers: `hack doctor` reports - that and migrates nothing, and never writes through the link. It can still remove - `env_file: .env` from the Compose file, and a service then stops receiving a value only that - file set, so move such values with `hack env set` first. A `.hack` or a layer linked outside the - project is refused with a closed error, and the link and outside files stay as they are; + directory or another non-file (`E_PATH_CONFLICT`). With a symlinked layer inside the project, + or with nothing to move, `hack doctor` reports that the repo already uses env layers and + migrates nothing, and it never writes through the link. It can still remove `.hack/.env*` + entries from `env_file` in the Compose file, and a service then stops receiving a value only + that file set, so move such values first with `hack env set --local`. A `.hack` or a layer + linked outside the project is refused with a closed error, and the link and outside files + stay as they are; - git tracks a machine-local layer it would write, or, in a git checkout, git cannot report whether it does (`E_PATH_CONFLICT`); - in a git checkout, git does not ignore a machine-local layer it would write, for From 5fafb7ed31a30dec94455c7b4f433a25212d29e2 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:57:25 -0500 Subject: [PATCH 46/55] fix(env): refuse a migrated value that replaces a scoped layer value A later env layer beats an earlier one whatever their scopes, so a migrated local global value silently replaced a service or host value in a tracked layer. The migration now resolves every target's value with the planned layers, in the base and every overlay, and refuses before writing. Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 203 ++++++++++++++++++++++- tests/env-legacy-service-scope.test.ts | 220 +++++++++++++++++++++++++ 2 files changed, 422 insertions(+), 1 deletion(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 84ada53e7..283288226 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6526,6 +6526,199 @@ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { } } +/** `layer` with the migration's planned global values added, as the merge will write it. */ +function layerWithPlannedGlobals(opts: { + readonly layer: ProjectEnvConfig | null; + readonly environment: string; + readonly planned: Readonly>; +}): ProjectEnvConfig | null { + if (Object.keys(opts.planned).length === 0) { + return opts.layer; + } + const layer = + opts.layer ?? + buildMigratedProjectEnvConfig({ + environment: opts.environment, + values: { global: {} }, + }); + return { + ...layer, + values: { + ...layer.values, + global: { ...layer.values.global, ...opts.planned }, + }, + }; +} + +/** `keys` whose scoped value or scoped deletion the planned layers replace for one target. */ +function scopedValuesReplacedForTarget(opts: { + readonly target: { + readonly name: string; + readonly scopeNames: readonly string[]; + readonly before: Readonly>; + }; + readonly keys: readonly string[]; + readonly envLayers: readonly (ProjectEnvConfig | null)[]; + readonly plannedLayers: readonly (ProjectEnvConfig | null)[]; + readonly envName: string | null; +}): readonly string[] { + const { target } = opts; + const globalBefore = resolveEffectiveStoredEntries({ + layers: opts.envLayers, + scopeNames: ["global"], + }); + const before = resolveEffectiveStoredEntries({ + layers: opts.envLayers, + scopeNames: target.scopeNames, + }); + const after = resolveEffectiveStoredEntries({ + layers: opts.plannedLayers, + scopeNames: target.scopeNames, + }); + return opts.keys.flatMap((key) => { + const entry = before[key]; + // A scoped value, or a scoped deletion that hides a global one. + const scoped = entry ? entry.scope !== "global" : key in globalBefore; + const value = after[key]?.value; + const replaced = + scoped && + value !== undefined && + !isProjectEnvSecretValue(value) && + String(value) !== target.before[key]; + return replaced + ? [ + `${JSON.stringify(key)} for ${JSON.stringify(entry?.scope ?? target.name)} in ${JSON.stringify(opts.envName ?? "default")}`, + ] + : []; + }); +} + +/** + * Which service and host values a planned local write would replace in one env. + * A later layer beats an earlier one whatever their scopes, so a local global + * value outranks a service or host value in a tracked layer, while a scoped value + * in the same local layer still wins. So this resolves the layers as they will be, + * not by comparing scopes. Returns names only. + */ +async function scopedValuesReplacedInEnv(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly envName: string | null; + readonly planned: ReadonlyMap< + string, + Readonly> + >; +}): Promise { + const layers = await readProjectEnvLayers(opts); + const resolved = await resolveProjectEnvConfig(opts); + if (!(layers && resolved)) { + return []; + } + const { selection, envLayers } = layers; + const plannedBase = opts.planned.get(selection.localDefaultPath) ?? {}; + const plannedOverlay = + (selection.localOverlayPath && + opts.planned.get(selection.localOverlayPath)) || + {}; + const keys = Object.keys({ ...plannedBase, ...plannedOverlay }); + if (keys.length === 0) { + return []; + } + // The two machine-local layers always come last (readProjectEnvLayers). + const plannedLayers = [ + ...envLayers.slice(0, -2), + layerWithPlannedGlobals({ + layer: envLayers.at(-2) ?? null, + environment: "default", + planned: plannedBase, + }), + layerWithPlannedGlobals({ + layer: envLayers.at(-1) ?? null, + environment: selection.effectiveEnv ?? "default", + planned: plannedOverlay, + }), + ]; + const projection = projectEnvProjection({ + layers, + serviceNames: opts.serviceNames, + }); + const targets = [ + ...projection.serviceTargets.flatMap((target) => [ + { + name: target.serviceName, + scopeNames: target.composeScopeNames, + before: resolved.serviceEnv[target.serviceName] ?? {}, + }, + { + name: target.serviceName, + scopeNames: target.hostScopeNames, + before: resolved.hostTargetEnv[target.serviceName] ?? {}, + }, + ]), + { + name: PROJECT_ENV_HOST_SCOPE as string, + scopeNames: projection.globalHostScopeNames, + before: resolved.hostTargetEnv.global ?? {}, + }, + ]; + const replaced = targets.flatMap((target) => + scopedValuesReplacedForTarget({ + target, + keys, + envLayers, + plannedLayers, + envName: opts.envName, + }) + ); + return [...new Set(replaced)]; +} + +/** + * Refuses a planned local write that would replace a service or host value from + * an env layer, in the base or in any overlay. That target would silently change + * value and stop receiving later team changes to the key. Names only. + */ +async function assertMigrationKeepsScopedValues(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly overlayNames: readonly string[]; + readonly localWrites: readonly PlannedEnvLayerWrite[]; +}): Promise { + if (opts.localWrites.length === 0) { + return; + } + const planned = new Map( + opts.localWrites.map((write) => [ + write.path, + write.config.values.global ?? {}, + ]) + ); + const overlays = new Set([ + ...opts.overlayNames, + ...(await listProjectEnvOverlayNames({ projectDir: opts.projectDir })), + ]); + const replaced: string[] = []; + for (const envName of [null, ...overlays]) { + replaced.push( + ...(await scopedValuesReplacedInEnv({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + serviceNames: opts.serviceNames, + envName, + planned, + })) + ); + } + if (replaced.length > 0) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `a legacy env file sets ${replaced.sort().join(", ")}, and its migrated value would replace what that env layer gives that target, because a machine-local global value outranks a service or host value in an earlier layer. Remove the key from the legacy file, or change that value in its env layer, then retry.`, + }); + } +} + /** * An existing machine-local layer the migration may add to, or null when absent. * Refuses one it can't read or parse, or that isn't a regular file: `.config` @@ -6775,7 +6968,8 @@ async function publishMigratedLegacyLayers(opts: { * machine-local layer, a Compose `env_file` on a file the migration leaves * out or on an overlay a launch without `--env` would not select, a Compose * `environment` or `env_file` entry the migrated value would override, a - * service that would gain values a legacy file gave only to others, an + * service that would gain values a legacy file gave only to others, a + * service or host value in an env layer that a migrated value would replace, an * `env_file` order that gives a key a different value from the layers, a Compose * input that hides `env_file` from this check, a line Compose reads differently * from Hack, a key that does not match `^[A-Z_][A-Z0-9_]*$` @@ -6929,6 +7123,13 @@ async function migrateUndeclaredLegacyDotenv(opts: { overlayNames, legacyValues, }); + await assertMigrationKeepsScopedValues({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + serviceNames: opts.serviceNames, + overlayNames, + localWrites, + }); const merges = await planLocalLayerMerges({ localWrites }); for (const write of localWrites) { await assertUntrackedMigrationLocalLayer({ diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 7713fcc20..ad423619e 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -3852,3 +3852,223 @@ test("a contract project with a layer linked outside the project refuses the mig expect(await hackDirContents(repo)).toEqual(before); expect(await readFile(target, "utf8")).toBe(LAYER); }); + +const SCOPED_LEGACY_VALUE = ["synthetic", "legacy", "third", "a1c4"].join("-"); +const SCOPED_GLOBAL_VALUE = ["synthetic", "tracked", "global", "a1c4"].join( + "-" +); +const SCOPED_VALUE = ["synthetic", "tracked", "scoped", "a1c4"].join("-"); + +function scopedLayer(environment: string, values: string): string { + return `version: 1\nenvironment: ${environment}\nsecretsprovider: project_key\nvalues:\n${values}`; +} + +/** What each target receives for every env, so a test compares resolved runtime values. */ +async function effectiveValues( + repo: Repo, + envNames: readonly (string | null)[] +): Promise> { + const out: Record = {}; + for (const envName of envNames) { + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + envName, + }); + out[envName ?? "default"] = { + serviceEnv: resolved?.serviceEnv, + hostTargetEnv: resolved?.hostTargetEnv, + }; + } + return out; +} + +for (const refused of [ + { + name: "a tracked service value", + files: { + "hack.env.default.yaml": ` global:\n SETTING: ${SCOPED_GLOBAL_VALUE}\n api:\n SETTING: ${SCOPED_VALUE}\n`, + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "api" in "default"', + }, + { + name: "a tracked service value equal to the global one", + files: { + "hack.env.default.yaml": ` global:\n SETTING: ${SCOPED_VALUE}\n api:\n SETTING: ${SCOPED_VALUE}\n`, + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "api" in "default"', + }, + { + name: "a tracked service value with no global one", + files: { + "hack.env.default.yaml": ` global: {}\n api:\n SETTING: ${SCOPED_VALUE}\n`, + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "api" in "default"', + }, + { + name: "a tracked host value", + files: { + "hack.env.default.yaml": ` global:\n SETTING: ${SCOPED_GLOBAL_VALUE}\n host:\n SETTING: ${SCOPED_VALUE}\n`, + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "host" in "default"', + }, + { + name: "a tracked service deletion", + files: { + "hack.env.default.yaml": ` global:\n SETTING: ${SCOPED_GLOBAL_VALUE}\n api:\n SETTING: null\n`, + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "api" in "default"', + }, + { + name: "a tracked overlay's service value, from the base file", + files: { + "hack.env.default.yaml": " global: {}\n", + "hack.env.staging.yaml": ` global: {}\n api:\n SETTING: ${SCOPED_VALUE}\n`, + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "api" in "staging"', + }, +] as const) { + test(`a migrated value that would replace ${refused.name} refuses, and every target keeps its value`, async () => { + const repo = await createLegacyRepo(); + for (const [name, values] of Object.entries(refused.files)) { + await writeFile( + resolve(repo.projectDir, name), + scopedLayer(name.includes("staging") ? "staging" : "default", values) + ); + } + for (const [name, text] of Object.entries(refused.legacy)) { + await writeFile(resolve(repo.projectDir, name), text); + } + const before = await effectiveValues(repo, [null, "staging"]); + + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: refused.reason, + hidden: [SCOPED_LEGACY_VALUE, SCOPED_GLOBAL_VALUE, SCOPED_VALUE], + }); + + expect(await effectiveValues(repo, [null, "staging"])).toEqual(before); + }); +} + +test("a legacy overlay's value that would replace a tracked service value in that overlay refuses", async () => { + const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer("default", " global: {}\n") + ); + await writeFile( + resolve(repo.projectDir, "hack.env.staging.yaml"), + scopedLayer( + "staging", + ` global: {}\n api:\n SETTING: ${SCOPED_VALUE}\n` + ) + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + const before = await effectiveValues(repo, ["staging"]); + + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"SETTING" for "api" in "staging"', + hidden: [SCOPED_LEGACY_VALUE, SCOPED_VALUE], + }); + + expect(await effectiveValues(repo, ["staging"])).toEqual(before); +}); + +test("a new overlay's migrated value that would replace a tracked base service value refuses", async () => { + const repo = await createLegacyRepo(); + await useStagingAsDefaultOverlay(repo); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer( + "default", + ` global: {}\n api:\n SETTING: ${SCOPED_VALUE}\n` + ) + ); + await writeFile( + resolve(repo.projectDir, ".env.staging"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"SETTING" for "api" in "staging"', + hidden: [SCOPED_LEGACY_VALUE, SCOPED_VALUE], + }); +}); + +test("a migrated value that doesn't change a scoped result still migrates", async () => { + // Equal to the scoped value, so the service keeps what it had. + const equal = await createLegacyRepo(); + await writeFile( + resolve(equal.projectDir, "hack.env.default.yaml"), + scopedLayer( + "default", + ` global: {}\n api:\n SETTING: ${SCOPED_LEGACY_VALUE}\n` + ) + ); + await writeFile( + resolve(equal.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + await migrateLegacyRepo(equal, { migrateUndeclaredDotenv: true }); + expect(await readLayerValues(equal, "hack.env.local.yaml")).toEqual({ + global: { SETTING: SCOPED_LEGACY_VALUE }, + }); + + // A scoped value in the same local layer outranks that layer's global value. + const sameLayer = await createLegacyRepo(); + await writeFile( + resolve(sameLayer.projectDir, "hack.env.default.yaml"), + scopedLayer("default", ` global:\n SETTING: ${SCOPED_GLOBAL_VALUE}\n`) + ); + await writeFile( + resolve(sameLayer.projectDir, "hack.env.local.yaml"), + scopedLayer( + "default", + ` global: {}\n api:\n SETTING: ${SCOPED_VALUE}\n` + ) + ); + await writeFile( + resolve(sameLayer.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + await migrateLegacyRepo(sameLayer, { migrateUndeclaredDotenv: true }); + const kept = await effectiveValues(sameLayer, [null]); + expect(kept.default).toMatchObject({ + serviceEnv: { api: { SETTING: SCOPED_VALUE } }, + hostTargetEnv: { global: { SETTING: SCOPED_LEGACY_VALUE } }, + }); + + // Another key's scoped value is untouched, and an unscoped target gets the new value. + const other = await createLegacyRepo(); + await writeFile( + resolve(other.projectDir, "hack.env.default.yaml"), + scopedLayer("default", ` global: {}\n api:\n OTHER: ${SCOPED_VALUE}\n`) + ); + await writeFile( + resolve(other.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + await migrateLegacyRepo(other, { migrateUndeclaredDotenv: true }); + const moved = await effectiveValues(other, [null]); + expect(moved.default).toMatchObject({ + serviceEnv: { api: { OTHER: SCOPED_VALUE, SETTING: SCOPED_LEGACY_VALUE } }, + hostTargetEnv: { global: { SETTING: SCOPED_LEGACY_VALUE } }, + }); +}); From 2155631b20660c13fe02594c7d33752a1095305b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:05:18 -0500 Subject: [PATCH 47/55] fix(env): check overlays a linked worktree inherits before migrating A migrated local value outranks a primary checkout's local overlay layer too, and those overlays weren't listed here. Document the new refusal. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 5 ++ src/lib/project-env-config.ts | 38 +++++++++-- tests/project-env-local-inheritance.test.ts | 71 +++++++++++++++++++++ 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/docs/env.md b/docs/env.md index 92096d8ec..030f32daa 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1118,6 +1118,11 @@ files, when: - in a git checkout, git does not ignore a machine-local layer it would write, for example because `.hack/.gitignore` is read-only and lacks Hack's managed block (`E_PATH_CONFLICT`). Make `.hack/.gitignore` writable, or ignore the path, and retry; +- a migrated value would replace a service or host value in an env layer, in the base or in + an overlay, including an overlay a linked worktree inherits from its primary checkout + (`E_CONFIG_INVALID`). A machine-local global value outranks a service or host value in an + earlier layer, so that target would silently switch. The error names the keys, targets and + envs, never values. Remove the key from the legacy file, or change that value in its layer; - a key the legacy files set is also set in a Compose service's own `environment` (`E_CONFIG_INVALID`). Compose ranks `environment` above `env_file`, but migrated values become global layer values that Hack applies after `environment`, so the service would diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 283288226..2c7ce18e6 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6478,6 +6478,38 @@ async function readValidatedLegacyDotenvFiles(opts: { return legacyValues; } +/** + * Overlays a migrated local value can affect: this checkout's, plus any with only a + * primary checkout's local layer, which a linked worktree inherits below its own. + */ +async function overlaysAffectedByMigration(opts: { + readonly projectRoot: string; + readonly projectDir: string; +}): Promise { + const names = new Set( + await listProjectEnvOverlayNames({ projectDir: opts.projectDir }) + ); + const primaryDir = await resolvePrimaryLocalProjectDir(opts); + if (primaryDir !== null) { + const suffix = `.${PROJECT_ENV_LOCAL_SEGMENT}${PROJECT_ENV_CONFIG_FILENAME_SUFFIX}`; + for (const entry of await readdir(primaryDir)) { + if ( + entry.startsWith(PROJECT_ENV_CONFIG_FILENAME_PREFIX) && + entry.endsWith(suffix) + ) { + const name = entry.slice( + PROJECT_ENV_CONFIG_FILENAME_PREFIX.length, + -suffix.length + ); + if (parseEnvConfigSelection(name) === name) { + names.add(name); + } + } + } + } + return [...names].sort((left, right) => left.localeCompare(right)); +} + /** * Refuses when a migrated base value would replace what a tracked overlay * delivers for a key its legacy overlay file doesn't set. The local base outranks @@ -6496,9 +6528,7 @@ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { ...opts, envName: null, }); - for (const overlayName of await listProjectEnvOverlayNames({ - projectDir: opts.projectDir, - })) { + for (const overlayName of await overlaysAffectedByMigration(opts)) { const delivered = await deliveredLayerValuesForMigration({ ...opts, envName: overlayName, @@ -6697,7 +6727,7 @@ async function assertMigrationKeepsScopedValues(opts: { ); const overlays = new Set([ ...opts.overlayNames, - ...(await listProjectEnvOverlayNames({ projectDir: opts.projectDir })), + ...(await overlaysAffectedByMigration(opts)), ]); const replaced: string[] = []; for (const envName of [null, ...overlays]) { diff --git a/tests/project-env-local-inheritance.test.ts b/tests/project-env-local-inheritance.test.ts index bfdf678c3..b9af32eeb 100644 --- a/tests/project-env-local-inheritance.test.ts +++ b/tests/project-env-local-inheritance.test.ts @@ -14,6 +14,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { ClosedCliError } from "../src/lib/cli-result.ts"; import { + migrateLegacyProjectEnv, resolveProjectEnvConfig, setProjectEnvValue, unsetProjectEnvValue, @@ -354,3 +355,73 @@ test("actual CLI masks inherited global secret overriding tracked service plaint } } }); + +async function legacyLinkedCheckout() { + const f = await fixture(); + await writeFile( + join(f.linked, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.20\n env_file:\n - path: .env\n required: false\n" + ); + await writeFile(join(f.linked, ".env"), "SETTING=legacy-value\n"); + const migrate = () => + migrateLegacyProjectEnv({ + projectRoot: f.linkedRoot, + projectDir: f.linked, + projectName: "env-fixture", + serviceNames: ["api"], + materialize: false, + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + const staging = () => + resolveProjectEnvConfig({ + projectRoot: f.linkedRoot, + projectDir: f.linked, + envName: "staging", + serviceNames: ["api"], + }); + return { ...f, migrate, staging }; +} + +for (const [name, values, reason] of [ + [ + "service", + { global: {}, api: { SETTING: "primary-api" } }, + '"SETTING" for "api" in "staging"', + ], + [ + "global", + { global: { SETTING: "primary-staging" } }, + '"SETTING" for "staging"', + ], +] as const) { + test(`a migrated value that would replace an inherited overlay's ${name} value refuses`, async () => { + const f = await legacyLinkedCheckout(); + await config(f.primary, "hack.env.staging.local.yaml", values); + const before = await f.staging(); + const entries = await readdir(f.linked); + + const result = await f.migrate(); + + expect((result as Error).message).toContain(reason); + expect((result as Error).message).not.toContain("legacy-value"); + expect((result as Error).message).not.toContain("primary-"); + expect(await readdir(f.linked)).toEqual(entries); + expect((await f.staging())?.serviceEnv).toEqual(before?.serviceEnv); + }); +} + +test("an inherited overlay that doesn't set the migrated key doesn't stop the migration", async () => { + const f = await legacyLinkedCheckout(); + await config(f.primary, "hack.env.staging.local.yaml", { + global: {}, + api: { OTHER: "primary-api" }, + }); + + const result = await f.migrate(); + + expect(result).not.toBeInstanceOf(Error); + expect((await f.staging())?.serviceEnv.api).toMatchObject({ + OTHER: "primary-api", + SETTING: "legacy-value", + }); +}); From 4602b49700ee47836ce06ed81a4192ae8fcaff47 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:10:55 -0500 Subject: [PATCH 48/55] fix(env): skip reserved names in the inherited overlay check A reserved overlay name in a primary checkout isn't an overlay, and the base refusal now leads with the remedy that always works. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 7 ++++--- src/lib/project-env-config.ts | 13 ++++++++----- tests/project-env-local-inheritance.test.ts | 9 +++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/env.md b/docs/env.md index 030f32daa..bae75b91b 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1162,9 +1162,10 @@ files, when: entry, a secret's `environment:`), or it is a `COMPOSE_*` setting (`E_CONFIG_INVALID`). Compose takes it from `.hack/.env` today, but the overlay value would apply after the migration. Make the two files agree or stop using the key in Compose; -- a migrated `.hack/.env` value would replace what a tracked overlay delivers for a key its legacy - overlay file doesn't set (`E_CONFIG_INVALID`), because a local base outranks tracked overlays. Set - the value you want in that legacy overlay file, or remove the key from `.hack/.env`; +- a migrated `.hack/.env` value would replace what an overlay's layers deliver for a key its legacy + overlay file doesn't set (`E_CONFIG_INVALID`), because a local base outranks tracked and inherited + overlay layers. Remove the key from `.hack/.env`, or set the value you want in that legacy overlay + file, creating it if needed; - `.hack/.env` has a state file, env layers exist, and its text isn't what `hack env materialize` would write (`E_PATH_CONFLICT`). The file is out of date or was edited by hand, and Hack can't tell which. If it's generated, move any value of your own with `hack env set`, then run diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 2c7ce18e6..8c0a04d65 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6501,7 +6501,10 @@ async function overlaysAffectedByMigration(opts: { PROJECT_ENV_CONFIG_FILENAME_PREFIX.length, -suffix.length ); - if (parseEnvConfigSelection(name) === name) { + if ( + parseEnvConfigSelection(name) === name && + !isReservedLegacyOverlayName(name) + ) { names.add(name); } } @@ -6511,9 +6514,9 @@ async function overlaysAffectedByMigration(opts: { } /** - * Refuses when a migrated base value would replace what a tracked overlay - * delivers for a key its legacy overlay file doesn't set. The local base outranks - * tracked overlays, so that env would silently change value and stop receiving + * Refuses when a migrated base value would replace what an overlay's layers + * deliver for a key its legacy overlay file doesn't set. The local base outranks + * tracked and inherited overlay layers, so that env would silently change value and stop receiving * later team changes to the key. Names keys and overlays, never values. */ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { @@ -6551,7 +6554,7 @@ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { if (conflicts.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `.hack/.env sets ${conflicts.sort().join(", ")}, and its migrated value would replace what that overlay's env layers deliver. Set the value you want in that legacy overlay file, or remove the key from .hack/.env, then retry.`, + reason: `.hack/.env sets ${conflicts.sort().join(", ")}, and its migrated value would replace what that overlay's env layers deliver. Remove the key from .hack/.env, or set the value you want in that legacy overlay file, creating it if needed, then retry.`, }); } } diff --git a/tests/project-env-local-inheritance.test.ts b/tests/project-env-local-inheritance.test.ts index b9af32eeb..6e0f37daa 100644 --- a/tests/project-env-local-inheritance.test.ts +++ b/tests/project-env-local-inheritance.test.ts @@ -425,3 +425,12 @@ test("an inherited overlay that doesn't set the migrated key doesn't stop the mi SETTING: "legacy-value", }); }); + +test("a reserved overlay name in the primary checkout isn't treated as an overlay", async () => { + const f = await legacyLinkedCheckout(); + await config(f.primary, "hack.env.local.local.yaml", { + global: { SETTING: "primary-reserved" }, + }); + + expect(await f.migrate()).not.toBeInstanceOf(Error); +}); From a62921a8796ab5baefa748a3d74683e1f32ff94f Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:18:56 -0500 Subject: [PATCH 49/55] test(env): cover host, secret and unknown scopes in the scoped-value check Also refuse a primary env directory that can't be listed on Linux, and say that Compose checks apply to a created legacy overlay file. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 2 +- src/lib/project-env-config.ts | 14 +- tests/env-legacy-service-scope.test.ts | 194 ++++++++++++++++++++ tests/project-env-local-inheritance.test.ts | 16 ++ 4 files changed, 223 insertions(+), 3 deletions(-) diff --git a/docs/env.md b/docs/env.md index bae75b91b..b4ae2ccb1 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1165,7 +1165,7 @@ files, when: - a migrated `.hack/.env` value would replace what an overlay's layers deliver for a key its legacy overlay file doesn't set (`E_CONFIG_INVALID`), because a local base outranks tracked and inherited overlay layers. Remove the key from `.hack/.env`, or set the value you want in that legacy overlay - file, creating it if needed; + file, creating it if needed; the Compose `env_file` checks above then apply to that file; - `.hack/.env` has a state file, env layers exist, and its text isn't what `hack env materialize` would write (`E_PATH_CONFLICT`). The file is out of date or was edited by hand, and Hack can't tell which. If it's generated, move any value of your own with `hack env set`, then run diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 8c0a04d65..abd24b931 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6492,7 +6492,17 @@ async function overlaysAffectedByMigration(opts: { const primaryDir = await resolvePrimaryLocalProjectDir(opts); if (primaryDir !== null) { const suffix = `.${PROJECT_ENV_LOCAL_SEGMENT}${PROJECT_ENV_CONFIG_FILENAME_SUFFIX}`; - for (const entry of await readdir(primaryDir)) { + // macOS drops inheritance before this when the directory can't be listed; Linux gets here. + let entries: readonly string[]; + try { + entries = await readdir(primaryDir); + } catch { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: `the primary checkout's env layers in ${primaryDir} can't be listed, so the migration can't tell which overlays it would change. Fix that directory's permissions, then retry.`, + }); + } + for (const entry of entries) { if ( entry.startsWith(PROJECT_ENV_CONFIG_FILENAME_PREFIX) && entry.endsWith(suffix) @@ -6554,7 +6564,7 @@ async function assertBaseDoesNotOutrankTrackedOverlays(opts: { if (conflicts.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_CONFIG_INVALID", - reason: `.hack/.env sets ${conflicts.sort().join(", ")}, and its migrated value would replace what that overlay's env layers deliver. Remove the key from .hack/.env, or set the value you want in that legacy overlay file, creating it if needed, then retry.`, + reason: `.hack/.env sets ${conflicts.sort().join(", ")}, and its migrated value would replace what that overlay's env layers deliver. Remove the key from .hack/.env, or set the value you want in that legacy overlay file, creating it if needed; the Compose env_file checks then apply to that file. Then retry.`, }); } } diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index ad423619e..57124350c 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -35,6 +35,7 @@ import { replaceLocalLayerPrivately, resolveProjectEnvConfig, selectProjectEnvValues, + setProjectEnvValue, } from "../src/lib/project-env-config.ts"; import { resolveEnvMeta } from "../src/lib/project-meta.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -4072,3 +4073,196 @@ test("a migrated value that doesn't change a scoped result still migrates", asyn hostTargetEnv: { global: { SETTING: SCOPED_LEGACY_VALUE } }, }); }); + +/** A repo with an empty tracked layer, and a secret set in `scope` through the real writer. */ +async function repoWithScopedSecret(opts: { + readonly plaintext: string; + readonly local: boolean; +}): Promise { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer("default", ` global:\n SETTING: ${SCOPED_GLOBAL_VALUE}\n`) + ); + await setProjectEnvValue({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + envName: null, + scope: "api", + key: "SETTING", + value: opts.plaintext, + secret: true, + local: opts.local, + }); + await writeFile( + resolve(repo.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + return repo; +} + +async function migrateWithServices( + repo: Repo, + serviceNames: readonly string[] +): Promise { + return await migrateLegacyProjectEnv({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + projectName: "env-legacy-scope", + serviceNames: [...serviceNames], + materialize: false, + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); +} + +test("scoped coverage: both a service and a host value in one layer are named, and every target keeps its value", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer( + "default", + ` global:\n SETTING: ${SCOPED_GLOBAL_VALUE}\n api:\n SETTING: ${SCOPED_VALUE}\n host:\n SETTING: host-${SCOPED_VALUE}\n` + ) + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + const before = await effectiveValues(repo, [null]); + + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: + '"SETTING" for "api" in "default", "SETTING" for "host" in "default"', + hidden: [SCOPED_LEGACY_VALUE, SCOPED_GLOBAL_VALUE, SCOPED_VALUE], + }); + + expect(await effectiveValues(repo, [null])).toEqual(before); + expect(before.default).toMatchObject({ + serviceEnv: { api: { SETTING: SCOPED_VALUE } }, + hostTargetEnv: { + api: { SETTING: `host-${SCOPED_VALUE}` }, + global: { SETTING: `host-${SCOPED_VALUE}` }, + }, + }); +}); + +test("scoped coverage: a Compose service named host is treated as a service, not the host scope", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n host:\n image: alpine:3.20\n env_file:\n - path: .env\n required: false\n" + ); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer( + "default", + ` global:\n SETTING: ${SCOPED_GLOBAL_VALUE}\n host:\n SETTING: ${SCOPED_VALUE}\n` + ) + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + + const result = await migrateWithServices(repo, ["host"]); + + expect(result).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((result as Error).message).toContain( + '"SETTING" for "host" in "default"' + ); + expect(await readdir(repo.projectDir)).not.toContain("hack.env.local.yaml"); +}); + +test("scoped coverage: a migrated value that would replace a scoped secret refuses, showing neither value", async () => { + const plaintext = `secret-${SCOPED_VALUE}`; + const repo = await repoWithScopedSecret({ plaintext, local: false }); + const layer = await readFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + "utf8" + ); + expect(layer).toContain("secure:"); + expect(layer).not.toContain(plaintext); + + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"SETTING" for "api" in "default"', + hidden: [plaintext, SCOPED_LEGACY_VALUE, "v1:"], + }); + + expect(await effectiveValues(repo, [null])).toMatchObject({ + default: { serviceEnv: { api: { SETTING: plaintext } } }, + }); +}); + +test("scoped coverage: a scoped secret that opens to the migrated value still migrates", async () => { + const repo = await repoWithScopedSecret({ + plaintext: SCOPED_LEGACY_VALUE, + local: false, + }); + + expect(await migrateWithServices(repo, ["api"])).not.toBeInstanceOf(Error); + expect(await effectiveValues(repo, [null])).toMatchObject({ + default: { serviceEnv: { api: { SETTING: SCOPED_LEGACY_VALUE } } }, + }); +}); + +test("scoped coverage: a scoped secret in the same local layer outranks the migrated global value", async () => { + const plaintext = `secret-${SCOPED_VALUE}`; + const repo = await repoWithScopedSecret({ plaintext, local: true }); + + expect(await migrateWithServices(repo, ["api"])).not.toBeInstanceOf(Error); + expect(await effectiveValues(repo, [null])).toMatchObject({ + default: { + serviceEnv: { api: { SETTING: plaintext } }, + hostTargetEnv: { global: { SETTING: SCOPED_LEGACY_VALUE } }, + }, + }); +}); + +test("scoped coverage: a value under a scope Compose doesn't know refuses, naming that scope", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer( + "default", + ` global: {}\n retired-worker:\n SETTING: ${SCOPED_VALUE}\n` + ) + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: '"SETTING" for "retired-worker" in "default"', + hidden: [SCOPED_LEGACY_VALUE, SCOPED_VALUE], + }); +}); + +test("scoped coverage: a layer with an invalid scope name refuses closed, before any write", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer( + "default", + ` global: {}\n "Bad Scope!":\n SETTING: ${SCOPED_VALUE}\n` + ) + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `SETTING=${SCOPED_LEGACY_VALUE}\n` + ); + const entries = await readdir(repo.projectDir); + + const result = await migrateWithServices(repo, ["api"]); + + expect(result).toBeInstanceOf(Error); + expect((result as Error).message).not.toContain(SCOPED_VALUE); + expect((result as Error).message).not.toContain(SCOPED_LEGACY_VALUE); + expect(await readdir(repo.projectDir)).toEqual(entries); +}); diff --git a/tests/project-env-local-inheritance.test.ts b/tests/project-env-local-inheritance.test.ts index 6e0f37daa..8f107e571 100644 --- a/tests/project-env-local-inheritance.test.ts +++ b/tests/project-env-local-inheritance.test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, test } from "bun:test"; import { + chmod, mkdir, mkdtemp, readdir, @@ -434,3 +435,18 @@ test("a reserved overlay name in the primary checkout isn't treated as an overla expect(await f.migrate()).not.toBeInstanceOf(Error); }); + +test.skipIf(process.platform !== "linux" || process.geteuid?.() === 0)( + "on Linux, a primary checkout whose env layers can't be listed refuses, before any write", + async () => { + const f = await legacyLinkedCheckout(); + const entries = await readdir(f.linked); + await chmod(f.primary, 0o300); + + const result = await f.migrate(); + await chmod(f.primary, 0o700); + + expect((result as Error).message).toContain("can't be listed"); + expect(await readdir(f.linked)).toEqual(entries); + } +); From eaa083dc47f7ff03893bbf1a5e205fb53502f64b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:25:38 -0500 Subject: [PATCH 50/55] test(env): pin what the host-named and invalid-scope tests actually show Co-Authored-By: Claude Opus 5.5 --- tests/env-legacy-service-scope.test.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 57124350c..8fcf349db 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -4166,6 +4166,17 @@ test("scoped coverage: a Compose service named host is treated as a service, not `SETTING=${SCOPED_LEGACY_VALUE}\n` ); + // The projection that differs: with a service named host, the global host target + // doesn't read the host scope. + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["host"], + envName: null, + }); + expect(resolved?.serviceEnv.host?.SETTING).toBe(SCOPED_VALUE); + expect(resolved?.hostTargetEnv.global?.SETTING).toBe(SCOPED_GLOBAL_VALUE); + const result = await migrateWithServices(repo, ["host"]); expect(result).toBeInstanceOf(LegacyDotenvMigrationRefusalError); @@ -4244,7 +4255,7 @@ test("scoped coverage: a value under a scope Compose doesn't know refuses, namin }); }); -test("scoped coverage: a layer with an invalid scope name refuses closed, before any write", async () => { +test("a layer with an invalid scope name refuses as unreadable, before the scoped-value check or any write", async () => { const repo = await createLegacyRepo(); await writeFile( resolve(repo.projectDir, "hack.env.default.yaml"), @@ -4261,7 +4272,10 @@ test("scoped coverage: a layer with an invalid scope name refuses closed, before const result = await migrateWithServices(repo, ["api"]); - expect(result).toBeInstanceOf(Error); + expect(result).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((result as Error).message).toContain( + "an env layer can't be read or resolved" + ); expect((result as Error).message).not.toContain(SCOPED_VALUE); expect((result as Error).message).not.toContain(SCOPED_LEGACY_VALUE); expect(await readdir(repo.projectDir)).toEqual(entries); From 8d804f7a6fa0cd29e74f0e04ecfa3dc4e742e34a Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:50:57 -0500 Subject: [PATCH 51/55] test(env): run the unlistable primary test on Linux in CI CI's test shards run on macOS, so this test always skipped. Run it in the existing Config compiler job's Linux legs, failing if it skips, and add a separate injected case for the refusal branch on every platform. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 7 +++++ tests/project-env-local-inheritance.test.ts | 34 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ad31b59f..1ccd5862c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,13 @@ jobs: run: bun test tests/native-compose-file-sources.test.ts tests/native-compose-file-owner.test.ts - name: Test explicit imported jobs and readiness with the matching compiler run: HACK_TEST_NATIVE_COMPILER_BINARY="$PWD/dist/hack-config-compiler" bun test tests/native-config-import-jobs.test.ts tests/native-config-import-readiness.test.ts + # The test shards run on macOS, where this test skips. Fail if it skips here too. + - name: Test the native Linux primary-checkout listing refusal + if: runner.os == 'Linux' + shell: bash + run: | + bun test tests/project-env-local-inheritance.test.ts --test-name-pattern 'on Linux, a primary checkout' 2>&1 | tee "$RUNNER_TEMP/listing-refusal.log" + grep -Eq '^ *1 pass$' "$RUNNER_TEMP/listing-refusal.log" - name: Independently validate generated schema shape corpus run: | python3 -m venv "$RUNNER_TEMP/config-schema" diff --git a/tests/project-env-local-inheritance.test.ts b/tests/project-env-local-inheritance.test.ts index 8f107e571..9c8c38b60 100644 --- a/tests/project-env-local-inheritance.test.ts +++ b/tests/project-env-local-inheritance.test.ts @@ -1,4 +1,5 @@ -import { afterEach, expect, test } from "bun:test"; +import { afterEach, expect, spyOn, test } from "bun:test"; +import * as fs from "node:fs/promises"; import { chmod, mkdir, @@ -15,6 +16,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { ClosedCliError } from "../src/lib/cli-result.ts"; import { + LegacyDotenvMigrationRefusalError, migrateLegacyProjectEnv, resolveProjectEnvConfig, setProjectEnvValue, @@ -450,3 +452,33 @@ test.skipIf(process.platform !== "linux" || process.geteuid?.() === 0)( expect(await readdir(f.linked)).toEqual(entries); } ); + +// Branch coverage only, on every platform: the failure is injected, so this says nothing +// about when a real permission reaches this code. The test above is the native evidence. +test("injected: a failed listing of the primary's env layers refuses, before any write", async () => { + const f = await legacyLinkedCheckout(); + const entries = await readdir(f.linked); + const original = fs.readdir; + const spy = spyOn(fs, "readdir").mockImplementation((( + ...args: Parameters + ) => { + if (String(args[0]) === f.primary) { + return Promise.reject( + Object.assign(new Error("EACCES"), { code: "EACCES" }) + ); + } + return original(...args); + }) as typeof original); + + let result: unknown; + try { + result = await f.migrate(); + } finally { + spy.mockRestore(); + } + + expect(result).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + expect((result as Error).message).toContain("can't be listed"); + expect((result as Error).message).toContain("Nothing was written"); + expect(await readdir(f.linked)).toEqual(entries); +}); From ebb7300152b6c63b97a84b564ca2084f9062b173 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:11:48 -0500 Subject: [PATCH 52/55] fix(env): compute retention on the no-contract migration route Cleanup assumed every selected legacy file's entries had reached a layer. A legacy local layer breaks that, so compare with what each env now receives and keep the file, naming the keys. Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 30 ++++++++++++-- tests/env-legacy-service-scope.test.ts | 54 ++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 01d85a93e..640d4399a 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -7016,6 +7016,27 @@ async function publishMigratedLegacyLayers(opts: { return wroteFiles; } +/** What each selected legacy file's env receives now, so cleanup can compare, not assume. */ +async function deliveredValuesPerLegacyFile(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly overlayNames: readonly string[]; +}): Promise>> { + const delivered = new Map>(); + for (const envName of [null, ...opts.overlayNames]) { + const fileName = + envName === null + ? PROJECT_ENV_FILENAME + : `${PROJECT_ENV_FILENAME}.${envName}`; + delivered.set( + resolve(opts.projectDir, fileName), + new Map(Object.entries(await deliveredLayerValues({ ...opts, envName }))) + ); + } + return delivered; +} + /** * Migrates a legacy project that has no `.hack/hack.env.json` contract, so * its values live only in `.hack/.env` and `.hack/.env.`. @@ -7256,9 +7277,12 @@ async function migrateUndeclaredLegacyDotenv(opts: { overlayNames, // Generated output wasn't read, so its values aren't in any layer this run wrote. materialize: opts.materialize || generatedBase, - // No retention check here: every entry in a selected file is in a layer by now, - // or the migration refused, and files it didn't select are never candidates. - deliveredValues: new Map(), + deliveredValues: await deliveredValuesPerLegacyFile({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + serviceNames: opts.serviceNames, + overlayNames, + }), configCleanupCandidates: [], composeEnvFileReferences, }); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 8fcf349db..ad792b722 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -4280,3 +4280,57 @@ test("a layer with an invalid scope name refuses as unreadable, before the scope expect((result as Error).message).not.toContain(SCOPED_LEGACY_VALUE); expect(await readdir(repo.projectDir)).toEqual(entries); }); + +test("a legacy file holding a value its env still doesn't receive after migrating is kept, naming the key", async () => { + // A local layer with `environment: local` is that env's overlay, outside a git + // checkout, so the base doesn't read a value merged into it. + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile( + resolve(repo.projectDir, "hack.env.local.yaml"), + LAYER_EMPTY.replace("environment: default", "environment: local") + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + + const envPath = resolve(repo.projectDir, ".env"); + expect(migrated.cleanupCandidates).not.toContain(envPath); + expect(migrated.retainedEnvFiles).toEqual([ + { path: envPath, keys: ["KEPT_SETTING"] }, + ]); + expect(JSON.stringify(migrated)).not.toContain(KEPT_VALUE); +}); + +test("a migrated legacy file whose values its env now receives is still offered for cleanup", async () => { + const repo = await writeTeammateCheckout(); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api:\n image: alpine:3.20\n" + ); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }).catch((caught: unknown) => caught); + + expect(migrated).toBeInstanceOf(LegacyDotenvMigrationRefusalError); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + LISTED_COMPOSE + ); + const repaired = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + expect(repaired.retainedEnvFiles).toEqual([]); + expect(repaired.blockedCleanupCandidates).toContain( + resolve(repo.projectDir, ".env") + ); +}); From 2bb4271237ee676e32e992d72c703fe024da86a6 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:00:58 -0500 Subject: [PATCH 53/55] fix(env): refuse a migrated value over a scoped deletion with no global value Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 41 +++++++++++++++++++++----- tests/env-legacy-service-scope.test.ts | 16 ++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 640d4399a..ba2d8beb9 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6617,6 +6617,28 @@ function layerWithPlannedGlobals(opts: { }; } +/** + * The scope whose deletion is the last word on `key`, in resolution order, whether + * or not a global value exists for it to hide. Null when the last word isn't a + * scoped deletion. Resolved entries can't show this: a deletion leaves no entry. + */ +function scopeDeletingKey(opts: { + readonly layers: readonly (ProjectEnvConfig | null)[]; + readonly scopeNames: readonly string[]; + readonly key: string; +}): string | null { + let deletedBy: string | null = null; + for (const layer of opts.layers) { + for (const scope of opts.scopeNames) { + const value = layer?.values[scope]?.[opts.key]; + if (value !== undefined) { + deletedBy = value === null && scope !== "global" ? scope : null; + } + } + } + return deletedBy; +} + /** `keys` whose scoped value or scoped deletion the planned layers replace for one target. */ function scopedValuesReplacedForTarget(opts: { readonly target: { @@ -6630,10 +6652,6 @@ function scopedValuesReplacedForTarget(opts: { readonly envName: string | null; }): readonly string[] { const { target } = opts; - const globalBefore = resolveEffectiveStoredEntries({ - layers: opts.envLayers, - scopeNames: ["global"], - }); const before = resolveEffectiveStoredEntries({ layers: opts.envLayers, scopeNames: target.scopeNames, @@ -6644,17 +6662,24 @@ function scopedValuesReplacedForTarget(opts: { }); return opts.keys.flatMap((key) => { const entry = before[key]; - // A scoped value, or a scoped deletion that hides a global one. - const scoped = entry ? entry.scope !== "global" : key in globalBefore; + // A scoped value, or a scoped deletion, with or without a global value to hide. + const scope = entry + ? entry.scope + : scopeDeletingKey({ + layers: opts.envLayers, + scopeNames: target.scopeNames, + key, + }); const value = after[key]?.value; const replaced = - scoped && + scope !== null && + scope !== "global" && value !== undefined && !isProjectEnvSecretValue(value) && String(value) !== target.before[key]; return replaced ? [ - `${JSON.stringify(key)} for ${JSON.stringify(entry?.scope ?? target.name)} in ${JSON.stringify(opts.envName ?? "default")}`, + `${JSON.stringify(key)} for ${JSON.stringify(scope)} in ${JSON.stringify(opts.envName ?? "default")}`, ] : []; }); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index ad792b722..9fac0bd05 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -3926,6 +3926,22 @@ for (const refused of [ legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, reason: '"SETTING" for "api" in "default"', }, + { + name: "a tracked service deletion with no global value", + files: { + "hack.env.default.yaml": " global: {}\n api:\n SETTING: null\n", + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "api" in "default"', + }, + { + name: "a tracked host deletion with no global value", + files: { + "hack.env.default.yaml": " global: {}\n host:\n SETTING: null\n", + }, + legacy: { ".env": `SETTING=${SCOPED_LEGACY_VALUE}\n` }, + reason: '"SETTING" for "host" in "default"', + }, { name: "a tracked overlay's service value, from the base file", files: { From e2e36fe01bfe043bd0c0d398ba20c06128dc8a9c Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:55:13 -0500 Subject: [PATCH 54/55] fix(env): refuse a no-contract migration into a local overlay slot Outside git, a hack.env.local.yaml with `environment: local` is the `local` overlay. Base values written there reach `--env local` and not the default env, past the scoped-value check, so the migration now refuses and writes nothing. Also corrects the symlink refusal's reason after the machine-local contract migration, and makes the cleanup test assert delivery and its candidate lists. Co-Authored-By: Claude Opus 5.5 --- docs/env.md | 4 ++ src/lib/project-env-config.ts | 34 ++++++++++- tests/env-legacy-service-scope.test.ts | 79 +++++++++++++------------- 3 files changed, 76 insertions(+), 41 deletions(-) diff --git a/docs/env.md b/docs/env.md index e90ab7d42..140abd591 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1119,6 +1119,10 @@ files, when: that file set, so move such values first with `hack env set --local`. A `.hack` or a layer linked outside the project is refused with a closed error, and the link and outside files stay as they are; +- outside a git checkout, `hack.env.local.yaml` sets `environment: local`, so it serves as the + `local` overlay and base values written there wouldn't reach the default env + (`E_CONFIG_INVALID`). If it's meant to be your machine-local layer, set its `environment` to + `default`, then retry; - git tracks a machine-local layer it would write, or, in a git checkout, git cannot report whether it does (`E_PATH_CONFLICT`); - in a git checkout, git does not ignore a machine-local layer it would write, for diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index a38404232..0a938db1f 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6434,7 +6434,7 @@ async function assertNoLegacyDotenvLinks(opts: { if (linked.length > 0) { throw new LegacyDotenvMigrationRefusalError({ code: "E_PATH_CONFLICT", - reason: `${linked.join(", ")} ${linked.length === 1 ? "is a symlink" : "are symlinks"}, and this migration writes plain_env values to tracked env layers, so another file's values could be committed. Replace the link with a regular file, or remove it, then retry.`, + reason: `${linked.join(", ")} ${linked.length === 1 ? "is a symlink" : "are symlinks"}, and this migration must not pull another file's values into this project's env layers. Replace the link with a regular file, or remove it, then retry.`, }); } } @@ -7204,6 +7204,37 @@ async function publishMigratedLegacyLayers(opts: { return wroteFiles; } +/** + * Refuses when `hack.env.local.yaml` serves as the `local` env overlay rather than the + * machine-local base. Base values written there would reach `--env local` and not the + * default env, past the scoped-value check, which models the machine-local base slot. + */ +async function assertLocalBaseIsTheDefaultEnvLayer(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly baseToMigrate: Readonly>; +}): Promise { + if (Object.keys(opts.baseToMigrate).length === 0) { + return; + } + const localBasePath = resolveProjectEnvLocalConfigPath({ + projectDir: opts.projectDir, + envName: null, + }); + const effectivePath = await resolveProjectEnvEffectiveLocalConfigPath({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + envName: null, + }); + if (effectivePath !== localBasePath) { + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: + "hack.env.local.yaml is this project's `local` env overlay, so migrated values would reach `--env local` and not the default env. If it's meant to be your machine-local layer, set its `environment` to `default`, then retry.", + }); + } +} + /** What each selected legacy file's env receives now, so cleanup can compare, not assume. */ async function deliveredValuesPerLegacyFile(opts: { readonly projectRoot: string; @@ -7325,6 +7356,7 @@ async function migrateUndeclaredLegacyDotenv(opts: { envName: null, }), }); + await assertLocalBaseIsTheDefaultEnvLayer({ ...opts, baseToMigrate }); const localWrites: PlannedEnvLayerWrite[] = []; const trackedWrites: PlannedEnvLayerWrite[] = [ { diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 9fac0bd05..7dfb45805 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -4297,56 +4297,55 @@ test("a layer with an invalid scope name refuses as unreadable, before the scope expect(await readdir(repo.projectDir)).toEqual(entries); }); -test("a legacy file holding a value its env still doesn't receive after migrating is kept, naming the key", async () => { - // A local layer with `environment: local` is that env's overlay, outside a git - // checkout, so the base doesn't read a value merged into it. - const repo = await createLegacyRepo(); - await writeFile( - resolve(repo.projectDir, "hack.env.default.yaml"), - LAYER_EMPTY - ); - await writeFile( - resolve(repo.projectDir, "hack.env.local.yaml"), - LAYER_EMPTY.replace("environment: default", "environment: local") - ); - await writeFile( - resolve(repo.projectDir, ".env"), - `KEPT_SETTING=${KEPT_VALUE}\n` - ); +for (const [name, defaultValues] of [ + ["a value", " global: {}\n"], + ["a scoped deletion", " global: {}\n api:\n KEPT_SETTING: null\n"], +] as const) { + test(`outside git, a local layer that is the \`local\` overlay refuses, with ${name} in the base layer`, async () => { + // Base values written to this file would reach `--env local`, not the default env. + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + scopedLayer("default", defaultValues) + ); + await writeFile( + resolve(repo.projectDir, "hack.env.local.yaml"), + LAYER_EMPTY.replace("environment: default", "environment: local") + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); - const migrated = await migrateLegacyRepo(repo, { - migrateUndeclaredDotenv: true, + await expectMigrationRefusal({ + repo, + code: "E_CONFIG_INVALID", + reason: "hack.env.local.yaml is this project's `local` env overlay", + hidden: [KEPT_VALUE], + }); }); +} - const envPath = resolve(repo.projectDir, ".env"); - expect(migrated.cleanupCandidates).not.toContain(envPath); - expect(migrated.retainedEnvFiles).toEqual([ - { path: envPath, keys: ["KEPT_SETTING"] }, - ]); - expect(JSON.stringify(migrated)).not.toContain(KEPT_VALUE); -}); - -test("a migrated legacy file whose values its env now receives is still offered for cleanup", async () => { +test("a migrated legacy file whose values its env now receives isn't retained, and waits on its Compose listing", async () => { const repo = await writeTeammateCheckout(); await writeFile( resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), - "services:\n api:\n image: alpine:3.20\n" + LISTED_COMPOSE ); const migrated = await migrateLegacyRepo(repo, { migrateUndeclaredDotenv: true, - }).catch((caught: unknown) => caught); + }); - expect(migrated).toBeInstanceOf(LegacyDotenvMigrationRefusalError); - await writeFile( - resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), - LISTED_COMPOSE - ); - const repaired = await migrateLegacyRepo(repo, { - migrateUndeclaredDotenv: true, + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + envName: null, }); - expect(repaired.retainedEnvFiles).toEqual([]); - expect(repaired.blockedCleanupCandidates).toContain( - resolve(repo.projectDir, ".env") - ); + expect(resolved?.hostTargetEnv.global?.KEPT_SETTING).toBe(KEPT_VALUE); + const envPath = resolve(repo.projectDir, ".env"); + expect(migrated.retainedEnvFiles).toEqual([]); + expect(migrated.blockedCleanupCandidates).toContain(envPath); + expect(migrated.cleanupCandidates).not.toContain(envPath); }); From 0c14155dc7cf6795e0361771d4a965532c2284ec Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:42:10 -0500 Subject: [PATCH 55/55] fix(env): apply the local overlay refusal outside git only Inside git, tracking decides what hack.env.local.yaml is, so a tracked layer got a refusal whose remedy didn't work. It now gets the tracked-layer refusal again. Adds tests for that case, for the remedy, and for cleanup keeping a file whose values its env doesn't receive. Co-Authored-By: Claude Opus 5.5 --- src/lib/project-env-config.ts | 9 ++- tests/env-legacy-service-scope.test.ts | 89 +++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 0a938db1f..5b347ebd7 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -7214,7 +7214,12 @@ async function assertLocalBaseIsTheDefaultEnvLayer(opts: { readonly projectDir: string; readonly baseToMigrate: Readonly>; }): Promise { - if (Object.keys(opts.baseToMigrate).length === 0) { + // Inside git, tracking decides what this file is, and the tracked-layer check + // refuses with the remedy that works there. + if ( + Object.keys(opts.baseToMigrate).length === 0 || + (await isInsideGitCheckoutMarker({ startDir: opts.projectRoot })) + ) { return; } const localBasePath = resolveProjectEnvLocalConfigPath({ @@ -7744,7 +7749,7 @@ async function readLegacyDotenvCleanupEntries(opts: { return parseDotEnv(source.text); } -async function collectLegacyProjectEnvCleanupCandidates(opts: { +export async function collectLegacyProjectEnvCleanupCandidates(opts: { readonly projectRoot: string; readonly projectDir: string; readonly overlayNames: readonly string[]; diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index 7dfb45805..09a85ed41 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -23,6 +23,7 @@ import { } from "../src/constants.ts"; import { ClosedCliError, HackCliError } from "../src/lib/cli-result.ts"; import { + collectLegacyProjectEnvCleanupCandidates, ENV_SCOPE_INVALID_MESSAGE, findUnmigratedLegacyDotenv, inspectProjectEnvMaterialization, @@ -4298,7 +4299,7 @@ test("a layer with an invalid scope name refuses as unreadable, before the scope }); for (const [name, defaultValues] of [ - ["a value", " global: {}\n"], + ["no value", " global: {}\n"], ["a scoped deletion", " global: {}\n api:\n KEPT_SETTING: null\n"], ] as const) { test(`outside git, a local layer that is the \`local\` overlay refuses, with ${name} in the base layer`, async () => { @@ -4326,6 +4327,92 @@ for (const [name, defaultValues] of [ }); } +test("outside git, setting that local layer's environment to default, as the refusal says, lets the migration deliver", async () => { + const repo = await createLegacyRepo(); + await writeFile( + resolve(repo.projectDir, "hack.env.default.yaml"), + LAYER_EMPTY + ); + await writeFile(resolve(repo.projectDir, "hack.env.local.yaml"), LAYER_EMPTY); + await writeFile( + resolve(repo.projectDir, PROJECT_COMPOSE_FILENAME), + LISTED_COMPOSE + ); + await writeFile( + resolve(repo.projectDir, ".env"), + `KEPT_SETTING=${KEPT_VALUE}\n` + ); + + const migrated = await migrateLegacyRepo(repo, { + migrateUndeclaredDotenv: true, + }); + + expect(migrated.retainedEnvFiles).toEqual([]); + const resolved = await resolveProjectEnvConfig({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + serviceNames: ["api"], + envName: null, + }); + expect(resolved?.hostTargetEnv.global?.KEPT_SETTING).toBe(KEPT_VALUE); +}); + +test("in git, a tracked local layer still in the working tree gets the tracked-layer refusal and its remedy", async () => { + const repo = await createLegacyRepo(); + await git({ repo, args: ["init", "-q"] }); + const localPath = resolve(repo.projectDir, "hack.env.local.yaml"); + await writeFile(localPath, LAYER); + await git({ repo, args: ["add", "-f", ".hack/hack.env.local.yaml"] }); + await writeFile( + resolve(repo.projectDir, ".env"), + `OTHER_SETTING=${KEPT_VALUE}\n` + ); + + await expectMigrationRefusal({ + repo, + code: "E_PATH_CONFLICT", + reason: `git tracks the machine-local env layer ${localPath}`, + hidden: [KEPT_VALUE], + }); +}, 60_000); + +test("cleanup keeps a legacy file holding a value its env doesn't receive, naming the key, and offers one it does", async () => { + const repo = await writeTeammateCheckout(); + const envPath = resolve(repo.projectDir, ".env"); + const collect = (delivered: string | null) => + collectLegacyProjectEnvCleanupCandidates({ + projectRoot: repo.projectRoot, + projectDir: repo.projectDir, + overlayNames: [], + materialize: false, + deliveredValues: new Map([ + [ + envPath, + new Map(delivered === null ? [] : [["KEPT_SETTING", delivered]]), + ], + ]), + configCleanupCandidates: [], + composeEnvFileReferences: [], + }); + + const undelivered = await collect(null); + expect(undelivered.retainedEnvFiles).toEqual([ + { path: envPath, keys: ["KEPT_SETTING"] }, + ]); + expect(undelivered.allowed).not.toContain(envPath); + expect(undelivered.blocked).not.toContain(envPath); + expect(JSON.stringify(undelivered)).not.toContain(KEPT_VALUE); + + const different = await collect("another-value"); + expect(different.retainedEnvFiles).toEqual([ + { path: envPath, keys: ["KEPT_SETTING"] }, + ]); + + const delivered = await collect(KEPT_VALUE); + expect(delivered.retainedEnvFiles).toEqual([]); + expect(delivered.allowed).toContain(envPath); +}); + test("a migrated legacy file whose values its env now receives isn't retained, and waits on its Compose listing", async () => { const repo = await writeTeammateCheckout(); await writeFile(