From e3814e87d88a2df1ecd1ca31a4c179a1e9884e3b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 02:02:37 -0400 Subject: [PATCH 1/2] fix(ci): read the test shard jobs in the Bun crash census and fix stale CI test job docs Since #442 the `test` job is only the required fan-in of the four `test shard i/4` jobs. It runs no Bun, but `ci:bun-crash-census` still read it by default and reported zero crashes without saying so, and `workflowJobBunVersion(ci.yml, "test")` found the fan-in block, which has no setup-bun. - With no --job, the census reads every `test shard i/n` job of a run, or the `test` job of a run that has no shard job (a run from before the split, whose `test` job ran the suite). - --job accepts `/` and a `*` pattern, so the documented `--job "test shard 1/4"` and `--job "test shard *"` work; before, both were refused as an invalid argument. - The workflow lookup matches a job by its `name:` template, so `test shard 2/4` finds the `test-shard` matrix job's Bun pin. - The report prints a closed `warning:` line (also on stderr, and in --json) when no job matched or a selected job shows no Bun setup in its log or workflow block. - docs/security/env-secrets-threat-model.md and docs/reference/shared-host-lifecycle.md now say the tests run in a `test shard i/4` job and that `test` is the fan-in. Refs HACK-1424 Co-Authored-By: Claude Opus 5.5 --- .ai/skills/hack-repo-verify/SKILL.md | 2 +- docs/reference/shared-host-lifecycle.md | 7 +- docs/security/env-secrets-threat-model.md | 2 +- scripts/ci-bun-crash-census.ts | 204 ++++++++++++++--- tests/ci-bun-crash-census.test.ts | 253 +++++++++++++++++++++- 5 files changed, 432 insertions(+), 36 deletions(-) diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index ed6106405..0faa9c9ff 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -98,7 +98,7 @@ untracked candidate additions. Keep private artifacts excluded from the index. | Native authored command slices | `tests/native-authored-command.test.ts` (unsupported matrix), `tests/native-authored-command-dispatch.test.ts`, `tests/native-authored-project-{start,run,observe,logs,exec,recovery}.test.ts` and the [command contract](../../../docs/reference/native-authored-commands.md); the dispatch "source CLI" cases need the prepared compiler (`bun run build:config-compiler` or absolute `HACK_CONFIG_COMPILER_BINARY`) and fail without it; `native-authored-project-start` and `native-authored-project-exec` are nondeterministic on a local macOS host, so a single local pass or failure is not proof; live acceptance runs host-serialized on one prepared macOS provider slot | | Native VM/routing/reclamation | [candidate guide](../../../docs/guides/native-candidate.md), relevant acceptance gate and isolated host fixture; Linux checks cannot qualify macOS effects | | Resource/performance claims | [performance skill](../hack-repo-performance/SKILL.md), matched workload and measured boundaries | -| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed `test` job logs; `--scan-passing` for all; since the suite moved to `test shard i/4` jobs, pass `--job "test shard i/4"` once per shard for newer runs). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | +| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed test-suite job logs: every `test shard i/n` job, or the `test` job of a run from before the split, since `test` is now the logless fan-in; `--scan-passing` for all; `--job NAME` selects by exact name or a `*` pattern such as `"test shard *"`; a `warning:` line means no job matched or a selected job ran no Bun, so its zero is not evidence). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | Read [CI](../../../.github/workflows/ci.yml) for current hosted gates. A local pass does not mean CI ran. diff --git a/docs/reference/shared-host-lifecycle.md b/docs/reference/shared-host-lifecycle.md index 0255d8138..587a8a172 100644 --- a/docs/reference/shared-host-lifecycle.md +++ b/docs/reference/shared-host-lifecycle.md @@ -226,9 +226,10 @@ Progress events are also closed: `lock-waiting`, `lock-acquired`, `claim-stale`, `bun test tests/shared-host-lifecycle.test.ts` runs portable local fixtures. Each fixture is a Bun process on a loopback port with a private HOME. No Docker, real -HOME, keychain or privileged effect is involved. The file runs in the hosted macOS -`test` job and is registered in the hosted Linux `linux-process-lifetime` job. A -Linux pass is a receipt only once that hosted job has run it. +HOME, keychain or privileged effect is involved. The file runs in one of the hosted +macOS `test shard i/4` jobs (the required `test` check passes only when every shard +passes) and is registered in the hosted Linux `linux-process-lifetime` job. A Linux +pass is a receipt only once that hosted job has run it. | Scenario | Asserted behavior | | --- | --- | diff --git a/docs/security/env-secrets-threat-model.md b/docs/security/env-secrets-threat-model.md index 7cff5ae79..a077f7de1 100644 --- a/docs/security/env-secrets-threat-model.md +++ b/docs/security/env-secrets-threat-model.md @@ -96,7 +96,7 @@ refer to `hack-dance/hack`. | A key copy ships in build output or a static site. | `hack doctor` reports `env-key-public-path` for copies under `public/`, `static/`, `dist/`, `build/`, `out/`, `www/`, `_site/`, `.next/` and `.output/` (#309). | | A weak, short or repetitive key. | `env-key-wrong-length` and `env-key-low-entropy` (#309). Generated keys are 32 random bytes in base64url. | | Materialized `.hack/.env` or local overrides are committed. | Hack owns a committed `.hack/.gitignore` block, and `hack doctor` flags tracked generated files (see [ignore rules](../env.md#ignore-rules-committed-hackgitignore)). | -| A secret leaks into a public artifact through a path nobody listed. | `tests/env-secret-sentinel-scan.test.ts` runs in the CI `test` job. It gives a primary checkout and a linked worktree random synthetic secrets through `env add --secret --value-stdin` (shared and `--local`), runs materialize, exec, up (Compose generation), env apply, logs, list, explain and doctor, then commits everything with `git add -A`. It requires zero occurrences of any value or the project key, raw or hex/base64/base64url encoded, in CLI output, recorded child argv, tracked files, every Git object, HACK_HOME, the private HOME and its caches, and generated Compose and runtime files. Plaintext may exist only in `.hack/.env`, and key text only in key files; each must be git-ignored, untracked and mode 0600. `.hack/.internal/compose.env.override.yml` must hold `${HACK_ENV_...}` references and no value, and must also be private and ignored. Values may reach a child's environment only for the exec'd command and `docker compose`. Positive controls prove delivery and that the scanner finds planted encodings in files and Git objects. Docker is a recording shim; no engine, keychain or installed Hack is reached. | +| A secret leaks into a public artifact through a path nobody listed. | `tests/env-secret-sentinel-scan.test.ts` runs in one of the hosted CI `test shard i/4` jobs; the required `test` check passes only when every shard passes. It gives a primary checkout and a linked worktree random synthetic secrets through `env add --secret --value-stdin` (shared and `--local`), runs materialize, exec, up (Compose generation), env apply, logs, list, explain and doctor, then commits everything with `git add -A`. It requires zero occurrences of any value or the project key, raw or hex/base64/base64url encoded, in CLI output, recorded child argv, tracked files, every Git object, HACK_HOME, the private HOME and its caches, and generated Compose and runtime files. Plaintext may exist only in `.hack/.env`, and key text only in key files; each must be git-ignored, untracked and mode 0600. `.hack/.internal/compose.env.override.yml` must hold `${HACK_ENV_...}` references and no value, and must also be private and ignored. Values may reach a child's environment only for the exec'd command and `docker compose`. Positive controls prove delivery and that the scanner finds planted encodings in files and Git objects. Docker is a recording shim; no engine, keychain or installed Hack is reached. | | A dependency vulnerability ships in Hack. | `bun install --frozen-lockfile`, and the read-only `scripts/dependabot-inventory.ts` lists open alerts (#309). Hack does not sandbox project dependencies that run under `hack host exec`; they receive the injected env by design. | ### Agent and MCP exfiltration diff --git a/scripts/ci-bun-crash-census.ts b/scripts/ci-bun-crash-census.ts index c6b853687..f04cce77c 100644 --- a/scripts/ci-bun-crash-census.ts +++ b/scripts/ci-bun-crash-census.ts @@ -3,10 +3,19 @@ * Read-only census of Bun crashes in hosted CI history. * * For a `--since`/`--until` window it lists every run of the CI workflow, reads - * the `test` job of every run attempt, downloads only failed-job logs (all + * the test-suite jobs of every run attempt, downloads only failed-job logs (all * completed logs with `--scan-passing`) into an owned temporary directory, and * classifies each with the shared classifier used by `scripts/bun-crash-loop.ts`. * + * By default the test-suite jobs are every `test shard i/n` job of a run. Since + * HACK-1381 the `test` job is only the required fan-in: it runs no Bun, so its + * log holds nothing to classify. A run with no shard job predates the split, so + * its `test` job ran the suite and is read instead. `--job NAME` selects jobs by + * exact name, or by a pattern where `*` matches any text + * (`--job "test shard *"`). The report warns when no job matched, or when a + * selected job shows no Bun setup in its log or workflow block, because a zero + * crash count from such a job is not evidence. + * * It uses the caller's existing `gh` authentication and never handles a token. * It is strictly read-only: no reruns, comments, labels or uploads. It prints * counts, closed tokens, versions and numeric run/job ids only, never log bodies. @@ -62,6 +71,18 @@ export interface JobRecord { readonly reason: BunRunReason | null; } +/** + * A closed warning. Each one means the census could not see the Bun output a + * zero crash count needs, so the totals are not evidence by themselves. + */ +export type CensusWarning = + | { readonly kind: "no_jobs_matched" } + | { + readonly kind: "no_bun_setup"; + readonly jobs: number; + readonly selected: number; + }; + export interface CrashingJob { readonly runId: number; readonly attempt: number; @@ -85,7 +106,15 @@ export interface VersionTotals { const ISO_INSTANT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/; const REPOSITORY = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/; const WORKFLOW_FILE = /^[A-Za-z0-9_.-]{1,100}\.ya?ml$/; -const JOB_NAME = /^[A-Za-z0-9_. -]{1,100}$/; +const JOB_NAME = /^[A-Za-z0-9_. /*-]{1,100}$/; +/** A shard job of the hosted test suite (`.github/workflows/ci.yml`, HACK-1381). */ +const TEST_SHARD_JOB = /^test shard \d+\/\d+$/; +/** Before the suite was sharded this job ran it; now it is the logless fan-in. */ +const LEGACY_TEST_JOB = "test"; +const WORKFLOW_JOB_KEY = /^ {2}([A-Za-z0-9_-]+):\s*$/; +const WORKFLOW_JOB_NAME = /^ {4}name:\s*(.+?)\s*$/; +const WORKFLOW_EXPRESSION = /\$\{\{.*?\}\}/; +const WORKFLOW_BUN_VERSION = /^\s+bun-version:\s*["']?([^"'\s]+)["']?\s*$/; const SHA = /^[0-9a-f]{40}$/; /** The runs endpoint returns at most 1000 results for one `created` query. */ const MAX_WINDOW_RUNS = 1000; @@ -107,24 +136,76 @@ const positiveInteger = (value: unknown): number | null => ? value : null; +const escapeRegExp = (text: string): string => + text.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); + +/** Whether a job display name matches a `--job` value (`*` matches any text). */ +export function jobNameMatches(pattern: string, name: string): boolean { + return new RegExp( + `^${pattern.split("*").map(escapeRegExp).join(".*")}$` + ).test(name); +} + /** - * The `bun-version` given to the named job in a workflow file, or null. The - * job block runs from ` :` to the next two-space-indented key. + * Whether a workflow job with key `key` and an optional `name:` template has + * the display name `name`. A `${{ ... }}` expression in the template matches + * any non-empty text, so `test shard ${{ matrix.shard }}/4` names + * `test shard 2/4`. + */ +function workflowJobNamed(opts: { + readonly key: string; + readonly template: string | null; + readonly name: string; +}): boolean { + if (opts.template === null) { + return opts.key === opts.name; + } + const template = opts.template.replace(/^(["'])(.*)\1$/, "$2"); + return new RegExp( + `^${template.split(WORKFLOW_EXPRESSION).map(escapeRegExp).join(".+")}$` + ).test(opts.name); +} + +/** + * The `bun-version` given to the job with display name `job` in a workflow + * file, or null. A job block runs from its two-space-indented key under + * `jobs:` to the next one. Its display name is its `name:` template, or its + * key when it has none, so `test shard 2/4` finds the `test-shard` matrix job. + * A job with no `bun-version`, such as the `test` fan-in, returns null. */ export function workflowJobBunVersion( workflow: string, job: string ): string | null { const lines = workflow.split(/\r?\n/); - const start = lines.findIndex((line) => line.trimEnd() === ` ${job}:`); - if (start === -1) { + const jobsAt = lines.findIndex((line) => line.trimEnd() === "jobs:"); + if (jobsAt === -1) { return null; } - for (const line of lines.slice(start + 1)) { - if (/^ {2}\S/.test(line)) { - return null; + const blocks: { readonly key: string; readonly lines: string[] }[] = []; + for (const line of lines.slice(jobsAt + 1)) { + if (/^\S/.test(line)) { + break; } - const match = /^\s+bun-version:\s*["']?([^"'\s]+)["']?\s*$/.exec(line); + const key = WORKFLOW_JOB_KEY.exec(line)?.[1]; + if (key) { + blocks.push({ key, lines: [] }); + } else { + blocks.at(-1)?.lines.push(line); + } + } + const block = blocks.find((candidate) => + workflowJobNamed({ + key: candidate.key, + template: + candidate.lines + .map((line) => WORKFLOW_JOB_NAME.exec(line)?.[1]) + .find((value) => value !== undefined) ?? null, + name: job, + }) + ); + for (const line of block?.lines ?? []) { + const match = WORKFLOW_BUN_VERSION.exec(line); if (match) { return closedBunVersion(match[1]); } @@ -132,6 +213,38 @@ export function workflowJobBunVersion( return null; } +/** + * Which job names of one run the census reads. With no `--job`, the run's + * `test shard i/n` jobs, or its `test` job when it has no shard job (a run + * from before the split, whose `test` job ran the suite). + */ +export function suiteJobSelector(opts: { + readonly job: string | null; + readonly runJobNames: readonly string[]; +}): (name: string) => boolean { + const { job } = opts; + if (job !== null) { + return (name) => jobNameMatches(job, name); + } + if (opts.runJobNames.some((name) => TEST_SHARD_JOB.test(name))) { + return (name) => TEST_SHARD_JOB.test(name); + } + return (name) => name === LEGACY_TEST_JOB; +} + +/** The warnings for a finished census; see {@link CensusWarning}. */ +export function censusWarnings(records: readonly JobRecord[]): CensusWarning[] { + if (records.length === 0) { + return [{ kind: "no_jobs_matched" }]; + } + const withoutBun = records.filter( + (entry) => entry.versionSource === "unknown" + ).length; + return withoutBun === 0 + ? [] + : [{ kind: "no_bun_setup", jobs: withoutBun, selected: records.length }]; +} + export function emptyTotals(bunVersion: string): VersionTotals { return { bunVersion, @@ -200,7 +313,8 @@ export function aggregateCensus( export interface CensusOptions { readonly repository: string; readonly workflow: string; - readonly job: string; + /** Null reads the test-suite jobs; see {@link suiteJobSelector}. */ + readonly job: string | null; readonly since: string; readonly until: string; readonly scanPassing: boolean; @@ -211,6 +325,7 @@ export interface CensusResult { readonly runs: number; readonly records: readonly JobRecord[]; readonly totals: readonly VersionTotals[]; + readonly warnings: readonly CensusWarning[]; } async function mapConcurrent( @@ -335,11 +450,15 @@ export async function runCensus( } const workflowVersions = new Map>(); - const workflowVersion = (sha: string | null): Promise => { + const workflowVersion = ( + sha: string | null, + job: string + ): Promise => { if (sha === null) { return Promise.resolve(null); } - const cached = workflowVersions.get(sha); + const cacheKey = `${sha}\n${job}`; + const cached = workflowVersions.get(cacheKey); if (cached) { return cached; } @@ -358,11 +477,11 @@ export async function runCensus( } return workflowJobBunVersion( Buffer.from(value.content, "base64").toString("utf8"), - options.job + job ); }) .catch(() => null); - workflowVersions.set(sha, pending); + workflowVersions.set(cacheKey, pending); return pending; }; @@ -372,16 +491,24 @@ export async function runCensus( async (run) => { const seen = new Set(); const records: JobRecord[] = []; - const lines = await gh.lines( - `repos/${repo}/actions/runs/${run.id}/jobs?filter=all&per_page=100`, - ".jobs[] | {id, name, run_attempt, status, conclusion}" - ); - for (const line of lines) { - const job = parseLine(line); + const jobs = ( + await gh.lines( + `repos/${repo}/actions/runs/${run.id}/jobs?filter=all&per_page=100`, + ".jobs[] | {id, name, run_attempt, status, conclusion}" + ) + ).map(parseLine); + const selected = suiteJobSelector({ + job: options.job, + runJobNames: jobs.flatMap((job) => + typeof job.name === "string" ? [job.name] : [] + ), + }); + for (const job of jobs) { const jobId = positiveInteger(job.id); const attempt = positiveInteger(job.run_attempt); if ( - job.name !== options.job || + typeof job.name !== "string" || + !selected(job.name) || job.status !== "completed" || job.conclusion === "skipped" || jobId === null || @@ -399,7 +526,7 @@ export async function runCensus( : null; const fromWorkflow = scanned?.bunVersion ? null - : await workflowVersion(run.headSha); + : await workflowVersion(run.headSha, job.name); records.push({ runId: run.id, attempt, @@ -414,9 +541,26 @@ export async function runCensus( } ); const records = perRun.flat(); - return { runs: runs.length, records, totals: aggregateCensus(records) }; + return { + runs: runs.length, + records, + totals: aggregateCensus(records), + warnings: censusWarnings(records), + }; +} + +/** One closed, value-free line for a warning. */ +export function renderCensusWarning(warning: CensusWarning): string { + if (warning.kind === "no_jobs_matched") { + return "warning: no completed job matched the job selector, so zero crashes is not evidence; check --job against the workflow's job names"; + } + return `warning: ${warning.jobs} of ${warning.selected} selected jobs show no Bun setup in their log or workflow, so their crash count is not evidence; the test job is a fan-in that runs no Bun since HACK-1381, so omit --job to read the test shard jobs`; } +/** The report's `job=` value: the quoted selector, or `default`. */ +const jobLabel = (job: string | null): string => + job === null ? "default" : JSON.stringify(job); + /** Closed, value-free text report. */ export function renderCensus(opts: { readonly options: CensusOptions; @@ -424,7 +568,7 @@ export function renderCensus(opts: { }): string { const { options, result } = opts; const lines = [ - `Bun crash census repo=${options.repository} workflow=${options.workflow} job=${options.job} since=${options.since} until=${options.until} runs=${result.runs} jobs=${result.records.length} scan_passing=${options.scanPassing}`, + `Bun crash census repo=${options.repository} workflow=${options.workflow} job=${jobLabel(options.job)} since=${options.since} until=${options.until} runs=${result.runs} jobs=${result.records.length} scan_passing=${options.scanPassing}`, ]; for (const totals of result.totals) { const sources = Object.entries(totals.versionSources) @@ -439,6 +583,7 @@ export function renderCensus(opts: { ); } } + lines.push(...result.warnings.map(renderCensusWarning)); return `${lines.join("\n")}\n`; } @@ -495,12 +640,12 @@ export function parseCensusArguments(argv: readonly string[]): CensusOptions & { } const repository = values.get("repo") ?? "hack-dance/hack"; const workflow = values.get("workflow") ?? "ci.yml"; - const job = values.get("job") ?? "test"; + const job = values.get("job") ?? null; if ( !( REPOSITORY.test(repository) && WORKFLOW_FILE.test(workflow) && - JOB_NAME.test(job) + (job === null || JOB_NAME.test(job)) ) ) { throw new CensusError("Invalid argument"); @@ -583,9 +728,12 @@ async function main(): Promise { const result = await runCensus(options, ghCliReader, workDir); process.stdout.write( json - ? `${JSON.stringify({ options, runs: result.runs, totals: result.totals }, null, 2)}\n` + ? `${JSON.stringify({ options, runs: result.runs, totals: result.totals, warnings: result.warnings }, null, 2)}\n` : renderCensus({ options, result }) ); + for (const warning of result.warnings) { + process.stderr.write(`${renderCensusWarning(warning)}\n`); + } } finally { process.off("SIGINT", interrupt); process.off("SIGTERM", interrupt); diff --git a/tests/ci-bun-crash-census.test.ts b/tests/ci-bun-crash-census.test.ts index ca037363d..acf751556 100644 --- a/tests/ci-bun-crash-census.test.ts +++ b/tests/ci-bun-crash-census.test.ts @@ -1,15 +1,17 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { aggregateCensus, type CensusOptions, type GhReader, type JobRecord, + jobNameMatches, parseCensusArguments, renderCensus, runCensus, + suiteJobSelector, workflowJobBunVersion, } from "../scripts/ci-bun-crash-census.ts"; import { @@ -407,12 +409,26 @@ test("census arguments are validated and closed", () => { ).toEqual({ repository: "hack-dance/hack", workflow: "ci.yml", - job: "test", + job: null, since: "2026-10-01T17:35:00Z", until: "2026-10-11T23:43:30Z", scanPassing: true, json: false, }); + const window = [ + "--since", + "2026-10-01T17:35:00Z", + "--until", + "2026-10-11T23:43:30Z", + ]; + // The verify skill documents `--job "test shard i/4"`; a `/` or `*` used to be refused. + expect(parseCensusArguments([...window, "--job", "test shard 1/4"]).job).toBe( + "test shard 1/4" + ); + expect(parseCensusArguments([...window, "--job=test shard *"]).job).toBe( + "test shard *" + ); + expect(parseCensusArguments([...window, "--job", "test"]).job).toBe("test"); for (const argv of [ [], ["--since", "2026-10-01", "--until", "2026-10-11T23:43:30Z"], @@ -440,7 +456,238 @@ test("census arguments are validated and closed", () => { "--json", "--json", ], + [...window, "--job", "test;shard"], + [...window, "--job", "t".repeat(101)], ]) { expect(() => parseCensusArguments(argv)).toThrow(); } }); + +describe("sharded test suite (HACK-1424)", () => { + const SHARDED_WORKFLOW = [ + "name: CI", + "on:", + " push:", + " pull_request:", + "jobs:", + " test-shard:", + " name: test shard ${{ matrix.shard }}/4", + " steps:", + " - uses: oven-sh/setup-bun@v1", + " with:", + ' bun-version: "1.4.2"', + " test:", + " needs: test-shard", + " if: ${{ always() }}", + " steps:", + " - name: Require every test shard to succeed", + " run: exit 0", + " quoted:", + ' name: "quoted ${{ matrix.os }}"', + " steps:", + " - with:", + " bun-version: 1.3.9", + ].join("\n"); + + test("a shard job's Bun version comes from its name template; the fan-in has none", () => { + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test shard 3/4")).toBe( + "1.4.2" + ); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test")).toBeNull(); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test shard /4")).toBeNull(); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test-shard")).toBeNull(); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "quoted macos")).toBe( + "1.3.9" + ); + // `on:` keys are not jobs. + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "push")).toBeNull(); + }); + + test("the real CI workflow gives every shard the pinned Bun and the test fan-in none", async () => { + const workflow = await readFile( + resolve(import.meta.dir, "../.github/workflows/ci.yml"), + "utf8" + ); + const manifest: unknown = JSON.parse( + await readFile(resolve(import.meta.dir, "../package.json"), "utf8") + ); + const pin = + typeof manifest === "object" && + manifest !== null && + "packageManager" in manifest && + typeof manifest.packageManager === "string" + ? manifest.packageManager.replace(/^bun@/, "") + : null; + expect(pin).toMatch(/^\d+\.\d+\.\d+$/); + for (const shard of [1, 2, 3, 4]) { + expect(workflowJobBunVersion(workflow, `test shard ${shard}/4`)).toBe( + pin + ); + } + expect(workflowJobBunVersion(workflow, "test")).toBeNull(); + }); + + test("job selectors: default reads shards, or the legacy test job when a run has none", () => { + const sharded = suiteJobSelector({ + job: null, + runJobNames: ["test shard 1/4", "test shard 2/4", "test", "docker-e2e"], + }); + expect( + ["test shard 1/4", "test shard 2/4", "test", "docker-e2e shard 1/4"].map( + sharded + ) + ).toEqual([true, true, false, false]); + const legacy = suiteJobSelector({ + job: null, + runJobNames: ["test", "runtime-images"], + }); + expect(["test", "runtime-images"].map(legacy)).toEqual([true, false]); + expect(jobNameMatches("test shard *", "test shard 4/4")).toBe(true); + expect(jobNameMatches("test shard *", "test")).toBe(false); + expect(jobNameMatches("test shard 1/4", "test shard 1/4")).toBe(true); + expect(jobNameMatches("test shard 1/4", "test shard 1/40")).toBe(false); + expect(jobNameMatches("test.shard", "testXshard")).toBe(false); + }); + + const FAN_IN_FAILURE_LOG = [ + stamp('##[group]Run if [ "$SHARD_RESULT" != "success" ]; then'), + stamp("test shards did not all succeed (result: failure)"), + stamp("##[error]Process completed with exit code 1."), + ].join("\n"); + const SHA_SHARDED = "c".repeat(40); + + function shardedGh() { + const downloads: string[] = []; + const logs: Record = { + 31: CRASH_LOG, + 33: FAN_IN_FAILURE_LOG, + 41: TEST_FAILURE_LOG, + }; + const job = (opts: { + readonly id: number; + readonly name: string; + readonly conclusion: string; + readonly attempt?: number; + }): string => + JSON.stringify({ + id: opts.id, + name: opts.name, + run_attempt: opts.attempt ?? 1, + status: "completed", + conclusion: opts.conclusion, + }); + const reader: GhReader = { + json: async (path) => { + if (path.includes("/runs?created=")) { + return { total_count: 2 }; + } + if (path.endsWith(`?ref=${SHA_SHARDED}`)) { + return { + content: Buffer.from(SHARDED_WORKFLOW).toString("base64"), + }; + } + if (path.endsWith(`?ref=${SHA_OLD}`)) { + return { content: workflowFor("1.3.9") }; + } + throw new Error("unexpected path"); + }, + lines: async (path) => { + if (path.includes("/runs?created=")) { + return [ + JSON.stringify({ id: 300, head_sha: SHA_SHARDED }), + JSON.stringify({ id: 400, head_sha: SHA_OLD }), + ]; + } + if (path.includes("/runs/300/jobs?filter=all")) { + return [ + job({ id: 31, name: "test shard 1/4", conclusion: "failure" }), + job({ id: 32, name: "test shard 2/4", conclusion: "success" }), + job({ id: 33, name: "test", conclusion: "failure" }), + job({ + id: 34, + name: "docker-e2e shard 1/4", + conclusion: "failure", + }), + ]; + } + if (path.includes("/runs/400/jobs?filter=all")) { + return [ + job({ id: 41, name: "test", conclusion: "failure" }), + job({ id: 42, name: "test", conclusion: "success", attempt: 2 }), + ]; + } + throw new Error("unexpected path"); + }, + download: async (path, destination) => { + const jobId = Number(/\/jobs\/(\d+)\/logs$/.exec(path)?.[1]); + downloads.push(path); + await Bun.write(destination, logs[jobId] ?? ""); + }, + }; + return { reader, downloads }; + } + + test("with no --job the census reads shard jobs, skips the fan-in, and still reads pre-split runs", async () => { + const dir = await workDir(); + const gh = shardedGh(); + const defaults: CensusOptions = { ...options, job: null }; + const result = await runCensus(defaults, gh.reader, dir); + expect(gh.downloads).toEqual([ + "repos/hack-dance/hack/actions/jobs/31/logs", + "repos/hack-dance/hack/actions/jobs/41/logs", + ]); + expect(await readdir(dir)).toEqual([]); + expect( + result.records.map((entry) => [ + entry.jobId, + entry.bunVersion, + entry.versionSource, + ]) + ).toEqual([ + [31, "1.3.9", "log"], + [32, "1.4.2", "workflow"], + [41, "1.4.2", "log"], + [42, "1.3.9", "workflow"], + ]); + expect(result.warnings).toEqual([]); + const text = renderCensus({ options: defaults, result }); + expect(text).toContain(" job=default "); + expect(text).toContain(" crash run=300 attempt=1 job=31 reason=banner"); + expect(text).not.toContain("warning:"); + }); + + test("an explicit --job test on sharded runs warns that the fan-in has no Bun output", async () => { + const dir = await workDir(); + const gh = shardedGh(); + const fanIn: CensusOptions = { ...options, job: "test" }; + const result = await runCensus(fanIn, gh.reader, dir); + expect(result.records.map((entry) => entry.jobId)).toEqual([33, 41, 42]); + expect(result.records[0]).toMatchObject({ + bunVersion: "unknown", + versionSource: "unknown", + scan: "failure", + }); + expect(result.warnings).toEqual([ + { kind: "no_bun_setup", jobs: 1, selected: 3 }, + ]); + const text = renderCensus({ options: fanIn, result }); + expect(text).toContain(' job="test" '); + expect(text).toContain( + "warning: 1 of 3 selected jobs show no Bun setup in their log or workflow" + ); + expect(text).not.toContain("SHARD_RESULT"); + }); + + test("a selector that matches no job warns instead of reporting a silent zero", async () => { + const dir = await workDir(); + const gh = shardedGh(); + const none: CensusOptions = { ...options, job: "test shard 9/9" }; + const result = await runCensus(none, gh.reader, dir); + expect(result.records).toEqual([]); + expect(gh.downloads).toEqual([]); + expect(result.warnings).toEqual([{ kind: "no_jobs_matched" }]); + expect(renderCensus({ options: none, result })).toContain( + "warning: no completed job matched the job selector" + ); + }); +}); From 159df1e029565b58deaa8d10d3843f663a5f3692 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 13:36:41 -0400 Subject: [PATCH 2/2] fix(ci): fetch each commit's workflow once in the Bun crash census and skip comment lines in jobs The census resolved a job's Bun version from the workflow file at the run's head commit through a cache keyed by (sha, job), so one commit cost one contents GET per distinct job name. It now caches the decoded workflow text per sha and keeps the per-(sha, job) version memo on top, with a comment on why the key must not be sha alone. workflowJobBunVersion ended the jobs scan at any column-0 line, so a column-0 YAML comment inside jobs: hid every later job. Blank lines and column-0 comments are now skipped; any other column-0 line still ends jobs. Tests pin the (sha, job) key with a wildcard selector over the fan-in and shards at one sha, count contents fetches across runs and concurrency, and cover comment, blank, commented-out-job and later top-level-key lines. Refs HACK-1441 Co-Authored-By: Claude Opus 5.5 --- scripts/ci-bun-crash-census.ts | 58 ++++++++----- tests/ci-bun-crash-census.test.ts | 139 ++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 20 deletions(-) diff --git a/scripts/ci-bun-crash-census.ts b/scripts/ci-bun-crash-census.ts index f04cce77c..bfc6d4657 100644 --- a/scripts/ci-bun-crash-census.ts +++ b/scripts/ci-bun-crash-census.ts @@ -21,7 +21,8 @@ * counts, closed tokens, versions and numeric run/job ids only, never log bodies. * * Bun version per job comes from the `Setup Bun` input in the job log when a log - * is read, and otherwise from the workflow file at the run's head commit. For a + * is read, and otherwise from the workflow file at the run's head commit, which + * is fetched once per commit and resolved per job name. For a * pull-request run the job actually used the merge commit's workflow, so a stale * pull-request head can attribute a job to the older pin; `versionSources` shows * how many jobs each source covers. @@ -172,6 +173,8 @@ function workflowJobNamed(opts: { * `jobs:` to the next one. Its display name is its `name:` template, or its * key when it has none, so `test shard 2/4` finds the `test-shard` matrix job. * A job with no `bun-version`, such as the `test` fan-in, returns null. + * Blank lines and column-0 `#` comments inside `jobs:` are skipped; any other + * column-0 line is the next top-level key and ends the jobs. */ export function workflowJobBunVersion( workflow: string, @@ -184,6 +187,9 @@ export function workflowJobBunVersion( } const blocks: { readonly key: string; readonly lines: string[] }[] = []; for (const line of lines.slice(jobsAt + 1)) { + if (line.trim() === "" || line.startsWith("#")) { + continue; + } if (/^\S/.test(line)) { break; } @@ -449,6 +455,34 @@ export async function runCensus( } } + // One contents GET per head sha: every job of every run at that sha resolves + // its Bun version from the same decoded workflow text. + const workflowTexts = new Map>(); + const workflowText = (sha: string): Promise => { + const cached = workflowTexts.get(sha); + if (cached) { + return cached; + } + const pending = gh + .json( + `repos/${repo}/contents/.github/workflows/${options.workflow}?ref=${sha}` + ) + .then((value) => + typeof value === "object" && + value !== null && + "content" in value && + typeof value.content === "string" + ? Buffer.from(value.content, "base64").toString("utf8") + : null + ) + .catch(() => null); + workflowTexts.set(sha, pending); + return pending; + }; + + // Keyed by (sha, job display name), never sha alone: jobs at one sha differ + // (the `test` fan-in has no Bun, each shard has the pin), so a sha-only key + // would hand the first job's version to every job a wildcard selector reads. const workflowVersions = new Map>(); const workflowVersion = ( sha: string | null, @@ -462,25 +496,9 @@ export async function runCensus( if (cached) { return cached; } - const pending = gh - .json( - `repos/${repo}/contents/.github/workflows/${options.workflow}?ref=${sha}` - ) - .then((value) => { - if ( - typeof value !== "object" || - value === null || - !("content" in value) || - typeof value.content !== "string" - ) { - return null; - } - return workflowJobBunVersion( - Buffer.from(value.content, "base64").toString("utf8"), - job - ); - }) - .catch(() => null); + const pending = workflowText(sha).then((text) => + text === null ? null : workflowJobBunVersion(text, job) + ); workflowVersions.set(cacheKey, pending); return pending; }; diff --git a/tests/ci-bun-crash-census.test.ts b/tests/ci-bun-crash-census.test.ts index acf751556..78ebca9a6 100644 --- a/tests/ci-bun-crash-census.test.ts +++ b/tests/ci-bun-crash-census.test.ts @@ -691,3 +691,142 @@ describe("sharded test suite (HACK-1424)", () => { ); }); }); + +describe("workflow reads (HACK-1441)", () => { + const WORKFLOW = [ + "name: CI", + "jobs:", + " lint:", + " steps:", + " - with:", + ' bun-version: "1.0.0"', + "", + "# The suite runs in four shards; `test` is their fan-in.", + " test-shard:", + " name: test shard ${{ matrix.shard }}/4", + " steps:", + " - with:", + ' bun-version: "1.4.2"', + "", + "# retired:", + "# bun-version: 0.9.0", + " test:", + " needs: test-shard", + " steps:", + " - run: exit 0", + "env:", + " late:", + " bun-version: 0.1.0", + ].join("\n"); + + test("blank and column-0 comment lines inside jobs: do not end the job scan", () => { + expect(workflowJobBunVersion(WORKFLOW, "test shard 2/4")).toBe("1.4.2"); + expect(workflowJobBunVersion(WORKFLOW, "lint")).toBe("1.0.0"); + expect(workflowJobBunVersion(WORKFLOW, "test")).toBeNull(); + // A commented-out job is not a job. + expect(workflowJobBunVersion(WORKFLOW, "retired")).toBeNull(); + // A column-0 key after `jobs:` still ends the jobs. + expect(workflowJobBunVersion(WORKFLOW, "late")).toBeNull(); + }); + + const SHA_SHARED = "d".repeat(40); + const SHA_OTHER = "e".repeat(40); + + /** Two runs at one sha and one at another; every job passed, so no log is read. */ + function countingGh() { + const contentReads: string[] = []; + const job = (id: number, name: string, attempt = 1): string => + JSON.stringify({ + id, + name, + run_attempt: attempt, + status: "completed", + conclusion: "success", + }); + const reader: GhReader = { + json: async (path) => { + if (path.includes("/runs?created=")) { + return { total_count: 3 }; + } + if (path.includes("/contents/.github/workflows/ci.yml?ref=")) { + contentReads.push(path); + return { content: Buffer.from(WORKFLOW).toString("base64") }; + } + throw new Error("unexpected path"); + }, + lines: async (path) => { + if (path.includes("/runs?created=")) { + return [ + JSON.stringify({ id: 500, head_sha: SHA_SHARED }), + JSON.stringify({ id: 600, head_sha: SHA_SHARED }), + JSON.stringify({ id: 700, head_sha: SHA_OTHER }), + ]; + } + // The fan-in is listed first, so the first version resolved at a sha is its null. + if (path.includes("/runs/500/jobs?filter=all")) { + return [ + job(50, "test"), + job(51, "test shard 1/4"), + job(52, "test shard 2/4"), + job(53, "test shard 3/4"), + job(54, "test shard 4/4"), + ]; + } + if (path.includes("/runs/600/jobs?filter=all")) { + return [job(61, "test shard 1/4"), job(62, "test shard 1/4", 2)]; + } + // Here a shard is first, so a sha-only key would give the fan-in the pin. + if (path.includes("/runs/700/jobs?filter=all")) { + return [job(71, "test shard 4/4"), job(70, "test")]; + } + throw new Error("unexpected path"); + }, + download: () => Promise.reject(new Error("no log should be read")), + }; + return { reader, contentReads }; + } + + test("each job at a sha gets its own workflow version under a wildcard selector", async () => { + const gh = countingGh(); + const result = await runCensus( + { ...options, job: "test*" }, + gh.reader, + await workDir() + ); + expect( + result.records.map((entry) => [ + entry.jobId, + entry.bunVersion, + entry.versionSource, + ]) + ).toEqual([ + [50, "unknown", "unknown"], + [51, "1.4.2", "workflow"], + [52, "1.4.2", "workflow"], + [53, "1.4.2", "workflow"], + [54, "1.4.2", "workflow"], + [61, "1.4.2", "workflow"], + [62, "1.4.2", "workflow"], + [71, "1.4.2", "workflow"], + [70, "unknown", "unknown"], + ]); + expect(result.warnings).toEqual([ + { kind: "no_bun_setup", jobs: 2, selected: 9 }, + ]); + }); + + test("one sha costs one workflow fetch however many jobs and runs resolve from it", async () => { + for (const concurrency of [1, 3]) { + const gh = countingGh(); + await runCensus( + { ...options, job: "test*", concurrency }, + gh.reader, + await workDir() + ); + expect(gh.contentReads.toSorted()).toEqual([ + `repos/hack-dance/hack/contents/.github/workflows/ci.yml?ref=${SHA_SHARED}`, + `repos/hack-dance/hack/contents/.github/workflows/ci.yml?ref=${SHA_OTHER}`, + ]); + } + }); +});