From 918e22b470d3363f1ff9eb8fd02453d895befb4b Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 09:43:44 +0000 Subject: [PATCH 1/9] fork: fast CI on GitHub-hosted runners (fork-tests.yml) Upstream's workflows are disabled in the fork, so fork pull requests ran no checks. fork-tests.yml now runs upstream ci.yml's checks on pull_request and push to main: lint (knip, generated GraphQL, vp check), build + typecheck + desktop launch + built web boot, package tests, web tests, 8 server shards, Rust, macOS mobile lint and release smoke. It keeps the workflow_call / workflow_dispatch `ref` interface, so a release run tests one commit with every Linux job and no path filter. Upstream's composite actions and change detector are used as they are. A new `drift` job fails, naming each job and step, when upstream's ci.yml or release.yml test jobs change and the copy did not follow. The Check job lists every job and the reason for every skip. Co-Authored-By: Claude Opus 5.5 --- .github/fork-tests-upstream.json | 452 ++++++++++++++ .github/scripts/fork-tests-drift.cjs | 176 ++++++ .github/scripts/fork-tests-drift.test.cjs | 68 +++ .github/workflows/fork-tests.yml | 707 ++++++++++++++++++++++ 4 files changed, 1403 insertions(+) create mode 100644 .github/fork-tests-upstream.json create mode 100644 .github/scripts/fork-tests-drift.cjs create mode 100644 .github/scripts/fork-tests-drift.test.cjs create mode 100644 .github/workflows/fork-tests.yml diff --git a/.github/fork-tests-upstream.json b/.github/fork-tests-upstream.json new file mode 100644 index 0000000000000..d91b62e23078f --- /dev/null +++ b/.github/fork-tests-upstream.json @@ -0,0 +1,452 @@ +{ + ".github/workflows/ci.yml": { + "changes": { + "header": "8ff211d87838fee8", + "steps": [ + { + "name": "Checkout", + "digest": "6e4f4d400d7166d3" + }, + { + "name": "Detect affected jobs", + "digest": "34a172066c8bd43c" + } + ] + }, + "lint": { + "header": "fb1d2080e81f407b", + "steps": [ + { + "name": "Checkout", + "digest": "b209d3ae4e2481dd" + }, + { + "name": "Reject repository-owned PR assets", + "digest": "a3b3fe56a20d10bc" + }, + { + "name": "Setup Vite+", + "digest": "26d8e10bd2d55595" + }, + { + "name": "Cache Electron download", + "digest": "353e0a76c1955ee0" + }, + { + "name": "Ensure Electron runtime is installed", + "digest": "b49682a395d1db97" + }, + { + "name": "Check unused code", + "digest": "1968f76ced131f47" + }, + { + "name": "Check generated GitHub GraphQL operations", + "digest": "7b0bc86822fffb65" + }, + { + "name": "Check generated Expo GraphQL operations", + "digest": "dfb39b82dec44c75" + }, + { + "name": "Check", + "digest": "b9e0beb00c16198e" + } + ] + }, + "build": { + "header": "7f62c17d569bc89a", + "steps": [ + { + "name": "Checkout", + "digest": "b209d3ae4e2481dd" + }, + { + "name": "./.github/actions/setup-apt-mirrors", + "digest": "0aeff75c5e98e418" + }, + { + "name": "Start installing browser secret helper build libraries", + "digest": "290064f86f52f6b7" + }, + { + "name": "Setup Vite+", + "digest": "26d8e10bd2d55595" + }, + { + "name": "Cache Electron download", + "digest": "353e0a76c1955ee0" + }, + { + "name": "Ensure Electron runtime is installed", + "digest": "154f683f77774ac2" + }, + { + "name": "Start typecheck", + "digest": "7e2b27816c22c788" + }, + { + "name": "Finish installing browser secret helper build libraries", + "digest": "f4047c9d5eb47c6f" + }, + { + "name": "Build desktop pipeline", + "digest": "aee0689d0bf38dc0" + }, + { + "name": "Verify preload bundle output", + "digest": "d70f3343194f687c" + }, + { + "name": "Open the seeded thread in the built desktop app", + "digest": "ef43acf89154cc0d" + }, + { + "name": "Upload desktop launch check", + "digest": "898b201dd5ae5f71" + }, + { + "name": "Resolve Playwright version", + "digest": "21bc211f7756f8f9" + }, + { + "name": "Cache Chromium", + "digest": "404b1c6c70f7eab0" + }, + { + "name": "Install Chromium for the built web client boot check", + "digest": "6bbf4f1dd3173918" + }, + { + "name": "Install missing Chromium system dependencies", + "digest": "1f3b26a2eb8d4245" + }, + { + "name": "Boot the built web client", + "digest": "2d647c22798cb29c" + }, + { + "name": "Typecheck", + "digest": "97be7ff701d1b8dd" + } + ] + }, + "test": { + "header": "0cae18bdc1ae5bb3", + "steps": [ + { + "name": "Checkout", + "digest": "b209d3ae4e2481dd" + }, + { + "name": "./.github/actions/setup-apt-mirrors", + "digest": "14f039b4273b5477" + }, + { + "name": "Start installing browser secret helper build libraries", + "digest": "3b68a19e5e1cf008" + }, + { + "name": "Setup Vite+", + "digest": "26d8e10bd2d55595" + }, + { + "name": "Cache Electron download", + "digest": "353e0a76c1955ee0" + }, + { + "name": "Ensure Electron runtime is installed", + "digest": "b49682a395d1db97" + }, + { + "name": "Test preview artifact validation", + "digest": "8ae0b86bebe56975" + }, + { + "name": "Test release checks", + "digest": "71e6a78fc8671f6f" + }, + { + "name": "Finish installing browser secret helper build libraries", + "digest": "939804bc6530a921" + }, + { + "name": "Test", + "digest": "801fc4dfe59fb6cd" + } + ] + }, + "test_web": { + "header": "0f5921c085423d52", + "steps": [ + { + "name": "Checkout", + "digest": "b209d3ae4e2481dd" + }, + { + "name": "Setup Vite+", + "digest": "26d8e10bd2d55595" + }, + { + "name": "Resolve Playwright version", + "digest": "287797def955482a" + }, + { + "name": "Cache Chromium", + "digest": "a26f01a6a03f3797" + }, + { + "name": "Install Chromium for web browser tests", + "digest": "6148d0bd5a6f738e" + }, + { + "name": "Install missing Chromium system dependencies", + "digest": "c2a97e6621813cb4" + }, + { + "name": "Test", + "digest": "b10e14843b443739" + } + ] + }, + "test_server": { + "header": "88f6bbaecaaa2d68", + "steps": [ + { + "name": "Checkout", + "digest": "b300e7178ac6e5a0" + }, + { + "name": "Detect affected jobs", + "digest": "5648db2117818e11" + }, + { + "name": "Setup Vite+", + "digest": "f939b7b648c8903f" + }, + { + "name": "Resolve Playwright version", + "digest": "5181e0fc26afb953" + }, + { + "name": "Cache Chromium", + "digest": "c11a7e038f53f2e0" + }, + { + "name": "Install Chromium for server browser integration tests", + "digest": "925ae79b31188f3a" + }, + { + "name": "Install missing Chromium system dependencies", + "digest": "862e4a563cb8a6b1" + }, + { + "name": "Test", + "digest": "1b94880171b914ed" + }, + { + "name": "Upload test cache measurements", + "digest": "10c0ad9a0337747b" + }, + { + "name": "Detect transfer budget report", + "digest": "6663d3329e713273" + }, + { + "name": "Publish transfer budget report", + "digest": "9e8eac20b2e9877f" + }, + { + "name": "Upload thread transfer result", + "digest": "f063a55279351e0c" + } + ] + }, + "rust": { + "header": "30bedddbe04fd44e", + "steps": [ + { + "name": "Checkout", + "digest": "b209d3ae4e2481dd" + }, + { + "name": "Setup Rust", + "digest": "7e4bade2fcac6c02" + }, + { + "name": "Check Rust formatting", + "digest": "e945fdc4fdd62992" + }, + { + "name": "Test Rust crates", + "digest": "0ecfc49333a39740" + } + ] + }, + "mobile_native_static_analysis": { + "header": "8b1472b3fce2c1bf", + "steps": [ + { + "name": "Checkout", + "digest": "3ac69f008422eb40" + }, + { + "name": "Setup Vite+", + "digest": "6e158a8bbd17c378" + }, + { + "name": "Install mobile native static analysis tools", + "digest": "2ce921f9eed13925" + }, + { + "name": "Lint mobile native sources", + "digest": "828af132b4613197" + } + ] + }, + "release_smoke": { + "header": "d8e29a4630f1113b", + "steps": [ + { + "name": "Checkout", + "digest": "b209d3ae4e2481dd" + }, + { + "name": "Setup Vite+", + "digest": "26d8e10bd2d55595" + }, + { + "name": "Exercise release-only workflow steps", + "digest": "40dae22ed3a65fd4" + } + ] + }, + "check": { + "header": "68e1588ae1870178", + "steps": [ + { + "name": "Require every job to pass", + "digest": "d7af266a25bf845d" + }, + { + "name": "Require transfer measurements", + "digest": "67ca47d5bd5ffc3d" + } + ] + } + }, + ".github/workflows/release.yml": { + "quality": { + "header": "64aee6ef731d6767", + "steps": [ + { + "name": "Checkout", + "digest": "6c2d53c2bffa53dd" + }, + { + "name": "Setup dependencies", + "digest": "7956327a0eceab8a" + }, + { + "name": "Ensure Electron runtime is installed", + "digest": "b49682a395d1db97" + }, + { + "name": "Check", + "digest": "b9e0beb00c16198e" + }, + { + "name": "Typecheck", + "digest": "a6830e957ddbd801" + } + ] + }, + "test": { + "header": "d304a3329d6f527f", + "steps": [ + { + "name": "Checkout", + "digest": "6c2d53c2bffa53dd" + }, + { + "name": "Setup dependencies", + "digest": "7956327a0eceab8a" + }, + { + "name": "Ensure Electron runtime is installed", + "digest": "b49682a395d1db97" + }, + { + "name": "./.github/actions/setup-apt-mirrors", + "digest": "14f039b4273b5477" + }, + { + "name": "Install browser secret helper build libraries", + "digest": "9ef6c7220ef74e0a" + }, + { + "name": "Test", + "digest": "af6646f21efe6013" + } + ] + }, + "test_web": { + "header": "fb7844322e849dba", + "steps": [ + { + "name": "Checkout", + "digest": "6c2d53c2bffa53dd" + }, + { + "name": "Setup dependencies", + "digest": "7956327a0eceab8a" + }, + { + "name": "Resolve Playwright version", + "digest": "287797def955482a" + }, + { + "name": "Cache Chromium", + "digest": "a26f01a6a03f3797" + }, + { + "name": "Install Chromium for web browser tests", + "digest": "6148d0bd5a6f738e" + }, + { + "name": "Install missing Chromium system dependencies", + "digest": "c2a97e6621813cb4" + }, + { + "name": "Test", + "digest": "b10e14843b443739" + }, + { + "name": "Boot the built web client", + "digest": "354fd8deb3f3b123" + } + ] + }, + "test_server": { + "header": "dcb7af07046ed9c0", + "steps": [ + { + "name": "Checkout", + "digest": "6c2d53c2bffa53dd" + }, + { + "name": "Setup dependencies", + "digest": "7956327a0eceab8a" + }, + { + "name": "Install Chromium for server browser integration tests", + "digest": "69d36de687d7ae89" + }, + { + "name": "Test", + "digest": "c5d091ce655d8187" + } + ] + } + } +} diff --git a/.github/scripts/fork-tests-drift.cjs b/.github/scripts/fork-tests-drift.cjs new file mode 100644 index 0000000000000..d67c383a0df90 --- /dev/null +++ b/.github/scripts/fork-tests-drift.cjs @@ -0,0 +1,176 @@ +// Fork only. .github/workflows/fork-tests.yml copies upstream's test jobs onto +// GitHub-hosted runners. This check fails when upstream's jobs change and the +// copy was not updated: it digests every job and step of the tracked upstream +// workflows and compares them with .github/fork-tests-upstream.json. +// +// node .github/scripts/fork-tests-drift.cjs check (CI) +// node .github/scripts/fork-tests-drift.cjs --record after updating fork-tests.yml +// +// Comment-only and blank-line changes are ignored. No dependencies: the +// workflows are parsed by indentation, which is all a job/step list needs. +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); + +const root = path.resolve(__dirname, "../.."); +const recordPath = path.join(root, ".github/fork-tests-upstream.json"); + +// Upstream file -> jobs the fork copies (null = every job in the file). +const TRACKED = { + ".github/workflows/ci.yml": null, + ".github/workflows/release.yml": ["quality", "test", "test_web", "test_server"], +}; + +const digest = (lines) => + crypto.createHash("sha256").update(lines.join("\n")).digest("hex").slice(0, 16); + +const indentOf = (line) => line.length - line.trimStart().length; + +function parseJobs(text) { + // Drop blank lines, full-line comments and trailing spaces. + const lines = text + .split("\n") + .map((line) => line.replace(/\s+$/, "")) + .filter((line) => line.trim() !== "" && !line.trim().startsWith("#")); + const start = lines.findIndex((line) => line === "jobs:"); + if (start === -1) throw new Error("no top-level `jobs:` key"); + + const jobs = {}; + let job = null; + let inSteps = false; + let stepsIndent = 0; + let step = null; + for (const line of lines.slice(start + 1)) { + if (indentOf(line) === 0) break; + const jobKey = line.match(/^ {2}([A-Za-z0-9_-]+):$/); + if (jobKey) { + job = jobs[jobKey[1]] = { header: [], steps: [] }; + inSteps = false; + step = null; + continue; + } + if (!job) continue; + if (inSteps && indentOf(line) <= stepsIndent && !line.trimStart().startsWith("- ")) { + inSteps = false; + step = null; + } + if (!inSteps && /^\s+steps:$/.test(line)) { + inSteps = true; + stepsIndent = indentOf(line); + continue; + } + if (!inSteps) { + job.header.push(line); + continue; + } + if (indentOf(line) <= stepsIndent + 2 && line.trimStart().startsWith("- ")) { + step = { lines: [] }; + job.steps.push(step); + } + step?.lines.push(line); + } + + const result = {}; + for (const [name, { header, steps }] of Object.entries(jobs)) { + const seen = new Map(); + result[name] = { + header: digest(header), + steps: steps.map((s) => { + const named = s.lines + .map((line) => line.trim().replace(/^- /, "")) + .map((line) => line.match(/^(?:name|uses):\s*(.+)$/)) + .find(Boolean); + let label = named ? named[1].replace(/^["']|["']$/g, "") : "(unnamed step)"; + // Repeated labels get a counter so each step keeps one identity. + const count = (seen.get(label) ?? 0) + 1; + seen.set(label, count); + if (count > 1) label = `${label} #${count}`; + return { name: label, digest: digest(s.lines) }; + }), + }; + } + return result; +} + +function current() { + const out = {}; + for (const [file, only] of Object.entries(TRACKED)) { + const jobs = parseJobs(fs.readFileSync(path.join(root, file), "utf8")); + if (only) { + const missing = only.filter((name) => !jobs[name]); + out[file] = Object.fromEntries(only.map((name) => [name, jobs[name] ?? null])); + if (missing.length) console.error(`${file}: tracked jobs missing: ${missing.join(", ")}`); + } else { + out[file] = jobs; + } + } + return out; +} + +function compare(recorded, now) { + const changes = []; + for (const file of new Set([...Object.keys(recorded), ...Object.keys(now)])) { + const before = recorded[file] ?? {}; + const after = now[file] ?? {}; + for (const job of new Set([...Object.keys(before), ...Object.keys(after)])) { + const a = before[job]; + const b = after[job]; + if (!a && b) { + changes.push(`${file}: job \`${job}\` added upstream`); + continue; + } + if (a && !b) { + changes.push(`${file}: job \`${job}\` removed upstream`); + continue; + } + if (a.header !== b.header) + changes.push(`${file}: job \`${job}\`: settings changed (runs-on, if, needs, matrix, env...)`); + const oldSteps = new Map(a.steps.map((s) => [s.name, s.digest])); + const newSteps = new Map(b.steps.map((s) => [s.name, s.digest])); + for (const [name, d] of newSteps) { + if (!oldSteps.has(name)) changes.push(`${file}: job \`${job}\`: step "${name}" added`); + else if (oldSteps.get(name) !== d) + changes.push(`${file}: job \`${job}\`: step "${name}" changed`); + } + for (const name of oldSteps.keys()) + if (!newSteps.has(name)) changes.push(`${file}: job \`${job}\`: step "${name}" removed`); + const oldOrder = a.steps.map((s) => s.name).filter((n) => newSteps.has(n)); + const newOrder = b.steps.map((s) => s.name).filter((n) => oldSteps.has(n)); + if (oldOrder.join("\0") !== newOrder.join("\0")) + changes.push(`${file}: job \`${job}\`: steps reordered`); + } + } + return changes; +} + +function main() { + const now = current(); + if (process.argv.includes("--record")) { + fs.writeFileSync(recordPath, `${JSON.stringify(now, null, 2)}\n`); + console.log(`Recorded ${path.relative(root, recordPath)}`); + return; + } + const recorded = JSON.parse(fs.readFileSync(recordPath, "utf8")); + const changes = compare(recorded, now); + const summary = process.env.GITHUB_STEP_SUMMARY; + if (changes.length === 0) { + const msg = "Upstream test jobs match the copy in fork-tests.yml (recorded digest)."; + console.log(msg); + if (summary) fs.appendFileSync(summary, `### Upstream drift\n\n${msg}\n`); + return; + } + const report = + "### Upstream drift\n\n" + + "Upstream's test jobs changed since `.github/workflows/fork-tests.yml` last followed them:\n\n" + + changes.map((c) => `- ${c}`).join("\n") + + "\n\nPort the change into fork-tests.yml (or decide it does not apply and say why in the PR), then run " + + "`node .github/scripts/fork-tests-drift.cjs --record` and commit `.github/fork-tests-upstream.json`.\n"; + console.error(report); + if (summary) fs.appendFileSync(summary, report); + for (const c of changes) console.log(`::error title=Upstream drift::${c}`); + process.exitCode = 1; +} + +module.exports = { parseJobs, compare }; + +if (require.main === module) main(); diff --git a/.github/scripts/fork-tests-drift.test.cjs b/.github/scripts/fork-tests-drift.test.cjs new file mode 100644 index 0000000000000..401ab9fd396c1 --- /dev/null +++ b/.github/scripts/fork-tests-drift.test.cjs @@ -0,0 +1,68 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); + +const { parseJobs, compare } = require("./fork-tests-drift.cjs"); + +const base = `name: CI +on: push +jobs: + lint: + runs-on: ubuntu-latest + steps: + # comment + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: | + /* + - uses: ./.github/actions/setup + - name: Check + run: | + vp check + test: + needs: [lint] + strategy: + matrix: + shard: [1, 2] + steps: + - name: Test + run: vp test +`; + +const wrap = (text) => ({ "ci.yml": parseJobs(text) }); + +test("parses jobs and steps, ignoring comments and blank lines", () => { + const jobs = parseJobs(base); + assert.deepEqual(Object.keys(jobs), ["lint", "test"]); + assert.deepEqual( + jobs.lint.steps.map((s) => s.name), + ["Checkout", "./.github/actions/setup", "Check"], + ); + const noisy = base.replace(" - name: Check", "\n # new comment\n - name: Check"); + assert.deepEqual(compare(wrap(base), wrap(noisy)), []); +}); + +test("reports added, removed, changed and reordered steps and jobs", () => { + const changed = base + .replace("vp check", "vp check --fix") + .replace(" - uses: ./.github/actions/setup\n", "") + .replace("shard: [1, 2]", "shard: [1, 2, 3]") + .concat(" rust:\n steps:\n - name: Cargo\n run: cargo test\n"); + assert.deepEqual(compare(wrap(base), wrap(changed)), [ + 'ci.yml: job `lint`: step "Check" changed', + 'ci.yml: job `lint`: step "./.github/actions/setup" removed', + "ci.yml: job `test`: settings changed (runs-on, if, needs, matrix, env...)", + "ci.yml: job `rust` added upstream", + ]); + const reordered = base.replace( + " - uses: ./.github/actions/setup\n - name: Check\n run: |\n vp check\n", + " - name: Check\n run: |\n vp check\n - uses: ./.github/actions/setup\n", + ); + assert.deepEqual(compare(wrap(base), wrap(reordered)), ["ci.yml: job `lint`: steps reordered"]); +}); + +test("repeated step names stay distinct", () => { + const twice = base.replace(" test:", " - name: Check\n run: echo again\n test:"); + const names = parseJobs(twice).lint.steps.map((s) => s.name); + assert.deepEqual(names.slice(-2), ["Check", "Check #2"]); +}); diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml new file mode 100644 index 0000000000000..a613f01bd4838 --- /dev/null +++ b/.github/workflows/fork-tests.yml @@ -0,0 +1,707 @@ +name: Fork tests + +# Fork only. Upstream's CI checks (ci.yml) on GitHub-hosted runners, because the +# fork has none of upstream's runners and upstream's workflows are disabled here. +# Commands follow ci.yml step for step; only the runners, time limits, apt setup +# and the private task cache differ. Upstream's composite actions and change +# detector (.github/actions, .github/scripts/ci-changes.cjs) are used as they +# are, so a rebase brings their changes. The `drift` job fails when upstream's +# ci.yml or release.yml test jobs change and this copy did not follow +# (.github/scripts/fork-tests-drift.cjs). +# +# Two modes: +# - pull_request / push to main: CI. Jobs a change cannot affect are skipped by +# upstream's detector; every skip is named with its reason in the run summary. +# - workflow_call / workflow_dispatch with `ref`: release. Every Linux job runs +# on that commit, no path filter. Covers release.yml's quality, test, test_web +# and test_server (ci.yml checks a superset of them). +on: + pull_request: + push: + branches: + - main + workflow_call: + inputs: + ref: + description: Commit to test + required: true + type: string + workflow_dispatch: + inputs: + ref: + description: Commit or branch to test + required: true + type: string + +permissions: + contents: read + +# A new push to a pull request cancels its older run. Main and release runs finish. +# The event name keeps a release's workflow_call apart from its caller's group. +concurrency: + group: fork-tests-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + changes: + name: Changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + typecheck: ${{ steps.release.outputs.typecheck || steps.detect.outputs.typecheck }} + build: ${{ steps.release.outputs.build || steps.detect.outputs.build }} + desktop_launch: ${{ steps.release.outputs.desktop_launch || steps.detect.outputs.desktop_launch }} + test: ${{ steps.release.outputs.test || steps.detect.outputs.test }} + test_packages: ${{ steps.release.outputs.test_packages || steps.detect.outputs.test_packages }} + test_web: ${{ steps.release.outputs.test_web || steps.detect.outputs.test_web }} + test_server: ${{ steps.release.outputs.test_server || steps.detect.outputs.test_server }} + rust: ${{ steps.release.outputs.rust || steps.detect.outputs.rust }} + release_smoke: ${{ steps.release.outputs.release_smoke || steps.detect.outputs.release_smoke }} + mobile_native: ${{ steps.release.outputs.mobile_native || steps.fork.outputs.mobile_native }} + steps: + - name: Checkout + if: inputs.ref == '' + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 2 + sparse-checkout: | + /pnpm-workspace.yaml + /.github/scripts/ci-changes.cjs + /.github/actions/ci-changes/ + /*/package.json + /*/*/package.json + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Detect affected jobs + id: detect + if: inputs.ref == '' + uses: ./.github/actions/ci-changes + + # ci-changes.cjs runs the macOS lint when upstream's ci.yml changes; the + # fork's copy of it counts the same. + - name: Fork workflow changes run the macOS lint + id: fork + if: inputs.ref == '' + env: + DETECTED: ${{ steps.detect.outputs.mobile_native }} + run: | + value="$DETECTED" + if grep -qxE '\.github/workflows/fork-tests\.yml' "$RUNNER_TEMP/changed-paths" 2>/dev/null; then + value=true + fi + echo "mobile_native=$value" >> "$GITHUB_OUTPUT" + + # Release runs test the whole commit. upstream's release.yml has no macOS lint. + - name: Release run checks everything + id: release + if: inputs.ref != '' + run: | + for gate in typecheck build desktop_launch test test_web test_server rust release_smoke; do + echo "$gate=true" >> "$GITHUB_OUTPUT" + done + echo "test_packages=" >> "$GITHUB_OUTPUT" + echo "mobile_native=false" >> "$GITHUB_OUTPUT" + + # The fork copy must follow upstream's test jobs. Fails with the list of + # jobs and steps upstream changed since the copy was last recorded. + drift: + name: Upstream drift + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /.github/ + sparse-checkout-cone-mode: false + + - name: Test the drift check + run: node --test .github/scripts/fork-tests-drift.test.cjs + + - name: Compare upstream's test jobs with the recorded copy + run: node .github/scripts/fork-tests-drift.cjs + + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /* + !/.repos/ + !/apps/mobile/modules/t3-terminal/Vendor/**/*.a + sparse-checkout-cone-mode: false + + - name: Reject repository-owned PR assets + run: | + files="$(git ls-files .github/pr-assets)" + if test -n "$files"; then + printf 'PR evidence must be uploaded to GitHub, not committed:\n%s\n' "$files" >&2 + exit 1 + fi + + - name: Setup Vite+ + uses: ./.github/actions/setup-ci-deps + + - name: Cache Electron download + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/electron + key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + + - name: Check unused code + run: vp run knip:check + + - name: Check generated GitHub GraphQL operations + run: vp run --filter @t3tools/source-control-github generate --check + + - name: Check generated Expo GraphQL operations + run: vp run --filter t3 generate --check + + - name: Check + run: vp check + + build: + name: Build and Typecheck + needs: changes + if: ${{ !cancelled() && (needs.changes.outputs.build != 'false' || needs.changes.outputs.typecheck != 'false') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /* + !/.repos/ + !/apps/mobile/modules/t3-terminal/Vendor/**/*.a + sparse-checkout-cone-mode: false + + # Runs in the background while Vite+ installs; the step before the build waits for it. + # Xvfb is for the desktop launch check below. + - name: Start installing browser secret helper build libraries + if: needs.changes.outputs.build != 'false' + run: | + ( + set +e + sudo apt-get update -o Acquire::Languages=none \ + -o Acquire::IndexTargets::deb::DEP-11::DefaultEnabled=false \ + -o Acquire::IndexTargets::deb::CNF::DefaultEnabled=false && + sudo apt-get install -y libsecret-1-dev pkg-config xvfb + echo "$?" > "$RUNNER_TEMP/apt-status" + ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null & + + - name: Setup Vite+ + uses: ./.github/actions/setup-ci-deps + with: + # Lint saves the pnpm store; the others only restore it. + save-pnpm-store: false + + - name: Cache Electron download + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/electron + key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} + + - name: Ensure Electron runtime is installed + id: electron + run: vp run --filter @t3tools/desktop ensure:electron + + # Runs in the background while the build runs, and reports in the last + # step: neither result hides the other. + - name: Start typecheck + id: typecheck + if: ${{ !cancelled() && steps.electron.outcome == 'success' && needs.changes.outputs.typecheck != 'false' }} + run: | + ( + set +e + vpr typecheck + echo "$?" > "$RUNNER_TEMP/typecheck-status" + ) > "$RUNNER_TEMP/typecheck.log" 2>&1 < /dev/null & + + - name: Finish installing browser secret helper build libraries + id: apt + if: ${{ !cancelled() && needs.changes.outputs.build != 'false' }} + timeout-minutes: 5 + run: | + until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done + cat "$RUNNER_TEMP/apt.log" + exit "$(cat "$RUNNER_TEMP/apt-status")" + + - name: Build desktop pipeline + id: build + if: ${{ !cancelled() && steps.electron.outcome == 'success' && steps.apt.outcome == 'success' && needs.changes.outputs.build != 'false' }} + run: vp run build:desktop + + - name: Verify preload bundle output + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: node apps/desktop/scripts/verify-preload-bundle.mjs + + - name: Open the seeded thread in the built desktop app + id: desktop_launch + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.changes.outputs.desktop_launch != 'false' }} + timeout-minutes: 4 + run: | + electron=$(node -p "require('./apps/desktop/node_modules/electron')") + xvfb-run -a node scripts/smoke-desktop-app.ts --executable "$electron" --app apps/desktop \ + --artifacts-dir "$RUNNER_TEMP/desktop-launch-check" + + - name: Upload desktop launch check + if: ${{ failure() && steps.desktop_launch.outcome == 'failure' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: desktop-launch-check + path: ${{ runner.temp }}/desktop-launch-check + if-no-files-found: ignore + + - name: Resolve Playwright version + id: playwright_version + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.changes.outputs.test_web != 'false' }} + run: echo "version=$(node -p "require('./apps/web/node_modules/playwright/package.json').version")" >> "$GITHUB_OUTPUT" + + - name: Cache Chromium + id: chromium_cache + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.changes.outputs.test_web != 'false' }} + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} + + - name: Install Chromium for the built web client boot check + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.changes.outputs.test_web != 'false' && steps.chromium_cache.outputs.cache-hit != 'true' }} + run: node apps/web/node_modules/playwright/cli.js install --only-shell chromium + + - name: Install missing Chromium system dependencies + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.changes.outputs.test_web != 'false' }} + run: | + browser=$(find "$HOME/.cache/ms-playwright" -name chrome-headless-shell -type f -print -quit) + test -n "$browser" + if ldd "$browser" | grep '=> not found'; then + node apps/web/node_modules/playwright/cli.js install-deps chromium + fi + + - name: Boot the built web client + if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.changes.outputs.test_web != 'false' }} + timeout-minutes: 3 + run: node apps/web/scripts/verify-built-boot.ts + + - name: Typecheck + if: ${{ !cancelled() && steps.typecheck.outcome == 'success' }} + timeout-minutes: 10 + run: | + until test -s "$RUNNER_TEMP/typecheck-status"; do sleep 1; done + cat "$RUNNER_TEMP/typecheck.log" + exit "$(cat "$RUNNER_TEMP/typecheck-status")" + + # Everything except `t3` (apps/server) and the web app, which have their own jobs. + test: + name: Test + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.test != 'false' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /* + !/.repos/ + !/apps/mobile/modules/t3-terminal/Vendor/**/*.a + sparse-checkout-cone-mode: false + + # The desktop tests compile the browser secret helper. The install runs in + # the background while Vite+ installs; the step before the tests waits for it. + - name: Start installing browser secret helper build libraries + run: | + ( + set +e + sudo apt-get update -o Acquire::Languages=none \ + -o Acquire::IndexTargets::deb::DEP-11::DefaultEnabled=false \ + -o Acquire::IndexTargets::deb::CNF::DefaultEnabled=false && + sudo apt-get install -y libsecret-1-dev pkg-config + echo "$?" > "$RUNNER_TEMP/apt-status" + ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null & + + - name: Setup Vite+ + uses: ./.github/actions/setup-ci-deps + with: + # Lint saves the pnpm store; the others only restore it. + save-pnpm-store: false + + - name: Cache Electron download + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/electron + key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + + - name: Test preview artifact validation + run: python3 -B .github/scripts/stage-preview-bundle.test.py + + - name: Test release checks + run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs + + - name: Finish installing browser secret helper build libraries + timeout-minutes: 5 + run: | + until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done + cat "$RUNNER_TEMP/apt.log" + exit "$(cat "$RUNNER_TEMP/apt-status")" + + # `changes` lists the affected packages, or nothing when all of them are. + - name: Test + env: + TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }} + run: | + set -o pipefail + filters=(--filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web') + if [[ -n "$TEST_PACKAGES" ]]; then + filters=() + for package in $TEST_PACKAGES; do filters+=(--filter "$package"); done + fi + # The runner's trace writer can crash on surrogate pairs in test output. + vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c ' + import sys + for line in sys.stdin: + sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii")) + ' + + test_web: + name: Test Web + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.test_web != 'false' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /* + !/.repos/ + !/apps/mobile/modules/t3-terminal/Vendor/**/*.a + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: ./.github/actions/setup-ci-deps + with: + # Lint saves the pnpm store; the others only restore it. + save-pnpm-store: false + + - name: Resolve Playwright version + id: playwright_version + run: echo "version=$(node -p "require('./apps/web/node_modules/playwright/package.json').version")" >> "$GITHUB_OUTPUT" + + - name: Cache Chromium + id: chromium_cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} + + - name: Install Chromium for web browser tests + if: steps.chromium_cache.outputs.cache-hit != 'true' + run: node apps/web/node_modules/playwright/cli.js install --only-shell chromium + + - name: Install missing Chromium system dependencies + run: | + browser=$(find "$HOME/.cache/ms-playwright" -name chrome-headless-shell -type f -print -quit) + test -n "$browser" + if ldd "$browser" | grep '=> not found'; then + node apps/web/node_modules/playwright/cli.js install-deps chromium + fi + + - name: Test + run: vp run --filter @t3tools/web test + + # apps/server runs its test files one at a time; shards spread them over + # separate runners. The shards gate themselves instead of waiting for + # `changes`; a shard the change cannot affect says so in its summary. + test_server: + name: Test Server ${{ matrix.shard }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + fetch-depth: 2 + sparse-checkout: | + /* + !/.repos/ + !/apps/mobile/modules/t3-terminal/Vendor/**/*.a + sparse-checkout-cone-mode: false + + - name: Detect affected jobs + id: changes + if: inputs.ref == '' + uses: ./.github/actions/ci-changes + + - name: Report skip + if: steps.changes.outputs.test_server == 'false' + run: | + msg="Skipped: upstream's change detector found no change that affects t3 (apps/server)." + echo "::notice title=Test Server ${{ matrix.shard }} skipped::$msg" + echo "$msg" >> "$GITHUB_STEP_SUMMARY" + + - name: Setup Vite+ + if: steps.changes.outputs.test_server != 'false' + uses: ./.github/actions/setup-ci-deps + with: + # Lint saves the pnpm store; the others only restore it. + save-pnpm-store: false + + - name: Resolve Playwright version + id: playwright_version + if: steps.changes.outputs.test_server != 'false' + run: echo "version=$(node -p "require('./apps/server/node_modules/playwright-core/package.json').version")" >> "$GITHUB_OUTPUT" + + - name: Cache Chromium + id: chromium_cache + if: steps.changes.outputs.test_server != 'false' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} + + - name: Install Chromium for server browser integration tests + if: steps.changes.outputs.test_server != 'false' && steps.chromium_cache.outputs.cache-hit != 'true' + run: node apps/server/node_modules/playwright-core/cli.js install --only-shell chromium + + - name: Install missing Chromium system dependencies + if: steps.changes.outputs.test_server != 'false' + run: | + browser=$(find "$HOME/.cache/ms-playwright" -name chrome-headless-shell -type f -print -quit) + test -n "$browser" + if ldd "$browser" | grep '=> not found'; then + node apps/server/node_modules/playwright-core/cli.js install-deps chromium + fi + + - name: Test + if: steps.changes.outputs.test_server != 'false' + env: + T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md + T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json + run: >- + vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }} + --uncached apps/server/integration/transferBudgetV2.integration.test.ts + --verify-hits 1 --report "${{ runner.temp }}/test-cache-${{ matrix.shard }}.json" apps/server + + - name: Upload test cache measurements + if: always() && steps.changes.outputs.test_server != 'false' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: test-cache-${{ matrix.shard }} + path: ${{ runner.temp }}/test-cache-${{ matrix.shard }}.json + if-no-files-found: error + + # integration/transferBudgetV2.integration.test.ts writes the report; exactly one shard + # produces these files. + - name: Detect transfer budget report + id: transfer_budget + if: always() + run: | + if test -f "${{ runner.temp }}/thread-transfer-result.json"; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + fi + + - name: Publish transfer budget report + if: always() && steps.transfer_budget.outputs.present == 'true' + run: | + if test -f "${{ runner.temp }}/t3code-transfer-budget.md"; then + tee -a "$GITHUB_STEP_SUMMARY" < "${{ runner.temp }}/t3code-transfer-budget.md" + else + echo "Transfer budget report was not produced." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload thread transfer result + if: always() && steps.transfer_budget.outputs.present == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: thread-transfer-results + path: ${{ runner.temp }}/thread-transfer-result.json + if-no-files-found: ignore + retention-days: 30 + + rust: + name: Rust + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.rust != 'false' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /* + !/.repos/ + !/apps/mobile/modules/t3-terminal/Vendor/**/*.a + sparse-checkout-cone-mode: false + + - name: Setup Rust + uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + components: rustfmt + + - name: Check Rust formatting + run: | + for crate in resource-monitor kde-snap-shot hyprland-snap-shot; do + cargo fmt --manifest-path "native/$crate/Cargo.toml" -- --check + done + + - name: Test Rust crates + run: | + for crate in resource-monitor kde-snap-shot hyprland-snap-shot; do + cargo test --locked --manifest-path "native/$crate/Cargo.toml" + done + + # Upstream runs this on macOS 26. GitHub-hosted macOS minutes are free on a + # public repository; it runs only for native mobile sources or a workflow change. + mobile_native_static_analysis: + name: Mobile Native Static Analysis + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.mobile_native != 'false' }} + runs-on: macos-26 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1 + with: + node-version-file: package.json + cache: true + run-install: | + args: + - --filter=@t3tools/scripts... + + - name: Install mobile native static analysis tools + run: brew bundle install --file apps/mobile/Brewfile + + - name: Lint mobile native sources + run: vp run lint:mobile + + release_smoke: + name: Release Smoke + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.release_smoke != 'false' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /* + !/.repos/ + !/apps/mobile/modules/t3-terminal/Vendor/**/*.a + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: ./.github/actions/setup-ci-deps + with: + # Lint saves the pnpm store; the others only restore it. + save-pnpm-store: false + + - name: Exercise release-only workflow steps + run: node scripts/release-smoke.ts + + # The one check to require. Fails when any job fails, is cancelled, or is + # skipped without `changes` saying it was not needed. The run summary lists + # every job, and every skip with its reason. + check: + name: Check + if: ${{ always() }} + needs: + [ + changes, + drift, + lint, + build, + test, + test_web, + test_server, + rust, + mobile_native_static_analysis, + release_smoke, + ] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require every job to pass + env: + RESULTS: ${{ toJSON(needs) }} + RELEASE_RUN: ${{ inputs.ref != '' }} + run: | + echo "$RESULTS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"' + { + echo "### Fork tests" + echo + echo "| Job | Result | Why skipped |" + echo "| --- | --- | --- |" + echo "$RESULTS" | jq -r --arg release "$RELEASE_RUN" '.changes.outputs as $g + | {build: "the change affects neither desktop, server nor web, and no package needs a typecheck", + test: "the change affects no package this job tests", + test_web: "the change affects nothing @t3tools/web depends on", + rust: "the change touches none of the native Rust crates", + mobile_native_static_analysis: (if $release == "true" + then "release run: upstream release.yml has no macOS lint" + else "no native mobile source, Brewfile or CI workflow changed" end), + release_smoke: "the change does not affect scripts/"} as $why + | to_entries[] + | "| \(.key) | \(.value.result) | \(if .value.result == "skipped" then ($why[.key] // "not explained") else "" end) |"' + if [[ "$(echo "$RESULTS" | jq -r '.changes.outputs.test_server')" == false ]]; then + echo + echo "Server shards passed without running tests: the change affects nothing t3 (apps/server) depends on." + fi + } >> "$GITHUB_STEP_SUMMARY" + # A job may skip only when `changes` said every gate it represents was + # not needed. Gates are keyed by job name, except merged Build and the + # macOS lint. + echo "$RESULTS" | jq -e '.changes.outputs as $gates + | {build: ["build", "typecheck", "desktop_launch"], mobile_native_static_analysis: ["mobile_native"]} as $renamed + | all(to_entries[]; + .value.result == "success" + or (.value.result == "skipped" + and (($renamed[.key] // [.key]) | all(.[]; $gates[.] == "false"))))' > /dev/null + + # A run whose server tests ran must have produced the transfer measurements. + - name: Require transfer measurements + if: needs.test_server.result == 'success' && needs.changes.outputs.test_server != 'false' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: thread-transfer-results + path: ${{ runner.temp }}/thread-transfer From 65243f1e075ec82187ab71fc69c039d862f814d8 Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 11:08:59 +0000 Subject: [PATCH 2/9] Fork CI: one fork-ci-result check, secret scan, actionlint, faster tests - fork-ci-result needs every job and fails on a skip without a reason; its summary lists each job, why it skipped, and job wall times. - Secret scan (gitleaks, checksum-pinned) of the commits a pull request or push adds; workflow lint (actionlint) next to the drift check. - Ubuntu 24.04 runners as upstream: ubuntu-latest (26.04) fails an apps/server test with its newer git. - Test is split across two runners by test file count; vp's task cache is kept between runs for the typecheck and the server shards. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-test-shards.cjs | 69 ++++++ .github/scripts/fork-test-shards.test.cjs | 34 +++ .github/scripts/fork-tests-drift.cjs | 4 +- .github/workflows/fork-tests.yml | 264 +++++++++++++++++----- 4 files changed, 319 insertions(+), 52 deletions(-) create mode 100644 .github/scripts/fork-test-shards.cjs create mode 100644 .github/scripts/fork-test-shards.test.cjs diff --git a/.github/scripts/fork-test-shards.cjs b/.github/scripts/fork-test-shards.cjs new file mode 100644 index 0000000000000..0719c050fb7d1 --- /dev/null +++ b/.github/scripts/fork-test-shards.cjs @@ -0,0 +1,69 @@ +// Fork only. Splits the Test job's packages across the shards of its matrix in +// .github/workflows/fork-tests.yml. Upstream runs the job on one large runner; +// a GitHub-hosted runner has 4 CPUs, so the packages go to several runners. +// +// node .github/scripts/fork-test-shards.cjs 1/2 prints shard 1's packages +// +// TEST_PACKAGES (from the `changes` job) limits the set; empty means every +// package the Test job covers. Packages are weighted by their test file count +// and placed, heaviest first, on the least loaded shard, so the split follows +// new packages and tests without a hand-kept list. +const childProcess = require("node:child_process"); +const fs = require("node:fs"); + +const { readWorkspace } = require("./ci-changes.cjs"); + +// Tested by their own jobs (Test Server, Test Web) or not a package to test. +const OWN_JOB = new Set(["t3", "@t3tools/web", "@t3tools/monorepo"]); +const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/; + +function assign(packages, count) { + const shards = Array.from({ length: count }, () => ({ weight: 0, packages: [] })); + const ordered = packages.toSorted((a, b) => b.weight - a.weight || a.name.localeCompare(b.name)); + for (const pkg of ordered) { + const lightest = shards.reduce((best, shard) => (shard.weight < best.weight ? shard : best)); + lightest.weight += pkg.weight; + lightest.packages.push(pkg); + } + return shards; +} + +function testFiles(dir) { + return childProcess + .execFileSync("git", ["ls-files", "-z", "--", dir], { encoding: "utf8", maxBuffer: 1 << 26 }) + .split("\0") + .filter((file) => TEST_FILE.test(file)).length; +} + +function main() { + const match = (process.argv[2] ?? "").match(/^(\d+)\/(\d+)$/); + if (!match || Number(match[1]) < 1 || Number(match[1]) > Number(match[2])) + throw new Error("usage: fork-test-shards.cjs /"); + const [index, count] = [Number(match[1]), Number(match[2])]; + const only = (process.env.TEST_PACKAGES ?? "").split(/\s+/).filter(Boolean); + const packages = readWorkspace() + .filter((pkg) => pkg.hasTest && !OWN_JOB.has(pkg.name)) + .filter((pkg) => only.length === 0 || only.includes(pkg.name)) + .map((pkg) => ({ name: pkg.name, weight: Math.max(1, testFiles(pkg.dir)) })); + const unknown = only.filter((name) => !packages.some((pkg) => pkg.name === name)); + if (unknown.length) + throw new Error(`TEST_PACKAGES names no tested package: ${unknown.join(" ")}`); + + const shard = assign(packages, count)[index - 1]; + for (const pkg of shard.packages) console.log(pkg.name); + const summary = process.env.GITHUB_STEP_SUMMARY; + if (summary) { + const list = shard.packages.map((pkg) => `\`${pkg.name}\` (${pkg.weight})`).join(", "); + fs.appendFileSync( + summary, + `### Test shard ${index}/${count}\n\n` + + (shard.packages.length + ? `${shard.packages.length} of ${packages.length} packages, ${shard.weight} test files: ${list}\n` + : `No packages: the change affects ${packages.length} tested package(s), all on other shards.\n`), + ); + } +} + +module.exports = { assign }; + +if (require.main === module) main(); diff --git a/.github/scripts/fork-test-shards.test.cjs b/.github/scripts/fork-test-shards.test.cjs new file mode 100644 index 0000000000000..e5ea56440d95a --- /dev/null +++ b/.github/scripts/fork-test-shards.test.cjs @@ -0,0 +1,34 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); + +const { assign } = require("./fork-test-shards.cjs"); + +const names = (shards) => shards.map((shard) => shard.packages.map((pkg) => pkg.name)); + +test("places the heaviest package first on the lightest shard", () => { + const packages = [ + { name: "small", weight: 1 }, + { name: "big", weight: 10 }, + { name: "mid", weight: 6 }, + { name: "mid2", weight: 5 }, + ]; + assert.deepEqual(names(assign(packages, 2)), [ + ["big", "small"], + ["mid", "mid2"], + ]); +}); + +test("ties break by name, so every shard computes the same split", () => { + const packages = [ + { name: "b", weight: 2 }, + { name: "a", weight: 2 }, + { name: "c", weight: 2 }, + ]; + assert.deepEqual(names(assign(packages, 2)), [["a", "c"], ["b"]]); + assert.deepEqual(names(assign(packages.toReversed(), 2)), [["a", "c"], ["b"]]); +}); + +test("every package lands on exactly one shard, and spare shards stay empty", () => { + const packages = [{ name: "only", weight: 3 }]; + assert.deepEqual(names(assign(packages, 3)), [["only"], [], []]); +}); diff --git a/.github/scripts/fork-tests-drift.cjs b/.github/scripts/fork-tests-drift.cjs index d67c383a0df90..31b1f07a85fec 100644 --- a/.github/scripts/fork-tests-drift.cjs +++ b/.github/scripts/fork-tests-drift.cjs @@ -124,7 +124,9 @@ function compare(recorded, now) { continue; } if (a.header !== b.header) - changes.push(`${file}: job \`${job}\`: settings changed (runs-on, if, needs, matrix, env...)`); + changes.push( + `${file}: job \`${job}\`: settings changed (runs-on, if, needs, matrix, env...)`, + ); const oldSteps = new Map(a.steps.map((s) => [s.name, s.digest])); const newSteps = new Map(b.steps.map((s) => [s.name, s.digest])); for (const [name, d] of newSteps) { diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index a613f01bd4838..cd9f09926e6d2 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -2,12 +2,22 @@ name: Fork tests # Fork only. Upstream's CI checks (ci.yml) on GitHub-hosted runners, because the # fork has none of upstream's runners and upstream's workflows are disabled here. -# Commands follow ci.yml step for step; only the runners, time limits, apt setup -# and the private task cache differ. Upstream's composite actions and change -# detector (.github/actions, .github/scripts/ci-changes.cjs) are used as they -# are, so a rebase brings their changes. The `drift` job fails when upstream's -# ci.yml or release.yml test jobs change and this copy did not follow -# (.github/scripts/fork-tests-drift.cjs). +# Commands follow ci.yml step for step; only the runners, time limits, apt +# setup, the task cache (kept with actions/cache instead of upstream's private +# remote cache) and the Test job's split across two runners differ. Upstream's +# composite actions and change detector (.github/actions, +# .github/scripts/ci-changes.cjs) are used as they are, so a rebase brings their +# changes. The `workflows` job lints the fork workflows (actionlint) and fails +# when upstream's ci.yml or release.yml test jobs change and this copy did not +# follow (.github/scripts/fork-tests-drift.cjs). +# +# Require only `fork-ci-result` in a ruleset: it needs every other job, so jobs +# can be added, split or resharded without touching the ruleset. +# +# Runners: GitHub-hosted only, Ubuntu 24.04 as upstream (ubuntu-latest is now +# 26.04, whose newer git fails an apps/server test upstream never sees). Never +# add a self-hosted runner to this workflow: this repository is public, so a +# pull request from any fork would run its code on that machine. # # Two modes: # - pull_request / push to main: CI. Jobs a change cannot affect are skipped by @@ -45,7 +55,7 @@ concurrency: jobs: changes: name: Changes - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: typecheck: ${{ steps.release.outputs.typecheck || steps.detect.outputs.typecheck }} @@ -103,11 +113,12 @@ jobs: echo "test_packages=" >> "$GITHUB_OUTPUT" echo "mobile_native=false" >> "$GITHUB_OUTPUT" - # The fork copy must follow upstream's test jobs. Fails with the list of - # jobs and steps upstream changed since the copy was last recorded. - drift: - name: Upstream drift - runs-on: ubuntu-latest + # A broken fork workflow is red on its own pull request, and the fork copy + # must follow upstream's test jobs: the drift check lists every job and step + # upstream changed since the copy was last recorded. + workflows: + name: Workflow lint and upstream drift + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Checkout @@ -118,15 +129,90 @@ jobs: /.github/ sparse-checkout-cone-mode: false - - name: Test the drift check - run: node --test .github/scripts/fork-tests-drift.test.cjs + # Pinned release, verified by checksum. Upstream's own workflows use runner + # labels this fork does not have, so only the fork's workflows are linted. + - name: Lint fork workflows (actionlint) + env: + VERSION: 1.7.12 + SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 + run: | + archive="actionlint_${VERSION}_linux_amd64.tar.gz" + curl -fsSL --retry 3 -o "$RUNNER_TEMP/$archive" \ + "https://github.com/rhysd/actionlint/releases/download/v${VERSION}/${archive}" + echo "$SHA256 $RUNNER_TEMP/$archive" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" actionlint + "$RUNNER_TEMP/actionlint" -color .github/workflows/fork-*.yml + + - name: Test the fork CI scripts + run: node --test .github/scripts/fork-tests-drift.test.cjs .github/scripts/fork-test-shards.test.cjs - name: Compare upstream's test jobs with the recorded copy run: node .github/scripts/fork-tests-drift.cjs + # Every commit a pull request or a push to main adds, history included: a + # secret committed and then deleted is still public. Release runs test a + # commit that already passed this on its way to main, so they skip it. + secrets: + name: Secret scan + if: ${{ inputs.ref == '' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + # A pull request fetches only its own commits (plus the parent of the + # oldest, below). A push fetches history without file contents; + # gitleaks then reads only the pushed commits. + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: ${{ github.event_name == 'pull_request' && github.event.pull_request.commits || 0 }} + filter: ${{ github.event_name != 'pull_request' && 'blob:none' || '' }} + + - name: Install gitleaks + env: + VERSION: 8.30.1 + SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb + run: | + archive="gitleaks_${VERSION}_linux_x64.tar.gz" + curl -fsSL --retry 3 -o "$RUNNER_TEMP/$archive" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${VERSION}/${archive}" + echo "$SHA256 $RUNNER_TEMP/$archive" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" gitleaks + + - name: Scan the added commits + env: + EVENT_NAME: ${{ github.event_name }} + COMMITS: ${{ github.event.pull_request.commits }} + BEFORE: ${{ github.event.before }} + run: | + if [[ "$EVENT_NAME" == pull_request ]]; then + # One more commit, so the oldest one is diffed against its parent + # instead of being read as a root that adds the whole tree. + git fetch --no-tags --quiet --deepen=1 origin "$(git rev-parse HEAD)" + range="--max-count=$COMMITS" + what="the pull request's $COMMITS commit(s)" + elif [[ -n "$BEFORE" && ! "$BEFORE" =~ ^0+$ ]] && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then + range="$BEFORE..$GITHUB_SHA" + what="the commits pushed after ${BEFORE:0:12}" + else + range="--max-count=1" + what="the pushed commit (no earlier commit to compare with)" + fi + echo "Scanning $what" + set +e + "$RUNNER_TEMP/gitleaks" git --no-banner --redact --verbose --log-opts="$range" . + status=$? + set -e + if [[ $status -eq 0 ]]; then + echo "### Secret scan"$'\n\n'"No secrets found in $what." >> "$GITHUB_STEP_SUMMARY" + else + echo "### Secret scan"$'\n\n'"gitleaks found possible secrets in $what (values redacted; see the job log). Rotate any real secret: it is already public." >> "$GITHUB_STEP_SUMMARY" + fi + exit $status + lint: name: Lint - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - name: Checkout @@ -175,7 +261,7 @@ jobs: name: Build and Typecheck needs: changes if: ${{ !cancelled() && (needs.changes.outputs.build != 'false' || needs.changes.outputs.typecheck != 'false') }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - name: Checkout @@ -218,6 +304,18 @@ jobs: id: electron run: vp run --filter @t3tools/desktop ensure:electron + # Upstream reads a private remote task cache (a secret the fork has not + # got). Here vp's local task cache is kept between runs instead: vp + # fingerprints each task's inputs, so a restored entry only replays a + # task whose inputs are unchanged. A new lockfile starts a fresh cache. + - name: Cache vp tasks + if: needs.changes.outputs.typecheck != 'false' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: node_modules/.vite/task-cache + key: vp-tasks-v1-typecheck-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }}-${{ github.sha }} + restore-keys: vp-tasks-v1-typecheck-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }}- + # Runs in the background while the build runs, and reports in the last # step: neither result hides the other. - name: Start typecheck @@ -305,12 +403,19 @@ jobs: exit "$(cat "$RUNNER_TEMP/typecheck-status")" # Everything except `t3` (apps/server) and the web app, which have their own jobs. + # Upstream runs this on one large runner. Here the packages are split across + # two 4-CPU runners by test file count (.github/scripts/fork-test-shards.cjs); + # one runner took 6 minutes, the longest job of the run. test: - name: Test + name: Test ${{ matrix.shard }} needs: changes if: ${{ !cancelled() && needs.changes.outputs.test != 'false' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 @@ -322,9 +427,20 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Shard 1 always gets the heaviest affected package; a shard left with + # none says so in the summary and stops here. + - name: Pick this shard's packages + id: shard + env: + TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }} + run: | + packages=$(node .github/scripts/fork-test-shards.cjs "${{ matrix.shard }}/${{ strategy.job-total }}" | tr '\n' ' ') + echo "packages=${packages% }" >> "$GITHUB_OUTPUT" + # The desktop tests compile the browser secret helper. The install runs in # the background while Vite+ installs; the step before the tests waits for it. - name: Start installing browser secret helper build libraries + if: steps.shard.outputs.packages != '' run: | ( set +e @@ -336,44 +452,47 @@ jobs: ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null & - name: Setup Vite+ + if: steps.shard.outputs.packages != '' uses: ./.github/actions/setup-ci-deps with: # Lint saves the pnpm store; the others only restore it. save-pnpm-store: false - name: Cache Electron download + if: steps.shard.outputs.packages != '' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/.cache/electron key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} - name: Ensure Electron runtime is installed + if: steps.shard.outputs.packages != '' run: vp run --filter @t3tools/desktop ensure:electron - name: Test preview artifact validation + if: matrix.shard == 1 run: python3 -B .github/scripts/stage-preview-bundle.test.py - name: Test release checks + if: matrix.shard == 1 run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs - name: Finish installing browser secret helper build libraries + if: steps.shard.outputs.packages != '' timeout-minutes: 5 run: | until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done cat "$RUNNER_TEMP/apt.log" exit "$(cat "$RUNNER_TEMP/apt-status")" - # `changes` lists the affected packages, or nothing when all of them are. - name: Test + if: steps.shard.outputs.packages != '' env: - TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }} + PACKAGES: ${{ steps.shard.outputs.packages }} run: | set -o pipefail - filters=(--filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web') - if [[ -n "$TEST_PACKAGES" ]]; then - filters=() - for package in $TEST_PACKAGES; do filters+=(--filter "$package"); done - fi + filters=() + for package in $PACKAGES; do filters+=(--filter "$package"); done # The runner's trace writer can crash on surrogate pairs in test output. vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c ' import sys @@ -385,7 +504,7 @@ jobs: name: Test Web needs: changes if: ${{ !cancelled() && needs.changes.outputs.test_web != 'false' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - name: Checkout @@ -435,7 +554,7 @@ jobs: # `changes`; a shard the change cannot affect says so in its summary. test_server: name: Test Server ${{ matrix.shard }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 20 strategy: fail-fast: false @@ -472,6 +591,18 @@ jobs: # Lint saves the pnpm store; the others only restore it. save-pnpm-store: false + # Upstream reads a private remote task cache (a secret the fork has not + # got). Here vp's local task cache is kept between runs instead: vp + # fingerprints each task's inputs, so a restored entry only replays a + # task whose inputs are unchanged. A new lockfile starts a fresh cache. + - name: Cache vp tasks + if: steps.changes.outputs.test_server != 'false' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: node_modules/.vite/task-cache + key: vp-tasks-v1-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }}-${{ github.sha }} + restore-keys: vp-tasks-v1-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }}- + - name: Resolve Playwright version id: playwright_version if: steps.changes.outputs.test_server != 'false' @@ -550,7 +681,7 @@ jobs: name: Rust needs: changes if: ${{ !cancelled() && needs.changes.outputs.rust != 'false' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - name: Checkout @@ -617,7 +748,7 @@ jobs: name: Release Smoke needs: changes if: ${{ !cancelled() && needs.changes.outputs.release_smoke != 'false' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - name: Checkout @@ -639,16 +770,20 @@ jobs: - name: Exercise release-only workflow steps run: node scripts/release-smoke.ts - # The one check to require. Fails when any job fails, is cancelled, or is - # skipped without `changes` saying it was not needed. The run summary lists - # every job, and every skip with its reason. - check: - name: Check + # The one check a ruleset requires. Fails when any job fails or is cancelled, + # or is skipped without a reason: `changes` said its gates were not needed, + # or (secret scan) it is a release run. The summary lists every job, the + # reason for every skip, and each job's wall time. + # `always()` rather than `!cancelled()`: a skipped job reports success to + # branch protection, so a cancelled run must still produce a failing result. + fork_ci_result: + name: fork-ci-result if: ${{ always() }} needs: [ changes, - drift, + workflows, + secrets, lint, build, test, @@ -658,7 +793,7 @@ jobs: mobile_native_static_analysis, release_smoke, ] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Require every job to pass @@ -667,36 +802,40 @@ jobs: RELEASE_RUN: ${{ inputs.ref != '' }} run: | echo "$RESULTS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"' + why='{ + build: "the change affects neither desktop, server nor web, and no package needs a typecheck", + test: "the change affects no package this job tests", + test_web: "the change affects nothing @t3tools/web depends on", + rust: "the change touches none of the native Rust crates", + mobile_native_static_analysis: (if $release == "true" + then "release run: upstream release.yml has no macOS lint" + else "no native mobile source, Brewfile or CI workflow changed" end), + release_smoke: "the change does not affect scripts/", + secrets: "release run: the commit was scanned on its way to main" + }' { - echo "### Fork tests" + echo "### fork-ci-result" echo echo "| Job | Result | Why skipped |" echo "| --- | --- | --- |" - echo "$RESULTS" | jq -r --arg release "$RELEASE_RUN" '.changes.outputs as $g - | {build: "the change affects neither desktop, server nor web, and no package needs a typecheck", - test: "the change affects no package this job tests", - test_web: "the change affects nothing @t3tools/web depends on", - rust: "the change touches none of the native Rust crates", - mobile_native_static_analysis: (if $release == "true" - then "release run: upstream release.yml has no macOS lint" - else "no native mobile source, Brewfile or CI workflow changed" end), - release_smoke: "the change does not affect scripts/"} as $why + echo "$RESULTS" | jq -r --arg release "$RELEASE_RUN" "$why as \$why | to_entries[] - | "| \(.key) | \(.value.result) | \(if .value.result == "skipped" then ($why[.key] // "not explained") else "" end) |"' + | \"| \(.key) | \(.value.result) | \(if .value.result == \"skipped\" then (\$why[.key] // \"no reason: fails\") else \"\" end) |\"" if [[ "$(echo "$RESULTS" | jq -r '.changes.outputs.test_server')" == false ]]; then echo echo "Server shards passed without running tests: the change affects nothing t3 (apps/server) depends on." fi } >> "$GITHUB_STEP_SUMMARY" # A job may skip only when `changes` said every gate it represents was - # not needed. Gates are keyed by job name, except merged Build and the - # macOS lint. - echo "$RESULTS" | jq -e '.changes.outputs as $gates + # not needed (gates are keyed by job name, except merged Build and the + # macOS lint), or, for the secret scan, in a release run. + echo "$RESULTS" | jq -e --arg release "$RELEASE_RUN" '.changes.outputs as $gates | {build: ["build", "typecheck", "desktop_launch"], mobile_native_static_analysis: ["mobile_native"]} as $renamed | all(to_entries[]; .value.result == "success" or (.value.result == "skipped" - and (($renamed[.key] // [.key]) | all(.[]; $gates[.] == "false"))))' > /dev/null + and (if .key == "secrets" then $release == "true" + else (($renamed[.key] // [.key]) | all(.[]; $gates[.] == "false")) end)))' > /dev/null # A run whose server tests ran must have produced the transfer measurements. - name: Require transfer measurements @@ -705,3 +844,26 @@ jobs: with: name: thread-transfer-results path: ${{ runner.temp }}/thread-transfer + + # Information only: read from the public run's job list, never fails the run. + - name: Job wall times + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + jobs=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" 2>/dev/null) || { + echo "Job timings unavailable." >> "$GITHUB_STEP_SUMMARY" + exit 0 + } + { + echo + echo "#### Job wall times" + echo + echo "| Job | Queued | Ran |" + echo "| --- | ---: | ---: |" + echo "$jobs" | jq -r '.jobs[] + | select(.completed_at != null and .started_at != null) + | ((.started_at | fromdateiso8601) - (.created_at | fromdateiso8601)) as $queued + | ((.completed_at | fromdateiso8601) - (.started_at | fromdateiso8601)) as $ran + | "| \(.name) | \($queued)s | \($ran / 60 | floor)m \($ran % 60)s |"' + } >> "$GITHUB_STEP_SUMMARY" From 9510d2fb0c3bebd09d8af935d55e14aa2ffcca63 Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 11:24:30 +0000 Subject: [PATCH 3/9] Fork CI: slice large test runs with vitest --shard; lint fixes - Test and Test Web run on two runners each. A package with 100+ test files (and Test Web's 548) runs a vitest --shard slice on every runner; the rest are placed whole by test file count. - Script header comments fit upstream's 4-line comment rule; the jq program in fork-ci-result passes shellcheck. - The Ubuntu 24.04 pin is about choosing OS upgrades, not git: the 24.04 image now ships git 2.56 as well. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-test-shards.cjs | 64 +++++++++++------- .github/scripts/fork-test-shards.test.cjs | 40 ++++++----- .github/scripts/fork-tests-drift.cjs | 14 ++-- .github/workflows/fork-tests.yml | 82 ++++++++++++++++------- 4 files changed, 123 insertions(+), 77 deletions(-) diff --git a/.github/scripts/fork-test-shards.cjs b/.github/scripts/fork-test-shards.cjs index 0719c050fb7d1..b345949dc32c7 100644 --- a/.github/scripts/fork-test-shards.cjs +++ b/.github/scripts/fork-test-shards.cjs @@ -1,29 +1,30 @@ -// Fork only. Splits the Test job's packages across the shards of its matrix in -// .github/workflows/fork-tests.yml. Upstream runs the job on one large runner; -// a GitHub-hosted runner has 4 CPUs, so the packages go to several runners. -// -// node .github/scripts/fork-test-shards.cjs 1/2 prints shard 1's packages -// -// TEST_PACKAGES (from the `changes` job) limits the set; empty means every -// package the Test job covers. Packages are weighted by their test file count -// and placed, heaviest first, on the least loaded shard, so the split follows -// new packages and tests without a hand-kept list. +// Fork only: prints one Test shard's work (`fork-test-shards.cjs 1/2`): `split +// ` for a large package every shard runs a vitest slice of, `whole ` +// for the rest, heaviest first (by test files) onto the lightest shard. +// TEST_PACKAGES from `changes` limits the set; empty means every package. const childProcess = require("node:child_process"); const fs = require("node:fs"); +const path = require("node:path"); const { readWorkspace } = require("./ci-changes.cjs"); // Tested by their own jobs (Test Server, Test Web) or not a package to test. const OWN_JOB = new Set(["t3", "@t3tools/web", "@t3tools/monorepo"]); const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/; +// One vitest run of this many files outlasts the rest of a shard. +const SPLIT_AT = 100; function assign(packages, count) { - const shards = Array.from({ length: count }, () => ({ weight: 0, packages: [] })); - const ordered = packages.toSorted((a, b) => b.weight - a.weight || a.name.localeCompare(b.name)); + const split = packages.filter((pkg) => pkg.splittable && pkg.weight >= SPLIT_AT); + const share = split.reduce((total, pkg) => total + pkg.weight, 0) / count; + const shards = Array.from({ length: count }, () => ({ weight: share, whole: [], split })); + const ordered = packages + .filter((pkg) => !split.includes(pkg)) + .toSorted((a, b) => b.weight - a.weight || a.name.localeCompare(b.name)); for (const pkg of ordered) { const lightest = shards.reduce((best, shard) => (shard.weight < best.weight ? shard : best)); lightest.weight += pkg.weight; - lightest.packages.push(pkg); + lightest.whole.push(pkg); } return shards; } @@ -41,29 +42,44 @@ function main() { throw new Error("usage: fork-test-shards.cjs /"); const [index, count] = [Number(match[1]), Number(match[2])]; const only = (process.env.TEST_PACKAGES ?? "").split(/\s+/).filter(Boolean); + const root = path.resolve(__dirname, "../.."); const packages = readWorkspace() .filter((pkg) => pkg.hasTest && !OWN_JOB.has(pkg.name)) .filter((pkg) => only.length === 0 || only.includes(pkg.name)) - .map((pkg) => ({ name: pkg.name, weight: Math.max(1, testFiles(pkg.dir)) })); + .map((pkg) => { + const manifest = JSON.parse( + fs.readFileSync(path.join(root, pkg.dir, "package.json"), "utf8"), + ); + return { + name: pkg.name, + dir: pkg.dir, + weight: Math.max(1, testFiles(pkg.dir)), + // Run in its directory with --shard, so the script must be vitest itself. + splittable: /^vp test run(\s|$)/.test(manifest.scripts.test), + }; + }); const unknown = only.filter((name) => !packages.some((pkg) => pkg.name === name)); if (unknown.length) throw new Error(`TEST_PACKAGES names no tested package: ${unknown.join(" ")}`); const shard = assign(packages, count)[index - 1]; - for (const pkg of shard.packages) console.log(pkg.name); + for (const pkg of shard.split) console.log(`split ${pkg.dir}`); + for (const pkg of shard.whole) console.log(`whole ${pkg.name}`); const summary = process.env.GITHUB_STEP_SUMMARY; if (summary) { - const list = shard.packages.map((pkg) => `\`${pkg.name}\` (${pkg.weight})`).join(", "); - fs.appendFileSync( - summary, - `### Test shard ${index}/${count}\n\n` + - (shard.packages.length - ? `${shard.packages.length} of ${packages.length} packages, ${shard.weight} test files: ${list}\n` - : `No packages: the change affects ${packages.length} tested package(s), all on other shards.\n`), - ); + const list = (pkgs) => pkgs.map((pkg) => `\`${pkg.name}\` (${pkg.weight})`).join(", "); + const lines = [`### Test shard ${index}/${count}`, ""]; + if (shard.split.length) lines.push(`Slice ${index}/${count} of ${list(shard.split)}.`, ""); + if (shard.whole.length) lines.push(`Whole: ${list(shard.whole)}.`, ""); + if (!shard.split.length && !shard.whole.length) + lines.push( + `Nothing to run: the ${packages.length} affected package(s) are on other shards.`, + "", + ); + fs.appendFileSync(summary, `${lines.join("\n")}\n`); } } -module.exports = { assign }; +module.exports = { assign, SPLIT_AT }; if (require.main === module) main(); diff --git a/.github/scripts/fork-test-shards.test.cjs b/.github/scripts/fork-test-shards.test.cjs index e5ea56440d95a..81310927ed4a5 100644 --- a/.github/scripts/fork-test-shards.test.cjs +++ b/.github/scripts/fork-test-shards.test.cjs @@ -1,34 +1,38 @@ const assert = require("node:assert/strict"); const test = require("node:test"); -const { assign } = require("./fork-test-shards.cjs"); +const { assign, SPLIT_AT } = require("./fork-test-shards.cjs"); -const names = (shards) => shards.map((shard) => shard.packages.map((pkg) => pkg.name)); +const pkg = (name, weight, splittable = true) => ({ + name, + dir: `packages/${name}`, + weight, + splittable, +}); +const whole = (shards) => shards.map((shard) => shard.whole.map((p) => p.name)); +const split = (shards) => shards.map((shard) => shard.split.map((p) => p.name)); test("places the heaviest package first on the lightest shard", () => { - const packages = [ - { name: "small", weight: 1 }, - { name: "big", weight: 10 }, - { name: "mid", weight: 6 }, - { name: "mid2", weight: 5 }, - ]; - assert.deepEqual(names(assign(packages, 2)), [ + const packages = [pkg("small", 1), pkg("big", 10), pkg("mid", 6), pkg("mid2", 5)]; + assert.deepEqual(whole(assign(packages, 2)), [ ["big", "small"], ["mid", "mid2"], ]); }); test("ties break by name, so every shard computes the same split", () => { - const packages = [ - { name: "b", weight: 2 }, - { name: "a", weight: 2 }, - { name: "c", weight: 2 }, - ]; - assert.deepEqual(names(assign(packages, 2)), [["a", "c"], ["b"]]); - assert.deepEqual(names(assign(packages.toReversed(), 2)), [["a", "c"], ["b"]]); + const packages = [pkg("b", 2), pkg("a", 2), pkg("c", 2)]; + assert.deepEqual(whole(assign(packages, 2)), [["a", "c"], ["b"]]); + assert.deepEqual(whole(assign(packages.toReversed(), 2)), [["a", "c"], ["b"]]); +}); + +test("a large vitest package is sliced on every shard, the rest stay whole", () => { + const packages = [pkg("large", SPLIT_AT + 50), pkg("script", SPLIT_AT, false), pkg("a", 3)]; + const shards = assign(packages, 2); + assert.deepEqual(split(shards), [["large"], ["large"]]); + assert.deepEqual(whole(shards), [["script"], ["a"]]); }); test("every package lands on exactly one shard, and spare shards stay empty", () => { - const packages = [{ name: "only", weight: 3 }]; - assert.deepEqual(names(assign(packages, 3)), [["only"], [], []]); + assert.deepEqual(whole(assign([pkg("only", 3)], 3)), [["only"], [], []]); }); diff --git a/.github/scripts/fork-tests-drift.cjs b/.github/scripts/fork-tests-drift.cjs index 31b1f07a85fec..d39e73267feee 100644 --- a/.github/scripts/fork-tests-drift.cjs +++ b/.github/scripts/fork-tests-drift.cjs @@ -1,13 +1,7 @@ -// Fork only. .github/workflows/fork-tests.yml copies upstream's test jobs onto -// GitHub-hosted runners. This check fails when upstream's jobs change and the -// copy was not updated: it digests every job and step of the tracked upstream -// workflows and compares them with .github/fork-tests-upstream.json. -// -// node .github/scripts/fork-tests-drift.cjs check (CI) -// node .github/scripts/fork-tests-drift.cjs --record after updating fork-tests.yml -// -// Comment-only and blank-line changes are ignored. No dependencies: the -// workflows are parsed by indentation, which is all a job/step list needs. +// Fork only: fails when upstream's ci.yml or release.yml test jobs change and +// .github/workflows/fork-tests.yml did not follow. Compares a digest of every +// job and step with .github/fork-tests-upstream.json; comments are ignored. +// After porting a change: node .github/scripts/fork-tests-drift.cjs --record const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index cd9f09926e6d2..ea873b8359181 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -2,10 +2,10 @@ name: Fork tests # Fork only. Upstream's CI checks (ci.yml) on GitHub-hosted runners, because the # fork has none of upstream's runners and upstream's workflows are disabled here. -# Commands follow ci.yml step for step; only the runners, time limits, apt -# setup, the task cache (kept with actions/cache instead of upstream's private -# remote cache) and the Test job's split across two runners differ. Upstream's -# composite actions and change detector (.github/actions, +# Commands follow ci.yml step for step. Only the runners, time limits, apt +# setup and the task cache differ (kept with actions/cache instead of upstream's +# private remote cache), and Test and Test Web split over two runners each. +# Upstream's composite actions and change detector (.github/actions, # .github/scripts/ci-changes.cjs) are used as they are, so a rebase brings their # changes. The `workflows` job lints the fork workflows (actionlint) and fails # when upstream's ci.yml or release.yml test jobs change and this copy did not @@ -14,8 +14,8 @@ name: Fork tests # Require only `fork-ci-result` in a ruleset: it needs every other job, so jobs # can be added, split or resharded without touching the ruleset. # -# Runners: GitHub-hosted only, Ubuntu 24.04 as upstream (ubuntu-latest is now -# 26.04, whose newer git fails an apps/server test upstream never sees). Never +# Runners: GitHub-hosted only, Ubuntu 24.04 like upstream's runners, so an OS +# upgrade is a reviewed change here, not a surprise from ubuntu-latest. Never # add a self-hosted runner to this workflow: this repository is public, so a # pull request from any fork would run its code on that machine. # @@ -403,9 +403,9 @@ jobs: exit "$(cat "$RUNNER_TEMP/typecheck-status")" # Everything except `t3` (apps/server) and the web app, which have their own jobs. - # Upstream runs this on one large runner. Here the packages are split across - # two 4-CPU runners by test file count (.github/scripts/fork-test-shards.cjs); - # one runner took 6 minutes, the longest job of the run. + # Upstream runs this on one large runner; one 4-CPU runner took 6 minutes. + # Two runners split it (.github/scripts/fork-test-shards.cjs): each runs a + # vitest slice of every large package and half of the others, whole. test: name: Test ${{ matrix.shard }} needs: changes @@ -427,20 +427,21 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false - # Shard 1 always gets the heaviest affected package; a shard left with - # none says so in the summary and stops here. + # A shard with nothing to run says so in the summary and stops here. - name: Pick this shard's packages id: shard env: TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }} run: | - packages=$(node .github/scripts/fork-test-shards.cjs "${{ matrix.shard }}/${{ strategy.job-total }}" | tr '\n' ' ') - echo "packages=${packages% }" >> "$GITHUB_OUTPUT" + work=$(node .github/scripts/fork-test-shards.cjs "${{ matrix.shard }}/${{ strategy.job-total }}") + whole=$(awk '$1 == "whole" { printf "%s ", $2 }' <<< "$work") + split=$(awk '$1 == "split" { printf "%s ", $2 }' <<< "$work") + { echo "whole=${whole% }"; echo "split=${split% }"; echo "any=$([[ -n "$work" ]] && echo true || echo false)"; } >> "$GITHUB_OUTPUT" # The desktop tests compile the browser secret helper. The install runs in # the background while Vite+ installs; the step before the tests waits for it. - name: Start installing browser secret helper build libraries - if: steps.shard.outputs.packages != '' + if: steps.shard.outputs.any == 'true' run: | ( set +e @@ -452,21 +453,21 @@ jobs: ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null & - name: Setup Vite+ - if: steps.shard.outputs.packages != '' + if: steps.shard.outputs.any == 'true' uses: ./.github/actions/setup-ci-deps with: # Lint saves the pnpm store; the others only restore it. save-pnpm-store: false - name: Cache Electron download - if: steps.shard.outputs.packages != '' + if: steps.shard.outputs.any == 'true' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/.cache/electron key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} - name: Ensure Electron runtime is installed - if: steps.shard.outputs.packages != '' + if: steps.shard.outputs.any == 'true' run: vp run --filter @t3tools/desktop ensure:electron - name: Test preview artifact validation @@ -478,34 +479,58 @@ jobs: run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs - name: Finish installing browser secret helper build libraries - if: steps.shard.outputs.packages != '' + if: steps.shard.outputs.any == 'true' timeout-minutes: 5 run: | until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done cat "$RUNNER_TEMP/apt.log" exit "$(cat "$RUNNER_TEMP/apt-status")" + # Whole packages as upstream runs them; a large package's slice runs its + # own test script in its directory with vitest's --shard (`vp run` does + # not pass arguments through to vitest). - name: Test - if: steps.shard.outputs.packages != '' + if: steps.shard.outputs.any == 'true' env: - PACKAGES: ${{ steps.shard.outputs.packages }} + WHOLE: ${{ steps.shard.outputs.whole }} + SPLIT: ${{ steps.shard.outputs.split }} + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} run: | set -o pipefail - filters=() - for package in $PACKAGES; do filters+=(--filter "$package"); done # The runner's trace writer can crash on surrogate pairs in test output. - vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c ' + ascii() { + python3 -u -c ' import sys for line in sys.stdin: sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii")) ' + } + status=0 + if [[ -n "$WHOLE" ]]; then + filters=() + for package in $WHOLE; do filters+=(--filter "$package"); done + vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | ascii || status=1 + fi + for dir in $SPLIT; do + echo "::group::$dir slice $SHARD" + script=$(jq -r .scripts.test "$dir/package.json") + (cd "$dir" && eval "$script --shard=$SHARD") 2>&1 | ascii || status=1 + echo "::endgroup::" + done + exit "$status" + # Upstream runs this on one large runner; on a 4-CPU runner the 548 files + # took 7.5 minutes, so two runners each take a vitest slice. test_web: - name: Test Web + name: Test Web ${{ matrix.shard }} needs: changes if: ${{ !cancelled() && needs.changes.outputs.test_web != 'false' }} runs-on: ubuntu-24.04 timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 @@ -546,8 +571,13 @@ jobs: node apps/web/node_modules/playwright/cli.js install-deps chromium fi + # Upstream: `vp run --filter @t3tools/web test`. Same script, run in its + # directory, because `vp run` does not pass --shard through to vitest. - name: Test - run: vp run --filter @t3tools/web test + env: + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} + working-directory: apps/web + run: eval "$(jq -r .scripts.test package.json) --shard=$SHARD" # apps/server runs its test files one at a time; shards spread them over # separate runners. The shards gate themselves instead of waiting for @@ -802,6 +832,8 @@ jobs: RELEASE_RUN: ${{ inputs.ref != '' }} run: | echo "$RESULTS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"' + # A jq program: $release is jq's variable, not the shell's. + # shellcheck disable=SC2016 why='{ build: "the change affects neither desktop, server nor web, and no package needs a typecheck", test: "the change affects no package this job tests", From 95935cef3bc93b9e5f2a19b6b92c6e9ddfd60bc6 Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 11:35:46 +0000 Subject: [PATCH 4/9] Fork CI: run every fork script test, checked against a list The workflows job runs each .github/scripts/fork-*.test.{sh,py,cjs} and checks them against .github/fork-ci-tests.txt. A present test that is not listed fails. A listed test that is missing fails once it has been in the tested commit's history; before that it is a note in the job summary, so a test whose pull request has not merged yet does not fail the run. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 10 +++ .github/scripts/fork-ci-tests.cjs | 99 ++++++++++++++++++++++++++ .github/scripts/fork-ci-tests.test.cjs | 30 ++++++++ .github/workflows/fork-tests.yml | 11 ++- 4 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 .github/fork-ci-tests.txt create mode 100644 .github/scripts/fork-ci-tests.cjs create mode 100644 .github/scripts/fork-ci-tests.test.cjs diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt new file mode 100644 index 0000000000000..005a330dbb333 --- /dev/null +++ b/.github/fork-ci-tests.txt @@ -0,0 +1,10 @@ +# Fork script tests the workflow job runs (.github/scripts/fork-ci-tests.cjs). +# Every fork-*.test.{sh,py,cjs} must be listed here. A listed file that is not +# present yet is a note; one that was present and then deleted fails. +.github/scripts/fork-ci-tests.test.cjs +.github/scripts/fork-follow.test.sh +.github/scripts/fork-release-workflow.test.py +.github/scripts/fork-release.test.sh +.github/scripts/fork-schema-check.test.cjs +.github/scripts/fork-test-shards.test.cjs +.github/scripts/fork-tests-drift.test.cjs diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs new file mode 100644 index 0000000000000..0a73f2e164a36 --- /dev/null +++ b/.github/scripts/fork-ci-tests.cjs @@ -0,0 +1,99 @@ +// Fork only: runs every `.github/scripts/fork-*.test.{sh,py,cjs}` and checks +// them against `.github/fork-ci-tests.txt`. A present test missing from the list +// fails; a listed test that is missing fails once it has been in the tested +// commit's history (deleted while still listed) and is a note before that. +const childProcess = require("node:child_process"); +const fs = require("node:fs"); +const path = require("node:path"); + +const ROOT = path.resolve(__dirname, "../.."); +const LIST = ".github/fork-ci-tests.txt"; +const TEST_FILE = /^fork-.+\.test\.(sh|py|cjs)$/; +const RUNNER = { sh: ["bash"], py: ["python3"], cjs: ["node", "--test"] }; + +// `ever(file)` says whether the file is in the tested commit's history. +function plan(listed, present, ever) { + const unlisted = present.filter((file) => !listed.includes(file)); + const missing = listed.filter((file) => !present.includes(file)); + const deleted = missing.filter((file) => ever(file)); + const pending = missing.filter((file) => !deleted.includes(file)); + return { run: present, unlisted, deleted, pending }; +} + +function readList(text) { + return text + .split("\n") + .map((line) => line.replace(/#.*/, "").trim()) + .filter(Boolean); +} + +function git(args) { + return childProcess.execFileSync("git", args, { cwd: ROOT, encoding: "utf8" }).trim(); +} + +// A shallow checkout first fetches the full history of HEAD, trees only. +function everInHistory(file) { + if (git(["rev-parse", "--is-shallow-repository"]) === "true") + git([ + "fetch", + "--quiet", + "--no-tags", + "--filter=blob:none", + "--unshallow", + "origin", + git(["rev-parse", "HEAD"]), + ]); + return git(["log", "-1", "--format=%h", "HEAD", "--", file]) !== ""; +} + +function main() { + const listed = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8")); + const present = fs + .readdirSync(path.join(ROOT, ".github/scripts")) + .filter((name) => TEST_FILE.test(name)) + .map((name) => `.github/scripts/${name}`) + .toSorted(); + const result = plan(listed, present, everInHistory); + const rows = []; + let failed = result.unlisted.length + result.deleted.length; + for (const file of result.run) { + const [command, ...args] = RUNNER[path.basename(file).match(TEST_FILE)[1]]; + const started = Date.now(); + console.log(`::group::${file}`); + const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" }); + console.log("::endgroup::"); + const seconds = ((Date.now() - started) / 1000).toFixed(1); + if (run.status !== 0) { + failed += 1; + console.log(`::error file=${file}::${file} failed (exit ${run.status ?? run.signal})`); + } + rows.push(`| \`${file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${seconds} s |`); + } + for (const file of result.unlisted) + console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`); + for (const file of result.deleted) + console.log(`::error::${file} is listed in ${LIST} but was deleted; restore it or unlist it`); + for (const file of result.pending) + rows.push(`| \`${file}\` | not present: its PR has not merged | |`); + for (const file of result.unlisted) rows.push(`| \`${file}\` | **not in ${LIST}** | |`); + for (const file of result.deleted) rows.push(`| \`${file}\` | **deleted while listed** | |`); + + const summary = process.env.GITHUB_STEP_SUMMARY; + if (summary) + fs.appendFileSync( + summary, + [ + "### Fork script tests", + "", + "| Test | Result | Time |", + "| --- | --- | --- |", + ...rows, + "", + ].join("\n") + "\n", + ); + if (failed) process.exitCode = 1; +} + +module.exports = { plan, readList }; + +if (require.main === module) main(); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs new file mode 100644 index 0000000000000..8703b0d841c84 --- /dev/null +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -0,0 +1,30 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); + +const { plan, readList } = require("./fork-ci-tests.cjs"); + +const never = () => false; + +test("runs every present test, listed or not", () => { + assert.deepEqual(plan(["a"], ["a", "b"], never).run, ["a", "b"]); +}); + +test("a present test missing from the list fails", () => { + assert.deepEqual(plan(["a"], ["a", "b"], never).unlisted, ["b"]); +}); + +test("a listed test never in history is pending, not a failure", () => { + const result = plan(["a", "later"], ["a"], never); + assert.deepEqual(result.pending, ["later"]); + assert.deepEqual(result.deleted, []); +}); + +test("a listed test deleted from history fails", () => { + const result = plan(["a", "gone"], ["a"], (file) => file === "gone"); + assert.deepEqual(result.deleted, ["gone"]); + assert.deepEqual(result.pending, []); +}); + +test("the list ignores comments and blank lines", () => { + assert.deepEqual(readList("# head\n\na.sh # from #2\n b.py\n"), ["a.sh", "b.py"]); +}); diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index ea873b8359181..4f74f45750f2e 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -143,8 +143,15 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" actionlint "$RUNNER_TEMP/actionlint" -color .github/workflows/fork-*.yml - - name: Test the fork CI scripts - run: node --test .github/scripts/fork-tests-drift.test.cjs .github/scripts/fork-test-shards.test.cjs + # Every fork script test, checked against .github/fork-ci-tests.txt: an + # unlisted test or a listed one deleted from history fails, a listed one + # whose pull request has not merged yet is a note in the summary. + - name: Fork script tests + run: | + if compgen -G '.github/scripts/fork-*.test.py' >/dev/null && ! python3 -c 'import yaml' 2>/dev/null; then + sudo apt-get update -qq && sudo apt-get install -y -qq python3-yaml + fi + node .github/scripts/fork-ci-tests.cjs - name: Compare upstream's test jobs with the recorded copy run: node .github/scripts/fork-tests-drift.cjs From 831e8888b526a2a401dd0f269a17dc589855376b Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 11:37:29 +0000 Subject: [PATCH 5/9] Fork CI: give fork-ci-result actions: read for the job wall times The job list API needs it; a failed call now warns in the log instead of failing silently. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fork-tests.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index 4f74f45750f2e..72aebda5e5e86 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -832,6 +832,9 @@ jobs: ] runs-on: ubuntu-24.04 timeout-minutes: 5 + # The job list API needs actions: read; nothing here checks out code. + permissions: + actions: read steps: - name: Require every job to pass env: @@ -884,16 +887,18 @@ jobs: name: thread-transfer-results path: ${{ runner.temp }}/thread-transfer - # Information only: read from the public run's job list, never fails the run. + # Information only: read from the run's job list, never fails the run. - name: Job wall times if: always() env: GH_TOKEN: ${{ github.token }} run: | - jobs=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" 2>/dev/null) || { + jobs=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100") || { + echo "::warning::Job timings unavailable: the job list API call failed." echo "Job timings unavailable." >> "$GITHUB_STEP_SUMMARY" exit 0 } + echo "Job timings for $(echo "$jobs" | jq '.jobs | length') jobs." { echo echo "#### Job wall times" From 993ccc8068ad3e93a7b832da18b4a6343cabc896 Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 11:49:24 +0000 Subject: [PATCH 6/9] scripts: move release-repository.test.ts out of scripts/lib apps/desktop's typecheck includes scripts/lib, and this test imports scripts/build-desktop-artifact.ts, which that project does not list (TS6307). In scripts/ the scripts project covers both files, and the scripts package still runs the test. Co-Authored-By: Claude Opus 5.5 --- scripts/{lib => }/release-repository.test.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{lib => }/release-repository.test.ts (100%) diff --git a/scripts/lib/release-repository.test.ts b/scripts/release-repository.test.ts similarity index 100% rename from scripts/lib/release-repository.test.ts rename to scripts/release-repository.test.ts From f9560594a21a62ef41191c9daf8cec0aa996f659 Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 11:55:55 +0000 Subject: [PATCH 7/9] scripts: fix release-repository.test.ts imports after the move Co-Authored-By: Claude Opus 5.5 --- scripts/release-repository.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release-repository.test.ts b/scripts/release-repository.test.ts index 46a285b29425e..fe17b731d0303 100644 --- a/scripts/release-repository.test.ts +++ b/scripts/release-repository.test.ts @@ -3,8 +3,8 @@ import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; import { describe, expect, it } from "vite-plus/test"; -import { resolveGitHubPublishConfig } from "../build-desktop-artifact.ts"; -import { releaseRepositoryDefine, resolveReleaseRepository } from "./release-repository.ts"; +import { resolveGitHubPublishConfig } from "./build-desktop-artifact.ts"; +import { releaseRepositoryDefine, resolveReleaseRepository } from "./lib/release-repository.ts"; const desktopFeedRepository = (env: Record) => resolveGitHubPublishConfig("latest").pipe( From 6d6e0b55d285fbd7aa00b57388e21db0865a41fc Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 12:23:16 +0000 Subject: [PATCH 8/9] fork-tests: fix round 1 (secret scan range, every test run, one commit per run, drift) Secret scan (.github/scripts/fork-secret-scan.sh, tests: fork-secret-scan.test.sh) - Pull requests scan base.sha..head.sha on a full blob-less clone instead of the newest N commits by date. After a merge of main, the PR's oldest commits fell outside --max-count and were never scanned; a squash merge never brings them to main, yet they stay public under refs/pull/N/head. - Merge commits are diffed with --remerge-diff. Plain `git log -p` shows merges with no diff, so a key added only in a merge commit was never read. - Push: before..after; a force push scans what the new tip has and the old one had not. A new branch, or an old tip that cannot be fetched, scans only the pushed commit and says so in a warning. - The Secret scan job runs the test against the pinned gitleaks before scanning. Every test file under .github/scripts runs somewhere (fork-ci-tests.cjs) - Finds *.test.* in subdirectories too. Runs .sh, .py, .cjs, .mjs and .js; a listed test with another extension fails instead of being skipped. - Upstream's tests are listed as " run by "; the file must name the test and cannot be a disabled upstream workflow. thread-transfer-report.test.cjs was run nowhere (upstream runs it from a workflow_run of its disabled CI); Test 1 runs it now with upstream's other release-check tests. One commit per release run - `changes` resolves a workflow_dispatch/workflow_call ref to a SHA once; every job checks that out, so a branch that moves mid-run cannot split a run over two commits. workflows, lint and the server shards now wait for `changes`. Upstream drift (fork-tests-drift.cjs) - Also digests the top-level env, defaults, concurrency and permissions. - --record refuses (exit 1) while a tracked job is missing instead of saving null, and an old null entry is reported instead of crashing. - --record prints the upstream changes it accepts; the commit and PR must say what was ported for each. Re-recorded .github/fork-tests-upstream.json. It accepts: - ci.yml top-level permissions (contents: read): fork-tests.yml has the same. - ci.yml top-level concurrency (group per PR/sha, cancel on pull_request): fork-tests.yml has the equivalent, with the event name in the group. - release.yml top-level concurrency and permissions: they apply to upstream's release run, not its test jobs. fork-release.yml has its own. No job or step changed upstream. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 22 +++- .github/fork-tests-upstream.json | 26 ++++ .github/scripts/fork-ci-tests.cjs | 111 +++++++++++++--- .github/scripts/fork-ci-tests.test.cjs | 80 ++++++++++-- .github/scripts/fork-secret-scan.sh | 89 +++++++++++++ .github/scripts/fork-secret-scan.test.sh | 150 ++++++++++++++++++++++ .github/scripts/fork-tests-drift.cjs | 115 ++++++++++++++--- .github/scripts/fork-tests-drift.test.cjs | 53 +++++++- .github/workflows/fork-tests.yml | 115 ++++++++++------- 9 files changed, 662 insertions(+), 99 deletions(-) create mode 100755 .github/scripts/fork-secret-scan.sh create mode 100755 .github/scripts/fork-secret-scan.test.sh diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 005a330dbb333..37279dfa8c754 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -1,10 +1,26 @@ -# Fork script tests the workflow job runs (.github/scripts/fork-ci-tests.cjs). -# Every fork-*.test.{sh,py,cjs} must be listed here. A listed file that is not -# present yet is a note; one that was present and then deleted fails. +# Every test file under .github/scripts (*.test.*, subdirectories included) and +# where it runs (.github/scripts/fork-ci-tests.cjs checks this list). +# run by the workflows job, by extension (sh, py, cjs, mjs, js) +# run by run by , which must name the test's path (not one +# of upstream's workflows: they are disabled in the fork) +# A present test that is not listed fails. A listed file that is not present yet +# is a note; one that was present and then deleted fails. .github/scripts/fork-ci-tests.test.cjs .github/scripts/fork-follow.test.sh .github/scripts/fork-release-workflow.test.py .github/scripts/fork-release.test.sh .github/scripts/fork-schema-check.test.cjs +.github/scripts/fork-secret-scan.test.sh .github/scripts/fork-test-shards.test.cjs .github/scripts/fork-tests-drift.test.cjs + +# Upstream's tests, run where fork-tests.yml follows upstream's ci.yml. +.github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml +.github/scripts/mobile-preview-comment.test.cjs run by .github/workflows/fork-tests.yml +.github/scripts/stage-preview-bundle.test.py run by .github/workflows/fork-tests.yml +.github/scripts/test-delta.test.cjs run by .github/workflows/fork-tests.yml +.github/scripts/vercel-deployment-output.test.cjs run by .github/workflows/fork-tests.yml +# Upstream runs this one from thread-transfer-report.yml (workflow_run of the disabled CI). +.github/scripts/thread-transfer-report.test.cjs run by .github/workflows/fork-tests.yml +# Release Smoke runs scripts/release-smoke.ts, which runs this one. +.github/scripts/relay-state-output.test.cjs run by scripts/release-smoke.ts diff --git a/.github/fork-tests-upstream.json b/.github/fork-tests-upstream.json index b2d826809a501..18c65f9f0286f 100644 --- a/.github/fork-tests-upstream.json +++ b/.github/fork-tests-upstream.json @@ -1,5 +1,18 @@ { ".github/workflows/ci.yml": { + "(top-level)": { + "header": "e3b0c44298fc1c14", + "steps": [ + { + "name": "permissions", + "digest": "893e619df25e8e99" + }, + { + "name": "concurrency", + "digest": "8782324e66667532" + } + ] + }, "changes": { "header": "8ff211d87838fee8", "steps": [ @@ -340,6 +353,19 @@ } }, ".github/workflows/release.yml": { + "(top-level)": { + "header": "e3b0c44298fc1c14", + "steps": [ + { + "name": "concurrency", + "digest": "7675d281fd1ce2db" + }, + { + "name": "permissions", + "digest": "2fa142cb2564a193" + } + ] + }, "quality": { "header": "64aee6ef731d6767", "steps": [ diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 0a73f2e164a36..ef9fe4f8fcf97 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -1,30 +1,86 @@ -// Fork only: runs every `.github/scripts/fork-*.test.{sh,py,cjs}` and checks -// them against `.github/fork-ci-tests.txt`. A present test missing from the list -// fails; a listed test that is missing fails once it has been in the tested -// commit's history (deleted while still listed) and is a note before that. +// Fork only: every test file under `.github/scripts` (`*.test.*`, subdirectories +// included) must be run somewhere, checked against `.github/fork-ci-tests.txt`. +// A plain entry is run here, by its extension (RUNNER). An entry +// ` run by ` is run by that file, which must name the test's path; it +// cannot be one of upstream's workflows, which are disabled in the fork. A +// present test missing from the list fails, and so does a test with no runner +// for its extension. A listed test that is missing fails once it has been in the +// tested commit's history (deleted while still listed) and is a note before that. const childProcess = require("node:child_process"); const fs = require("node:fs"); const path = require("node:path"); const ROOT = path.resolve(__dirname, "../.."); const LIST = ".github/fork-ci-tests.txt"; -const TEST_FILE = /^fork-.+\.test\.(sh|py|cjs)$/; -const RUNNER = { sh: ["bash"], py: ["python3"], cjs: ["node", "--test"] }; +const SCRIPTS = ".github/scripts"; +const TEST_FILE = /\.test\.([^./]+)$/; +const RUNNER = { + sh: ["bash"], + py: ["python3"], + cjs: ["node", "--test"], + mjs: ["node", "--test"], + js: ["node", "--test"], +}; +const UPSTREAM_WORKFLOW = /^\.github\/workflows\/(?!fork-)[^/]+\.ya?ml$/; -// `ever(file)` says whether the file is in the tested commit's history. -function plan(listed, present, ever) { +// `entries`: [{ file, runBy }]; `present`: test files in the tree; +// `ever(file)` says whether the file is in the tested commit's history; +// `read(file)` returns a runner file's text, or null when it does not exist. +function plan(entries, present, ever, read = () => null) { + const listed = entries.map((entry) => entry.file); const unlisted = present.filter((file) => !listed.includes(file)); const missing = listed.filter((file) => !present.includes(file)); const deleted = missing.filter((file) => ever(file)); const pending = missing.filter((file) => !deleted.includes(file)); - return { run: present, unlisted, deleted, pending }; + const here = entries.filter((entry) => !entry.runBy && present.includes(entry.file)); + const noRunner = here + .filter((entry) => !RUNNER[entry.file.match(TEST_FILE)?.[1]]) + .map((entry) => entry.file); + const elsewhere = []; + const notRun = []; + for (const { file, runBy } of entries) { + if (!runBy || !present.includes(file)) continue; + let why = null; + if (UPSTREAM_WORKFLOW.test(runBy)) why = `${runBy} is an upstream workflow, disabled in the fork`; + else { + const text = read(runBy); + if (text === null) why = `${runBy} does not exist`; + else if (!text.includes(file)) why = `${runBy} does not name ${file}`; + } + if (why) notRun.push({ file, why }); + else elsewhere.push({ file, runBy }); + } + return { + run: here.map((entry) => entry.file).filter((file) => !noRunner.includes(file)), + unlisted, + deleted, + pending, + noRunner, + elsewhere, + notRun, + }; } function readList(text) { return text .split("\n") .map((line) => line.replace(/#.*/, "").trim()) - .filter(Boolean); + .filter(Boolean) + .map((line) => { + const match = line.match(/^(\S+)(?:\s+run by\s+(\S+))?$/); + if (!match) throw new Error(`${LIST}: cannot read "${line}" (expected "" or " run by ")`); + return { file: match[1], runBy: match[2] ?? null }; + }); +} + +function findTests(dir, root = ROOT) { + const out = []; + for (const entry of fs.readdirSync(path.join(root, dir), { withFileTypes: true })) { + const rel = `${dir}/${entry.name}`; + if (entry.isDirectory()) out.push(...findTests(rel, root)); + else if (TEST_FILE.test(entry.name)) out.push(rel); + } + return out.toSorted(); } function git(args) { @@ -46,18 +102,23 @@ function everInHistory(file) { return git(["log", "-1", "--format=%h", "HEAD", "--", file]) !== ""; } +function readFile(file) { + try { + return fs.readFileSync(path.join(ROOT, file), "utf8"); + } catch (error) { + if (error.code === "ENOENT") return null; + throw error; + } +} + function main() { - const listed = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8")); - const present = fs - .readdirSync(path.join(ROOT, ".github/scripts")) - .filter((name) => TEST_FILE.test(name)) - .map((name) => `.github/scripts/${name}`) - .toSorted(); - const result = plan(listed, present, everInHistory); + const entries = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8")); + const result = plan(entries, findTests(SCRIPTS), everInHistory, readFile); const rows = []; - let failed = result.unlisted.length + result.deleted.length; + let failed = + result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length; for (const file of result.run) { - const [command, ...args] = RUNNER[path.basename(file).match(TEST_FILE)[1]]; + const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]]; const started = Date.now(); console.log(`::group::${file}`); const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" }); @@ -73,10 +134,20 @@ function main() { console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`); for (const file of result.deleted) console.log(`::error::${file} is listed in ${LIST} but was deleted; restore it or unlist it`); + for (const file of result.noRunner) + console.log( + `::error file=${file}::${file}: no runner for this extension (${Object.keys(RUNNER).join(", ")}); ` + + `rename it or list it as "run by "`, + ); + for (const { file, why } of result.notRun) + console.log(`::error file=${file}::${file} is not run: ${why}`); + for (const { file, runBy } of result.elsewhere) rows.push(`| \`${file}\` | run by \`${runBy}\` | |`); for (const file of result.pending) rows.push(`| \`${file}\` | not present: its PR has not merged | |`); for (const file of result.unlisted) rows.push(`| \`${file}\` | **not in ${LIST}** | |`); for (const file of result.deleted) rows.push(`| \`${file}\` | **deleted while listed** | |`); + for (const file of result.noRunner) rows.push(`| \`${file}\` | **no runner for its extension** | |`); + for (const { file, why } of result.notRun) rows.push(`| \`${file}\` | **not run: ${why}** | |`); const summary = process.env.GITHUB_STEP_SUMMARY; if (summary) @@ -94,6 +165,6 @@ function main() { if (failed) process.exitCode = 1; } -module.exports = { plan, readList }; +module.exports = { plan, readList, findTests }; if (require.main === module) main(); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index 8703b0d841c84..bf32edbf3f47f 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -1,30 +1,90 @@ const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); const test = require("node:test"); -const { plan, readList } = require("./fork-ci-tests.cjs"); +const { plan, readList, findTests } = require("./fork-ci-tests.cjs"); const never = () => false; +const here = (...files) => files.map((file) => ({ file, runBy: null })); -test("runs every present test, listed or not", () => { - assert.deepEqual(plan(["a"], ["a", "b"], never).run, ["a", "b"]); +test("runs every listed present test with a runner", () => { + assert.deepEqual(plan(here("a.test.sh", "b.test.mjs"), ["a.test.sh", "b.test.mjs"], never).run, [ + "a.test.sh", + "b.test.mjs", + ]); }); test("a present test missing from the list fails", () => { - assert.deepEqual(plan(["a"], ["a", "b"], never).unlisted, ["b"]); + assert.deepEqual(plan(here("a.test.sh"), ["a.test.sh", "b.test.js"], never).unlisted, [ + "b.test.js", + ]); +}); + +test("a listed test with no runner for its extension fails and is not run", () => { + const result = plan(here("a.test.ts"), ["a.test.ts"], never); + assert.deepEqual(result.noRunner, ["a.test.ts"]); + assert.deepEqual(result.run, []); }); test("a listed test never in history is pending, not a failure", () => { - const result = plan(["a", "later"], ["a"], never); - assert.deepEqual(result.pending, ["later"]); + const result = plan(here("a.test.sh", "later.test.sh"), ["a.test.sh"], never); + assert.deepEqual(result.pending, ["later.test.sh"]); assert.deepEqual(result.deleted, []); }); test("a listed test deleted from history fails", () => { - const result = plan(["a", "gone"], ["a"], (file) => file === "gone"); - assert.deepEqual(result.deleted, ["gone"]); + const result = plan(here("a.test.sh", "gone.test.sh"), ["a.test.sh"], (f) => f === "gone.test.sh"); + assert.deepEqual(result.deleted, ["gone.test.sh"]); assert.deepEqual(result.pending, []); }); -test("the list ignores comments and blank lines", () => { - assert.deepEqual(readList("# head\n\na.sh # from #2\n b.py\n"), ["a.sh", "b.py"]); +test("a test run elsewhere must be named by its runner, which is not an upstream workflow", () => { + const files = { + ".github/workflows/fork-tests.yml": "run: node --test x/named.test.cjs", + ".github/workflows/ci.yml": "run: node --test x/upstream.test.cjs", + }; + const entries = [ + { file: "x/named.test.cjs", runBy: ".github/workflows/fork-tests.yml" }, + { file: "x/unnamed.test.cjs", runBy: ".github/workflows/fork-tests.yml" }, + { file: "x/upstream.test.cjs", runBy: ".github/workflows/ci.yml" }, + { file: "x/nowhere.test.cjs", runBy: "scripts/missing.ts" }, + ]; + const result = plan( + entries, + entries.map((e) => e.file), + never, + (file) => files[file] ?? null, + ); + assert.deepEqual(result.run, []); + assert.deepEqual(result.elsewhere, [ + { file: "x/named.test.cjs", runBy: ".github/workflows/fork-tests.yml" }, + ]); + assert.deepEqual( + result.notRun.map((n) => n.file), + ["x/unnamed.test.cjs", "x/upstream.test.cjs", "x/nowhere.test.cjs"], + ); + assert.match(result.notRun[1].why, /upstream workflow/); +}); + +test("the list ignores comments and blank lines and reads `run by`", () => { + assert.deepEqual(readList("# head\n\na.test.sh # from #2\n b.test.py run by c.yml\n"), [ + { file: "a.test.sh", runBy: null }, + { file: "b.test.py", runBy: "c.yml" }, + ]); + assert.throws(() => readList("a.test.sh run c.yml\n"), /cannot read/); +}); + +test("finds every *.test.* file, subdirectories included", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fork-ci-tests-")); + try { + for (const file of ["s/a.test.cjs", "s/b.cjs", "s/sub/c.test.mjs", "s/sub/deep/d.test.ts"]) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), ""); + } + assert.deepEqual(findTests("s", root), ["s/a.test.cjs", "s/sub/c.test.mjs", "s/sub/deep/d.test.ts"]); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } }); diff --git a/.github/scripts/fork-secret-scan.sh b/.github/scripts/fork-secret-scan.sh new file mode 100755 index 0000000000000..d9405f39c9618 --- /dev/null +++ b/.github/scripts/fork-secret-scan.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Fork only: the Secret scan job of fork-tests.yml. Picks the commits a pull request or a push adds +# and runs gitleaks over them. Tests: fork-secret-scan.test.sh. +# +# Environment: +# GITLEAKS gitleaks binary +# EVENT_NAME pull_request or push +# BASE_SHA pull_request: the base branch commit the PR is compared with +# HEAD_SHA pull_request: the PR head +# BEFORE push: the branch tip before the push (all zeros when the branch is new) +# AFTER push: the branch tip after the push +# +# Ranges: +# pull_request BASE_SHA..HEAD_SHA: every commit the PR adds, merges of the base branch +# included, whatever their dates. A squash merge never brings the PR's own commits +# to main, but they stay public under refs/pull/N/head, so they are scanned here. +# push BEFORE..AFTER. After a force push that is every commit the new tip has and the +# old one had not; commits both share were scanned when they were first pushed. +# push, no BEFORE a new branch (BEFORE all zeros) or an old tip that can no longer be fetched: +# only AFTER itself, with a warning. Everything before it is history this fork +# already had (upstream's, or scanned by earlier runs), and there is no older tip +# to tell new commits from old ones. +# Merge commits are diffed with --remerge-diff: only what the merge itself changed beyond an +# automatic merge of its parents (a conflict resolution, or a file added in the merge). Without it +# git log shows merges with no diff at all, so gitleaks would never read them. +set -euo pipefail + +have() { git cat-file -e "$1^{commit}" 2>/dev/null; } +# Commits only, file contents on demand: the checkout is a blob-less clone. +fetch() { + have "$1" || git fetch --no-tags --quiet --filter=blob:none origin "$1" 2>/dev/null || true + have "$1" +} +zero() { [[ -z "$1" || "$1" =~ ^0+$ ]]; } +summary() { if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then printf '%s\n' "$@" >> "$GITHUB_STEP_SUMMARY"; fi; } + +warning="" +case "$EVENT_NAME" in + pull_request) + for sha in "$BASE_SHA" "$HEAD_SHA"; do + if ! fetch "$sha"; then + echo "::error::Secret scan: cannot fetch commit $sha" >&2 + exit 2 + fi + done + range="$BASE_SHA..$HEAD_SHA" + opts="--remerge-diff $range" + what="the pull request's $(git rev-list --count "$range") commit(s) after base ${BASE_SHA:0:12}, merges included" + ;; + push) + if ! zero "$BEFORE" && fetch "$BEFORE"; then + range="$BEFORE..$AFTER" + opts="--remerge-diff $range" + if git merge-base --is-ancestor "$BEFORE" "$AFTER"; then + what="the $(git rev-list --count "$range") commit(s) pushed after ${BEFORE:0:12}" + else + what="force push: the $(git rev-list --count "$range") commit(s) ${AFTER:0:12} has that the replaced tip ${BEFORE:0:12} had not" + fi + else + range="--max-count=1 $AFTER" + opts="--remerge-diff $range" + if zero "$BEFORE"; then + warning="a new branch has no earlier tip, so only its newest commit ${AFTER:0:12} is scanned" + else + warning="the earlier tip ${BEFORE:0:12} cannot be fetched (force push?), so only the newest commit ${AFTER:0:12} is scanned" + fi + what="the pushed commit ${AFTER:0:12}" + fi + ;; + *) + echo "::error::Secret scan: unexpected event $EVENT_NAME" >&2 + exit 2 + ;; +esac + +echo "Scanning $what ($range)" +if [[ -n "$warning" ]]; then echo "::warning title=Secret scan::$warning"; fi +set +e +"$GITLEAKS" git --no-banner --redact --verbose --log-opts="$opts" . +status=$? +set -e +summary "### Secret scan" "" +if [[ -n "$warning" ]]; then summary "Warning: $warning." ""; fi +if [[ $status -eq 0 ]]; then + summary "No secrets found in $what." +else + summary "gitleaks found possible secrets in $what (values redacted; see the job log). Rotate any real secret: it is already public." +fi +exit "$status" diff --git a/.github/scripts/fork-secret-scan.test.sh b/.github/scripts/fork-secret-scan.test.sh new file mode 100755 index 0000000000000..4b85fb13b1c9c --- /dev/null +++ b/.github/scripts/fork-secret-scan.test.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# Tests for fork-secret-scan.sh against local git repositories; no network. +# bash .github/scripts/fork-secret-scan.test.sh +# GITLEAKS_BIN=path runs the real gitleaks; without it a stand-in greps the scanned commits' diffs +# for an AWS access key id. FORK_SECRET_SCAN_SH picks another script. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script="$(realpath "${FORK_SECRET_SCAN_SH:-$here/fork-secret-scan.sh}")" +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT + +export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$root/gitconfig" +git config --global user.name test +git config --global user.email test@example.invalid +git config --global init.defaultBranch main +git config --global advice.detachedHead false +unset GITHUB_STEP_SUMMARY GITHUB_OUTPUT + +if [[ -n "${GITLEAKS_BIN:-}" ]]; then + gitleaks="$(realpath "$GITLEAKS_BIN")" +else + gitleaks="$root/fake-gitleaks" + export TMPDIR="$root" + cat > "$gitleaks" <<'FAKE' +#!/usr/bin/env bash +# Stand-in: `gitleaks git ... --log-opts=OPTS .` fails when the diffs of `git log OPTS` add a key id. +set -euo pipefail +opts="" +for arg in "$@"; do [[ "$arg" == --log-opts=* ]] && opts="${arg#--log-opts=}"; done +# Into a file first: `grep -q` stops reading early, and pipefail would read git's SIGPIPE as no match. +# shellcheck disable=SC2086 +git log -p -U0 $opts > "$TMPDIR/fake-gitleaks.log" +if grep -qE '^\+.*(AKIA)[A-Z2-7]{16}' "$TMPDIR/fake-gitleaks.log"; then echo "leaks found: 1"; exit 1; fi +echo "no leaks found" +FAKE + chmod +x "$gitleaks" +fi + +# A made-up AWS access key id, built at run time so this file holds no key. +key="AK""IA""QX7WMB3KZLRT5NHP" +date=1700000000 +commit() { # commit FILE CONTENT MESSAGE (each commit one minute newer) + date=$((date + 60)) + printf '%s\n' "$2" > "$1" + git add -A + GIT_AUTHOR_DATE="@$date" GIT_COMMITTER_DATE="@$date" git commit -qm "$3" +} +remove() { + date=$((date + 60)) + git rm -q "$1" + GIT_AUTHOR_DATE="@$date" GIT_COMMITTER_DATE="@$date" git commit -qm "$2" +} +merge() { + date=$((date + 60)) + GIT_AUTHOR_DATE="@$date" GIT_COMMITTER_DATE="@$date" git merge -q --no-ff -m "$2" "$1" +} + +failures=0 +check() { # check NAME EXPECTED_STATUS [GREP_PATTERN] -- runs the scan in $clone with the env given + local name="$1" expected="$2" pattern="${3:-}" out status + set +e + out="$(cd "$clone" && GITLEAKS="$gitleaks" GITHUB_STEP_SUMMARY="$root/summary" "$script" 2>&1)" + status=$? + set -e + if [[ "$status" -ne "$expected" ]]; then + echo "FAIL $name: exit $status, expected $expected" + printf '%s\n' "$out" | sed 's/^/ /' + failures=$((failures + 1)) + elif [[ -n "$pattern" ]] && ! grep -qE "$pattern" <<< "$out"; then + echo "FAIL $name: output lacks /$pattern/" + printf '%s\n' "$out" | sed 's/^/ /' + failures=$((failures + 1)) + else + echo "ok $name" + fi + rm -f "$root/summary" +} + +# origin: main with an old key on it (already public, not this PR's business), then two PR +# branches that each merge three newer main commits. +git init -q --bare "$root/origin.git" +git -C "$root/origin.git" config uploadpack.allowFilter true +git -C "$root/origin.git" config uploadpack.allowAnySHA1InWant true +git init -q "$root/work" +cd "$root/work" +git remote add origin "$root/origin.git" +commit app.txt "app v1" "main 1" +commit old.txt "old=$key" "main 2: an old key" +remove old.txt "main 3: drop the old key" +fork_point=$(git rev-parse HEAD) + +git checkout -q -b leaky +commit config.txt "aws_access_key_id=$key" "PR A: add a key" +remove config.txt "PR B: delete it again" +git checkout -q -b clean "$fork_point" +commit notes.txt "notes" "PR A': no key" + +git checkout -q main +for n in 4 5 6; do commit "app$n.txt" "app v$n" "main $n"; done +base=$(git rev-parse HEAD) + +for branch in leaky clean; do + git checkout -q "$branch" + merge main "Merge main into $branch" + commit "$branch-tail.txt" "tail" "PR C: after the merge" +done +# A key that only the merge commit adds (on top of an automatic merge of main). +git checkout -q -b evil "$fork_point" +commit notes2.txt "notes" "PR A'': no key" +git merge -q --no-ff --no-commit main >/dev/null 2>&1 +printf '%s\n' "token=$key" > merged.txt +git add merged.txt +date=$((date + 60)) +GIT_AUTHOR_DATE="@$date" GIT_COMMITTER_DATE="@$date" git commit -qm "Merge main into evil" +git push -q origin main leaky clean evil +leaky=$(git rev-parse leaky) +clean=$(git rev-parse clean) +evil=$(git rev-parse evil) + +# The workflow's checkout: blob-less, only the PR head's history (no main ref). +clone="$root/clone" +git clone -q --filter=blob:none --no-checkout --single-branch --branch leaky "file://$root/origin.git" "$clone" + +export EVENT_NAME=pull_request BASE_SHA="$base" BEFORE="" AFTER="" +# The count GitHub reports for the PR, for scripts that use it. +COMMITS=$(git rev-list --count "$base..$leaky") +export COMMITS +HEAD_SHA="$leaky" check "pull request: a key added then deleted before a merge of main is found" 1 +HEAD_SHA="$clean" check "pull request: main's own old key is not the PR's" 0 +HEAD_SHA="$leaky" check "pull request: the range is base..head" 1 "\\.\\.$leaky" +HEAD_SHA="$evil" check "pull request: a key added only in a merge commit is found" 1 + +export EVENT_NAME=push BASE_SHA="" HEAD_SHA="" +BEFORE="$fork_point" AFTER="$leaky" check "push: the pushed commits are scanned" 1 "pushed after" +BEFORE="$(git rev-parse leaky~1)" AFTER="$leaky" check "push: only commits after the old tip" 0 +BEFORE="$clean" AFTER="$leaky" check "force push: commits the old tip had not" 1 "force push" +BEFORE="$fork_point" AFTER="$evil" check "push: a key added only in a merge commit is found" 1 +BEFORE="0000000000000000000000000000000000000000" AFTER="$leaky" \ + check "push of a new branch: only its newest commit, with a warning" 0 "::warning.*new branch" +BEFORE="1111111111111111111111111111111111111111" AFTER="$leaky" \ + check "push: an old tip that cannot be fetched warns" 0 "::warning.*cannot be fetched" + +EVENT_NAME=workflow_dispatch check "an unknown event fails" 2 + +if [[ "$failures" -ne 0 ]]; then + echo "$failures check(s) failed" + exit 1 +fi +echo "all checks passed" diff --git a/.github/scripts/fork-tests-drift.cjs b/.github/scripts/fork-tests-drift.cjs index d39e73267feee..86f5e8b3940ed 100644 --- a/.github/scripts/fork-tests-drift.cjs +++ b/.github/scripts/fork-tests-drift.cjs @@ -1,7 +1,14 @@ -// Fork only: fails when upstream's ci.yml or release.yml test jobs change and -// .github/workflows/fork-tests.yml did not follow. Compares a digest of every -// job and step with .github/fork-tests-upstream.json; comments are ignored. -// After porting a change: node .github/scripts/fork-tests-drift.cjs --record +// Fork only: fails when upstream's ci.yml or release.yml test jobs change since +// the copy in .github/workflows/fork-tests.yml last followed them. Compares a +// digest of every job and step, and of the top-level keys that change how every +// job runs (TOP_LEVEL), with .github/fork-tests-upstream.json; comments are ignored. +// +// Limit: the check compares upstream with its own record, not with +// fork-tests.yml. Recording without porting passes, so --record prints what +// changed, and the commit and PR that update the record must say, for each line, +// what was ported or why it does not apply. +// node .github/scripts/fork-tests-drift.cjs check +// node .github/scripts/fork-tests-drift.cjs --record after porting: print the changes, save the record const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); @@ -15,6 +22,10 @@ const TRACKED = { ".github/workflows/release.yml": ["quality", "test", "test_web", "test_server"], }; +// Top-level keys digested as the pseudo-job TOP (job ids cannot contain parentheses). +const TOP_LEVEL = ["env", "defaults", "concurrency", "permissions"]; +const TOP = "(top-level)"; + const digest = (lines) => crypto.createHash("sha256").update(lines.join("\n")).digest("hex").slice(0, 16); @@ -29,6 +40,18 @@ function parseJobs(text) { const start = lines.findIndex((line) => line === "jobs:"); if (start === -1) throw new Error("no top-level `jobs:` key"); + // Top-level keys, wherever they are in the file: one "step" per key present. + const top = []; + let topKey = null; + for (const line of lines) { + if (indentOf(line) === 0) { + const key = line.match(/^([A-Za-z0-9_-]+):/)?.[1]; + topKey = TOP_LEVEL.includes(key) ? { name: key, lines: [] } : null; + if (topKey) top.push(topKey); + } + topKey?.lines.push(line); + } + const jobs = {}; let job = null; let inSteps = false; @@ -65,6 +88,11 @@ function parseJobs(text) { } const result = {}; + if (top.length) + result[TOP] = { + header: digest([]), + steps: top.map((key) => ({ name: key.name, digest: digest(key.lines) })), + }; for (const [name, { header, steps }] of Object.entries(jobs)) { const seen = new Map(); result[name] = { @@ -86,19 +114,22 @@ function parseJobs(text) { return result; } +// Returns the digests and the tracked jobs upstream no longer has. function current() { const out = {}; + const missing = []; for (const [file, only] of Object.entries(TRACKED)) { const jobs = parseJobs(fs.readFileSync(path.join(root, file), "utf8")); if (only) { - const missing = only.filter((name) => !jobs[name]); - out[file] = Object.fromEntries(only.map((name) => [name, jobs[name] ?? null])); - if (missing.length) console.error(`${file}: tracked jobs missing: ${missing.join(", ")}`); + for (const name of only) if (!jobs[name]) missing.push(`${file}: job \`${name}\``); + out[file] = Object.fromEntries( + [TOP, ...only].filter((name) => jobs[name]).map((name) => [name, jobs[name]]), + ); } else { out[file] = jobs; } } - return out; + return { now: out, missing }; } function compare(recorded, now) { @@ -107,13 +138,29 @@ function compare(recorded, now) { const before = recorded[file] ?? {}; const after = now[file] ?? {}; for (const job of new Set([...Object.keys(before), ...Object.keys(after)])) { - const a = before[job]; - const b = after[job]; - if (!a && b) { + // A record saved before --record refused missing jobs may hold null. + const a = before[job] ?? null; + const b = after[job] ?? null; + if (job === TOP) { + const oldKeys = new Map((a?.steps ?? []).map((s) => [s.name, s.digest])); + const newKeys = new Map((b?.steps ?? []).map((s) => [s.name, s.digest])); + for (const [key, d] of newKeys) { + if (!oldKeys.has(key)) changes.push(`${file}: top-level \`${key}:\` added`); + else if (oldKeys.get(key) !== d) changes.push(`${file}: top-level \`${key}:\` changed`); + } + for (const key of oldKeys.keys()) + if (!newKeys.has(key)) changes.push(`${file}: top-level \`${key}:\` removed`); + continue; + } + if (!a && !b) { + changes.push(`${file}: job \`${job}\` missing upstream (renamed or removed?)`); + continue; + } + if (!a) { changes.push(`${file}: job \`${job}\` added upstream`); continue; } - if (a && !b) { + if (!b) { changes.push(`${file}: job \`${job}\` removed upstream`); continue; } @@ -139,13 +186,45 @@ function compare(recorded, now) { return changes; } -function main() { - const now = current(); - if (process.argv.includes("--record")) { - fs.writeFileSync(recordPath, `${JSON.stringify(now, null, 2)}\n`); - console.log(`Recorded ${path.relative(root, recordPath)}`); +function readRecord(file) { + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + if (error.code === "ENOENT") return {}; + throw error; + } +} + +// Refuses while a tracked job is missing: the copy in fork-tests.yml then follows +// a job upstream no longer has, and TRACKED must be updated first. +function record(now, missing, file = recordPath) { + if (missing.length) { + console.error( + `Not recorded: tracked jobs missing upstream: ${missing.join(", ")}.\n` + + "Find what replaced them, port that into fork-tests.yml and update TRACKED in " + + "fork-tests-drift.cjs, then record again.", + ); + process.exitCode = 1; return; } + const changes = compare(readRecord(file), now); + fs.writeFileSync(file, `${JSON.stringify(now, null, 2)}\n`); + console.log(`Recorded ${path.relative(root, file)}.`); + if (changes.length === 0) { + console.log("Nothing changed upstream since the last record."); + return; + } + console.log( + `\nUpstream changes this record accepts (${changes.length}):\n` + + changes.map((c) => `- ${c}`).join("\n") + + "\n\nSay in the commit message and the PR body, for each line, what was ported " + + "into fork-tests.yml or why it does not apply. The check cannot tell; reviewers rely on it.", + ); +} + +function main() { + const { now, missing } = current(); + if (process.argv.includes("--record")) return record(now, missing); const recorded = JSON.parse(fs.readFileSync(recordPath, "utf8")); const changes = compare(recorded, now); const summary = process.env.GITHUB_STEP_SUMMARY; @@ -167,6 +246,6 @@ function main() { process.exitCode = 1; } -module.exports = { parseJobs, compare }; +module.exports = { parseJobs, compare, record, TOP }; if (require.main === module) main(); diff --git a/.github/scripts/fork-tests-drift.test.cjs b/.github/scripts/fork-tests-drift.test.cjs index 401ab9fd396c1..ee56bff886ebe 100644 --- a/.github/scripts/fork-tests-drift.test.cjs +++ b/.github/scripts/fork-tests-drift.test.cjs @@ -1,7 +1,7 @@ const assert = require("node:assert/strict"); const test = require("node:test"); -const { parseJobs, compare } = require("./fork-tests-drift.cjs"); +const { parseJobs, compare, record } = require("./fork-tests-drift.cjs"); const base = `name: CI on: push @@ -66,3 +66,54 @@ test("repeated step names stay distinct", () => { const names = parseJobs(twice).lint.steps.map((s) => s.name); assert.deepEqual(names.slice(-2), ["Check", "Check #2"]); }); + +test("top-level env, defaults, concurrency and permissions count, wherever they are", () => { + const changed = `env:\n FOO: bar\n${base}defaults:\n run:\n shell: bash\n`; + assert.deepEqual(compare(wrap(base), wrap(changed)), [ + "ci.yml: top-level `env:` added", + "ci.yml: top-level `defaults:` added", + ]); + const withPerms = base.replace("jobs:", "permissions:\n contents: read\njobs:"); + assert.deepEqual(compare(wrap(withPerms), wrap(withPerms.replace(": read", ": write"))), [ + "ci.yml: top-level `permissions:` changed", + ]); + assert.deepEqual(compare(wrap(withPerms), wrap(base)), ["ci.yml: top-level `permissions:` removed"]); + // `on:` and `name:` are not digested. + assert.deepEqual(compare(wrap(base), wrap(base.replace("on: push", "on: pull_request"))), []); +}); + +test("a tracked job missing on both sides is reported, not a crash", () => { + const recorded = { "release.yml": { test_web: null } }; + assert.deepEqual(compare(recorded, { "release.yml": {} }), [ + "release.yml: job `test_web` missing upstream (renamed or removed?)", + ]); +}); + +test("--record refuses while a tracked job is missing and writes nothing", () => { + const fs = require("node:fs"); + const os = require("node:os"); + const path = require("node:path"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "fork-drift-")); + const file = path.join(dir, "record.json"); + const error = console.error; + const log = console.log; + const said = []; + console.error = console.log = (line) => said.push(line); + try { + record(wrap(base), ["release.yml: job `test_web`"], file); + assert.equal(process.exitCode, 1); + assert.equal(fs.existsSync(file), false); + assert.match(said.join("\n"), /Not recorded: tracked jobs missing upstream/); + process.exitCode = undefined; + said.length = 0; + record(wrap(base), [], file); + assert.equal(process.exitCode, undefined); + assert.deepEqual(JSON.parse(fs.readFileSync(file, "utf8")), wrap(base)); + assert.match(said.join("\n"), /ci.yml: job `lint` added upstream[\s\S]*what was ported/); + } finally { + console.error = error; + console.log = log; + process.exitCode = undefined; + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index 1f6566c9690a1..8900a8530c46a 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -39,7 +39,7 @@ on: workflow_dispatch: inputs: ref: - description: Commit or branch to test + description: Commit or branch to test (resolved to one commit for every job) required: true type: string @@ -58,6 +58,9 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: + # Release runs: the commit every job checks out. Empty on pull_request and + # push, where each checkout already takes the run's own commit. + sha: ${{ steps.resolve.outputs.sha }} typecheck: ${{ steps.release.outputs.typecheck || steps.detect.outputs.typecheck }} build: ${{ steps.release.outputs.build || steps.detect.outputs.build }} desktop_launch: ${{ steps.release.outputs.desktop_launch || steps.detect.outputs.desktop_launch }} @@ -102,6 +105,28 @@ jobs: fi echo "mobile_native=$value" >> "$GITHUB_OUTPUT" + # A branch name is resolved once, here: jobs start minutes apart, and a + # branch that moves in between would otherwise test two commits in one run. + - name: Checkout the commit to test + if: inputs.ref != '' + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + /package.json + sparse-checkout-cone-mode: false + + - name: Resolve the commit to test + id: resolve + if: inputs.ref != '' + env: + REF: ${{ inputs.ref }} + run: | + sha=$(git rev-parse HEAD) + echo "Testing $sha (from \`$REF\`) in every job." + echo "Release run: every job tests \`$sha\` (from \`$REF\`)." >> "$GITHUB_STEP_SUMMARY" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + # Release runs test the whole commit. upstream's release.yml has no macOS lint. - name: Release run checks everything id: release @@ -118,15 +143,18 @@ jobs: # upstream changed since the copy was last recorded. workflows: name: Workflow lint and upstream drift + needs: changes runs-on: ubuntu-24.04 timeout-minutes: 5 steps: + # scripts/: some tests are run by a script there (.github/fork-ci-tests.txt). - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /.github/ + /scripts/ sparse-checkout-cone-mode: false # Pinned release, verified by checksum. Upstream's own workflows use runner @@ -143,9 +171,10 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" actionlint "$RUNNER_TEMP/actionlint" -color .github/workflows/fork-*.yml - # Every fork script test, checked against .github/fork-ci-tests.txt: an - # unlisted test or a listed one deleted from history fails, a listed one - # whose pull request has not merged yet is a note in the summary. + # Every test under .github/scripts, checked against .github/fork-ci-tests.txt: + # the fork's tests run here, the others must be named by the job or script + # that runs them. An unlisted test, or a listed one deleted from history, + # fails; a listed one whose pull request has not merged yet is a note. - name: Fork script tests run: | if compgen -G '.github/scripts/fork-*.test.py' >/dev/null && ! python3 -c 'import yaml' 2>/dev/null; then @@ -159,21 +188,21 @@ jobs: # Every commit a pull request or a push to main adds, history included: a # secret committed and then deleted is still public. Release runs test a # commit that already passed this on its way to main, so they skip it. + # The ranges, and why: .github/scripts/fork-secret-scan.sh. secrets: name: Secret scan if: ${{ inputs.ref == '' }} runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - # A pull request fetches only its own commits (plus the parent of the - # oldest, below). A push fetches history without file contents; - # gitleaks then reads only the pushed commits. + # Full history without file contents; gitleaks then fetches the contents + # of only the scanned commits. - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - fetch-depth: ${{ github.event_name == 'pull_request' && github.event.pull_request.commits || 0 }} - filter: ${{ github.event_name != 'pull_request' && 'blob:none' || '' }} + fetch-depth: 0 + filter: blob:none - name: Install gitleaks env: @@ -186,46 +215,33 @@ jobs: echo "$SHA256 $RUNNER_TEMP/$archive" | sha256sum -c - tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" gitleaks + # The range rules against this gitleaks: a key added and deleted again + # before a merge of main, a key only a merge commit adds, push ranges. + - name: Test the scan ranges + env: + GITLEAKS_BIN: ${{ runner.temp }}/gitleaks + run: bash .github/scripts/fork-secret-scan.test.sh + - name: Scan the added commits env: + GITLEAKS: ${{ runner.temp }}/gitleaks EVENT_NAME: ${{ github.event_name }} - COMMITS: ${{ github.event.pull_request.commits }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} BEFORE: ${{ github.event.before }} - run: | - if [[ "$EVENT_NAME" == pull_request ]]; then - # One more commit, so the oldest one is diffed against its parent - # instead of being read as a root that adds the whole tree. - git fetch --no-tags --quiet --deepen=1 origin "$(git rev-parse HEAD)" - range="--max-count=$COMMITS" - what="the pull request's $COMMITS commit(s)" - elif [[ -n "$BEFORE" && ! "$BEFORE" =~ ^0+$ ]] && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then - range="$BEFORE..$GITHUB_SHA" - what="the commits pushed after ${BEFORE:0:12}" - else - range="--max-count=1" - what="the pushed commit (no earlier commit to compare with)" - fi - echo "Scanning $what" - set +e - "$RUNNER_TEMP/gitleaks" git --no-banner --redact --verbose --log-opts="$range" . - status=$? - set -e - if [[ $status -eq 0 ]]; then - echo "### Secret scan"$'\n\n'"No secrets found in $what." >> "$GITHUB_STEP_SUMMARY" - else - echo "### Secret scan"$'\n\n'"gitleaks found possible secrets in $what (values redacted; see the job log). Rotate any real secret: it is already public." >> "$GITHUB_STEP_SUMMARY" - fi - exit $status + AFTER: ${{ github.sha }} + run: bash .github/scripts/fork-secret-scan.sh lint: name: Lint + needs: changes runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /* !/.repos/ @@ -274,7 +290,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /* !/.repos/ @@ -427,7 +443,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /* !/.repos/ @@ -481,9 +497,11 @@ jobs: if: matrix.shard == 1 run: python3 -B .github/scripts/stage-preview-bundle.test.py + # Upstream's ci.yml runs the first four; the fork adds thread-transfer-report, + # which upstream runs from a workflow_run of its CI (disabled in the fork). - name: Test release checks if: matrix.shard == 1 - run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs .github/scripts/test-delta.test.cjs + run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs .github/scripts/test-delta.test.cjs .github/scripts/thread-transfer-report.test.cjs - name: Finish installing browser secret helper build libraries if: steps.shard.outputs.any == 'true' @@ -542,7 +560,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /* !/.repos/ @@ -589,10 +607,13 @@ jobs: run: eval "$(jq -r .scripts.test package.json) --shard=$SHARD" # apps/server runs its test files one at a time; shards spread them over - # separate runners. The shards gate themselves instead of waiting for - # `changes`; a shard the change cannot affect says so in its summary. + # separate runners. As upstream, the shards run the change detector + # themselves, so the steps follow ci.yml one for one; a shard the change + # cannot affect says so in its summary. They wait for `changes` only for a + # release run's resolved commit (seconds; every other job waits for it too). test_server: name: Test Server ${{ matrix.shard }} + needs: changes runs-on: ubuntu-24.04 timeout-minutes: 20 strategy: @@ -603,7 +624,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} fetch-depth: 2 sparse-checkout: | /* @@ -735,7 +756,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /* !/.repos/ @@ -771,7 +792,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /* !/.repos/ @@ -802,7 +823,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.changes.outputs.sha }} sparse-checkout: | /* !/.repos/ From 675886adf1b7280f9bd24a0b063fac0689f9096e Mon Sep 17 00:00:00 2001 From: t3code-fork Date: Thu, 12 Nov 2026 12:43:54 +0000 Subject: [PATCH 9/9] fork-tests: fix round 2 (secret scan reads every commit offline, Lint) Secret scan (.github/scripts/fork-secret-scan.sh, tests: fork-secret-scan.test.sh) - The checkout is blob-less, so gitleaks fetched file contents on demand. gitleaks 8.30.1 stops at the first git error (a failed content fetch), logs "0 commits scanned" and exits 0, and the summary said "No secrets found". The script now runs gitleaks' own `git log -p -U0 ` first: it fetches every blob the scan reads and fails the job (exit 2) when a fetch fails. gitleaks then runs with GIT_NO_LAZY_FETCH=1. - The job also fails when gitleaks scanned fewer commits than that git log has commits with a hunk in a file they do not delete (what gitleaks counts; a pure rename or a clean merge has none, so it can be below rev-list --count). - New tests: contents unreachable after the clone, gitleaks stopping early, and a base or head commit that cannot be fetched. Lint (vp check) - Formatting of fork-ci-tests.cjs, fork-ci-tests.test.cjs, fork-tests-drift.test.cjs. - The header comments of fork-ci-tests.cjs and fork-tests-drift.cjs cut to 4 lines (t3code/no-low-value-comments). Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.cjs | 26 ++++++------ .github/scripts/fork-ci-tests.test.cjs | 12 +++++- .github/scripts/fork-secret-scan.sh | 38 ++++++++++++++++- .github/scripts/fork-secret-scan.test.sh | 50 +++++++++++++++++++++-- .github/scripts/fork-tests-drift.cjs | 15 ++----- .github/scripts/fork-tests-drift.test.cjs | 4 +- .github/workflows/fork-tests.yml | 5 ++- 7 files changed, 117 insertions(+), 33 deletions(-) diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index ef9fe4f8fcf97..2bca87ed6b6eb 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -1,11 +1,7 @@ -// Fork only: every test file under `.github/scripts` (`*.test.*`, subdirectories -// included) must be run somewhere, checked against `.github/fork-ci-tests.txt`. -// A plain entry is run here, by its extension (RUNNER). An entry -// ` run by ` is run by that file, which must name the test's path; it -// cannot be one of upstream's workflows, which are disabled in the fork. A -// present test missing from the list fails, and so does a test with no runner -// for its extension. A listed test that is missing fails once it has been in the -// tested commit's history (deleted while still listed) and is a note before that. +// Fork only: every `*.test.*` under .github/scripts must be listed in .github/fork-ci-tests.txt. +// A plain entry runs here by its extension (RUNNER); ` run by ` is run by that file, +// which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less +// tests fail; a listed test that is gone fails once the tested history had it, a note before. const childProcess = require("node:child_process"); const fs = require("node:fs"); const path = require("node:path"); @@ -41,7 +37,8 @@ function plan(entries, present, ever, read = () => null) { for (const { file, runBy } of entries) { if (!runBy || !present.includes(file)) continue; let why = null; - if (UPSTREAM_WORKFLOW.test(runBy)) why = `${runBy} is an upstream workflow, disabled in the fork`; + if (UPSTREAM_WORKFLOW.test(runBy)) + why = `${runBy} is an upstream workflow, disabled in the fork`; else { const text = read(runBy); if (text === null) why = `${runBy} does not exist`; @@ -68,7 +65,10 @@ function readList(text) { .filter(Boolean) .map((line) => { const match = line.match(/^(\S+)(?:\s+run by\s+(\S+))?$/); - if (!match) throw new Error(`${LIST}: cannot read "${line}" (expected "" or " run by ")`); + if (!match) + throw new Error( + `${LIST}: cannot read "${line}" (expected "" or " run by ")`, + ); return { file: match[1], runBy: match[2] ?? null }; }); } @@ -141,12 +141,14 @@ function main() { ); for (const { file, why } of result.notRun) console.log(`::error file=${file}::${file} is not run: ${why}`); - for (const { file, runBy } of result.elsewhere) rows.push(`| \`${file}\` | run by \`${runBy}\` | |`); + for (const { file, runBy } of result.elsewhere) + rows.push(`| \`${file}\` | run by \`${runBy}\` | |`); for (const file of result.pending) rows.push(`| \`${file}\` | not present: its PR has not merged | |`); for (const file of result.unlisted) rows.push(`| \`${file}\` | **not in ${LIST}** | |`); for (const file of result.deleted) rows.push(`| \`${file}\` | **deleted while listed** | |`); - for (const file of result.noRunner) rows.push(`| \`${file}\` | **no runner for its extension** | |`); + for (const file of result.noRunner) + rows.push(`| \`${file}\` | **no runner for its extension** | |`); for (const { file, why } of result.notRun) rows.push(`| \`${file}\` | **not run: ${why}** | |`); const summary = process.env.GITHUB_STEP_SUMMARY; diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index bf32edbf3f47f..28c72efa155b6 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -35,7 +35,11 @@ test("a listed test never in history is pending, not a failure", () => { }); test("a listed test deleted from history fails", () => { - const result = plan(here("a.test.sh", "gone.test.sh"), ["a.test.sh"], (f) => f === "gone.test.sh"); + const result = plan( + here("a.test.sh", "gone.test.sh"), + ["a.test.sh"], + (f) => f === "gone.test.sh", + ); assert.deepEqual(result.deleted, ["gone.test.sh"]); assert.deepEqual(result.pending, []); }); @@ -83,7 +87,11 @@ test("finds every *.test.* file, subdirectories included", () => { fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); fs.writeFileSync(path.join(root, file), ""); } - assert.deepEqual(findTests("s", root), ["s/a.test.cjs", "s/sub/c.test.mjs", "s/sub/deep/d.test.ts"]); + assert.deepEqual(findTests("s", root), [ + "s/a.test.cjs", + "s/sub/c.test.mjs", + "s/sub/deep/d.test.ts", + ]); } finally { fs.rmSync(root, { recursive: true, force: true }); } diff --git a/.github/scripts/fork-secret-scan.sh b/.github/scripts/fork-secret-scan.sh index d9405f39c9618..b6411894d226a 100755 --- a/.github/scripts/fork-secret-scan.sh +++ b/.github/scripts/fork-secret-scan.sh @@ -23,6 +23,15 @@ # Merge commits are diffed with --remerge-diff: only what the merge itself changed beyond an # automatic merge of its parents (a conflict resolution, or a file added in the merge). Without it # git log shows merges with no diff at all, so gitleaks would never read them. +# +# Offline, and counted: the checkout is a blob-less clone, so file contents are fetched on demand. +# gitleaks 8.30.1 stops reading at the first git error (a blob fetch that fails: network, GitHub +# 5xx), logs "0 commits scanned" and still exits 0. So the script runs gitleaks' own git log first, +# which fetches every blob the scan reads and fails the job if a fetch fails, then runs gitleaks with +# GIT_NO_LAZY_FETCH=1, and fails when gitleaks scanned fewer commits than that git log holds. +# gitleaks counts a commit when a file it neither deletes nor turns binary has a hunk: a pure +# rename, a mode change, an empty new file or a merge with nothing beyond the automatic merge has +# none, so the count can be below `git rev-list --count`. set -euo pipefail have() { git cat-file -e "$1^{commit}" 2>/dev/null; } @@ -75,14 +84,39 @@ esac echo "Scanning $what ($range)" if [[ -n "$warning" ]]; then echo "::warning title=Secret scan::$warning"; fi + +# gitleaks runs `git log -p -U0 `. The same command here fetches what it reads; counts the +# commits gitleaks will scan (see the header); and fails the job (pipefail) when a fetch fails. +# shellcheck disable=SC2086 +if ! expected="$(git log -p -U0 $opts | awk ' + /^commit / { sha = $2; next } + /^diff --git / { deleted = 0; next } + /^deleted file mode / { deleted = 1; next } + /^@@ / { if (!deleted && sha != "" && !(sha in seen)) { seen[sha] = 1; n++ } } + END { print n + 0 }')"; then + echo "::error::Secret scan: cannot read the commits to scan (a file content fetch failed?)" >&2 + summary "### Secret scan" "" "Failed: the contents of $what could not be read, so nothing was scanned." + exit 2 +fi + +log="$(mktemp)" +trap 'rm -f "$log"' EXIT set +e -"$GITLEAKS" git --no-banner --redact --verbose --log-opts="$opts" . +GIT_NO_LAZY_FETCH=1 "$GITLEAKS" git --no-banner --no-color --redact --verbose --log-opts="$opts" . > "$log" 2>&1 status=$? set -e +cat "$log" +scanned="$(sed -E 's/\x1b\[[0-9;]*m//g' "$log" | sed -nE 's/.* ([0-9]+) commits scanned.*/\1/p' | tail -n 1)" +echo "gitleaks scanned ${scanned:-no} commit(s); $expected commit(s) in range have changes it reads" summary "### Secret scan" "" if [[ -n "$warning" ]]; then summary "Warning: $warning." ""; fi +if [[ $status -eq 0 && ( -z "$scanned" || "$scanned" -lt "$expected" ) ]]; then + echo "::error::Secret scan: gitleaks scanned ${scanned:-no} commit(s), but $expected commit(s) in range have changes (a git error stopped it?)" >&2 + summary "Failed: gitleaks scanned ${scanned:-no} of the $expected commit(s) with changes in $what, so the rest were not checked." + exit 2 +fi if [[ $status -eq 0 ]]; then - summary "No secrets found in $what." + summary "No secrets found in $what ($scanned commit(s) with changes scanned)." else summary "gitleaks found possible secrets in $what (values redacted; see the job log). Rotate any real secret: it is already public." fi diff --git a/.github/scripts/fork-secret-scan.test.sh b/.github/scripts/fork-secret-scan.test.sh index 4b85fb13b1c9c..e5a0c191a1291 100755 --- a/.github/scripts/fork-secret-scan.test.sh +++ b/.github/scripts/fork-secret-scan.test.sh @@ -24,13 +24,24 @@ else export TMPDIR="$root" cat > "$gitleaks" <<'FAKE' #!/usr/bin/env bash -# Stand-in: `gitleaks git ... --log-opts=OPTS .` fails when the diffs of `git log OPTS` add a key id. -set -euo pipefail +# Stand-in: `gitleaks git ... --log-opts=OPTS .` fails when the diffs of `git log -p -U0 OPTS` add a +# key id, and logs how many commits it scanned. Like gitleaks 8.30.1, a git error (a file content +# fetch that fails) stops it with "0 commits scanned" and exit 0. +set -uo pipefail opts="" for arg in "$@"; do [[ "$arg" == --log-opts=* ]] && opts="${arg#--log-opts=}"; done # Into a file first: `grep -q` stops reading early, and pipefail would read git's SIGPIPE as no match. # shellcheck disable=SC2086 -git log -p -U0 $opts > "$TMPDIR/fake-gitleaks.log" +if ! git log -p -U0 $opts > "$TMPDIR/fake-gitleaks.log" 2> "$TMPDIR/fake-gitleaks.err"; then + echo "ERR [git] $(head -n 1 "$TMPDIR/fake-gitleaks.err")" + echo "INF 0 commits scanned." + echo "no leaks found" + exit 0 +fi +# A commit counts when a file it does not delete has a hunk, as in gitleaks. +scanned="$(awk '/^commit / { c = $2 } /^diff --git / { d = 0 } /^deleted file mode / { d = 1 } + /^@@ / && !d && !(c in s) { s[c] = 1; n++ } END { print n + 0 }' "$TMPDIR/fake-gitleaks.log")" +echo "INF $scanned commits scanned." if grep -qE '^\+.*(AKIA)[A-Z2-7]{16}' "$TMPDIR/fake-gitleaks.log"; then echo "leaks found: 1"; exit 1; fi echo "no leaks found" FAKE @@ -143,6 +154,39 @@ BEFORE="1111111111111111111111111111111111111111" AFTER="$leaky" \ EVENT_NAME=workflow_dispatch check "an unknown event fails" 2 +export EVENT_NAME=pull_request BEFORE="" AFTER="" +BASE_SHA="1111111111111111111111111111111111111111" HEAD_SHA="$leaky" \ + check "pull request: a base commit that cannot be fetched fails" 2 "cannot fetch commit 1111" +BASE_SHA="$base" HEAD_SHA="2222222222222222222222222222222222222222" \ + check "pull request: a head commit that cannot be fetched fails" 2 "cannot fetch commit 2222" + +# gitleaks stopping early (a git error part-way) without failing: it scans only the newest commit. +real_gitleaks="$gitleaks" +gitleaks="$root/early-stop-gitleaks" +cat > "$gitleaks" < [--record]`. const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); diff --git a/.github/scripts/fork-tests-drift.test.cjs b/.github/scripts/fork-tests-drift.test.cjs index ee56bff886ebe..7d869a3e49e62 100644 --- a/.github/scripts/fork-tests-drift.test.cjs +++ b/.github/scripts/fork-tests-drift.test.cjs @@ -77,7 +77,9 @@ test("top-level env, defaults, concurrency and permissions count, wherever they assert.deepEqual(compare(wrap(withPerms), wrap(withPerms.replace(": read", ": write"))), [ "ci.yml: top-level `permissions:` changed", ]); - assert.deepEqual(compare(wrap(withPerms), wrap(base)), ["ci.yml: top-level `permissions:` removed"]); + assert.deepEqual(compare(wrap(withPerms), wrap(base)), [ + "ci.yml: top-level `permissions:` removed", + ]); // `on:` and `name:` are not digested. assert.deepEqual(compare(wrap(base), wrap(base.replace("on: push", "on: pull_request"))), []); }); diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index 8900a8530c46a..29063b2b940ce 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -195,8 +195,9 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - # Full history without file contents; gitleaks then fetches the contents - # of only the scanned commits. + # Full history without file contents. The scan script fetches the contents + # of only the scanned commits (a failed fetch fails the job), then runs + # gitleaks offline. - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: