From fd9d2f35bb9b6538f69fe5f75765a839753eab95 Mon Sep 17 00:00:00 2001 From: irparent <134547608+irparent@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:13:05 -0700 Subject: [PATCH 1/3] ci: a dropped native SQLite module fails as an install problem, and e2e installs its system libraries from a cache Two ways CI went red for reasons outside the code under test: - better-sqlite3 is optional, so when its prebuilt download and its source build both fail, npm drops it and reports success. On 2026-10-24 the nightly worker-exit run's macOS Node 22 cell did exactly that, then failed all 50 search-worker runs on the module's absence. Every job that needs the native driver now runs scripts/ci/native-sqlite-installed.sh after npm ci: it opens a database with the module and, if that fails, installs once more with the install scripts' output in the log, then fails with that cause if the module still does not load. - On 2026-10-22 the Ubuntu mirror served apt's index at 9.7 MB/s and the 54 MB of browser system libraries at about 50 KB/s, so the e2e step hit its 10-minute bound on main and on three pull requests. apt now downloads into a folder actions/cache keeps per runner image and Playwright version; with it restored, apt reads only the index from the mirror. Checked in an ubuntu:24.04 container with the mirror's hosts blocked: "Need to get 0 B", every package installed from the folder. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 41 +++++++++++++++++++++++++-- .github/workflows/worker-exit.yml | 4 +++ scripts/ci/native-sqlite-installed.sh | 32 +++++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 scripts/ci/native-sqlite-installed.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42220ba7..a011409b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,8 @@ jobs: cache: npm cache-dependency-path: package-lock.json - run: npm ci + # npm drops this optional native module silently when its download fails; the script says why it checks. + - run: bash scripts/ci/native-sqlite-installed.sh # Coverage gate: enforces the ratchet floors in vitest.config.ts # (a plain `npm test` run would leave the thresholds unenforced). - run: npm run test:coverage @@ -200,6 +202,9 @@ jobs: cache: npm cache-dependency-path: package-lock.json - run: npm ci + # npm drops this optional native module silently when its download fails; the script says why it checks. + - if: matrix.driver == 'native' + run: bash scripts/ci/native-sqlite-installed.sh - run: npx vitest run tests/unit/storage/migration-015.test.ts tests/unit/storage/trace-search.test.ts tests/unit/storage/search-query.test.ts tests/unit/tools/get-traces-search.test.ts tests/unit/storage/stepped-maintenance.test.ts tests/unit/storage/checkpointer.test.ts tests/unit/storage/driver.test.ts tests/unit/storage/erasure.test.ts # The retention sweep, the merge it owes, the erasure of a retired search @@ -229,6 +234,9 @@ jobs: cache: npm cache-dependency-path: package-lock.json - run: npm ci + # npm drops this optional native module silently when its download fails; the script says why it checks. + - if: matrix.driver == 'native' + run: bash scripts/ci/native-sqlite-installed.sh - run: npx vitest run --config tests/stall/vitest.config.ts # The test jobs install better-sqlite3's prebuilt binary. When that download @@ -1000,9 +1008,14 @@ jobs: - run: npm run build # The browsers Playwright downloads, kept per Playwright version: the # version is in the key, so a bump downloads the matching browsers once. - - name: Playwright version + - name: Playwright version, and the runner image id: playwright - run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" + run: | + { + echo "version=$(node -p "require('@playwright/test/package.json').version")" + echo "os=${ImageOS}" + echo "image=${ImageVersion}" + } >> "$GITHUB_OUTPUT" - name: Cache the Playwright browsers uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -1017,6 +1030,21 @@ jobs: # downloads from Playwright's server, which can be slow, so it has # no bound of its own. run: npx playwright install chromium firefox + # The .deb files the browsers' system libraries come in, kept per runner + # image and Playwright version. On 2026-10-22 the mirror served apt's + # package index at 9.7 MB/s and the 54 MB of packages at about 50 KB/s, + # and the step below hit its 10-minute bound on main and on three pull + # requests. With the packages cached, apt reads only the index from the + # mirror and installs from here ("Need to get 0 B"). A new runner image + # restores the previous image's packages and downloads only what changed. + - name: Cache the system libraries' packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/apt-archives + key: apt-${{ steps.playwright.outputs.os }}-${{ steps.playwright.outputs.image }}-playwright-${{ steps.playwright.outputs.version }} + restore-keys: | + apt-${{ steps.playwright.outputs.os }}-${{ steps.playwright.outputs.image }}- + apt-${{ steps.playwright.outputs.os }}- - name: Install the browsers' system libraries # These come through apt from the runner's mirror list. On 2026-10-07 # the first mirror stopped answering and apt waited on it for 39 @@ -1024,10 +1052,17 @@ jobs: # apt now gives up on a silent mirror after 30 seconds and moves to # the next, and the step stops at 10 minutes, so a bad mirror day # fails in minutes with a clear cause instead of hanging the job. + # apt downloads into the cached folder above; autoclean drops the + # packages a newer image no longer offers, so the cache stays small. timeout-minutes: 10 run: | - printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' | sudo tee /etc/apt/apt.conf.d/80-ci-timeouts > /dev/null + mkdir -p ~/.cache/apt-archives/partial + printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nDir::Cache::Archives "%s/.cache/apt-archives/";\n' "$HOME" | sudo tee /etc/apt/apt.conf.d/80-ci > /dev/null npx playwright install-deps chromium firefox + sudo apt-get autoclean -qq + # apt wrote the packages as root; the cache step saves them as this user. + sudo chown -R "$(id -u):$(id -g)" ~/.cache/apt-archives + echo "$(find ~/.cache/apt-archives -maxdepth 1 -name '*.deb' | wc -l) packages in the apt cache" - name: Run Playwright E2E (Chromium + Firefox) run: npm run test:e2e - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/worker-exit.yml b/.github/workflows/worker-exit.yml index 9810bf95..262edf14 100644 --- a/.github/workflows/worker-exit.yml +++ b/.github/workflows/worker-exit.yml @@ -28,6 +28,7 @@ on: - 'tests/unit/storage/search-worker.test.ts' - 'package-lock.json' - '.github/workflows/worker-exit.yml' + - 'scripts/ci/native-sqlite-installed.sh' push: branches: [main] paths: @@ -36,6 +37,7 @@ on: - 'tests/unit/storage/search-worker.test.ts' - 'package-lock.json' - '.github/workflows/worker-exit.yml' + - 'scripts/ci/native-sqlite-installed.sh' schedule: - cron: '23 5 * * *' workflow_dispatch: @@ -88,6 +90,8 @@ jobs: cache: npm cache-dependency-path: package-lock.json - run: npm ci + # npm drops this optional native module silently when its download fails; the script says why it checks. + - run: bash scripts/ci/native-sqlite-installed.sh - run: npm run build:server - name: Workers started and ended every way, on both drivers, raw and as Iris's two workers env: diff --git a/scripts/ci/native-sqlite-installed.sh b/scripts/ci/native-sqlite-installed.sh new file mode 100644 index 00000000..26bbb4e8 --- /dev/null +++ b/scripts/ci/native-sqlite-installed.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Run after `npm ci` in a job that needs the native SQLite driver. +# +# better-sqlite3 is an optional dependency. When its prebuilt download fails +# and the source build fails too, npm drops it and still reports success, and +# every test that opens a store with the native driver then fails on its +# absence, which says nothing about the code under test. On 2026-10-24 the +# nightly worker-exit run's macOS Node 22 cell printed "added 310 packages", +# then failed all 50 of its search-worker runs on "The native SQLite module +# (better-sqlite3) is not installed". +# +# This loads the module and opens a database. If that fails, it installs once +# more with the install scripts' output in the log, so the cause is visible +# and one failed download does not fail the job; if the module still does not +# load, the job fails here, with that log above it. +set -euo pipefail + +loads() { node -e "new (require('better-sqlite3'))(':memory:').close()"; } + +if loads 2>/dev/null; then + echo "better-sqlite3 $(node -p "require('better-sqlite3/package.json').version") loads on Node $(node -v)" + exit 0 +fi + +echo "::warning::npm ci did not leave a working better-sqlite3 (it is optional, so npm drops it when its download and its source build both fail). Installing once more, with the install scripts' output shown." +npm ci --foreground-scripts + +if ! loads; then + echo "::error::better-sqlite3 still does not load on this runner; the install log above says why." + exit 1 +fi +echo "better-sqlite3 loads after the second install" From aab3655728ff8a693fa31a1b847814b9a95ebdbe Mon Sep 17 00:00:00 2001 From: irparent <134547608+irparent@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:13:06 -0700 Subject: [PATCH 2/3] test(upgrade): the retention sweep's cutoff no longer moves with the day the test runs The upgrade test has the previous release sweep traces older than 30 days from a fixture whose traces are dated 2025-09-24 and 2026-09-24. From 2026-10-24 the sweep also took a 2026-09-24 trace, so "swept: 2" became 3 on every operating system. The cutoff is now 2026-01-01 whatever the day: past the 2025 trace, before the 2026 ones. Co-Authored-By: Claude Opus 5.5 --- tests/upgrade/from-previous-release.test.ts | 5 ++++- tests/upgrade/previous-writer.mjs | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/upgrade/from-previous-release.test.ts b/tests/upgrade/from-previous-release.test.ts index fd50168c..d4a3715d 100644 --- a/tests/upgrade/from-previous-release.test.ts +++ b/tests/upgrade/from-previous-release.test.ts @@ -326,7 +326,10 @@ describe(`the later migrations are safe for ${PREVIOUS} to write through`, () => raw.prepare("DELETE FROM _iris_migrations WHERE id > '014-trace-session'").run(); raw.close(); - const writer = await run([join(repoRoot, 'tests', 'upgrade', 'previous-writer.mjs'), join(root, 'previous', 'node_modules', '@iris-eval', 'mcp-server'), file, estimated.trace_id]); + // The sweep's cutoff is 2026-01-01 on whatever day this runs: past the fixture's 2025 trace, before its + // traces of 2026-09-24. It was a fixed 30 days, and from 2026-10-24 that also swept a 2026-09-24 trace. + const retentionDays = Math.floor((Date.now() - Date.parse('2026-01-01T00:00:00.000Z')) / 86_400_000); + const writer = await run([join(repoRoot, 'tests', 'upgrade', 'previous-writer.mjs'), join(root, 'previous', 'node_modules', '@iris-eval', 'mcp-server'), file, estimated.trace_id, String(retentionDays)]); expect(writer.code, writer.stderr).toBe(0); // The patch and the delete found their rows; the sweep took the expired trace it stored and the fixture's trace from 2025. expect(JSON.parse(writer.stdout)).toEqual({ patched: true, deleted: true, swept: 2 }); diff --git a/tests/upgrade/previous-writer.mjs b/tests/upgrade/previous-writer.mjs index f215e684..1353c052 100644 --- a/tests/upgrade/previous-writer.mjs +++ b/tests/upgrade/previous-writer.mjs @@ -2,13 +2,13 @@ // process of its own (its native SQLite module is then never loaded into the // test runner, where Windows would keep the file locked). // -// node previous-writer.mjs +// node previous-writer.mjs // // Prints one JSON line with what each call returned. import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; -const [pkg, file, estimatedId] = process.argv.slice(2); +const [pkg, file, estimatedId, retentionDays] = process.argv.slice(2); const { SqliteAdapter } = await import(pathToFileURL(join(pkg, 'dist', 'storage', 'sqlite-adapter.js')).href); const now = new Date().toISOString(); const store = new SqliteAdapter(file); @@ -20,6 +20,6 @@ await store.insertTraces('local', [ ]); const patched = await store.updateTraceMetadata('local', estimatedId, { patched: 'by the previous release' }); const deleted = await store.deleteTrace('local', '0190a000000000000000000000000001'); -const swept = await store.deleteTracesOlderThan('local', 30); +const swept = await store.deleteTracesOlderThan('local', Number(retentionDays)); await store.close(); process.stdout.write(`${JSON.stringify({ patched, deleted, swept })}\n`); From 9e24034b11b69e2407d588e784613f09e09c5a7e Mon Sep 17 00:00:00 2001 From: irparent <134547608+irparent@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:19:45 -0700 Subject: [PATCH 3/3] test(preflight): the native SQLite install check is a runner's install step, not a check the preflight mirrors The preflight mirrors the jobs whose checks can run locally, and a test requires it to run every script those jobs run. The new install check is not a check of the tree: it confirms a runner's fresh npm ci kept the optional native module and, when it did not, runs npm ci again, which must never run in a checkout whose node_modules other worktrees share. It is now named with that reason. Co-Authored-By: Claude Opus 5.5 --- tests/preflight-mirrors-ci.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/preflight-mirrors-ci.test.ts b/tests/preflight-mirrors-ci.test.ts index 34efc26f..aa3221aa 100644 --- a/tests/preflight-mirrors-ci.test.ts +++ b/tests/preflight-mirrors-ci.test.ts @@ -83,12 +83,17 @@ describe('the preflight says what the pull-request workflows say', () => { 'test:coverage': 'the preflight runs the same vitest --coverage, adding a JSON report the capture reads', 'test:integration': 'tests/integration is inside the root suite the preflight runs', }; + // Steps that finish a runner's fresh install rather than check the tree, and why the preflight leaves them out. + const installStep: Record = { + 'scripts/ci/native-sqlite-installed.sh': + "confirms the runner's npm ci kept the optional native SQLite module and, when it did not, runs npm ci again; a local checkout already has its install, and npm ci must not run in one whose node_modules other worktrees share", + }; const missing: string[] = []; for (const job of mirrored) { const body = jobs.get(job) ?? ''; for (const m of body.matchAll(/npm run(?: -s)? ([\w:-]+)|(?:node|bash) (scripts\/[\w./-]+)|--config (tests\/[\w./-]+)/g)) { const script = m[1] ?? m[2] ?? m[3]; - if (script in sameCheck) continue; + if (script in sameCheck || script in installStep) continue; const ran = m[1] ? runsScript(script) : runs.includes(script); if (!ran) missing.push(`${job}: ${m[0]}`); }