From 1759b104a22cd6c1acd115b6cd1759050415e36c Mon Sep 17 00:00:00 2001 From: Yamonwan Justin DAH-KENANGNON Date: Fri, 23 Oct 2026 06:01:10 +0100 Subject: [PATCH 1/4] fix(ci): improve tutorial verification reliability Signed-off-by: Yamonwan Justin DAH-KENANGNON --- .github/workflows/quality.yml | 119 +++++++--- p4-editorial-publishing/docs/tutorials.md | 2 +- p4-editorial-publishing/package.json | 2 +- p4-editorial-publishing/pnpm-lock.yaml | 82 +++---- shared/tools/repository-automation/README.md | 27 +++ .../changed-projects.mjs | 62 ++++-- .../changed-projects.test.mjs | 148 ++++++++++++- .../tools/repository-automation/ci-report.mjs | 49 +++++ .../repository-automation/ci-report.test.mjs | 31 +++ .../tools/repository-automation/ci-result.mjs | 56 +++++ .../repository-automation/ci-result.test.mjs | 88 ++++++++ .../repository-automation/inspect-ci.mjs | 109 +++++++++ .../repository-automation/inspect-ci.test.mjs | 80 +++++++ .../tools/repository-automation/package.json | 5 +- .../repository-automation/pnpm-lock.yaml | 10 +- .../repository-automation/setup-browser.mjs | 206 ++++++++++++++++++ .../setup-browser.test.mjs | 123 +++++++++++ shared/tools/step/README.md | 2 +- shared/tools/step/run.mjs | 67 +++--- shared/tools/step/source.mjs | 115 ++++++++++ shared/tools/step/source.test.mjs | 141 ++++++++++++ 21 files changed, 1387 insertions(+), 137 deletions(-) create mode 100644 shared/tools/repository-automation/README.md create mode 100644 shared/tools/repository-automation/ci-report.mjs create mode 100644 shared/tools/repository-automation/ci-report.test.mjs create mode 100644 shared/tools/repository-automation/ci-result.mjs create mode 100644 shared/tools/repository-automation/ci-result.test.mjs create mode 100644 shared/tools/repository-automation/inspect-ci.mjs create mode 100644 shared/tools/repository-automation/inspect-ci.test.mjs create mode 100644 shared/tools/repository-automation/setup-browser.mjs create mode 100644 shared/tools/repository-automation/setup-browser.test.mjs create mode 100644 shared/tools/step/source.mjs create mode 100644 shared/tools/step/source.test.mjs diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 1dc8625..c666451 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -19,35 +19,43 @@ concurrency: jobs: step: name: Tutorial helper (${{ matrix.os }}) - runs-on: ${{ matrix.os }} + needs: changes + if: needs.changes.outputs.helper == 'true' + runs-on: ${{ matrix.os == 'ubuntu-latest' && 'ubuntu-24.04' || matrix.os }} timeout-minutes: 5 strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v7.0.1 - - uses: actions/setup-node@v7.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 - name: Check tutorial step helper - run: node --test shared/tools/step/run.test.mjs + id: check + run: node --test shared/tools/step/run.test.mjs shared/tools/step/source.test.mjs changes: name: Select projects - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 5 outputs: projects: ${{ steps.select.outputs.projects }} compose: ${{ steps.select.outputs.compose }} + helper: ${{ steps.select.outputs.helper }} steps: - - uses: actions/checkout@v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - - uses: pnpm/action-setup@v6.1.0 + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 10.17.1 run_install: false - - uses: actions/setup-node@v7.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 cache: pnpm @@ -58,6 +66,10 @@ jobs: - name: Check repository automation working-directory: shared/tools/repository-automation run: pnpm check + - name: Audit repository automation + timeout-minutes: 2 + working-directory: shared/tools/repository-automation + run: pnpm audit --audit-level low - id: select name: Detect affected projects run: node shared/tools/repository-automation/changed-projects.mjs @@ -71,8 +83,8 @@ jobs: name: ${{ matrix.project }} (${{ matrix.os }}) needs: changes if: needs.changes.outputs.projects != '[]' - runs-on: ${{ matrix.os }} - timeout-minutes: 10 + runs-on: ${{ matrix.os == 'ubuntu-latest' && 'ubuntu-24.04' || matrix.os }} + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -80,64 +92,96 @@ jobs: project: ${{ fromJSON(needs.changes.outputs.projects) }} steps: - - uses: actions/checkout@v7.0.1 - - uses: pnpm/action-setup@v6.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 10.17.1 run_install: false - - uses: actions/setup-node@v7.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 cache: pnpm - cache-dependency-path: ${{ matrix.project }}/pnpm-lock.yaml + cache-dependency-path: | + ${{ matrix.project }}/pnpm-lock.yaml + shared/identity-provider/pnpm-lock.yaml + - name: Runner identity + run: node shared/tools/repository-automation/ci-report.mjs runner - name: Install + id: install + timeout-minutes: 3 working-directory: ${{ matrix.project }} run: pnpm install --frozen-lockfile - name: Prepare shared identity provider + id: identity + timeout-minutes: 2 if: matrix.project != 'shared/identity-provider' working-directory: shared/identity-provider run: | pnpm install --frozen-lockfile pnpm build - name: Install browser and Linux dependencies + id: browser-linux + timeout-minutes: 7 if: (matrix.project == 'p1-task-manager' || matrix.project == 'p4-editorial-publishing' || matrix.project == 'p5-dataguard' || matrix.project == 'p6-field-inspection' || matrix.project == 'p7-collaborative-docs' || matrix.project == 'p10-warehouse-workloads' || matrix.project == 'p11-saas-workspace' || matrix.project == 'p14-capital-district' || matrix.project == 'p15-marketplace') && runner.os == 'Linux' working-directory: ${{ matrix.project }} - run: pnpm exec playwright install --with-deps chromium + run: node ../shared/tools/repository-automation/setup-browser.mjs - name: Install browser + id: browser + timeout-minutes: 3 if: (matrix.project == 'p1-task-manager' || matrix.project == 'p4-editorial-publishing' || matrix.project == 'p5-dataguard' || matrix.project == 'p6-field-inspection' || matrix.project == 'p7-collaborative-docs' || matrix.project == 'p10-warehouse-workloads' || matrix.project == 'p14-capital-district' || matrix.project == 'p15-marketplace') && runner.os != 'Linux' working-directory: ${{ matrix.project }} run: pnpm exec playwright install chromium + env: + PLAYWRIGHT_DOWNLOAD_CONNECTION_TIMEOUT: 30000 - name: Check + id: check + timeout-minutes: 5 working-directory: ${{ matrix.project }} run: pnpm check - name: Run Linux integration scenario + id: integration + timeout-minutes: 5 if: runner.os == 'Linux' && (matrix.project == 'p8-cedarfile' || matrix.project == 'p9-cedarrealtime' || matrix.project == 'p11-saas-workspace' || matrix.project == 'p12-hr-access-governance' || matrix.project == 'p13-student-records' || matrix.project == 'p15-marketplace') working-directory: ${{ matrix.project }} run: pnpm test:e2e - name: Audit dependencies + id: audit + timeout-minutes: 2 working-directory: ${{ matrix.project }} run: pnpm audit --audit-level low + - name: Verification summary + if: always() + run: node shared/tools/repository-automation/ci-report.mjs stages + env: + STEPS_JSON: ${{ toJSON(steps) }} + p3-sidecar: name: P3 signed-token sidecar workflow needs: changes if: contains(fromJSON(needs.changes.outputs.projects), 'p3-mcp-capability-governance') - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 defaults: run: working-directory: p3-mcp-capability-governance steps: - - uses: actions/checkout@v7.0.1 - - uses: pnpm/action-setup@v6.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 10.17.1 run_install: false - - uses: actions/setup-node@v7.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 cache: pnpm - cache-dependency-path: p3-mcp-capability-governance/pnpm-lock.yaml + cache-dependency-path: | + p3-mcp-capability-governance/pnpm-lock.yaml + shared/identity-provider/pnpm-lock.yaml - run: pnpm install --frozen-lockfile - run: pnpm test:e2e @@ -145,15 +189,17 @@ jobs: name: Compose startup (${{ matrix.project }}) needs: changes if: needs.changes.outputs.compose != '[]' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 20 strategy: fail-fast: false matrix: project: ${{ fromJSON(needs.changes.outputs.compose) }} steps: - - uses: actions/checkout@v7.0.1 - - uses: actions/setup-node@v7.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 - name: Validate Compose @@ -185,11 +231,13 @@ jobs: name: Concurrent project stacks needs: changes if: needs.changes.outputs.compose != '[]' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 25 steps: - - uses: actions/checkout@v7.0.1 - - uses: actions/setup-node@v7.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 - name: Start independent stacks @@ -208,3 +256,22 @@ jobs: docker compose -f p1-task-manager/compose.yaml down --volumes docker compose -f p4-editorial-publishing/compose.yaml down --volumes docker compose -f p10-warehouse-workloads/compose.yaml down --volumes + + result: + name: Quality result + if: always() + needs: + [changes, step, project, p3-sidecar, compose-config, compose-concurrency] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24.21.0 + - name: Check selected results + run: node shared/tools/repository-automation/ci-result.mjs + env: + NEEDS_JSON: ${{ toJSON(needs) }} diff --git a/p4-editorial-publishing/docs/tutorials.md b/p4-editorial-publishing/docs/tutorials.md index 1eb6198..ca0607a 100644 --- a/p4-editorial-publishing/docs/tutorials.md +++ b/p4-editorial-publishing/docs/tutorials.md @@ -384,7 +384,7 @@ From the P4 directory, let's install Cedarling and the tools that package and validate our policy store, along with the Next.js security update: ```bash -pnpm add --save-exact @janssenproject/cedarling_wasm@0.0.468 fflate@0.8.3 next@16.3.8 +pnpm add --save-exact @janssenproject/cedarling_wasm@0.0.468 fflate@0.8.3 next@16.4.1 pnpm add --save-dev --save-exact @cedar-policy/cedar-wasm@4.12.0 @types/node@24.19.0 vitest@4.1.12 ``` diff --git a/p4-editorial-publishing/package.json b/p4-editorial-publishing/package.json index c82a1f5..243382f 100644 --- a/p4-editorial-publishing/package.json +++ b/p4-editorial-publishing/package.json @@ -26,7 +26,7 @@ "@janssenproject/cedarling_wasm": "0.0.468", "better-sqlite3": "13.0.3", "fflate": "0.8.3", - "next": "16.3.8", + "next": "16.4.1", "openid-client": "6.8.8", "react": "19.3.0", "react-dom": "19.3.0", diff --git a/p4-editorial-publishing/pnpm-lock.yaml b/p4-editorial-publishing/pnpm-lock.yaml index 552f0b2..b244814 100644 --- a/p4-editorial-publishing/pnpm-lock.yaml +++ b/p4-editorial-publishing/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: specifier: 0.8.3 version: 0.8.3 next: - specifier: 16.3.8 - version: 16.3.8(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + specifier: 16.4.1 + version: 16.4.1(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) openid-client: specifier: 6.8.8 version: 6.8.8 @@ -278,53 +278,53 @@ packages: '@jridgewell/sourcemap-codec@1.6.0': resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} - '@next/env@16.3.8': - resolution: {integrity: sha512-Al9zqHVV7TJv0eFuOU4U7Lvv74PTih4Ch63sk2xCIpSTkE3udFnaOcnzP2lQVymiL7yS9Cj2iClUXlR3EQ5sEw==} + '@next/env@16.4.1': + resolution: {integrity: sha512-5sUUrrSvs/2/2LnLoesybMGx12EtPB46fdZ6RhAw0XDGSPxU2LFyQBB7yuEiOFFJPKmBY6+q2jTr1/AMbnp0/g==} - '@next/swc-darwin-arm64@16.3.8': - resolution: {integrity: sha512-2JPRMh2nmQG5CiL7cXGL9AGwnPWJQ//cTtAUCT+w511QHk79SYz3LGv/pc5X643B/WEO0rvu3Yww0hqwt3kgeA==} + '@next/swc-darwin-arm64@16.4.1': + resolution: {integrity: sha512-U+hQ5Gj3fpzf4fveTIF3dpUFYhaSW8vHhdAnbiIomzXzg7boreYFgBqprepXFtW4sw+aTttiNmg0gew/iFNhbg==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] - '@next/swc-darwin-x64@16.3.8': - resolution: {integrity: sha512-GZtCCOBKJ4leVIT/Th0llWKhD1ca92lzbQiS5R5ON9QkoiFnilFsebDae1JU2a3HWoKMEmEZWGs1AGLavVM72Q==} + '@next/swc-darwin-x64@16.4.1': + resolution: {integrity: sha512-/hqC3V1XLUjhMASf/WridKdO9FXoEgN3HBWmtKbj00MPgnltutfraWJ3bYTpCRIgEIpbjJyOVLpYgAhh/BkdcQ==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] - '@next/swc-linux-arm64-gnu@16.3.8': - resolution: {integrity: sha512-O659ygeQYqneJ1fBKMpFxIFqYkYswu8IAS1OCKK/4f3ZgJJm1dRz4fVJZRi/kLLWjnBKnebOePA4WNv+sV1pVA==} + '@next/swc-linux-arm64-gnu@16.4.1': + resolution: {integrity: sha512-GHzncHbcaS9g95kcvTNtKQW3aCLoR+xc3Jm7qrkpk1R+D2BjMHIPyvQGRPKCdMX95BsH/421jBUK06q9tEwzjw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - '@next/swc-linux-arm64-musl@16.3.8': - resolution: {integrity: sha512-dSjKSyWpzxoO1d3DIZZcP4XJcNaKeLmxQMFOiYl5vuBRMmweIqnAhty8tAmRsvTss779cK1FtYnDMj40e4TQlg==} + '@next/swc-linux-arm64-musl@16.4.1': + resolution: {integrity: sha512-1SuJyDxW/PR9dHJat/ZRgcYLhuSZ+1psP4DlKb2tKIuXbjygqpLrJWRSq+onZQBv8LcmU67EIrmb6c3XTaJLjw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - '@next/swc-linux-x64-gnu@16.3.8': - resolution: {integrity: sha512-lbqOuz3RPRcv+o9msNsJw5x4+Y1ZwPTs6vmL6DCf7i0fZfvng/F59wyeDwqHIvV0mK//RBy/jJkZ+nCKsSMXjQ==} + '@next/swc-linux-x64-gnu@16.4.1': + resolution: {integrity: sha512-Istng1hMGp9TGO4lUqeKLd70MPN/OOkucVTP2Nmj+gLkwBgmPqUw8CJYBCC7nbl/B/23bBCcjCI5Hsh6hazC1A==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - '@next/swc-linux-x64-musl@16.3.8': - resolution: {integrity: sha512-+316WswI8ScVgZeUd+1KGaXkHhaYQzCjvH/05TZSpJ8zBizb1a4G7DtO7F12jcBIqMOtsz9ji1t48fmKtzqsGA==} + '@next/swc-linux-x64-musl@16.4.1': + resolution: {integrity: sha512-aSC9wMNiZRRFVpPkPWBzRwLQxIGm/CiQcXyFF1XNcgodb+uMxdk7aSSGGpDXCbnvZdf4itjHxpOd5h7i7zaiYw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - '@next/swc-win32-arm64-msvc@16.3.8': - resolution: {integrity: sha512-ji0gd4kMYUxO+1fJBIbiBVRCjzG/lloiyCccnlebvb1ZJ5qXCPZqYg4Jl1DrrixnWNMKylzgpmMWx0yNDYXlzw==} + '@next/swc-win32-arm64-msvc@16.4.1': + resolution: {integrity: sha512-t/myvwFDlypMUjRvbBf8qLPyuaB0iXYYdBtwMt6DM6f+60wtGr4bSYholdPqNeV5VUXZhrpMwfza829kmhS3iA==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] - '@next/swc-win32-x64-msvc@16.3.8': - resolution: {integrity: sha512-WcTlaKt/TWkh5kUjdJcUmB1XgZ+1c6fz4Y9fDHL73YNSdGaUWjceeWrrlwF0nv19iABYWC4iAq1oX1w4Bn0vfg==} + '@next/swc-win32-x64-msvc@16.4.1': + resolution: {integrity: sha512-6ICqPo9awCkrtYNPhwkx1uU7rlFWrZRrIboDHS2LU+jv088UBYQtmJ25h6pCp27/+FZjF6BQ4Te4ErugyMeuBg==} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -629,8 +629,8 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - next@16.3.8: - resolution: {integrity: sha512-U7QEZaTini6wKrb8A8hqLLqYQyCetegKjCpJOyxk642vWoMoU1x5PyZCJFvgYgiptA8xc5j/9xYlZFO7w9Sjmw==} + next@16.4.1: + resolution: {integrity: sha512-uc657V62E34fOI6W5NtG9IzwsPcms0Z60grz2MJGtglgD+pTFnNt4Yw2PzaU622tfX4vhrDgLZzicE2YlhI20Q==} engines: {node: '>=20.9.0'} hasBin: true peerDependencies: @@ -1022,30 +1022,30 @@ snapshots: '@jridgewell/sourcemap-codec@1.6.0': {} - '@next/env@16.3.8': {} + '@next/env@16.4.1': {} - '@next/swc-darwin-arm64@16.3.8': + '@next/swc-darwin-arm64@16.4.1': optional: true - '@next/swc-darwin-x64@16.3.8': + '@next/swc-darwin-x64@16.4.1': optional: true - '@next/swc-linux-arm64-gnu@16.3.8': + '@next/swc-linux-arm64-gnu@16.4.1': optional: true - '@next/swc-linux-arm64-musl@16.3.8': + '@next/swc-linux-arm64-musl@16.4.1': optional: true - '@next/swc-linux-x64-gnu@16.3.8': + '@next/swc-linux-x64-gnu@16.4.1': optional: true - '@next/swc-linux-x64-musl@16.3.8': + '@next/swc-linux-x64-musl@16.4.1': optional: true - '@next/swc-win32-arm64-msvc@16.3.8': + '@next/swc-win32-arm64-msvc@16.4.1': optional: true - '@next/swc-win32-x64-msvc@16.3.8': + '@next/swc-win32-x64-msvc@16.4.1': optional: true '@oxc-project/types@0.155.0': {} @@ -1267,9 +1267,9 @@ snapshots: nanoid@3.3.19: {} - next@16.3.8(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): + next@16.4.1(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): dependencies: - '@next/env': 16.3.8 + '@next/env': 16.4.1 '@swc/helpers': 0.5.23 baseline-browser-mapping: 2.11.24 caniuse-lite: 1.0.30001810 @@ -1278,14 +1278,14 @@ snapshots: react-dom: 19.3.0(react@19.3.0) styled-jsx: 5.1.6(react@19.3.0) optionalDependencies: - '@next/swc-darwin-arm64': 16.3.8 - '@next/swc-darwin-x64': 16.3.8 - '@next/swc-linux-arm64-gnu': 16.3.8 - '@next/swc-linux-arm64-musl': 16.3.8 - '@next/swc-linux-x64-gnu': 16.3.8 - '@next/swc-linux-x64-musl': 16.3.8 - '@next/swc-win32-arm64-msvc': 16.3.8 - '@next/swc-win32-x64-msvc': 16.3.8 + '@next/swc-darwin-arm64': 16.4.1 + '@next/swc-darwin-x64': 16.4.1 + '@next/swc-linux-arm64-gnu': 16.4.1 + '@next/swc-linux-arm64-musl': 16.4.1 + '@next/swc-linux-x64-gnu': 16.4.1 + '@next/swc-linux-x64-musl': 16.4.1 + '@next/swc-win32-arm64-msvc': 16.4.1 + '@next/swc-win32-x64-msvc': 16.4.1 '@playwright/test': 1.63.0 sharp: 0.35.5(@types/node@24.19.0) transitivePeerDependencies: diff --git a/shared/tools/repository-automation/README.md b/shared/tools/repository-automation/README.md new file mode 100644 index 0000000..00f754d --- /dev/null +++ b/shared/tools/repository-automation/README.md @@ -0,0 +1,27 @@ +# Repository checks + +`pnpm check` tests the CI tools and validates every published tutorial's metadata +and referenced images. Application changes run the affected projects on Linux, +macOS, and Windows, with separate Docker checks. Tutorial text, images, and README +changes run source validation. Shared runtime or workflow changes run the full +portfolio, as do scheduled and manually dispatched checks. + +The Linux browser setup uses Ubuntu 24.04 and official HTTPS package sources. +Each attempt has a three-minute budget. A transport failure can trigger one +retry; signature, package-configuration, and timeout failures stop setup. +The final Quality result check requires every selected job to succeed. + +## Inspect a slow run + +With the GitHub CLI authenticated, run this from this directory: + +```sh +pnpm inspect:ci +``` + +The command separates queue time from execution time and lists slow or failed +steps. It only reads workflow metadata. If every step, including Complete job, +has finished but GitHub still reports the job as running, it flags the unresolved +status and exits unsuccessfully. Check GitHub Status before retrying that job. +An application failure or security-audit finding needs investigation even when +another job has a stale status. diff --git a/shared/tools/repository-automation/changed-projects.mjs b/shared/tools/repository-automation/changed-projects.mjs index 0aecf9d..d41e872 100644 --- a/shared/tools/repository-automation/changed-projects.mjs +++ b/shared/tools/repository-automation/changed-projects.mjs @@ -1,7 +1,7 @@ // Selects the tutorial projects that CI must verify for the current change. -import { appendFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; export const tutorialProjects = [ @@ -21,8 +21,6 @@ export const tutorialProjects = [ "p14-capital-district", "p15-marketplace", ]; - - export const projects = [...tutorialProjects, "shared/identity-provider"]; export const composeProjects = new Set(tutorialProjects); @@ -38,32 +36,58 @@ function changedFiles(environment = process.env) { const separator = event === "pull_request" ? "..." : ".."; return execFileSync( "git", - ["diff", "--name-only", `${base}${separator}${head}`], + [ + "diff", + "--name-only", + "-z", + "--no-renames", + `${base}${separator}${head}`, + "--", + ], { encoding: "utf8", }, ) - .trim() - .split("\n") + .split("\0") .filter(Boolean); } export function selectProjects(files) { - const affectsAll = - files === null || - files.some( - (file) => - file.startsWith(".github/") || - file.startsWith("shared/"), - ); - const selected = affectsAll - ? projects - : projects.filter((project) => - files.some((file) => file.startsWith(`${project}/`)), - ); + const all = () => ({ projects, compose: [...composeProjects], helper: true }); + if (files === null) return all(); + const affected = new Set(); + let helper = false; + for (const file of files) { + if ( + [ + "README.md", + "LICENSE", + "shared/tools/repository-automation/README.md", + ].includes(file) + ) + continue; + if (file.startsWith("shared/tools/step/")) { + helper = true; + continue; + } + const project = projects.find((name) => file.startsWith(`${name}/`)); + if (!project) return all(); + const path = file.slice(project.length + 1); + if ( + path === "README.md" || + path === "docs/tutorials.md" || + /^docs\/assets\/[A-Za-z0-9_./-]+\.(avif|jpe?g|png|svg|webp)$/.test(path) + ) + continue; + // Shared runtime changes affect every application using that runtime. + if (project === "shared/identity-provider") return all(); + affected.add(project); + } + const selected = projects.filter((project) => affected.has(project)); return { projects: selected, compose: selected.filter((project) => composeProjects.has(project)), + helper, }; } @@ -78,7 +102,7 @@ function main(environment = process.env) { files = null; } const selected = selectProjects(files); - const output = `projects=${JSON.stringify(selected.projects)}\ncompose=${JSON.stringify(selected.compose)}\n`; + const output = `projects=${JSON.stringify(selected.projects)}\ncompose=${JSON.stringify(selected.compose)}\nhelper=${selected.helper}\n`; if (environment.GITHUB_OUTPUT) appendFileSync(environment.GITHUB_OUTPUT, output); else process.stdout.write(output); diff --git a/shared/tools/repository-automation/changed-projects.test.mjs b/shared/tools/repository-automation/changed-projects.test.mjs index 16e1cc3..2aba4ac 100644 --- a/shared/tools/repository-automation/changed-projects.test.mjs +++ b/shared/tools/repository-automation/changed-projects.test.mjs @@ -1,7 +1,10 @@ // Verifies CI project selection for project, shared, and repository changes. import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; import { @@ -15,17 +18,25 @@ const selectorScript = fileURLToPath( ); test("selects only the changed project", () => { - assert.deepEqual(selectProjects(["p7-collaborative-docs/src/server/app.ts"]), { - projects: ["p7-collaborative-docs"], - compose: ["p7-collaborative-docs"], - }); + assert.deepEqual( + selectProjects(["p7-collaborative-docs/src/server/app.ts"]), + { + projects: ["p7-collaborative-docs"], + compose: ["p7-collaborative-docs"], + helper: false, + }, + ); }); test("selects Capital District for a P14 game change", () => { - assert.deepEqual(selectProjects(["p14-capital-district/src/server/game/service.ts"]), { - projects: ["p14-capital-district"], - compose: ["p14-capital-district"], - }); + assert.deepEqual( + selectProjects(["p14-capital-district/src/server/game/service.ts"]), + { + projects: ["p14-capital-district"], + compose: ["p14-capital-district"], + helper: false, + }, + ); }); test("selects P3 Compose without a separate shared-provider stack", () => { @@ -34,6 +45,7 @@ test("selects P3 Compose without a separate shared-provider stack", () => { { projects: ["p3-mcp-capability-governance"], compose: ["p3-mcp-capability-governance"], + helper: false, }, ); assert.equal(composeProjects.has("shared/identity-provider"), false); @@ -44,7 +56,10 @@ test("selects the complete portfolio for shared and workflow changes", () => { selectProjects(["shared/identity-provider/src/config.ts"]).projects, projects, ); - assert.deepEqual(selectProjects(["shared/dev-supervisor.mjs"]).projects, projects); + assert.deepEqual( + selectProjects(["shared/dev-supervisor.mjs"]).projects, + projects, + ); assert.deepEqual( selectProjects([".github/workflows/quality.yml"]).projects, projects, @@ -56,7 +71,62 @@ test("does not run project jobs for root documentation alone", () => { assert.deepEqual(selectProjects(["README.md"]), { projects: [], compose: [], + helper: false, + }); +}); + +test("tutorial and README changes need validation without application matrices", () => { + assert.deepEqual( + selectProjects([ + "p1-task-manager/docs/tutorials.md", + "p4-editorial-publishing/docs/assets/social-card.webp", + "p6-field-inspection/README.md", + "shared/tools/repository-automation/README.md", + ]), + { projects: [], compose: [], helper: false }, + ); +}); + +test("helper changes select its portable tests without unrelated applications", () => { + assert.deepEqual(selectProjects(["shared/tools/step/run.mjs"]), { + projects: [], + compose: [], + helper: true, }); + assert.deepEqual( + selectProjects([ + "shared/tools/step/projects/p1.json", + "p1-task-manager/src/server/app.ts", + ]), + { + projects: ["p1-task-manager"], + compose: ["p1-task-manager"], + helper: true, + }, + ); +}); + +test("unknown paths and scheduled runs conservatively select everything", () => { + for (const files of [ + null, + ["scripts/new-tool.mjs"], + [".nvmrc"], + ["new-project/app.ts"], + ]) { + assert.deepEqual(selectProjects(files), { + projects, + compose: [...composeProjects], + helper: true, + }); + } + assert.deepEqual( + selectProjects(["p1-task-manager/docs/execute.ts"]).projects, + ["p1-task-manager"], + ); + assert.deepEqual( + selectProjects(["p1-task-manager/src/odd\nname.ts"]).projects, + ["p1-task-manager"], + ); }); test("reports when project detection falls back to the full matrix", () => { @@ -80,3 +150,61 @@ test("reports when project detection falls back to the full matrix", () => { ); assert.ok(result.stdout.includes(`projects=${JSON.stringify(projects)}`)); }); + +test("CLI accounts for deleted files, source-to-doc renames, and embedded newlines", async (t) => { + const root = await mkdtemp(join(tmpdir(), "ci-selection-")); + t.after(() => rm(root, { recursive: true, force: true })); + const git = (...args) => + execFileSync("git", ["-C", root, ...args], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + git("init", "-b", "main"); + const put = async (path) => { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), "example\n"); + }; + await put("p1-task-manager/src/old.ts"); + await put("p7-collaborative-docs/src/odd\nname.ts"); + git("add", "."); + const commit = () => + git( + "-c", + "commit.gpgsign=false", + "-c", + "user.name=CI test", + "-c", + "user.email=test@example.invalid", + "commit", + "-m", + "fixture", + ); + commit(); + const base = git("rev-parse", "HEAD"); + await mkdir(join(root, "p1-task-manager/docs")); + await rename( + join(root, "p1-task-manager/src/old.ts"), + join(root, "p1-task-manager/docs/tutorials.md"), + ); + await rm(join(root, "p7-collaborative-docs/src/odd\nname.ts")); + git("add", "-A"); + commit(); + const env = { + ...process.env, + EVENT_NAME: "push", + BEFORE_SHA: base, + HEAD_SHA: git("rev-parse", "HEAD"), + }; + delete env.GITHUB_OUTPUT; + const result = spawnSync(process.execPath, [selectorScript], { + cwd: root, + env, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.match( + result.stdout, + /projects=\["p1-task-manager","p7-collaborative-docs"\]/, + ); + assert.match(result.stdout, /helper=false/); +}); diff --git a/shared/tools/repository-automation/ci-report.mjs b/shared/tools/repository-automation/ci-report.mjs new file mode 100644 index 0000000..e066b16 --- /dev/null +++ b/shared/tools/repository-automation/ci-report.mjs @@ -0,0 +1,49 @@ +// Writes runner identity and step outcomes without environment dumps or step outputs. +import { appendFile } from "node:fs/promises"; +import { arch, platform } from "node:os"; + +const stages = { + install: "Install", + identity: "Shared identity provider", + "browser-linux": "Linux browser setup", + browser: "Browser download", + check: "Checks", + integration: "Linux integration", + audit: "Dependency audit", +}; + +export function stageReport(steps) { + return `### Verification stages\n\n| Stage | Result |\n|---|---|\n${Object.entries( + stages, + ) + .map(([id, label]) => { + const outcome = steps?.[id]?.outcome; + const result = ["success", "failure", "cancelled", "skipped"].includes( + outcome, + ) + ? outcome + : "not run"; + return `| ${label} | ${result} |`; + }) + .join("\n")}\n`; +} + +if (import.meta.main) { + try { + const mode = process.argv[2]; + let text; + if (mode === "runner") { + const value = (input) => + String(input ?? "unknown").replace(/[^A-Za-z0-9_.:/+-]/g, ""); + text = `Runner: ${platform()}/${arch()}, ${value(process.env.ImageOS)} ${value(process.env.ImageVersion)}, Node ${process.version}.\n`; + } else if (mode === "stages") + text = stageReport(JSON.parse(process.env.STEPS_JSON ?? "{}")); + else throw new Error("Usage: node ci-report.mjs runner|stages"); + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile(process.env.GITHUB_STEP_SUMMARY, text); + console.info(text); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/ci-report.test.mjs b/shared/tools/repository-automation/ci-report.test.mjs new file mode 100644 index 0000000..4ecdf09 --- /dev/null +++ b/shared/tools/repository-automation/ci-report.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stageReport } from "./ci-report.mjs"; + +test("stage reports distinguish failure, cancellation, and intentional skips", () => { + const report = stageReport({ + install: { outcome: "success" }, + check: { outcome: "failure" }, + integration: { outcome: "skipped" }, + browser: { outcome: "cancelled" }, + }); + assert.match(report, /Checks \| failure/); + assert.match(report, /Browser download \| cancelled/); + assert.match(report, /Linux integration \| skipped/); + assert.match(report, /Dependency audit \| not run/); +}); + +test("reports publish known outcomes without step outputs or arbitrary fields", () => { + const report = stageReport({ + install: { + outcome: "success", + outputs: { credentials: "sensitive-fixture" }, + }, + custom: { outcome: "unsafe-fixture" }, + audit: { outcome: "unexpected-fixture" }, + }); + assert.doesNotMatch( + report, + /sensitive-fixture|unsafe-fixture|unexpected-fixture/, + ); +}); diff --git a/shared/tools/repository-automation/ci-result.mjs b/shared/tools/repository-automation/ci-result.mjs new file mode 100644 index 0000000..4399f8b --- /dev/null +++ b/shared/tools/repository-automation/ci-result.mjs @@ -0,0 +1,56 @@ +// Fails the workflow when selected verification did not complete successfully. +import { appendFile } from "node:fs/promises"; +import { composeProjects, projects } from "./changed-projects.mjs"; + +export function qualityResult(needs) { + if (needs?.changes?.result !== "success") + throw new Error("Project selection did not succeed"); + const outputs = needs.changes.outputs; + const selected = JSON.parse(outputs.projects); + const compose = JSON.parse(outputs.compose); + if ( + !Array.isArray(selected) || + new Set(selected).size !== selected.length || + !selected.every((id) => projects.includes(id)) || + !Array.isArray(compose) || + !compose.every((id) => composeProjects.has(id) && selected.includes(id)) || + JSON.stringify(compose) !== + JSON.stringify(selected.filter((id) => composeProjects.has(id))) || + !["true", "false"].includes(outputs.helper) + ) + throw new Error("Invalid CI selection outputs"); + const expected = { + step: outputs.helper === "true", + project: selected.length > 0, + "p3-sidecar": selected.includes("p3-mcp-capability-governance"), + "compose-config": compose.length > 0, + "compose-concurrency": compose.length > 0, + }; + const rows = Object.entries(expected).map(([job, required]) => { + const result = needs[job]?.result ?? "missing"; + return { + job, + required, + result, + valid: result === "success" || (!required && result === "skipped"), + }; + }); + return { selected, compose, rows, success: rows.every((row) => row.valid) }; +} + +if (import.meta.main) { + try { + const result = qualityResult(JSON.parse(process.env.NEEDS_JSON ?? "null")); + const text = `## Quality result\n\nProjects: ${result.selected.join(", ") || "none"}.\n\n| Job | Selected | Result |\n|---|---|---|\n${result.rows.map((row) => `| ${row.job} | ${row.required ? "yes" : "no"} | ${row.result} |`).join("\n")}\n`; + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile(process.env.GITHUB_STEP_SUMMARY, text); + console.info(text); + if (!result.success) + throw new Error( + "Selected verification failed, was cancelled, or did not run", + ); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/ci-result.test.mjs b/shared/tools/repository-automation/ci-result.test.mjs new file mode 100644 index 0000000..73aa1c2 --- /dev/null +++ b/shared/tools/repository-automation/ci-result.test.mjs @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { parse } from "yaml"; +import { qualityResult } from "./ci-result.mjs"; + +test("the final workflow result waits for every verification job", async () => { + const workflow = parse( + await readFile( + new URL("../../../.github/workflows/quality.yml", import.meta.url), + "utf8", + ), + ); + assert.deepEqual( + new Set(workflow.jobs.result.needs), + new Set(Object.keys(workflow.jobs).filter((id) => id !== "result")), + ); + assert.equal(workflow.jobs.result.if, "always()"); +}); + +function needs(selected = [], compose = [], helper = false) { + return { + changes: { + result: "success", + outputs: { + projects: JSON.stringify(selected), + compose: JSON.stringify(compose), + helper: String(helper), + }, + }, + step: { result: helper ? "success" : "skipped" }, + project: { result: selected.length ? "success" : "skipped" }, + "p3-sidecar": { + result: selected.includes("p3-mcp-capability-governance") + ? "success" + : "skipped", + }, + "compose-config": { result: compose.length ? "success" : "skipped" }, + "compose-concurrency": { result: compose.length ? "success" : "skipped" }, + }; +} + +test("quality accepts intentional skips and complete selected verification", () => { + for (const scenario of [ + needs(), + needs([], [], true), + needs(["p1-task-manager"], ["p1-task-manager"]), + needs( + ["p3-mcp-capability-governance"], + ["p3-mcp-capability-governance"], + true, + ), + ]) { + assert.equal(qualityResult(scenario).success, true); + } +}); + +test("quality rejects failed selection, malformed outputs, and absent job results", () => { + assert.throws(() => qualityResult({}), /selection/); + const scenario = needs(); + scenario.changes.outputs.helper = ""; + assert.throws(() => qualityResult(scenario), /outputs/); + const missing = needs(); + delete missing.step; + assert.equal(qualityResult(missing).success, false); + assert.throws(() => qualityResult(needs(["p1-task-manager"], [])), /outputs/); + assert.throws(() => qualityResult(needs(["unknown-project"], [])), /outputs/); +}); + +test("every selected verification must succeed, including sidecar and concurrency", () => { + for (const job of [ + "step", + "project", + "p3-sidecar", + "compose-config", + "compose-concurrency", + ]) { + for (const result of ["failure", "cancelled", "skipped"]) { + const scenario = needs( + ["p3-mcp-capability-governance"], + ["p3-mcp-capability-governance"], + true, + ); + scenario[job].result = result; + assert.equal(qualityResult(scenario).success, false, `${job}: ${result}`); + } + } +}); diff --git a/shared/tools/repository-automation/inspect-ci.mjs b/shared/tools/repository-automation/inspect-ci.mjs new file mode 100644 index 0000000..f18d973 --- /dev/null +++ b/shared/tools/repository-automation/inspect-ci.mjs @@ -0,0 +1,109 @@ +// Reads workflow metadata only; it never cancels, reruns, or changes a check. +import { execFileSync } from "node:child_process"; + +const repository = "GluuFederation/cedarling-tutorials"; +const seconds = (start, end) => { + const duration = (Date.parse(end) - Date.parse(start)) / 1000; + return Number.isFinite(duration) && duration >= 0 + ? Math.round(duration) + : null; +}; + +export function inspectRun(run, jobs) { + const rows = jobs.map((job) => { + const steps = job.steps ?? []; + const last = steps.at(-1); + const stale = + job.status !== "completed" && + last?.name === "Complete job" && + last.status === "completed"; + return { + name: job.name, + status: job.status, + conclusion: job.conclusion, + queuedSeconds: seconds(job.created_at, job.started_at), + executionSeconds: seconds( + job.started_at, + stale ? last.completed_at : job.completed_at, + ), + stale, + slowSteps: steps + .map((step) => ({ + name: step.name, + conclusion: step.conclusion, + seconds: seconds(step.started_at, step.completed_at), + })) + .filter( + (step) => + step.seconds >= 30 || + ["failure", "cancelled"].includes(step.conclusion), + ), + }; + }); + return { + status: run.status, + conclusion: run.conclusion, + jobs: rows, + success: + rows.length > 0 && + run.status === "completed" && + run.conclusion === "success" && + rows.every( + (job) => + job.status === "completed" && + ["success", "skipped"].includes(job.conclusion), + ), + }; +} + +export function fetchRun(id, execute = execFileSync) { + if (!/^[1-9][0-9]{0,15}$/.test(String(id))) + throw new Error("Usage: pnpm inspect:ci "); + const get = (path, paginate = false) => + JSON.parse( + execute( + "gh", + ["api", path, ...(paginate ? ["--paginate", "--slurp"] : [])], + { + encoding: "utf8", + timeout: 60_000, + maxBuffer: 32 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }, + ), + ); + const run = get(`repos/${repository}/actions/runs/${id}`); + const pages = get( + `repos/${repository}/actions/runs/${id}/jobs?filter=latest&per_page=100`, + true, + ); + return inspectRun( + run, + pages.flatMap((page) => page.jobs), + ); +} + +if (import.meta.main) { + try { + const args = process.argv.slice(2).filter((value) => value !== "--"); + if (args.length !== 1) throw new Error("Usage: pnpm inspect:ci "); + const result = fetchRun(args[0]); + console.info( + `Workflow: ${result.status}; result: ${result.conclusion ?? "pending"}`, + ); + for (const job of result.jobs) { + console.info( + `${job.name}: ${job.conclusion ?? job.status}; queued ${job.queuedSeconds ?? "?"}s; execution ${job.executionSeconds ?? "?"}s${job.stale ? "; final step completed, GitHub status is unresolved" : ""}`, + ); + for (const step of job.slowSteps) + console.info( + ` ${step.name}: ${step.seconds ?? "?"}s; ${step.conclusion ?? "pending"}`, + ); + } + if (!result.success) process.exitCode = 1; + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/inspect-ci.test.mjs b/shared/tools/repository-automation/inspect-ci.test.mjs new file mode 100644 index 0000000..0b1f19b --- /dev/null +++ b/shared/tools/repository-automation/inspect-ci.test.mjs @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { fetchRun, inspectRun } from "./inspect-ci.mjs"; + +const complete = { status: "completed", conclusion: "success" }; +const job = { + ...complete, + name: "example", + created_at: "2026-10-22T18:00:00Z", + started_at: "2026-10-22T18:01:00Z", + completed_at: "2026-10-22T18:03:00Z", + steps: [ + { + name: "Install", + ...complete, + started_at: "2026-10-22T18:01:00Z", + completed_at: "2026-10-22T18:02:30Z", + }, + ], +}; + +test("inspection separates queue and execution durations and identifies slow stages", () => { + const result = inspectRun(complete, [job]); + assert.equal(result.success, true); + assert.equal(result.jobs[0].queuedSeconds, 60); + assert.equal(result.jobs[0].executionSeconds, 120); + assert.equal(result.jobs[0].slowSteps[0].seconds, 90); +}); + +test("completed steps never turn an unresolved or cancelled job green", () => { + const stale = { + ...job, + status: "in_progress", + conclusion: null, + completed_at: null, + steps: [ + ...job.steps, + { name: "Complete job", ...complete, completed_at: job.completed_at }, + ], + }; + const result = inspectRun(complete, [stale]); + assert.equal(result.jobs[0].stale, true); + assert.equal(result.jobs[0].executionSeconds, 120); + assert.equal(result.success, false); + assert.equal( + inspectRun(complete, [{ ...job, conclusion: "cancelled" }]).success, + false, + ); + assert.equal( + inspectRun({ status: "in_progress", conclusion: null }, [job]).success, + false, + ); +}); + +test("inspection handles queued jobs without inventing execution timings", () => { + const result = inspectRun({ status: "queued" }, [ + { name: "waiting", status: "queued", steps: [] }, + ]); + assert.equal(result.jobs[0].executionSeconds, null); + assert.equal(result.jobs[0].queuedSeconds, null); + assert.equal(result.success, false); +}); + +test("inspection reads every jobs page with GET-only gh calls", () => { + const calls = []; + const result = fetchRun("123", (_file, args) => { + calls.push(args); + return JSON.stringify( + args[1].endsWith("/123") + ? complete + : [{ jobs: [job] }, { jobs: [{ ...job, name: "second" }] }], + ); + }); + assert.equal(result.jobs.length, 2); + assert.equal( + calls.every((args) => args[0] === "api" && !args.includes("--method")), + true, + ); + assert.throws(() => fetchRun("123;unsafe"), /Usage/); +}); diff --git a/shared/tools/repository-automation/package.json b/shared/tools/repository-automation/package.json index c42943a..e897f0e 100644 --- a/shared/tools/repository-automation/package.json +++ b/shared/tools/repository-automation/package.json @@ -9,11 +9,12 @@ "pnpm": ">=10.17.1 <11" }, "scripts": { - "check": "node --test changed-projects.test.mjs tutorial-contract.test.mjs && pnpm validate:tutorials", + "check": "node --test changed-projects.test.mjs tutorial-contract.test.mjs setup-browser.test.mjs ci-result.test.mjs ci-report.test.mjs inspect-ci.test.mjs && pnpm validate:tutorials", + "inspect:ci": "node inspect-ci.mjs", "validate:tutorials": "node validate-tutorials.mjs --repository ../../.." }, "devDependencies": { - "dompurify": "3.4.16", + "dompurify": "3.4.17", "jsdom": "30.1.1", "mdast-util-from-markdown": "2.0.3", "sharp": "0.35.5", diff --git a/shared/tools/repository-automation/pnpm-lock.yaml b/shared/tools/repository-automation/pnpm-lock.yaml index b8290f7..be67bbd 100644 --- a/shared/tools/repository-automation/pnpm-lock.yaml +++ b/shared/tools/repository-automation/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: dompurify: - specifier: 3.4.16 - version: 3.4.16 + specifier: 3.4.17 + version: 3.4.17 jsdom: specifier: 30.1.1 version: 30.1.1 @@ -290,8 +290,8 @@ packages: devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} - dompurify@3.4.16: - resolution: {integrity: sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ==} + dompurify@3.4.17: + resolution: {integrity: sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w==} entities@8.1.0: resolution: {integrity: sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==} @@ -698,7 +698,7 @@ snapshots: dependencies: dequal: 2.0.3 - dompurify@3.4.16: + dompurify@3.4.17: optionalDependencies: '@types/trusted-types': 2.0.7 diff --git a/shared/tools/repository-automation/setup-browser.mjs b/shared/tools/repository-automation/setup-browser.mjs new file mode 100644 index 0000000..8957282 --- /dev/null +++ b/shared/tools/repository-automation/setup-browser.mjs @@ -0,0 +1,206 @@ +// Installs Chromium on disposable GitHub Ubuntu runners with bounded network setup. +import { execFileSync, spawn } from "node:child_process"; +import { appendFile, readFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join, resolve } from "node:path"; + +export function hardenSources(text) { + return text + .replace( + /https?:\/\/(?:azure\.archive|archive)\.ubuntu\.com\/ubuntu(?=[/\s]|$)/g, + "https://archive.ubuntu.com/ubuntu", + ) + .replace( + /http:\/\/security\.ubuntu\.com\/ubuntu(?=[/\s]|$)/g, + "https://security.ubuntu.com/ubuntu", + ); +} + +function transient(error) { + const text = error.message; + if ( + /signature|NO_PUBKEY|BADSIG|EXPKEYSIG|Hash Sum mismatch|dpkg|certificate|CERT_|timed out|cancelled|Unable to locate|no installation candidate/i.test( + text, + ) + ) + return false; + return /Temporary failure resolving|Could not resolve|Could not connect|Connection (?:reset|failed|refused)|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENETUNREACH|socket hang up|\b(?:429|50[234])\b/.test( + text, + ); +} + +export async function installBrowser({ install, report = () => {} }) { + let dependenciesReady = false; + for (let attempt = 1; attempt <= 2; attempt++) { + const started = performance.now(); + for (const phase of dependenciesReady + ? ["browser"] + : ["dependencies", "browser"]) { + const phaseStarted = performance.now(); + try { + const remaining = 180_000 - (performance.now() - started); + if (remaining <= 0) throw new Error("Browser setup timed out"); + await install(phase, remaining); + if (phase === "dependencies") dependenciesReady = true; + report({ + phase, + attempt, + result: "success", + seconds: Math.ceil((performance.now() - phaseStarted) / 1000), + }); + } catch (error) { + report({ + phase, + attempt, + result: "failure", + seconds: Math.ceil((performance.now() - phaseStarted) / 1000), + }); + if (attempt === 2 || !transient(error)) throw error; + break; + } + if (phase === "browser") return; + } + } +} + +async function prepareUbuntu() { + let found = false; + for (const path of [ + "/etc/apt/sources.list.d/ubuntu.sources", + "/etc/apt/sources.list", + ]) { + let before; + try { + before = await readFile(path, "utf8"); + } catch (error) { + if (error.code === "ENOENT") continue; + throw error; + } + if ( + /https?:\/\/(?:azure\.archive|archive|security)\.ubuntu\.com\/ubuntu/.test( + before, + ) + ) + found = true; + const after = hardenSources(before); + if (after !== before) + execFileSync("sudo", ["-n", "tee", path], { + input: after, + stdio: ["pipe", "ignore", "pipe"], + }); + } + if (!found) + throw new Error("Expected official Ubuntu package sources on this runner"); + execFileSync("sudo", ["-n", "tee", "/etc/apt/apt.conf.d/99tutorial-ci"], { + input: + 'Acquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAcquire::Retries "0";\nDpkg::Use-Pty "0";\n', + stdio: ["pipe", "ignore", "pipe"], + }); +} + +export function installPhase( + phase, + budget, + { cli = resolve("node_modules/@playwright/test/cli.js"), signal } = {}, +) { + if (signal?.aborted) + return Promise.reject(new Error("Browser setup cancelled")); + const timeout = [ + "timeout", + "--signal=TERM", + "--kill-after=10s", + `${Math.max(1, Math.floor(budget / 1000))}s`, + process.execPath, + cli, + ]; + const command = phase === "dependencies" ? "sudo" : timeout.shift(); + const args = + phase === "dependencies" + ? ["-n", ...timeout, "install-deps", "chromium"] + : [...timeout, "install", "chromium"]; + return new Promise((done, reject) => { + // Root's timeout owns the APT process group; the browser download runs as the runner user. + const child = spawn(command, args, { + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + PLAYWRIGHT_BROWSERS_PATH: + process.env.PLAYWRIGHT_BROWSERS_PATH || + join(homedir(), ".cache", "ms-playwright"), + PLAYWRIGHT_DOWNLOAD_CONNECTION_TIMEOUT: "30000", + }, + }); + let output = ""; + let cancelled = false; + const stop = () => { + cancelled = true; + child.kill("SIGTERM"); + }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + signal?.addEventListener("abort", stop, { once: true }); + const clean = () => { + process.off("SIGINT", stop); + process.off("SIGTERM", stop); + signal?.removeEventListener("abort", stop); + }; + const copy = (stream) => (bytes) => { + output = (output + bytes.toString()).slice(-128 * 1024); + stream.write(bytes); + }; + child.stdout.on("data", copy(process.stdout)); + child.stderr.on("data", copy(process.stderr)); + child.once("error", (error) => { + clean(); + reject(error); + }); + child.once("close", (code, signal) => { + clean(); + if (cancelled) reject(new Error("Browser setup cancelled")); + else if (code === 124 || code === 137) + reject(new Error("Browser setup timed out")); + else if (code !== 0) + reject( + new Error( + `Browser ${phase} setup failed (${code ?? signal}): ${output}`, + ), + ); + else done(); + }); + }); +} + +if (import.meta.main) { + try { + if ( + process.env.GITHUB_ACTIONS !== "true" || + process.env.RUNNER_OS !== "Linux" || + process.platform !== "linux" + ) + throw new Error( + "Browser system setup is restricted to disposable GitHub Linux runners", + ); + await prepareUbuntu(); + const rows = []; + try { + await installBrowser({ + install: installPhase, + report: (row) => { + rows.push(row); + console.info( + `Browser setup: ${row.phase}, attempt ${row.attempt}, ${row.result}, ${row.seconds}s`, + ); + }, + }); + } finally { + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `### Browser setup\n\n| Phase | Attempt | Result | Seconds |\n|---|---:|---|---:|\n${rows.map((row) => `| ${row.phase} | ${row.attempt} | ${row.result} | ${row.seconds} |`).join("\n")}\n`, + ); + } + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/setup-browser.test.mjs b/shared/tools/repository-automation/setup-browser.test.mjs new file mode 100644 index 0000000..48b2c97 --- /dev/null +++ b/shared/tools/repository-automation/setup-browser.test.mjs @@ -0,0 +1,123 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { + hardenSources, + installBrowser, + installPhase, +} from "./setup-browser.mjs"; + +test("Ubuntu sources use official HTTPS endpoints and retain signing configuration", () => { + const source = + "URIs: http://azure.archive.ubuntu.com/ubuntu/\nSuites: noble noble-updates\nComponents: main universe\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n"; + assert.equal( + hardenSources(source), + source.replace( + "http://azure.archive.ubuntu.com", + "https://archive.ubuntu.com", + ), + ); + assert.equal( + hardenSources("deb http://security.ubuntu.com/ubuntu noble-security main"), + "deb https://security.ubuntu.com/ubuntu noble-security main", + ); + assert.equal( + hardenSources("URIs: https://packages.microsoft.com/repos/code"), + "URIs: https://packages.microsoft.com/repos/code", + ); +}); + +test( + "the installer enforces real process deadlines and cancellation", + { skip: process.platform !== "linux", timeout: 15_000 }, + async (t) => { + const root = await mkdtemp(join(tmpdir(), "ci-installer-")); + t.after(() => rm(root, { recursive: true, force: true })); + const cli = join(root, "fixture.mjs"); + await writeFile(cli, "setInterval(() => {}, 1000);\n"); + await assert.rejects(installPhase("browser", 1000, { cli }), /timed out/); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 100); + try { + await assert.rejects( + installPhase("browser", 10_000, { cli, signal: controller.signal }), + /cancelled/, + ); + } finally { + clearTimeout(timer); + } + }, +); + +test("browser setup retries a transport failure once and keeps successful dependency setup", async () => { + const calls = []; + let downloads = 0; + await installBrowser({ + install: async (phase) => { + calls.push(phase); + if (phase === "browser" && ++downloads === 1) + throw new Error("ECONNRESET"); + }, + }); + assert.deepEqual(calls, ["dependencies", "browser", "browser"]); +}); + +test("browser setup fails after its single network retry", async () => { + let calls = 0; + await assert.rejects( + installBrowser({ + install: async () => { + calls++; + throw new Error("Temporary failure resolving archive.ubuntu.com"); + }, + }), + /Temporary failure/, + ); + assert.equal(calls, 2); +}); + +test("browser setup does not retry verification, package, timeout, or cancellation failures", async () => { + for (const message of [ + "NO_PUBKEY; Failed to fetch", + "Hash Sum mismatch", + "dpkg was interrupted", + "Unable to locate package", + "certificate verification failed", + "setup timed out", + "setup cancelled", + "unrecognized failure", + ]) { + let calls = 0; + await assert.rejects( + installBrowser({ + install: async () => { + calls++; + throw new Error(message); + }, + }), + new RegExp(message), + ); + assert.equal(calls, 1, message); + } +}); + +test("setup reports each phase and a finite remaining attempt budget", async () => { + const reports = []; + await installBrowser({ + install: async (_phase, budget) => + assert.ok(budget > 0 && budget <= 180_000), + report: (row) => reports.push(row), + }); + assert.deepEqual( + reports.map((row) => row.phase), + ["dependencies", "browser"], + ); + assert.ok( + reports.every( + (row) => + row.attempt === 1 && row.result === "success" && row.seconds >= 0, + ), + ); +}); diff --git a/shared/tools/step/README.md b/shared/tools/step/README.md index beaf6cc..c57e168 100644 --- a/shared/tools/step/README.md +++ b/shared/tools/step/README.md @@ -39,5 +39,5 @@ Replay the steps in a separate checkout of that baseline before publishing. Run the helper tests from the repository root: ```sh -node --test shared/tools/step/run.test.mjs +node --test shared/tools/step/run.test.mjs shared/tools/step/source.test.mjs ``` diff --git a/shared/tools/step/run.mjs b/shared/tools/step/run.mjs index 243ca1d..86f2db3 100644 --- a/shared/tools/step/run.mjs +++ b/shared/tools/step/run.mjs @@ -5,14 +5,15 @@ import { lstat, mkdir, open, - readFile, readdir, + readFile, realpath, rename, unlink, } from "node:fs/promises"; import { join, relative, sep } from "node:path"; import { createInterface } from "node:readline/promises"; +import { readSources } from "./source.mjs"; const toolPath = "shared/tools/step"; const statePath = ".local/step"; @@ -196,24 +197,6 @@ function requireCommit(root, sha) { } } -function sourceFile(root, commit, path) { - const entry = git(root, ["ls-tree", "-z", commit, "--", path]).toString( - "utf8", - ); - if (!entry) return null; - const match = /^(100644|100755) blob ([0-9a-f]{40})\t([^\0]+)\0$/.exec(entry); - requireThat( - match && match[3] === path, - `Source must be a regular file: ${path}`, - ); - const size = Number(git(root, ["cat-file", "-s", match[2]]).toString()); - requireThat(size <= maxFileBytes, `Source file exceeds 2 MiB: ${path}`); - return { - bytes: git(root, ["cat-file", "blob", match[2]]), - mode: match[1] === "100755" ? 0o755 : 0o644, - }; -} - // Check every existing component, including ignored state directories. async function safePath(root, path, directory = false) { const parts = path.split("/"); @@ -271,7 +254,7 @@ async function writeAtomic(root, path, bytes, mode = 0o600) { } } -async function verifyTools(root, source) { +async function verifyTools(root, source, sourceFile) { const entries = git(root, [ "ls-tree", "-r", @@ -293,7 +276,7 @@ async function verifyTools(root, source) { requireThat( equal( await currentFile(root, path), - sourceFile(root, source, path)?.bytes ?? null, + sourceFile(source, path)?.bytes ?? null, ), `Helper differs from its pinned source: ${path}. Use a fresh checkout and the matching helper.`, ); @@ -306,15 +289,15 @@ function packageScripts(bytes, label) { return value; } -async function prepare(root, manifest, source, step) { +async function prepare(root, manifest, source, step, sourceFile) { const changes = []; for (const [paths, removing] of [ [step.files, false], [step.remove, true], ]) { for (const path of paths) { - const baseline = sourceFile(root, manifest.baseline, path); - const target = sourceFile(root, source, path); + const baseline = sourceFile(manifest.baseline, path); + const target = sourceFile(source, path); requireThat( removing ? baseline && !target : target, `Missing or inconsistent source operation: ${path}`, @@ -337,10 +320,10 @@ async function prepare(root, manifest, source, step) { requireThat(before !== null, `Missing ${path}`); const current = packageScripts(before, path); const baseline = packageScripts( - sourceFile(root, manifest.baseline, path)?.bytes, + sourceFile(manifest.baseline, path)?.bytes, path, ); - const target = packageScripts(sourceFile(root, source, path)?.bytes, path); + const target = packageScripts(sourceFile(source, path)?.bytes, path); requireThat( current.name === baseline.name && target.name === baseline.name, `Unexpected package name: ${path}`, @@ -517,9 +500,19 @@ export async function runTutorial( ); requireCommit(root, source); requireCommit(root, manifest.baseline); - await verifyTools(root, source); + const sourceFile = readSources( + root, + [source, manifest.baseline], + [ + toolPath, + path, + `${manifest.directory}/package.json`, + ...manifest.steps.flatMap((step) => [...step.files, ...step.remove]), + ], + ); + await verifyTools(root, source, sourceFile); requireThat( - equal(bytes, sourceFile(root, source, path)?.bytes ?? null), + equal(bytes, sourceFile(source, path)?.bytes ?? null), "Project manifest is not registered in the pinned source", ); @@ -561,12 +554,18 @@ export async function runTutorial( } // Check the declared starting files, including the files the integration will delete. for (const step of manifest.steps) { - for (const change of await prepare(root, manifest, source, step)) { + for (const change of await prepare( + root, + manifest, + source, + step, + sourceFile, + )) { if (!change.scripts) requireThat( equal( change.before, - sourceFile(root, manifest.baseline, change.path)?.bytes ?? null, + sourceFile(manifest.baseline, change.path)?.bytes ?? null, ), `Start from the declared baseline; ${change.path} is already changed`, ); @@ -595,7 +594,13 @@ export async function runTutorial( ); let changes; try { - changes = await prepare(root, manifest, source, manifest.steps[index]); + changes = await prepare( + root, + manifest, + source, + manifest.steps[index], + sourceFile, + ); } catch (error) { if (!(error instanceof LocalEditsError)) throw error; const next = manifest.steps[state.completed.length]?.id; diff --git a/shared/tools/step/source.mjs b/shared/tools/step/source.mjs new file mode 100644 index 0000000..263e88a --- /dev/null +++ b/shared/tools/step/source.mjs @@ -0,0 +1,115 @@ +// Reads only the declared immutable Git files for one helper invocation. +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; + +const maxFileBytes = 2 * 1024 * 1024; +const batchBytes = 8 * 1024 * 1024; + +export function readSources( + root, + commits, + paths, + { execute = execFileSync } = {}, +) { + const trees = new Map(); + const objects = new Map(); + const requested = [...new Set(paths)]; + if (!requested.length) throw new Error("No source paths requested"); + const git = (args, input) => + execute("git", ["--literal-pathspecs", "-C", root, ...args], { + input, + maxBuffer: 16 * 1024 * 1024, + stdio: [input ? "pipe" : "ignore", "pipe", "pipe"], + windowsHide: true, + }); + for (const commit of new Set(commits)) { + const entries = new Map(); + // Keep argument lists within Windows' command-line limit too. + const chunks = []; + let chunk = []; + let argumentBytes = 0; + for (const path of requested) { + if (chunk.length && argumentBytes + path.length + 1 > 8192) { + chunks.push(chunk); + chunk = []; + argumentBytes = 0; + } + chunk.push(path); + argumentBytes += path.length + 1; + } + chunks.push(chunk); + for (const paths of chunks) { + const listing = git([ + "ls-tree", + "-r", + "-l", + "-z", + commit, + "--", + ...paths, + ]).toString("utf8"); + for (const line of listing.split("\0").filter(Boolean)) { + const match = + /^(100644|100755) blob ([0-9a-f]{40})\s+(\d+)\t([^\0]+)$/.exec(line); + if (!match) throw new Error("Source must be a regular file"); + const [, mode, oid, length, path] = match; + const size = Number(length); + if (size > maxFileBytes) + throw new Error(`Source file exceeds 2 MiB: ${path}`); + entries.set(path, { oid, mode: mode === "100755" ? 0o755 : 0o644 }); + objects.set(oid, size); + } + } + trees.set(commit, entries); + } + const blobs = new Map(); + let cachedBytes = 0; + function load(batch) { + if (!batch.length) return; + const output = git( + ["cat-file", "--batch"], + `${batch.map(([oid]) => oid).join("\n")}\n`, + ); + let offset = 0; + for (const [oid, size] of batch) { + const end = output.indexOf(10, offset); + const header = output.subarray(offset, end).toString("ascii"); + if ( + end < offset || + header !== `${oid} blob ${size}` || + output.length < end + size + 2 + ) + throw new Error("Invalid or truncated source blob response"); + const bytes = output.subarray(end + 1, end + 1 + size); + const actual = createHash("sha1") + .update(`blob ${size}\0`) + .update(bytes) + .digest("hex"); + offset = end + size + 2; + if (output[offset - 1] !== 10 || actual !== oid) + throw new Error("Corrupt source blob response"); + while (cachedBytes + size > batchBytes && blobs.size) { + const oldest = blobs.keys().next().value; + cachedBytes -= blobs.get(oldest).length; + blobs.delete(oldest); + } + blobs.set(oid, bytes); + cachedBytes += size; + } + if (offset !== output.length) + throw new Error("Unexpected source blob response"); + } + // Small source sets fit in one batch. Larger sets are read on demand so a + // future project's manifest cannot turn the snapshot into an unbounded cache. + if ( + objects.size <= 256 && + [...objects.values()].reduce((sum, size) => sum + size, 0) <= batchBytes + ) + load([...objects]); + return (commit, path) => { + const entry = trees.get(commit)?.get(path); + if (!entry) return null; + if (!blobs.has(entry.oid)) load([[entry.oid, objects.get(entry.oid)]]); + return { bytes: blobs.get(entry.oid), mode: entry.mode }; + }; +} diff --git a/shared/tools/step/source.test.mjs b/shared/tools/step/source.test.mjs new file mode 100644 index 0000000..9cbe491 --- /dev/null +++ b/shared/tools/step/source.test.mjs @@ -0,0 +1,141 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import { readSources } from "./source.mjs"; + +const commit = "a".repeat(40); +const blob = Buffer.from([0, 10, 255, 13, 0]); +const oid = createHash("sha1") + .update(`blob ${blob.length}\0`) + .update(blob) + .digest("hex"); +const tree = (mode = "100755", size = blob.length) => + Buffer.from(`${mode} blob ${oid} ${size}\tapp/[id].js\0`); +const batch = () => + Buffer.concat([ + Buffer.from(`${oid} blob ${blob.length}\n`), + blob, + Buffer.from("\n"), + ]); +const execute = (listing, content) => (_command, args) => + args.includes("ls-tree") ? listing : content; + +test("source snapshots preserve binary bytes, modes, literal paths, and missing entries", () => { + const read = readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(tree(), batch()), + }); + assert.deepEqual(read(commit, "app/[id].js"), { bytes: blob, mode: 0o755 }); + assert.equal(read(commit, "missing.js"), null); +}); + +test("source snapshots reject non-regular and oversized files before reading blobs", () => { + for (const listing of [tree("120000"), tree("100644", 2 * 1024 * 1024 + 1)]) { + assert.throws( + () => + readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(listing, batch()), + }), + /regular file|exceeds 2 MiB/, + ); + } +}); + +test("source snapshots reject truncated, missing, corrupt, and unsolicited blob responses", () => { + for (const content of [ + batch().subarray(0, -2), + Buffer.from(`${oid} missing\n`), + Buffer.concat([ + Buffer.from(`${oid} blob ${blob.length}\n`), + Buffer.alloc(blob.length), + Buffer.from("\n"), + ]), + Buffer.concat([batch(), Buffer.from("extra")]), + ]) { + assert.throws( + () => + readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(tree(), content), + }), + /source blob/i, + ); + } +}); + +test("each invocation reads its own source snapshot", () => { + const first = readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(tree(), batch()), + }); + const second = readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(Buffer.alloc(0), Buffer.alloc(0)), + }); + assert.deepEqual(first(commit, "app/[id].js").bytes, blob); + assert.equal(second(commit, "app/[id].js"), null); +}); + +test("large source sets remain available after cache eviction", () => { + const files = Array.from({ length: 5 }, (_, index) => { + const bytes = Buffer.alloc(2 * 1024 * 1024, index); + const oid = createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + return { path: `app/file${index}.bin`, oid, bytes }; + }); + const objects = new Map(files.map((file) => [file.oid, file])); + const read = readSources( + "/fixture", + [commit], + files.map((file) => file.path), + { + execute: (_command, args, options) => + args.includes("ls-tree") + ? Buffer.from( + files + .map( + (file) => + `100644 blob ${file.oid} ${file.bytes.length}\t${file.path}\0`, + ) + .join(""), + ) + : Buffer.concat( + options.input + .trim() + .split("\n") + .map((oid) => { + const file = objects.get(oid); + return Buffer.concat([ + Buffer.from(`${oid} blob ${file.bytes.length}\n`), + file.bytes, + Buffer.from("\n"), + ]); + }), + ), + }, + ); + for (const file of [...files, files[0]]) + assert.deepEqual(read(commit, file.path).bytes, file.bytes); +}); + +test("source sets exceeding Windows' argument limit retain every literal path", () => { + const paths = Array.from( + { length: 256 }, + (_, index) => `app/${"x".repeat(180)}-${index}.js`, + ); + const read = readSources("/fixture", [commit], paths, { + execute: (_command, args) => { + assert.ok( + args.join(" ").length < 32767, + "Windows command-line limit exceeded", + ); + if (!args.includes("ls-tree")) return batch(); + return Buffer.from( + args + .slice(args.indexOf("--") + 1) + .map((path) => `100644 blob ${oid} ${blob.length}\t${path}\0`) + .join(""), + ); + }, + }); + for (const path of paths) assert.deepEqual(read(commit, path).bytes, blob); + assert.throws(() => readSources("/fixture", [commit], []), /No source paths/); +}); From 247d9fa17641693db403b36942353173275eecc5 Mon Sep 17 00:00:00 2001 From: Yamonwan Justin DAH-KENANGNON Date: Fri, 23 Oct 2026 06:16:37 +0100 Subject: [PATCH 2/4] fix(ci): handle GitHub Ubuntu mirror-list sources GitHub's Ubuntu 24.04 image uses mirror+file:/etc/apt/apt-mirrors.txt rather than direct archive URLs. Normalize this known source before validation while preserving repository signing settings, and cover preparation with hosted and legacy source fixtures. Signed-off-by: Yamonwan Justin DAH-KENANGNON --- .../repository-automation/setup-browser.mjs | 31 ++++++---- .../setup-browser.test.mjs | 60 +++++++++++++++++++ 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/shared/tools/repository-automation/setup-browser.mjs b/shared/tools/repository-automation/setup-browser.mjs index 8957282..638e36e 100644 --- a/shared/tools/repository-automation/setup-browser.mjs +++ b/shared/tools/repository-automation/setup-browser.mjs @@ -6,6 +6,10 @@ import { join, resolve } from "node:path"; export function hardenSources(text) { return text + .replace( + /mirror\+file:\/etc\/apt\/apt-mirrors\.txt(?=\s|$)/g, + "https://archive.ubuntu.com/ubuntu", + ) .replace( /https?:\/\/(?:azure\.archive|archive)\.ubuntu\.com\/ubuntu(?=[/\s]|$)/g, "https://archive.ubuntu.com/ubuntu", @@ -63,35 +67,40 @@ export async function installBrowser({ install, report = () => {} }) { } } -async function prepareUbuntu() { +export async function prepareUbuntu({ + read = readFile, + execute = execFileSync, +} = {}) { let found = false; + const updates = []; for (const path of [ "/etc/apt/sources.list.d/ubuntu.sources", "/etc/apt/sources.list", ]) { let before; try { - before = await readFile(path, "utf8"); + before = await read(path, "utf8"); } catch (error) { if (error.code === "ENOENT") continue; throw error; } + const after = hardenSources(before); if ( - /https?:\/\/(?:azure\.archive|archive|security)\.ubuntu\.com\/ubuntu/.test( - before, + /https:\/\/(?:archive|security)\.ubuntu\.com\/ubuntu(?=[/\s]|$)/.test( + after, ) ) found = true; - const after = hardenSources(before); - if (after !== before) - execFileSync("sudo", ["-n", "tee", path], { - input: after, - stdio: ["pipe", "ignore", "pipe"], - }); + if (after !== before) updates.push({ path, after }); } if (!found) throw new Error("Expected official Ubuntu package sources on this runner"); - execFileSync("sudo", ["-n", "tee", "/etc/apt/apt.conf.d/99tutorial-ci"], { + for (const { path, after } of updates) + execute("sudo", ["-n", "tee", path], { + input: after, + stdio: ["pipe", "ignore", "pipe"], + }); + execute("sudo", ["-n", "tee", "/etc/apt/apt.conf.d/99tutorial-ci"], { input: 'Acquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAcquire::Retries "0";\nDpkg::Use-Pty "0";\n', stdio: ["pipe", "ignore", "pipe"], diff --git a/shared/tools/repository-automation/setup-browser.test.mjs b/shared/tools/repository-automation/setup-browser.test.mjs index 48b2c97..f017fcb 100644 --- a/shared/tools/repository-automation/setup-browser.test.mjs +++ b/shared/tools/repository-automation/setup-browser.test.mjs @@ -7,8 +7,68 @@ import { hardenSources, installBrowser, installPhase, + prepareUbuntu, } from "./setup-browser.mjs"; +test("Ubuntu preparation supports the hosted runner mirror list and preserves repository signing", async () => { + const path = "/etc/apt/sources.list.d/ubuntu.sources"; + const source = + "Types: deb\nURIs: mirror+file:/etc/apt/apt-mirrors.txt\nSuites: noble noble-updates noble-security\nComponents: main restricted universe multiverse\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n"; + const writes = new Map(); + await prepareUbuntu({ + read: async (file) => { + if (file === path) return source; + throw Object.assign(new Error("missing fixture"), { code: "ENOENT" }); + }, + execute: (command, args, options) => { + assert.equal(command, "sudo"); + assert.deepEqual(args.slice(0, 2), ["-n", "tee"]); + writes.set(args[2], options.input); + }, + }); + assert.equal( + writes.get(path), + source.replace( + "mirror+file:/etc/apt/apt-mirrors.txt", + "https://archive.ubuntu.com/ubuntu", + ), + ); + assert.match( + writes.get("/etc/apt/apt.conf.d/99tutorial-ci"), + /Acquire::https::Timeout "30"/, + ); + assert.equal(writes.size, 2); +}); + +test("Ubuntu preparation also handles legacy direct sources", async () => { + const path = "/etc/apt/sources.list"; + const source = + "deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://azure.archive.ubuntu.com/ubuntu/ noble main\n"; + const writes = new Map(); + await prepareUbuntu({ + read: async (file) => { + if (file === path) return source; + throw Object.assign(new Error("missing fixture"), { code: "ENOENT" }); + }, + execute: (_command, args, options) => writes.set(args[2], options.input), + }); + assert.equal( + writes.get(path), + source.replace("http://azure.archive", "https://archive"), + ); + assert.equal(writes.size, 2); +}); + +test("Ubuntu preparation rejects unknown mirror files without changing configuration", async () => { + await assert.rejects( + prepareUbuntu({ + read: async () => "URIs: mirror+file:/tmp/untrusted-mirrors.txt\n", + execute: () => assert.fail("unsupported sources must not be written"), + }), + /Expected official Ubuntu package sources/, + ); +}); + test("Ubuntu sources use official HTTPS endpoints and retain signing configuration", () => { const source = "URIs: http://azure.archive.ubuntu.com/ubuntu/\nSuites: noble noble-updates\nComponents: main universe\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n"; From 89bf63fe1f6362ce10ac3efdc46e890df3adf7e1 Mon Sep 17 00:00:00 2001 From: Yamonwan Justin DAH-KENANGNON Date: Fri, 23 Oct 2026 06:16:38 +0100 Subject: [PATCH 3/4] fix(deps): patch Fastify header validation Update the six affected applications from Fastify 5.12.5 to 5.12.6 to address GHSA-389g-gv7q-r4x8 without changing their baseline commits or authorization behavior. Signed-off-by: Yamonwan Justin DAH-KENANGNON --- p1-task-manager/package.json | 2 +- p1-task-manager/pnpm-lock.yaml | 10 +++++----- p10-warehouse-workloads/package.json | 2 +- p10-warehouse-workloads/pnpm-lock.yaml | 10 +++++----- p14-capital-district/package.json | 2 +- p14-capital-district/pnpm-lock.yaml | 10 +++++----- p2-tenantrag/package.json | 2 +- p2-tenantrag/pnpm-lock.yaml | 10 +++++----- p6-field-inspection/package.json | 2 +- p6-field-inspection/pnpm-lock.yaml | 10 +++++----- p7-collaborative-docs/package.json | 2 +- p7-collaborative-docs/pnpm-lock.yaml | 10 +++++----- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/p1-task-manager/package.json b/p1-task-manager/package.json index 50a3285..3cc22a9 100644 --- a/p1-task-manager/package.json +++ b/p1-task-manager/package.json @@ -28,7 +28,7 @@ "@fastify/swagger": "9.9.2", "@janssenproject/cedarling_wasm": "0.0.468", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "fflate": "0.8.3", "openid-client": "6.8.8", "react": "19.3.1", diff --git a/p1-task-manager/pnpm-lock.yaml b/p1-task-manager/pnpm-lock.yaml index 6290a27..6fa1b3d 100644 --- a/p1-task-manager/pnpm-lock.yaml +++ b/p1-task-manager/pnpm-lock.yaml @@ -27,8 +27,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 fflate: specifier: 0.8.3 version: 0.8.3 @@ -885,8 +885,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -2279,7 +2279,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p10-warehouse-workloads/package.json b/p10-warehouse-workloads/package.json index 45499cf..a5531ca 100644 --- a/p10-warehouse-workloads/package.json +++ b/p10-warehouse-workloads/package.json @@ -25,7 +25,7 @@ "@fastify/helmet": "13.2.0", "@fastify/static": "10.1.6", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "jose": "6.2.2", "openid-client": "6.8.8", "react": "19.3.1", diff --git a/p10-warehouse-workloads/pnpm-lock.yaml b/p10-warehouse-workloads/pnpm-lock.yaml index b4c896e..69e951d 100644 --- a/p10-warehouse-workloads/pnpm-lock.yaml +++ b/p10-warehouse-workloads/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 jose: specifier: 6.2.2 version: 6.2.2 @@ -507,8 +507,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1391,7 +1391,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p14-capital-district/package.json b/p14-capital-district/package.json index d27db2d..bceb32e 100644 --- a/p14-capital-district/package.json +++ b/p14-capital-district/package.json @@ -27,7 +27,7 @@ "@fastify/helmet": "13.1.1", "@fastify/static": "10.1.5", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "openid-client": "6.8.8", "react": "19.3.0", "react-dom": "19.3.0" diff --git a/p14-capital-district/pnpm-lock.yaml b/p14-capital-district/pnpm-lock.yaml index c37b3a5..b788665 100644 --- a/p14-capital-district/pnpm-lock.yaml +++ b/p14-capital-district/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 openid-client: specifier: 6.8.8 version: 6.8.8 @@ -438,8 +438,8 @@ packages: fastify-plugin@6.0.1: resolution: {integrity: sha512-4wW4RxSmvPXLrh5mgQEiu43qU3bw44N2ss4FTkTkQIbprUhE+/s43QX0j/ttcztdWK7PEkHqVthdc5GdEGDvjA==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1160,7 +1160,7 @@ snapshots: fastify-plugin@6.0.1: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.7 '@fastify/error': 4.2.1 diff --git a/p2-tenantrag/package.json b/p2-tenantrag/package.json index c94667b..9f1f342 100644 --- a/p2-tenantrag/package.json +++ b/p2-tenantrag/package.json @@ -25,7 +25,7 @@ "dependencies": { "@janssenproject/cedarling_wasm": "0.0.468", "@orama/orama": "3.1.18", - "fastify": "5.12.5", + "fastify": "5.12.6", "fflate": "0.8.3", "jose": "6.2.10", "pdfjs-dist": "6.2.108", diff --git a/p2-tenantrag/pnpm-lock.yaml b/p2-tenantrag/pnpm-lock.yaml index a835a27..92931e4 100644 --- a/p2-tenantrag/pnpm-lock.yaml +++ b/p2-tenantrag/pnpm-lock.yaml @@ -15,8 +15,8 @@ importers: specifier: 3.1.18 version: 3.1.18 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 fflate: specifier: 0.8.3 version: 0.8.3 @@ -772,8 +772,8 @@ packages: fast-uri@4.1.5: resolution: {integrity: sha512-vZeoMRB4epNr7QfdHxel7te/RcX16CxyXI07JCCTFWZA2s4v1azGNESRj+2EoaHSaWFL/Z3GmKT2jF6A202jLg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -1917,7 +1917,7 @@ snapshots: fast-uri@4.1.5: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p6-field-inspection/package.json b/p6-field-inspection/package.json index 58f0e9a..15ae14e 100644 --- a/p6-field-inspection/package.json +++ b/p6-field-inspection/package.json @@ -26,7 +26,7 @@ "@fastify/helmet": "13.2.0", "@fastify/static": "10.1.6", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "openid-client": "6.8.8", "react": "19.3.1", "react-dom": "19.3.1" diff --git a/p6-field-inspection/pnpm-lock.yaml b/p6-field-inspection/pnpm-lock.yaml index 95aaa2b..c009c3c 100644 --- a/p6-field-inspection/pnpm-lock.yaml +++ b/p6-field-inspection/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 openid-client: specifier: 6.8.8 version: 6.8.8 @@ -514,8 +514,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1402,7 +1402,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p7-collaborative-docs/package.json b/p7-collaborative-docs/package.json index ace7287..ea49a14 100644 --- a/p7-collaborative-docs/package.json +++ b/p7-collaborative-docs/package.json @@ -26,7 +26,7 @@ "@fastify/helmet": "13.2.0", "@fastify/static": "10.1.6", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "openid-client": "6.8.8", "react": "19.3.1", "react-dom": "19.3.1" diff --git a/p7-collaborative-docs/pnpm-lock.yaml b/p7-collaborative-docs/pnpm-lock.yaml index 4eff895..213e233 100644 --- a/p7-collaborative-docs/pnpm-lock.yaml +++ b/p7-collaborative-docs/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 openid-client: specifier: 6.8.8 version: 6.8.8 @@ -600,8 +600,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1437,7 +1437,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 From 4591e17787e2b57c31b1977a7618fbfd45519156 Mon Sep 17 00:00:00 2001 From: Yamonwan Justin DAH-KENANGNON Date: Fri, 23 Oct 2026 07:10:37 +0100 Subject: [PATCH 4/4] perf(ci): streamline browser setup and stack verification Signed-off-by: Yamonwan Justin DAH-KENANGNON --- .github/workflows/quality.yml | 5 +- shared/tools/repository-automation/README.md | 6 +- .../changed-projects.mjs | 19 ++++- .../changed-projects.test.mjs | 76 +++++++++++++++---- .../tools/repository-automation/ci-result.mjs | 11 ++- .../repository-automation/ci-result.test.mjs | 33 +++++++- .../repository-automation/setup-browser.mjs | 2 +- .../setup-browser.test.mjs | 25 +++++- 8 files changed, 151 insertions(+), 26 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index c666451..346a008 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -46,6 +46,7 @@ jobs: projects: ${{ steps.select.outputs.projects }} compose: ${{ steps.select.outputs.compose }} helper: ${{ steps.select.outputs.helper }} + concurrent: ${{ steps.select.outputs.concurrent }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -132,7 +133,7 @@ jobs: timeout-minutes: 3 if: (matrix.project == 'p1-task-manager' || matrix.project == 'p4-editorial-publishing' || matrix.project == 'p5-dataguard' || matrix.project == 'p6-field-inspection' || matrix.project == 'p7-collaborative-docs' || matrix.project == 'p10-warehouse-workloads' || matrix.project == 'p14-capital-district' || matrix.project == 'p15-marketplace') && runner.os != 'Linux' working-directory: ${{ matrix.project }} - run: pnpm exec playwright install chromium + run: pnpm exec playwright install --only-shell chromium env: PLAYWRIGHT_DOWNLOAD_CONNECTION_TIMEOUT: 30000 - name: Check @@ -230,7 +231,7 @@ jobs: compose-concurrency: name: Concurrent project stacks needs: changes - if: needs.changes.outputs.compose != '[]' + if: needs.changes.outputs.concurrent == 'true' runs-on: ubuntu-24.04 timeout-minutes: 25 steps: diff --git a/shared/tools/repository-automation/README.md b/shared/tools/repository-automation/README.md index 00f754d..118175b 100644 --- a/shared/tools/repository-automation/README.md +++ b/shared/tools/repository-automation/README.md @@ -6,7 +6,11 @@ macOS, and Windows, with separate Docker checks. Tutorial text, images, and READ changes run source validation. Shared runtime or workflow changes run the full portfolio, as do scheduled and manually dispatched checks. -The Linux browser setup uses Ubuntu 24.04 and official HTTPS package sources. +The concurrent-stack scenario runs when P1, P4, or P10 is affected, or when the +full portfolio is selected. Other project changes run their own Compose checks. + +Browser checks install Chromium's headless shell. Linux setup uses Ubuntu 24.04 +and official HTTPS package sources. Each attempt has a three-minute budget. A transport failure can trigger one retry; signature, package-configuration, and timeout failures stop setup. The final Quality result check requires every selected job to succeed. diff --git a/shared/tools/repository-automation/changed-projects.mjs b/shared/tools/repository-automation/changed-projects.mjs index d41e872..6a5d715 100644 --- a/shared/tools/repository-automation/changed-projects.mjs +++ b/shared/tools/repository-automation/changed-projects.mjs @@ -23,6 +23,15 @@ export const tutorialProjects = [ ]; export const projects = [...tutorialProjects, "shared/identity-provider"]; export const composeProjects = new Set(tutorialProjects); +const concurrentProjects = new Set([ + "p1-task-manager", + "p4-editorial-publishing", + "p10-warehouse-workloads", +]); + +export function needsConcurrentStacks(selected) { + return selected.some((project) => concurrentProjects.has(project)); +} function changedFiles(environment = process.env) { const event = environment.EVENT_NAME; @@ -53,7 +62,12 @@ function changedFiles(environment = process.env) { } export function selectProjects(files) { - const all = () => ({ projects, compose: [...composeProjects], helper: true }); + const all = () => ({ + projects, + compose: [...composeProjects], + helper: true, + concurrent: true, + }); if (files === null) return all(); const affected = new Set(); let helper = false; @@ -88,6 +102,7 @@ export function selectProjects(files) { projects: selected, compose: selected.filter((project) => composeProjects.has(project)), helper, + concurrent: needsConcurrentStacks(selected), }; } @@ -102,7 +117,7 @@ function main(environment = process.env) { files = null; } const selected = selectProjects(files); - const output = `projects=${JSON.stringify(selected.projects)}\ncompose=${JSON.stringify(selected.compose)}\nhelper=${selected.helper}\n`; + const output = `projects=${JSON.stringify(selected.projects)}\ncompose=${JSON.stringify(selected.compose)}\nhelper=${selected.helper}\nconcurrent=${selected.concurrent}\n`; if (environment.GITHUB_OUTPUT) appendFileSync(environment.GITHUB_OUTPUT, output); else process.stdout.write(output); diff --git a/shared/tools/repository-automation/changed-projects.test.mjs b/shared/tools/repository-automation/changed-projects.test.mjs index 2aba4ac..204290c 100644 --- a/shared/tools/repository-automation/changed-projects.test.mjs +++ b/shared/tools/repository-automation/changed-projects.test.mjs @@ -24,6 +24,7 @@ test("selects only the changed project", () => { projects: ["p7-collaborative-docs"], compose: ["p7-collaborative-docs"], helper: false, + concurrent: false, }, ); }); @@ -35,6 +36,7 @@ test("selects Capital District for a P14 game change", () => { projects: ["p14-capital-district"], compose: ["p14-capital-district"], helper: false, + concurrent: false, }, ); }); @@ -46,25 +48,52 @@ test("selects P3 Compose without a separate shared-provider stack", () => { projects: ["p3-mcp-capability-governance"], compose: ["p3-mcp-capability-governance"], helper: false, + concurrent: false, }, ); assert.equal(composeProjects.has("shared/identity-provider"), false); }); test("selects the complete portfolio for shared and workflow changes", () => { - assert.deepEqual( - selectProjects(["shared/identity-provider/src/config.ts"]).projects, - projects, - ); - assert.deepEqual( - selectProjects(["shared/dev-supervisor.mjs"]).projects, - projects, - ); - assert.deepEqual( - selectProjects([".github/workflows/quality.yml"]).projects, - projects, + for (const path of [ + "shared/identity-provider/src/config.ts", + "shared/dev-supervisor.mjs", + "shared/tools/repository-automation/setup-browser.mjs", + ".github/workflows/quality.yml", + ]) { + assert.deepEqual(selectProjects([path]), { + projects, + compose: [...composeProjects], + helper: true, + concurrent: true, + }); + } +}); + +test("selects concurrent stacks only when their applications are affected", () => { + for (const project of [ + "p1-task-manager", + "p4-editorial-publishing", + "p10-warehouse-workloads", + ]) { + const selected = selectProjects([`${project}/src/server/app.ts`]); + assert.deepEqual(selected.projects, [project]); + assert.deepEqual(selected.compose, [project]); + assert.equal(selected.concurrent, true); + } + for (const project of ["p2-tenantrag", "p6-field-inspection"]) { + const selected = selectProjects([`${project}/compose.yaml`]); + assert.deepEqual(selected.projects, [project]); + assert.deepEqual(selected.compose, [project]); + assert.equal(selected.concurrent, false); + } + assert.equal( + selectProjects([ + "p2-tenantrag/src/app.ts", + "p4-editorial-publishing/compose.yaml", + ]).concurrent, + true, ); - assert.deepEqual(selectProjects(null).projects, projects); }); test("does not run project jobs for root documentation alone", () => { @@ -72,6 +101,7 @@ test("does not run project jobs for root documentation alone", () => { projects: [], compose: [], helper: false, + concurrent: false, }); }); @@ -83,7 +113,7 @@ test("tutorial and README changes need validation without application matrices", "p6-field-inspection/README.md", "shared/tools/repository-automation/README.md", ]), - { projects: [], compose: [], helper: false }, + { projects: [], compose: [], helper: false, concurrent: false }, ); }); @@ -92,6 +122,7 @@ test("helper changes select its portable tests without unrelated applications", projects: [], compose: [], helper: true, + concurrent: false, }); assert.deepEqual( selectProjects([ @@ -102,6 +133,7 @@ test("helper changes select its portable tests without unrelated applications", projects: ["p1-task-manager"], compose: ["p1-task-manager"], helper: true, + concurrent: true, }, ); }); @@ -117,6 +149,7 @@ test("unknown paths and scheduled runs conservatively select everything", () => projects, compose: [...composeProjects], helper: true, + concurrent: true, }); } assert.deepEqual( @@ -149,6 +182,22 @@ test("reports when project detection falls back to the full matrix", () => { /Changed-project detection failed; selecting the full project matrix/, ); assert.ok(result.stdout.includes(`projects=${JSON.stringify(projects)}`)); + assert.match(result.stdout, /concurrent=true/); +}); + +test("scheduled and manual runs select concurrent stacks with the full portfolio", () => { + for (const event of ["schedule", "workflow_dispatch"]) { + const env = { ...process.env, EVENT_NAME: event }; + delete env.GITHUB_OUTPUT; + const result = spawnSync(process.execPath, [selectorScript], { + encoding: "utf8", + env, + }); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.stdout.includes(`projects=${JSON.stringify(projects)}`)); + assert.match(result.stdout, /concurrent=true/); + assert.equal(result.stderr, ""); + } }); test("CLI accounts for deleted files, source-to-doc renames, and embedded newlines", async (t) => { @@ -207,4 +256,5 @@ test("CLI accounts for deleted files, source-to-doc renames, and embedded newlin /projects=\["p1-task-manager","p7-collaborative-docs"\]/, ); assert.match(result.stdout, /helper=false/); + assert.match(result.stdout, /concurrent=true/); }); diff --git a/shared/tools/repository-automation/ci-result.mjs b/shared/tools/repository-automation/ci-result.mjs index 4399f8b..8d3c8ba 100644 --- a/shared/tools/repository-automation/ci-result.mjs +++ b/shared/tools/repository-automation/ci-result.mjs @@ -1,6 +1,10 @@ // Fails the workflow when selected verification did not complete successfully. import { appendFile } from "node:fs/promises"; -import { composeProjects, projects } from "./changed-projects.mjs"; +import { + composeProjects, + needsConcurrentStacks, + projects, +} from "./changed-projects.mjs"; export function qualityResult(needs) { if (needs?.changes?.result !== "success") @@ -16,7 +20,8 @@ export function qualityResult(needs) { !compose.every((id) => composeProjects.has(id) && selected.includes(id)) || JSON.stringify(compose) !== JSON.stringify(selected.filter((id) => composeProjects.has(id))) || - !["true", "false"].includes(outputs.helper) + !["true", "false"].includes(outputs.helper) || + outputs.concurrent !== String(needsConcurrentStacks(selected)) ) throw new Error("Invalid CI selection outputs"); const expected = { @@ -24,7 +29,7 @@ export function qualityResult(needs) { project: selected.length > 0, "p3-sidecar": selected.includes("p3-mcp-capability-governance"), "compose-config": compose.length > 0, - "compose-concurrency": compose.length > 0, + "compose-concurrency": outputs.concurrent === "true", }; const rows = Object.entries(expected).map(([job, required]) => { const result = needs[job]?.result ?? "missing"; diff --git a/shared/tools/repository-automation/ci-result.test.mjs b/shared/tools/repository-automation/ci-result.test.mjs index 73aa1c2..01976ef 100644 --- a/shared/tools/repository-automation/ci-result.test.mjs +++ b/shared/tools/repository-automation/ci-result.test.mjs @@ -16,9 +16,24 @@ test("the final workflow result waits for every verification job", async () => { new Set(Object.keys(workflow.jobs).filter((id) => id !== "result")), ); assert.equal(workflow.jobs.result.if, "always()"); + assert.equal( + workflow.jobs["compose-concurrency"].if, + "needs.changes.outputs.concurrent == 'true'", + ); + assert.equal( + workflow.jobs.changes.outputs.concurrent, + `\${{ steps.select.outputs.concurrent }}`, + ); }); function needs(selected = [], compose = [], helper = false) { + const concurrent = selected.some((id) => + [ + "p1-task-manager", + "p4-editorial-publishing", + "p10-warehouse-workloads", + ].includes(id), + ); return { changes: { result: "success", @@ -26,6 +41,7 @@ function needs(selected = [], compose = [], helper = false) { projects: JSON.stringify(selected), compose: JSON.stringify(compose), helper: String(helper), + concurrent: String(concurrent), }, }, step: { result: helper ? "success" : "skipped" }, @@ -36,7 +52,7 @@ function needs(selected = [], compose = [], helper = false) { : "skipped", }, "compose-config": { result: compose.length ? "success" : "skipped" }, - "compose-concurrency": { result: compose.length ? "success" : "skipped" }, + "compose-concurrency": { result: concurrent ? "success" : "skipped" }, }; } @@ -67,6 +83,17 @@ test("quality rejects failed selection, malformed outputs, and absent job result assert.throws(() => qualityResult(needs(["unknown-project"], [])), /outputs/); }); +test("quality rejects missing or inconsistent concurrent-stack selection", () => { + const required = needs(["p1-task-manager"], ["p1-task-manager"]); + for (const value of [undefined, "", "false", "TRUE"]) { + required.changes.outputs.concurrent = value; + assert.throws(() => qualityResult(required), /outputs/); + } + const unrelated = needs(["p2-tenantrag"], ["p2-tenantrag"]); + unrelated.changes.outputs.concurrent = "true"; + assert.throws(() => qualityResult(unrelated), /outputs/); +}); + test("every selected verification must succeed, including sidecar and concurrency", () => { for (const job of [ "step", @@ -77,8 +104,8 @@ test("every selected verification must succeed, including sidecar and concurrenc ]) { for (const result of ["failure", "cancelled", "skipped"]) { const scenario = needs( - ["p3-mcp-capability-governance"], - ["p3-mcp-capability-governance"], + ["p1-task-manager", "p3-mcp-capability-governance"], + ["p1-task-manager", "p3-mcp-capability-governance"], true, ); scenario[job].result = result; diff --git a/shared/tools/repository-automation/setup-browser.mjs b/shared/tools/repository-automation/setup-browser.mjs index 638e36e..a9d6a6a 100644 --- a/shared/tools/repository-automation/setup-browser.mjs +++ b/shared/tools/repository-automation/setup-browser.mjs @@ -126,7 +126,7 @@ export function installPhase( const args = phase === "dependencies" ? ["-n", ...timeout, "install-deps", "chromium"] - : [...timeout, "install", "chromium"]; + : [...timeout, "install", "--only-shell", "chromium"]; return new Promise((done, reject) => { // Root's timeout owns the APT process group; the browser download runs as the runner user. const child = spawn(command, args, { diff --git a/shared/tools/repository-automation/setup-browser.test.mjs b/shared/tools/repository-automation/setup-browser.test.mjs index f017fcb..3ce5ed5 100644 --- a/shared/tools/repository-automation/setup-browser.test.mjs +++ b/shared/tools/repository-automation/setup-browser.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -10,6 +10,29 @@ import { prepareUbuntu, } from "./setup-browser.mjs"; +test( + "the browser installer requests Chromium's headless shell", + { skip: process.platform !== "linux", timeout: 15_000 }, + async (t) => { + const root = await mkdtemp(join(tmpdir(), "ci-browser-args-")); + t.after(() => rm(root, { recursive: true, force: true })); + const cli = join(root, "fixture.mjs"); + const output = join(root, "args.json"); + await writeFile( + cli, + 'import { writeFileSync } from "node:fs"; writeFileSync(' + + JSON.stringify(output) + + ", JSON.stringify(process.argv.slice(2)));\n", + ); + await installPhase("browser", 10_000, { cli }); + assert.deepEqual(JSON.parse(await readFile(output, "utf8")), [ + "install", + "--only-shell", + "chromium", + ]); + }, +); + test("Ubuntu preparation supports the hosted runner mirror list and preserves repository signing", async () => { const path = "/etc/apt/sources.list.d/ubuntu.sources"; const source =