diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 1dc8625..346a008 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -19,35 +19,44 @@ concurrency: jobs: step: name: Tutorial helper (${{ matrix.os }}) - runs-on: ${{ matrix.os }} + needs: changes + if: needs.changes.outputs.helper == 'true' + runs-on: ${{ matrix.os == 'ubuntu-latest' && 'ubuntu-24.04' || matrix.os }} timeout-minutes: 5 strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v7.0.1 - - uses: actions/setup-node@v7.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 - name: Check tutorial step helper - run: node --test shared/tools/step/run.test.mjs + id: check + run: node --test shared/tools/step/run.test.mjs shared/tools/step/source.test.mjs changes: name: Select projects - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 5 outputs: projects: ${{ steps.select.outputs.projects }} compose: ${{ steps.select.outputs.compose }} + helper: ${{ steps.select.outputs.helper }} + concurrent: ${{ steps.select.outputs.concurrent }} steps: - - uses: actions/checkout@v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - - uses: pnpm/action-setup@v6.1.0 + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 10.17.1 run_install: false - - uses: actions/setup-node@v7.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 cache: pnpm @@ -58,6 +67,10 @@ jobs: - name: Check repository automation working-directory: shared/tools/repository-automation run: pnpm check + - name: Audit repository automation + timeout-minutes: 2 + working-directory: shared/tools/repository-automation + run: pnpm audit --audit-level low - id: select name: Detect affected projects run: node shared/tools/repository-automation/changed-projects.mjs @@ -71,8 +84,8 @@ jobs: name: ${{ matrix.project }} (${{ matrix.os }}) needs: changes if: needs.changes.outputs.projects != '[]' - runs-on: ${{ matrix.os }} - timeout-minutes: 10 + runs-on: ${{ matrix.os == 'ubuntu-latest' && 'ubuntu-24.04' || matrix.os }} + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -80,64 +93,96 @@ jobs: project: ${{ fromJSON(needs.changes.outputs.projects) }} steps: - - uses: actions/checkout@v7.0.1 - - uses: pnpm/action-setup@v6.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 10.17.1 run_install: false - - uses: actions/setup-node@v7.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 cache: pnpm - cache-dependency-path: ${{ matrix.project }}/pnpm-lock.yaml + cache-dependency-path: | + ${{ matrix.project }}/pnpm-lock.yaml + shared/identity-provider/pnpm-lock.yaml + - name: Runner identity + run: node shared/tools/repository-automation/ci-report.mjs runner - name: Install + id: install + timeout-minutes: 3 working-directory: ${{ matrix.project }} run: pnpm install --frozen-lockfile - name: Prepare shared identity provider + id: identity + timeout-minutes: 2 if: matrix.project != 'shared/identity-provider' working-directory: shared/identity-provider run: | pnpm install --frozen-lockfile pnpm build - name: Install browser and Linux dependencies + id: browser-linux + timeout-minutes: 7 if: (matrix.project == 'p1-task-manager' || matrix.project == 'p4-editorial-publishing' || matrix.project == 'p5-dataguard' || matrix.project == 'p6-field-inspection' || matrix.project == 'p7-collaborative-docs' || matrix.project == 'p10-warehouse-workloads' || matrix.project == 'p11-saas-workspace' || matrix.project == 'p14-capital-district' || matrix.project == 'p15-marketplace') && runner.os == 'Linux' working-directory: ${{ matrix.project }} - run: pnpm exec playwright install --with-deps chromium + run: node ../shared/tools/repository-automation/setup-browser.mjs - name: Install browser + id: browser + timeout-minutes: 3 if: (matrix.project == 'p1-task-manager' || matrix.project == 'p4-editorial-publishing' || matrix.project == 'p5-dataguard' || matrix.project == 'p6-field-inspection' || matrix.project == 'p7-collaborative-docs' || matrix.project == 'p10-warehouse-workloads' || matrix.project == 'p14-capital-district' || matrix.project == 'p15-marketplace') && runner.os != 'Linux' working-directory: ${{ matrix.project }} - run: pnpm exec playwright install chromium + run: pnpm exec playwright install --only-shell chromium + env: + PLAYWRIGHT_DOWNLOAD_CONNECTION_TIMEOUT: 30000 - name: Check + id: check + timeout-minutes: 5 working-directory: ${{ matrix.project }} run: pnpm check - name: Run Linux integration scenario + id: integration + timeout-minutes: 5 if: runner.os == 'Linux' && (matrix.project == 'p8-cedarfile' || matrix.project == 'p9-cedarrealtime' || matrix.project == 'p11-saas-workspace' || matrix.project == 'p12-hr-access-governance' || matrix.project == 'p13-student-records' || matrix.project == 'p15-marketplace') working-directory: ${{ matrix.project }} run: pnpm test:e2e - name: Audit dependencies + id: audit + timeout-minutes: 2 working-directory: ${{ matrix.project }} run: pnpm audit --audit-level low + - name: Verification summary + if: always() + run: node shared/tools/repository-automation/ci-report.mjs stages + env: + STEPS_JSON: ${{ toJSON(steps) }} + p3-sidecar: name: P3 signed-token sidecar workflow needs: changes if: contains(fromJSON(needs.changes.outputs.projects), 'p3-mcp-capability-governance') - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 defaults: run: working-directory: p3-mcp-capability-governance steps: - - uses: actions/checkout@v7.0.1 - - uses: pnpm/action-setup@v6.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 10.17.1 run_install: false - - uses: actions/setup-node@v7.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 cache: pnpm - cache-dependency-path: p3-mcp-capability-governance/pnpm-lock.yaml + cache-dependency-path: | + p3-mcp-capability-governance/pnpm-lock.yaml + shared/identity-provider/pnpm-lock.yaml - run: pnpm install --frozen-lockfile - run: pnpm test:e2e @@ -145,15 +190,17 @@ jobs: name: Compose startup (${{ matrix.project }}) needs: changes if: needs.changes.outputs.compose != '[]' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 20 strategy: fail-fast: false matrix: project: ${{ fromJSON(needs.changes.outputs.compose) }} steps: - - uses: actions/checkout@v7.0.1 - - uses: actions/setup-node@v7.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 - name: Validate Compose @@ -184,12 +231,14 @@ jobs: compose-concurrency: name: Concurrent project stacks needs: changes - if: needs.changes.outputs.compose != '[]' - runs-on: ubuntu-latest + if: needs.changes.outputs.concurrent == 'true' + runs-on: ubuntu-24.04 timeout-minutes: 25 steps: - - uses: actions/checkout@v7.0.1 - - uses: actions/setup-node@v7.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24.21.0 - name: Start independent stacks @@ -208,3 +257,22 @@ jobs: docker compose -f p1-task-manager/compose.yaml down --volumes docker compose -f p4-editorial-publishing/compose.yaml down --volumes docker compose -f p10-warehouse-workloads/compose.yaml down --volumes + + result: + name: Quality result + if: always() + needs: + [changes, step, project, p3-sidecar, compose-config, compose-concurrency] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24.21.0 + - name: Check selected results + run: node shared/tools/repository-automation/ci-result.mjs + env: + NEEDS_JSON: ${{ toJSON(needs) }} diff --git a/p1-task-manager/package.json b/p1-task-manager/package.json index 50a3285..3cc22a9 100644 --- a/p1-task-manager/package.json +++ b/p1-task-manager/package.json @@ -28,7 +28,7 @@ "@fastify/swagger": "9.9.2", "@janssenproject/cedarling_wasm": "0.0.468", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "fflate": "0.8.3", "openid-client": "6.8.8", "react": "19.3.1", diff --git a/p1-task-manager/pnpm-lock.yaml b/p1-task-manager/pnpm-lock.yaml index 6290a27..6fa1b3d 100644 --- a/p1-task-manager/pnpm-lock.yaml +++ b/p1-task-manager/pnpm-lock.yaml @@ -27,8 +27,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 fflate: specifier: 0.8.3 version: 0.8.3 @@ -885,8 +885,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -2279,7 +2279,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p10-warehouse-workloads/package.json b/p10-warehouse-workloads/package.json index 45499cf..a5531ca 100644 --- a/p10-warehouse-workloads/package.json +++ b/p10-warehouse-workloads/package.json @@ -25,7 +25,7 @@ "@fastify/helmet": "13.2.0", "@fastify/static": "10.1.6", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "jose": "6.2.2", "openid-client": "6.8.8", "react": "19.3.1", diff --git a/p10-warehouse-workloads/pnpm-lock.yaml b/p10-warehouse-workloads/pnpm-lock.yaml index b4c896e..69e951d 100644 --- a/p10-warehouse-workloads/pnpm-lock.yaml +++ b/p10-warehouse-workloads/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 jose: specifier: 6.2.2 version: 6.2.2 @@ -507,8 +507,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1391,7 +1391,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p14-capital-district/package.json b/p14-capital-district/package.json index d27db2d..bceb32e 100644 --- a/p14-capital-district/package.json +++ b/p14-capital-district/package.json @@ -27,7 +27,7 @@ "@fastify/helmet": "13.1.1", "@fastify/static": "10.1.5", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "openid-client": "6.8.8", "react": "19.3.0", "react-dom": "19.3.0" diff --git a/p14-capital-district/pnpm-lock.yaml b/p14-capital-district/pnpm-lock.yaml index c37b3a5..b788665 100644 --- a/p14-capital-district/pnpm-lock.yaml +++ b/p14-capital-district/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 openid-client: specifier: 6.8.8 version: 6.8.8 @@ -438,8 +438,8 @@ packages: fastify-plugin@6.0.1: resolution: {integrity: sha512-4wW4RxSmvPXLrh5mgQEiu43qU3bw44N2ss4FTkTkQIbprUhE+/s43QX0j/ttcztdWK7PEkHqVthdc5GdEGDvjA==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1160,7 +1160,7 @@ snapshots: fastify-plugin@6.0.1: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.7 '@fastify/error': 4.2.1 diff --git a/p2-tenantrag/package.json b/p2-tenantrag/package.json index c94667b..9f1f342 100644 --- a/p2-tenantrag/package.json +++ b/p2-tenantrag/package.json @@ -25,7 +25,7 @@ "dependencies": { "@janssenproject/cedarling_wasm": "0.0.468", "@orama/orama": "3.1.18", - "fastify": "5.12.5", + "fastify": "5.12.6", "fflate": "0.8.3", "jose": "6.2.10", "pdfjs-dist": "6.2.108", diff --git a/p2-tenantrag/pnpm-lock.yaml b/p2-tenantrag/pnpm-lock.yaml index a835a27..92931e4 100644 --- a/p2-tenantrag/pnpm-lock.yaml +++ b/p2-tenantrag/pnpm-lock.yaml @@ -15,8 +15,8 @@ importers: specifier: 3.1.18 version: 3.1.18 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 fflate: specifier: 0.8.3 version: 0.8.3 @@ -772,8 +772,8 @@ packages: fast-uri@4.1.5: resolution: {integrity: sha512-vZeoMRB4epNr7QfdHxel7te/RcX16CxyXI07JCCTFWZA2s4v1azGNESRj+2EoaHSaWFL/Z3GmKT2jF6A202jLg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -1917,7 +1917,7 @@ snapshots: fast-uri@4.1.5: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p4-editorial-publishing/docs/tutorials.md b/p4-editorial-publishing/docs/tutorials.md index 1eb6198..ca0607a 100644 --- a/p4-editorial-publishing/docs/tutorials.md +++ b/p4-editorial-publishing/docs/tutorials.md @@ -384,7 +384,7 @@ From the P4 directory, let's install Cedarling and the tools that package and validate our policy store, along with the Next.js security update: ```bash -pnpm add --save-exact @janssenproject/cedarling_wasm@0.0.468 fflate@0.8.3 next@16.3.8 +pnpm add --save-exact @janssenproject/cedarling_wasm@0.0.468 fflate@0.8.3 next@16.4.1 pnpm add --save-dev --save-exact @cedar-policy/cedar-wasm@4.12.0 @types/node@24.19.0 vitest@4.1.12 ``` diff --git a/p4-editorial-publishing/package.json b/p4-editorial-publishing/package.json index c82a1f5..243382f 100644 --- a/p4-editorial-publishing/package.json +++ b/p4-editorial-publishing/package.json @@ -26,7 +26,7 @@ "@janssenproject/cedarling_wasm": "0.0.468", "better-sqlite3": "13.0.3", "fflate": "0.8.3", - "next": "16.3.8", + "next": "16.4.1", "openid-client": "6.8.8", "react": "19.3.0", "react-dom": "19.3.0", diff --git a/p4-editorial-publishing/pnpm-lock.yaml b/p4-editorial-publishing/pnpm-lock.yaml index 552f0b2..b244814 100644 --- a/p4-editorial-publishing/pnpm-lock.yaml +++ b/p4-editorial-publishing/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: specifier: 0.8.3 version: 0.8.3 next: - specifier: 16.3.8 - version: 16.3.8(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + specifier: 16.4.1 + version: 16.4.1(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) openid-client: specifier: 6.8.8 version: 6.8.8 @@ -278,53 +278,53 @@ packages: '@jridgewell/sourcemap-codec@1.6.0': resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} - '@next/env@16.3.8': - resolution: {integrity: sha512-Al9zqHVV7TJv0eFuOU4U7Lvv74PTih4Ch63sk2xCIpSTkE3udFnaOcnzP2lQVymiL7yS9Cj2iClUXlR3EQ5sEw==} + '@next/env@16.4.1': + resolution: {integrity: sha512-5sUUrrSvs/2/2LnLoesybMGx12EtPB46fdZ6RhAw0XDGSPxU2LFyQBB7yuEiOFFJPKmBY6+q2jTr1/AMbnp0/g==} - '@next/swc-darwin-arm64@16.3.8': - resolution: {integrity: sha512-2JPRMh2nmQG5CiL7cXGL9AGwnPWJQ//cTtAUCT+w511QHk79SYz3LGv/pc5X643B/WEO0rvu3Yww0hqwt3kgeA==} + '@next/swc-darwin-arm64@16.4.1': + resolution: {integrity: sha512-U+hQ5Gj3fpzf4fveTIF3dpUFYhaSW8vHhdAnbiIomzXzg7boreYFgBqprepXFtW4sw+aTttiNmg0gew/iFNhbg==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] - '@next/swc-darwin-x64@16.3.8': - resolution: {integrity: sha512-GZtCCOBKJ4leVIT/Th0llWKhD1ca92lzbQiS5R5ON9QkoiFnilFsebDae1JU2a3HWoKMEmEZWGs1AGLavVM72Q==} + '@next/swc-darwin-x64@16.4.1': + resolution: {integrity: sha512-/hqC3V1XLUjhMASf/WridKdO9FXoEgN3HBWmtKbj00MPgnltutfraWJ3bYTpCRIgEIpbjJyOVLpYgAhh/BkdcQ==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] - '@next/swc-linux-arm64-gnu@16.3.8': - resolution: {integrity: sha512-O659ygeQYqneJ1fBKMpFxIFqYkYswu8IAS1OCKK/4f3ZgJJm1dRz4fVJZRi/kLLWjnBKnebOePA4WNv+sV1pVA==} + '@next/swc-linux-arm64-gnu@16.4.1': + resolution: {integrity: sha512-GHzncHbcaS9g95kcvTNtKQW3aCLoR+xc3Jm7qrkpk1R+D2BjMHIPyvQGRPKCdMX95BsH/421jBUK06q9tEwzjw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - '@next/swc-linux-arm64-musl@16.3.8': - resolution: {integrity: sha512-dSjKSyWpzxoO1d3DIZZcP4XJcNaKeLmxQMFOiYl5vuBRMmweIqnAhty8tAmRsvTss779cK1FtYnDMj40e4TQlg==} + '@next/swc-linux-arm64-musl@16.4.1': + resolution: {integrity: sha512-1SuJyDxW/PR9dHJat/ZRgcYLhuSZ+1psP4DlKb2tKIuXbjygqpLrJWRSq+onZQBv8LcmU67EIrmb6c3XTaJLjw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - '@next/swc-linux-x64-gnu@16.3.8': - resolution: {integrity: sha512-lbqOuz3RPRcv+o9msNsJw5x4+Y1ZwPTs6vmL6DCf7i0fZfvng/F59wyeDwqHIvV0mK//RBy/jJkZ+nCKsSMXjQ==} + '@next/swc-linux-x64-gnu@16.4.1': + resolution: {integrity: sha512-Istng1hMGp9TGO4lUqeKLd70MPN/OOkucVTP2Nmj+gLkwBgmPqUw8CJYBCC7nbl/B/23bBCcjCI5Hsh6hazC1A==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - '@next/swc-linux-x64-musl@16.3.8': - resolution: {integrity: sha512-+316WswI8ScVgZeUd+1KGaXkHhaYQzCjvH/05TZSpJ8zBizb1a4G7DtO7F12jcBIqMOtsz9ji1t48fmKtzqsGA==} + '@next/swc-linux-x64-musl@16.4.1': + resolution: {integrity: sha512-aSC9wMNiZRRFVpPkPWBzRwLQxIGm/CiQcXyFF1XNcgodb+uMxdk7aSSGGpDXCbnvZdf4itjHxpOd5h7i7zaiYw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - '@next/swc-win32-arm64-msvc@16.3.8': - resolution: {integrity: sha512-ji0gd4kMYUxO+1fJBIbiBVRCjzG/lloiyCccnlebvb1ZJ5qXCPZqYg4Jl1DrrixnWNMKylzgpmMWx0yNDYXlzw==} + '@next/swc-win32-arm64-msvc@16.4.1': + resolution: {integrity: sha512-t/myvwFDlypMUjRvbBf8qLPyuaB0iXYYdBtwMt6DM6f+60wtGr4bSYholdPqNeV5VUXZhrpMwfza829kmhS3iA==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] - '@next/swc-win32-x64-msvc@16.3.8': - resolution: {integrity: sha512-WcTlaKt/TWkh5kUjdJcUmB1XgZ+1c6fz4Y9fDHL73YNSdGaUWjceeWrrlwF0nv19iABYWC4iAq1oX1w4Bn0vfg==} + '@next/swc-win32-x64-msvc@16.4.1': + resolution: {integrity: sha512-6ICqPo9awCkrtYNPhwkx1uU7rlFWrZRrIboDHS2LU+jv088UBYQtmJ25h6pCp27/+FZjF6BQ4Te4ErugyMeuBg==} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -629,8 +629,8 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - next@16.3.8: - resolution: {integrity: sha512-U7QEZaTini6wKrb8A8hqLLqYQyCetegKjCpJOyxk642vWoMoU1x5PyZCJFvgYgiptA8xc5j/9xYlZFO7w9Sjmw==} + next@16.4.1: + resolution: {integrity: sha512-uc657V62E34fOI6W5NtG9IzwsPcms0Z60grz2MJGtglgD+pTFnNt4Yw2PzaU622tfX4vhrDgLZzicE2YlhI20Q==} engines: {node: '>=20.9.0'} hasBin: true peerDependencies: @@ -1022,30 +1022,30 @@ snapshots: '@jridgewell/sourcemap-codec@1.6.0': {} - '@next/env@16.3.8': {} + '@next/env@16.4.1': {} - '@next/swc-darwin-arm64@16.3.8': + '@next/swc-darwin-arm64@16.4.1': optional: true - '@next/swc-darwin-x64@16.3.8': + '@next/swc-darwin-x64@16.4.1': optional: true - '@next/swc-linux-arm64-gnu@16.3.8': + '@next/swc-linux-arm64-gnu@16.4.1': optional: true - '@next/swc-linux-arm64-musl@16.3.8': + '@next/swc-linux-arm64-musl@16.4.1': optional: true - '@next/swc-linux-x64-gnu@16.3.8': + '@next/swc-linux-x64-gnu@16.4.1': optional: true - '@next/swc-linux-x64-musl@16.3.8': + '@next/swc-linux-x64-musl@16.4.1': optional: true - '@next/swc-win32-arm64-msvc@16.3.8': + '@next/swc-win32-arm64-msvc@16.4.1': optional: true - '@next/swc-win32-x64-msvc@16.3.8': + '@next/swc-win32-x64-msvc@16.4.1': optional: true '@oxc-project/types@0.155.0': {} @@ -1267,9 +1267,9 @@ snapshots: nanoid@3.3.19: {} - next@16.3.8(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): + next@16.4.1(@playwright/test@1.63.0)(@types/node@24.19.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): dependencies: - '@next/env': 16.3.8 + '@next/env': 16.4.1 '@swc/helpers': 0.5.23 baseline-browser-mapping: 2.11.24 caniuse-lite: 1.0.30001810 @@ -1278,14 +1278,14 @@ snapshots: react-dom: 19.3.0(react@19.3.0) styled-jsx: 5.1.6(react@19.3.0) optionalDependencies: - '@next/swc-darwin-arm64': 16.3.8 - '@next/swc-darwin-x64': 16.3.8 - '@next/swc-linux-arm64-gnu': 16.3.8 - '@next/swc-linux-arm64-musl': 16.3.8 - '@next/swc-linux-x64-gnu': 16.3.8 - '@next/swc-linux-x64-musl': 16.3.8 - '@next/swc-win32-arm64-msvc': 16.3.8 - '@next/swc-win32-x64-msvc': 16.3.8 + '@next/swc-darwin-arm64': 16.4.1 + '@next/swc-darwin-x64': 16.4.1 + '@next/swc-linux-arm64-gnu': 16.4.1 + '@next/swc-linux-arm64-musl': 16.4.1 + '@next/swc-linux-x64-gnu': 16.4.1 + '@next/swc-linux-x64-musl': 16.4.1 + '@next/swc-win32-arm64-msvc': 16.4.1 + '@next/swc-win32-x64-msvc': 16.4.1 '@playwright/test': 1.63.0 sharp: 0.35.5(@types/node@24.19.0) transitivePeerDependencies: diff --git a/p6-field-inspection/package.json b/p6-field-inspection/package.json index 58f0e9a..15ae14e 100644 --- a/p6-field-inspection/package.json +++ b/p6-field-inspection/package.json @@ -26,7 +26,7 @@ "@fastify/helmet": "13.2.0", "@fastify/static": "10.1.6", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "openid-client": "6.8.8", "react": "19.3.1", "react-dom": "19.3.1" diff --git a/p6-field-inspection/pnpm-lock.yaml b/p6-field-inspection/pnpm-lock.yaml index 95aaa2b..c009c3c 100644 --- a/p6-field-inspection/pnpm-lock.yaml +++ b/p6-field-inspection/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 openid-client: specifier: 6.8.8 version: 6.8.8 @@ -514,8 +514,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1402,7 +1402,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/p7-collaborative-docs/package.json b/p7-collaborative-docs/package.json index ace7287..ea49a14 100644 --- a/p7-collaborative-docs/package.json +++ b/p7-collaborative-docs/package.json @@ -26,7 +26,7 @@ "@fastify/helmet": "13.2.0", "@fastify/static": "10.1.6", "better-sqlite3": "13.0.3", - "fastify": "5.12.5", + "fastify": "5.12.6", "openid-client": "6.8.8", "react": "19.3.1", "react-dom": "19.3.1" diff --git a/p7-collaborative-docs/pnpm-lock.yaml b/p7-collaborative-docs/pnpm-lock.yaml index 4eff895..213e233 100644 --- a/p7-collaborative-docs/pnpm-lock.yaml +++ b/p7-collaborative-docs/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 13.0.3 version: 13.0.3 fastify: - specifier: 5.12.5 - version: 5.12.5 + specifier: 5.12.6 + version: 5.12.6 openid-client: specifier: 6.8.8 version: 6.8.8 @@ -600,8 +600,8 @@ packages: fastify-plugin@6.0.0: resolution: {integrity: sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==} - fastify@5.12.5: - resolution: {integrity: sha512-OB2k1dlxs5/NAABqeKV2FUHkSD2BbENsCak8yULVcymn3fHIPDVa9TI3SDnJSWYSllZmSYuZXy2gTnsT+Sut1A==} + fastify@5.12.6: + resolution: {integrity: sha512-XeErl1DDsxdbhE9RPRJ05iNnfMhHRHHewVNNAJXaGTOaCY4wHb6rEizphhgzBq5YQe4JyqQWPpGd+hLh2PXXJA==} fastq@1.20.3: resolution: {integrity: sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==} @@ -1437,7 +1437,7 @@ snapshots: fastify-plugin@6.0.0: {} - fastify@5.12.5: + fastify@5.12.6: dependencies: '@fastify/ajv-compiler': 4.0.6 '@fastify/error': 4.2.0 diff --git a/shared/tools/repository-automation/README.md b/shared/tools/repository-automation/README.md new file mode 100644 index 0000000..118175b --- /dev/null +++ b/shared/tools/repository-automation/README.md @@ -0,0 +1,31 @@ +# Repository checks + +`pnpm check` tests the CI tools and validates every published tutorial's metadata +and referenced images. Application changes run the affected projects on Linux, +macOS, and Windows, with separate Docker checks. Tutorial text, images, and README +changes run source validation. Shared runtime or workflow changes run the full +portfolio, as do scheduled and manually dispatched checks. + +The concurrent-stack scenario runs when P1, P4, or P10 is affected, or when the +full portfolio is selected. Other project changes run their own Compose checks. + +Browser checks install Chromium's headless shell. Linux setup uses Ubuntu 24.04 +and official HTTPS package sources. +Each attempt has a three-minute budget. A transport failure can trigger one +retry; signature, package-configuration, and timeout failures stop setup. +The final Quality result check requires every selected job to succeed. + +## Inspect a slow run + +With the GitHub CLI authenticated, run this from this directory: + +```sh +pnpm inspect:ci +``` + +The command separates queue time from execution time and lists slow or failed +steps. It only reads workflow metadata. If every step, including Complete job, +has finished but GitHub still reports the job as running, it flags the unresolved +status and exits unsuccessfully. Check GitHub Status before retrying that job. +An application failure or security-audit finding needs investigation even when +another job has a stale status. diff --git a/shared/tools/repository-automation/changed-projects.mjs b/shared/tools/repository-automation/changed-projects.mjs index 0aecf9d..6a5d715 100644 --- a/shared/tools/repository-automation/changed-projects.mjs +++ b/shared/tools/repository-automation/changed-projects.mjs @@ -1,7 +1,7 @@ // Selects the tutorial projects that CI must verify for the current change. -import { appendFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; export const tutorialProjects = [ @@ -21,10 +21,17 @@ export const tutorialProjects = [ "p14-capital-district", "p15-marketplace", ]; - - export const projects = [...tutorialProjects, "shared/identity-provider"]; export const composeProjects = new Set(tutorialProjects); +const concurrentProjects = new Set([ + "p1-task-manager", + "p4-editorial-publishing", + "p10-warehouse-workloads", +]); + +export function needsConcurrentStacks(selected) { + return selected.some((project) => concurrentProjects.has(project)); +} function changedFiles(environment = process.env) { const event = environment.EVENT_NAME; @@ -38,32 +45,64 @@ function changedFiles(environment = process.env) { const separator = event === "pull_request" ? "..." : ".."; return execFileSync( "git", - ["diff", "--name-only", `${base}${separator}${head}`], + [ + "diff", + "--name-only", + "-z", + "--no-renames", + `${base}${separator}${head}`, + "--", + ], { encoding: "utf8", }, ) - .trim() - .split("\n") + .split("\0") .filter(Boolean); } export function selectProjects(files) { - const affectsAll = - files === null || - files.some( - (file) => - file.startsWith(".github/") || - file.startsWith("shared/"), - ); - const selected = affectsAll - ? projects - : projects.filter((project) => - files.some((file) => file.startsWith(`${project}/`)), - ); + const all = () => ({ + projects, + compose: [...composeProjects], + helper: true, + concurrent: true, + }); + if (files === null) return all(); + const affected = new Set(); + let helper = false; + for (const file of files) { + if ( + [ + "README.md", + "LICENSE", + "shared/tools/repository-automation/README.md", + ].includes(file) + ) + continue; + if (file.startsWith("shared/tools/step/")) { + helper = true; + continue; + } + const project = projects.find((name) => file.startsWith(`${name}/`)); + if (!project) return all(); + const path = file.slice(project.length + 1); + if ( + path === "README.md" || + path === "docs/tutorials.md" || + /^docs\/assets\/[A-Za-z0-9_./-]+\.(avif|jpe?g|png|svg|webp)$/.test(path) + ) + continue; + // Shared runtime changes affect every application using that runtime. + if (project === "shared/identity-provider") return all(); + affected.add(project); + } + const selected = projects.filter((project) => affected.has(project)); return { projects: selected, compose: selected.filter((project) => composeProjects.has(project)), + helper, + concurrent: needsConcurrentStacks(selected), }; } @@ -78,7 +117,7 @@ function main(environment = process.env) { files = null; } const selected = selectProjects(files); - const output = `projects=${JSON.stringify(selected.projects)}\ncompose=${JSON.stringify(selected.compose)}\n`; + const output = `projects=${JSON.stringify(selected.projects)}\ncompose=${JSON.stringify(selected.compose)}\nhelper=${selected.helper}\nconcurrent=${selected.concurrent}\n`; if (environment.GITHUB_OUTPUT) appendFileSync(environment.GITHUB_OUTPUT, output); else process.stdout.write(output); diff --git a/shared/tools/repository-automation/changed-projects.test.mjs b/shared/tools/repository-automation/changed-projects.test.mjs index 16e1cc3..204290c 100644 --- a/shared/tools/repository-automation/changed-projects.test.mjs +++ b/shared/tools/repository-automation/changed-projects.test.mjs @@ -1,7 +1,10 @@ // Verifies CI project selection for project, shared, and repository changes. import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; import { @@ -15,17 +18,27 @@ const selectorScript = fileURLToPath( ); test("selects only the changed project", () => { - assert.deepEqual(selectProjects(["p7-collaborative-docs/src/server/app.ts"]), { - projects: ["p7-collaborative-docs"], - compose: ["p7-collaborative-docs"], - }); + assert.deepEqual( + selectProjects(["p7-collaborative-docs/src/server/app.ts"]), + { + projects: ["p7-collaborative-docs"], + compose: ["p7-collaborative-docs"], + helper: false, + concurrent: false, + }, + ); }); test("selects Capital District for a P14 game change", () => { - assert.deepEqual(selectProjects(["p14-capital-district/src/server/game/service.ts"]), { - projects: ["p14-capital-district"], - compose: ["p14-capital-district"], - }); + assert.deepEqual( + selectProjects(["p14-capital-district/src/server/game/service.ts"]), + { + projects: ["p14-capital-district"], + compose: ["p14-capital-district"], + helper: false, + concurrent: false, + }, + ); }); test("selects P3 Compose without a separate shared-provider stack", () => { @@ -34,29 +47,119 @@ test("selects P3 Compose without a separate shared-provider stack", () => { { projects: ["p3-mcp-capability-governance"], compose: ["p3-mcp-capability-governance"], + helper: false, + concurrent: false, }, ); assert.equal(composeProjects.has("shared/identity-provider"), false); }); test("selects the complete portfolio for shared and workflow changes", () => { - assert.deepEqual( - selectProjects(["shared/identity-provider/src/config.ts"]).projects, - projects, - ); - assert.deepEqual(selectProjects(["shared/dev-supervisor.mjs"]).projects, projects); - assert.deepEqual( - selectProjects([".github/workflows/quality.yml"]).projects, - projects, + for (const path of [ + "shared/identity-provider/src/config.ts", + "shared/dev-supervisor.mjs", + "shared/tools/repository-automation/setup-browser.mjs", + ".github/workflows/quality.yml", + ]) { + assert.deepEqual(selectProjects([path]), { + projects, + compose: [...composeProjects], + helper: true, + concurrent: true, + }); + } +}); + +test("selects concurrent stacks only when their applications are affected", () => { + for (const project of [ + "p1-task-manager", + "p4-editorial-publishing", + "p10-warehouse-workloads", + ]) { + const selected = selectProjects([`${project}/src/server/app.ts`]); + assert.deepEqual(selected.projects, [project]); + assert.deepEqual(selected.compose, [project]); + assert.equal(selected.concurrent, true); + } + for (const project of ["p2-tenantrag", "p6-field-inspection"]) { + const selected = selectProjects([`${project}/compose.yaml`]); + assert.deepEqual(selected.projects, [project]); + assert.deepEqual(selected.compose, [project]); + assert.equal(selected.concurrent, false); + } + assert.equal( + selectProjects([ + "p2-tenantrag/src/app.ts", + "p4-editorial-publishing/compose.yaml", + ]).concurrent, + true, ); - assert.deepEqual(selectProjects(null).projects, projects); }); test("does not run project jobs for root documentation alone", () => { assert.deepEqual(selectProjects(["README.md"]), { projects: [], compose: [], + helper: false, + concurrent: false, + }); +}); + +test("tutorial and README changes need validation without application matrices", () => { + assert.deepEqual( + selectProjects([ + "p1-task-manager/docs/tutorials.md", + "p4-editorial-publishing/docs/assets/social-card.webp", + "p6-field-inspection/README.md", + "shared/tools/repository-automation/README.md", + ]), + { projects: [], compose: [], helper: false, concurrent: false }, + ); +}); + +test("helper changes select its portable tests without unrelated applications", () => { + assert.deepEqual(selectProjects(["shared/tools/step/run.mjs"]), { + projects: [], + compose: [], + helper: true, + concurrent: false, }); + assert.deepEqual( + selectProjects([ + "shared/tools/step/projects/p1.json", + "p1-task-manager/src/server/app.ts", + ]), + { + projects: ["p1-task-manager"], + compose: ["p1-task-manager"], + helper: true, + concurrent: true, + }, + ); +}); + +test("unknown paths and scheduled runs conservatively select everything", () => { + for (const files of [ + null, + ["scripts/new-tool.mjs"], + [".nvmrc"], + ["new-project/app.ts"], + ]) { + assert.deepEqual(selectProjects(files), { + projects, + compose: [...composeProjects], + helper: true, + concurrent: true, + }); + } + assert.deepEqual( + selectProjects(["p1-task-manager/docs/execute.ts"]).projects, + ["p1-task-manager"], + ); + assert.deepEqual( + selectProjects(["p1-task-manager/src/odd\nname.ts"]).projects, + ["p1-task-manager"], + ); }); test("reports when project detection falls back to the full matrix", () => { @@ -79,4 +182,79 @@ test("reports when project detection falls back to the full matrix", () => { /Changed-project detection failed; selecting the full project matrix/, ); assert.ok(result.stdout.includes(`projects=${JSON.stringify(projects)}`)); + assert.match(result.stdout, /concurrent=true/); +}); + +test("scheduled and manual runs select concurrent stacks with the full portfolio", () => { + for (const event of ["schedule", "workflow_dispatch"]) { + const env = { ...process.env, EVENT_NAME: event }; + delete env.GITHUB_OUTPUT; + const result = spawnSync(process.execPath, [selectorScript], { + encoding: "utf8", + env, + }); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.stdout.includes(`projects=${JSON.stringify(projects)}`)); + assert.match(result.stdout, /concurrent=true/); + assert.equal(result.stderr, ""); + } +}); + +test("CLI accounts for deleted files, source-to-doc renames, and embedded newlines", async (t) => { + const root = await mkdtemp(join(tmpdir(), "ci-selection-")); + t.after(() => rm(root, { recursive: true, force: true })); + const git = (...args) => + execFileSync("git", ["-C", root, ...args], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + git("init", "-b", "main"); + const put = async (path) => { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), "example\n"); + }; + await put("p1-task-manager/src/old.ts"); + await put("p7-collaborative-docs/src/odd\nname.ts"); + git("add", "."); + const commit = () => + git( + "-c", + "commit.gpgsign=false", + "-c", + "user.name=CI test", + "-c", + "user.email=test@example.invalid", + "commit", + "-m", + "fixture", + ); + commit(); + const base = git("rev-parse", "HEAD"); + await mkdir(join(root, "p1-task-manager/docs")); + await rename( + join(root, "p1-task-manager/src/old.ts"), + join(root, "p1-task-manager/docs/tutorials.md"), + ); + await rm(join(root, "p7-collaborative-docs/src/odd\nname.ts")); + git("add", "-A"); + commit(); + const env = { + ...process.env, + EVENT_NAME: "push", + BEFORE_SHA: base, + HEAD_SHA: git("rev-parse", "HEAD"), + }; + delete env.GITHUB_OUTPUT; + const result = spawnSync(process.execPath, [selectorScript], { + cwd: root, + env, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.match( + result.stdout, + /projects=\["p1-task-manager","p7-collaborative-docs"\]/, + ); + assert.match(result.stdout, /helper=false/); + assert.match(result.stdout, /concurrent=true/); }); diff --git a/shared/tools/repository-automation/ci-report.mjs b/shared/tools/repository-automation/ci-report.mjs new file mode 100644 index 0000000..e066b16 --- /dev/null +++ b/shared/tools/repository-automation/ci-report.mjs @@ -0,0 +1,49 @@ +// Writes runner identity and step outcomes without environment dumps or step outputs. +import { appendFile } from "node:fs/promises"; +import { arch, platform } from "node:os"; + +const stages = { + install: "Install", + identity: "Shared identity provider", + "browser-linux": "Linux browser setup", + browser: "Browser download", + check: "Checks", + integration: "Linux integration", + audit: "Dependency audit", +}; + +export function stageReport(steps) { + return `### Verification stages\n\n| Stage | Result |\n|---|---|\n${Object.entries( + stages, + ) + .map(([id, label]) => { + const outcome = steps?.[id]?.outcome; + const result = ["success", "failure", "cancelled", "skipped"].includes( + outcome, + ) + ? outcome + : "not run"; + return `| ${label} | ${result} |`; + }) + .join("\n")}\n`; +} + +if (import.meta.main) { + try { + const mode = process.argv[2]; + let text; + if (mode === "runner") { + const value = (input) => + String(input ?? "unknown").replace(/[^A-Za-z0-9_.:/+-]/g, ""); + text = `Runner: ${platform()}/${arch()}, ${value(process.env.ImageOS)} ${value(process.env.ImageVersion)}, Node ${process.version}.\n`; + } else if (mode === "stages") + text = stageReport(JSON.parse(process.env.STEPS_JSON ?? "{}")); + else throw new Error("Usage: node ci-report.mjs runner|stages"); + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile(process.env.GITHUB_STEP_SUMMARY, text); + console.info(text); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/ci-report.test.mjs b/shared/tools/repository-automation/ci-report.test.mjs new file mode 100644 index 0000000..4ecdf09 --- /dev/null +++ b/shared/tools/repository-automation/ci-report.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stageReport } from "./ci-report.mjs"; + +test("stage reports distinguish failure, cancellation, and intentional skips", () => { + const report = stageReport({ + install: { outcome: "success" }, + check: { outcome: "failure" }, + integration: { outcome: "skipped" }, + browser: { outcome: "cancelled" }, + }); + assert.match(report, /Checks \| failure/); + assert.match(report, /Browser download \| cancelled/); + assert.match(report, /Linux integration \| skipped/); + assert.match(report, /Dependency audit \| not run/); +}); + +test("reports publish known outcomes without step outputs or arbitrary fields", () => { + const report = stageReport({ + install: { + outcome: "success", + outputs: { credentials: "sensitive-fixture" }, + }, + custom: { outcome: "unsafe-fixture" }, + audit: { outcome: "unexpected-fixture" }, + }); + assert.doesNotMatch( + report, + /sensitive-fixture|unsafe-fixture|unexpected-fixture/, + ); +}); diff --git a/shared/tools/repository-automation/ci-result.mjs b/shared/tools/repository-automation/ci-result.mjs new file mode 100644 index 0000000..8d3c8ba --- /dev/null +++ b/shared/tools/repository-automation/ci-result.mjs @@ -0,0 +1,61 @@ +// Fails the workflow when selected verification did not complete successfully. +import { appendFile } from "node:fs/promises"; +import { + composeProjects, + needsConcurrentStacks, + projects, +} from "./changed-projects.mjs"; + +export function qualityResult(needs) { + if (needs?.changes?.result !== "success") + throw new Error("Project selection did not succeed"); + const outputs = needs.changes.outputs; + const selected = JSON.parse(outputs.projects); + const compose = JSON.parse(outputs.compose); + if ( + !Array.isArray(selected) || + new Set(selected).size !== selected.length || + !selected.every((id) => projects.includes(id)) || + !Array.isArray(compose) || + !compose.every((id) => composeProjects.has(id) && selected.includes(id)) || + JSON.stringify(compose) !== + JSON.stringify(selected.filter((id) => composeProjects.has(id))) || + !["true", "false"].includes(outputs.helper) || + outputs.concurrent !== String(needsConcurrentStacks(selected)) + ) + throw new Error("Invalid CI selection outputs"); + const expected = { + step: outputs.helper === "true", + project: selected.length > 0, + "p3-sidecar": selected.includes("p3-mcp-capability-governance"), + "compose-config": compose.length > 0, + "compose-concurrency": outputs.concurrent === "true", + }; + const rows = Object.entries(expected).map(([job, required]) => { + const result = needs[job]?.result ?? "missing"; + return { + job, + required, + result, + valid: result === "success" || (!required && result === "skipped"), + }; + }); + return { selected, compose, rows, success: rows.every((row) => row.valid) }; +} + +if (import.meta.main) { + try { + const result = qualityResult(JSON.parse(process.env.NEEDS_JSON ?? "null")); + const text = `## Quality result\n\nProjects: ${result.selected.join(", ") || "none"}.\n\n| Job | Selected | Result |\n|---|---|---|\n${result.rows.map((row) => `| ${row.job} | ${row.required ? "yes" : "no"} | ${row.result} |`).join("\n")}\n`; + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile(process.env.GITHUB_STEP_SUMMARY, text); + console.info(text); + if (!result.success) + throw new Error( + "Selected verification failed, was cancelled, or did not run", + ); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/ci-result.test.mjs b/shared/tools/repository-automation/ci-result.test.mjs new file mode 100644 index 0000000..01976ef --- /dev/null +++ b/shared/tools/repository-automation/ci-result.test.mjs @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { parse } from "yaml"; +import { qualityResult } from "./ci-result.mjs"; + +test("the final workflow result waits for every verification job", async () => { + const workflow = parse( + await readFile( + new URL("../../../.github/workflows/quality.yml", import.meta.url), + "utf8", + ), + ); + assert.deepEqual( + new Set(workflow.jobs.result.needs), + new Set(Object.keys(workflow.jobs).filter((id) => id !== "result")), + ); + assert.equal(workflow.jobs.result.if, "always()"); + assert.equal( + workflow.jobs["compose-concurrency"].if, + "needs.changes.outputs.concurrent == 'true'", + ); + assert.equal( + workflow.jobs.changes.outputs.concurrent, + `\${{ steps.select.outputs.concurrent }}`, + ); +}); + +function needs(selected = [], compose = [], helper = false) { + const concurrent = selected.some((id) => + [ + "p1-task-manager", + "p4-editorial-publishing", + "p10-warehouse-workloads", + ].includes(id), + ); + return { + changes: { + result: "success", + outputs: { + projects: JSON.stringify(selected), + compose: JSON.stringify(compose), + helper: String(helper), + concurrent: String(concurrent), + }, + }, + step: { result: helper ? "success" : "skipped" }, + project: { result: selected.length ? "success" : "skipped" }, + "p3-sidecar": { + result: selected.includes("p3-mcp-capability-governance") + ? "success" + : "skipped", + }, + "compose-config": { result: compose.length ? "success" : "skipped" }, + "compose-concurrency": { result: concurrent ? "success" : "skipped" }, + }; +} + +test("quality accepts intentional skips and complete selected verification", () => { + for (const scenario of [ + needs(), + needs([], [], true), + needs(["p1-task-manager"], ["p1-task-manager"]), + needs( + ["p3-mcp-capability-governance"], + ["p3-mcp-capability-governance"], + true, + ), + ]) { + assert.equal(qualityResult(scenario).success, true); + } +}); + +test("quality rejects failed selection, malformed outputs, and absent job results", () => { + assert.throws(() => qualityResult({}), /selection/); + const scenario = needs(); + scenario.changes.outputs.helper = ""; + assert.throws(() => qualityResult(scenario), /outputs/); + const missing = needs(); + delete missing.step; + assert.equal(qualityResult(missing).success, false); + assert.throws(() => qualityResult(needs(["p1-task-manager"], [])), /outputs/); + assert.throws(() => qualityResult(needs(["unknown-project"], [])), /outputs/); +}); + +test("quality rejects missing or inconsistent concurrent-stack selection", () => { + const required = needs(["p1-task-manager"], ["p1-task-manager"]); + for (const value of [undefined, "", "false", "TRUE"]) { + required.changes.outputs.concurrent = value; + assert.throws(() => qualityResult(required), /outputs/); + } + const unrelated = needs(["p2-tenantrag"], ["p2-tenantrag"]); + unrelated.changes.outputs.concurrent = "true"; + assert.throws(() => qualityResult(unrelated), /outputs/); +}); + +test("every selected verification must succeed, including sidecar and concurrency", () => { + for (const job of [ + "step", + "project", + "p3-sidecar", + "compose-config", + "compose-concurrency", + ]) { + for (const result of ["failure", "cancelled", "skipped"]) { + const scenario = needs( + ["p1-task-manager", "p3-mcp-capability-governance"], + ["p1-task-manager", "p3-mcp-capability-governance"], + true, + ); + scenario[job].result = result; + assert.equal(qualityResult(scenario).success, false, `${job}: ${result}`); + } + } +}); diff --git a/shared/tools/repository-automation/inspect-ci.mjs b/shared/tools/repository-automation/inspect-ci.mjs new file mode 100644 index 0000000..f18d973 --- /dev/null +++ b/shared/tools/repository-automation/inspect-ci.mjs @@ -0,0 +1,109 @@ +// Reads workflow metadata only; it never cancels, reruns, or changes a check. +import { execFileSync } from "node:child_process"; + +const repository = "GluuFederation/cedarling-tutorials"; +const seconds = (start, end) => { + const duration = (Date.parse(end) - Date.parse(start)) / 1000; + return Number.isFinite(duration) && duration >= 0 + ? Math.round(duration) + : null; +}; + +export function inspectRun(run, jobs) { + const rows = jobs.map((job) => { + const steps = job.steps ?? []; + const last = steps.at(-1); + const stale = + job.status !== "completed" && + last?.name === "Complete job" && + last.status === "completed"; + return { + name: job.name, + status: job.status, + conclusion: job.conclusion, + queuedSeconds: seconds(job.created_at, job.started_at), + executionSeconds: seconds( + job.started_at, + stale ? last.completed_at : job.completed_at, + ), + stale, + slowSteps: steps + .map((step) => ({ + name: step.name, + conclusion: step.conclusion, + seconds: seconds(step.started_at, step.completed_at), + })) + .filter( + (step) => + step.seconds >= 30 || + ["failure", "cancelled"].includes(step.conclusion), + ), + }; + }); + return { + status: run.status, + conclusion: run.conclusion, + jobs: rows, + success: + rows.length > 0 && + run.status === "completed" && + run.conclusion === "success" && + rows.every( + (job) => + job.status === "completed" && + ["success", "skipped"].includes(job.conclusion), + ), + }; +} + +export function fetchRun(id, execute = execFileSync) { + if (!/^[1-9][0-9]{0,15}$/.test(String(id))) + throw new Error("Usage: pnpm inspect:ci "); + const get = (path, paginate = false) => + JSON.parse( + execute( + "gh", + ["api", path, ...(paginate ? ["--paginate", "--slurp"] : [])], + { + encoding: "utf8", + timeout: 60_000, + maxBuffer: 32 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }, + ), + ); + const run = get(`repos/${repository}/actions/runs/${id}`); + const pages = get( + `repos/${repository}/actions/runs/${id}/jobs?filter=latest&per_page=100`, + true, + ); + return inspectRun( + run, + pages.flatMap((page) => page.jobs), + ); +} + +if (import.meta.main) { + try { + const args = process.argv.slice(2).filter((value) => value !== "--"); + if (args.length !== 1) throw new Error("Usage: pnpm inspect:ci "); + const result = fetchRun(args[0]); + console.info( + `Workflow: ${result.status}; result: ${result.conclusion ?? "pending"}`, + ); + for (const job of result.jobs) { + console.info( + `${job.name}: ${job.conclusion ?? job.status}; queued ${job.queuedSeconds ?? "?"}s; execution ${job.executionSeconds ?? "?"}s${job.stale ? "; final step completed, GitHub status is unresolved" : ""}`, + ); + for (const step of job.slowSteps) + console.info( + ` ${step.name}: ${step.seconds ?? "?"}s; ${step.conclusion ?? "pending"}`, + ); + } + if (!result.success) process.exitCode = 1; + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/inspect-ci.test.mjs b/shared/tools/repository-automation/inspect-ci.test.mjs new file mode 100644 index 0000000..0b1f19b --- /dev/null +++ b/shared/tools/repository-automation/inspect-ci.test.mjs @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { fetchRun, inspectRun } from "./inspect-ci.mjs"; + +const complete = { status: "completed", conclusion: "success" }; +const job = { + ...complete, + name: "example", + created_at: "2026-10-22T18:00:00Z", + started_at: "2026-10-22T18:01:00Z", + completed_at: "2026-10-22T18:03:00Z", + steps: [ + { + name: "Install", + ...complete, + started_at: "2026-10-22T18:01:00Z", + completed_at: "2026-10-22T18:02:30Z", + }, + ], +}; + +test("inspection separates queue and execution durations and identifies slow stages", () => { + const result = inspectRun(complete, [job]); + assert.equal(result.success, true); + assert.equal(result.jobs[0].queuedSeconds, 60); + assert.equal(result.jobs[0].executionSeconds, 120); + assert.equal(result.jobs[0].slowSteps[0].seconds, 90); +}); + +test("completed steps never turn an unresolved or cancelled job green", () => { + const stale = { + ...job, + status: "in_progress", + conclusion: null, + completed_at: null, + steps: [ + ...job.steps, + { name: "Complete job", ...complete, completed_at: job.completed_at }, + ], + }; + const result = inspectRun(complete, [stale]); + assert.equal(result.jobs[0].stale, true); + assert.equal(result.jobs[0].executionSeconds, 120); + assert.equal(result.success, false); + assert.equal( + inspectRun(complete, [{ ...job, conclusion: "cancelled" }]).success, + false, + ); + assert.equal( + inspectRun({ status: "in_progress", conclusion: null }, [job]).success, + false, + ); +}); + +test("inspection handles queued jobs without inventing execution timings", () => { + const result = inspectRun({ status: "queued" }, [ + { name: "waiting", status: "queued", steps: [] }, + ]); + assert.equal(result.jobs[0].executionSeconds, null); + assert.equal(result.jobs[0].queuedSeconds, null); + assert.equal(result.success, false); +}); + +test("inspection reads every jobs page with GET-only gh calls", () => { + const calls = []; + const result = fetchRun("123", (_file, args) => { + calls.push(args); + return JSON.stringify( + args[1].endsWith("/123") + ? complete + : [{ jobs: [job] }, { jobs: [{ ...job, name: "second" }] }], + ); + }); + assert.equal(result.jobs.length, 2); + assert.equal( + calls.every((args) => args[0] === "api" && !args.includes("--method")), + true, + ); + assert.throws(() => fetchRun("123;unsafe"), /Usage/); +}); diff --git a/shared/tools/repository-automation/package.json b/shared/tools/repository-automation/package.json index c42943a..e897f0e 100644 --- a/shared/tools/repository-automation/package.json +++ b/shared/tools/repository-automation/package.json @@ -9,11 +9,12 @@ "pnpm": ">=10.17.1 <11" }, "scripts": { - "check": "node --test changed-projects.test.mjs tutorial-contract.test.mjs && pnpm validate:tutorials", + "check": "node --test changed-projects.test.mjs tutorial-contract.test.mjs setup-browser.test.mjs ci-result.test.mjs ci-report.test.mjs inspect-ci.test.mjs && pnpm validate:tutorials", + "inspect:ci": "node inspect-ci.mjs", "validate:tutorials": "node validate-tutorials.mjs --repository ../../.." }, "devDependencies": { - "dompurify": "3.4.16", + "dompurify": "3.4.17", "jsdom": "30.1.1", "mdast-util-from-markdown": "2.0.3", "sharp": "0.35.5", diff --git a/shared/tools/repository-automation/pnpm-lock.yaml b/shared/tools/repository-automation/pnpm-lock.yaml index b8290f7..be67bbd 100644 --- a/shared/tools/repository-automation/pnpm-lock.yaml +++ b/shared/tools/repository-automation/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: dompurify: - specifier: 3.4.16 - version: 3.4.16 + specifier: 3.4.17 + version: 3.4.17 jsdom: specifier: 30.1.1 version: 30.1.1 @@ -290,8 +290,8 @@ packages: devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} - dompurify@3.4.16: - resolution: {integrity: sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ==} + dompurify@3.4.17: + resolution: {integrity: sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w==} entities@8.1.0: resolution: {integrity: sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==} @@ -698,7 +698,7 @@ snapshots: dependencies: dequal: 2.0.3 - dompurify@3.4.16: + dompurify@3.4.17: optionalDependencies: '@types/trusted-types': 2.0.7 diff --git a/shared/tools/repository-automation/setup-browser.mjs b/shared/tools/repository-automation/setup-browser.mjs new file mode 100644 index 0000000..a9d6a6a --- /dev/null +++ b/shared/tools/repository-automation/setup-browser.mjs @@ -0,0 +1,215 @@ +// Installs Chromium on disposable GitHub Ubuntu runners with bounded network setup. +import { execFileSync, spawn } from "node:child_process"; +import { appendFile, readFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join, resolve } from "node:path"; + +export function hardenSources(text) { + return text + .replace( + /mirror\+file:\/etc\/apt\/apt-mirrors\.txt(?=\s|$)/g, + "https://archive.ubuntu.com/ubuntu", + ) + .replace( + /https?:\/\/(?:azure\.archive|archive)\.ubuntu\.com\/ubuntu(?=[/\s]|$)/g, + "https://archive.ubuntu.com/ubuntu", + ) + .replace( + /http:\/\/security\.ubuntu\.com\/ubuntu(?=[/\s]|$)/g, + "https://security.ubuntu.com/ubuntu", + ); +} + +function transient(error) { + const text = error.message; + if ( + /signature|NO_PUBKEY|BADSIG|EXPKEYSIG|Hash Sum mismatch|dpkg|certificate|CERT_|timed out|cancelled|Unable to locate|no installation candidate/i.test( + text, + ) + ) + return false; + return /Temporary failure resolving|Could not resolve|Could not connect|Connection (?:reset|failed|refused)|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENETUNREACH|socket hang up|\b(?:429|50[234])\b/.test( + text, + ); +} + +export async function installBrowser({ install, report = () => {} }) { + let dependenciesReady = false; + for (let attempt = 1; attempt <= 2; attempt++) { + const started = performance.now(); + for (const phase of dependenciesReady + ? ["browser"] + : ["dependencies", "browser"]) { + const phaseStarted = performance.now(); + try { + const remaining = 180_000 - (performance.now() - started); + if (remaining <= 0) throw new Error("Browser setup timed out"); + await install(phase, remaining); + if (phase === "dependencies") dependenciesReady = true; + report({ + phase, + attempt, + result: "success", + seconds: Math.ceil((performance.now() - phaseStarted) / 1000), + }); + } catch (error) { + report({ + phase, + attempt, + result: "failure", + seconds: Math.ceil((performance.now() - phaseStarted) / 1000), + }); + if (attempt === 2 || !transient(error)) throw error; + break; + } + if (phase === "browser") return; + } + } +} + +export async function prepareUbuntu({ + read = readFile, + execute = execFileSync, +} = {}) { + let found = false; + const updates = []; + for (const path of [ + "/etc/apt/sources.list.d/ubuntu.sources", + "/etc/apt/sources.list", + ]) { + let before; + try { + before = await read(path, "utf8"); + } catch (error) { + if (error.code === "ENOENT") continue; + throw error; + } + const after = hardenSources(before); + if ( + /https:\/\/(?:archive|security)\.ubuntu\.com\/ubuntu(?=[/\s]|$)/.test( + after, + ) + ) + found = true; + if (after !== before) updates.push({ path, after }); + } + if (!found) + throw new Error("Expected official Ubuntu package sources on this runner"); + for (const { path, after } of updates) + execute("sudo", ["-n", "tee", path], { + input: after, + stdio: ["pipe", "ignore", "pipe"], + }); + execute("sudo", ["-n", "tee", "/etc/apt/apt.conf.d/99tutorial-ci"], { + input: + 'Acquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAcquire::Retries "0";\nDpkg::Use-Pty "0";\n', + stdio: ["pipe", "ignore", "pipe"], + }); +} + +export function installPhase( + phase, + budget, + { cli = resolve("node_modules/@playwright/test/cli.js"), signal } = {}, +) { + if (signal?.aborted) + return Promise.reject(new Error("Browser setup cancelled")); + const timeout = [ + "timeout", + "--signal=TERM", + "--kill-after=10s", + `${Math.max(1, Math.floor(budget / 1000))}s`, + process.execPath, + cli, + ]; + const command = phase === "dependencies" ? "sudo" : timeout.shift(); + const args = + phase === "dependencies" + ? ["-n", ...timeout, "install-deps", "chromium"] + : [...timeout, "install", "--only-shell", "chromium"]; + return new Promise((done, reject) => { + // Root's timeout owns the APT process group; the browser download runs as the runner user. + const child = spawn(command, args, { + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + PLAYWRIGHT_BROWSERS_PATH: + process.env.PLAYWRIGHT_BROWSERS_PATH || + join(homedir(), ".cache", "ms-playwright"), + PLAYWRIGHT_DOWNLOAD_CONNECTION_TIMEOUT: "30000", + }, + }); + let output = ""; + let cancelled = false; + const stop = () => { + cancelled = true; + child.kill("SIGTERM"); + }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + signal?.addEventListener("abort", stop, { once: true }); + const clean = () => { + process.off("SIGINT", stop); + process.off("SIGTERM", stop); + signal?.removeEventListener("abort", stop); + }; + const copy = (stream) => (bytes) => { + output = (output + bytes.toString()).slice(-128 * 1024); + stream.write(bytes); + }; + child.stdout.on("data", copy(process.stdout)); + child.stderr.on("data", copy(process.stderr)); + child.once("error", (error) => { + clean(); + reject(error); + }); + child.once("close", (code, signal) => { + clean(); + if (cancelled) reject(new Error("Browser setup cancelled")); + else if (code === 124 || code === 137) + reject(new Error("Browser setup timed out")); + else if (code !== 0) + reject( + new Error( + `Browser ${phase} setup failed (${code ?? signal}): ${output}`, + ), + ); + else done(); + }); + }); +} + +if (import.meta.main) { + try { + if ( + process.env.GITHUB_ACTIONS !== "true" || + process.env.RUNNER_OS !== "Linux" || + process.platform !== "linux" + ) + throw new Error( + "Browser system setup is restricted to disposable GitHub Linux runners", + ); + await prepareUbuntu(); + const rows = []; + try { + await installBrowser({ + install: installPhase, + report: (row) => { + rows.push(row); + console.info( + `Browser setup: ${row.phase}, attempt ${row.attempt}, ${row.result}, ${row.seconds}s`, + ); + }, + }); + } finally { + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `### Browser setup\n\n| Phase | Attempt | Result | Seconds |\n|---|---:|---|---:|\n${rows.map((row) => `| ${row.phase} | ${row.attempt} | ${row.result} | ${row.seconds} |`).join("\n")}\n`, + ); + } + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/shared/tools/repository-automation/setup-browser.test.mjs b/shared/tools/repository-automation/setup-browser.test.mjs new file mode 100644 index 0000000..3ce5ed5 --- /dev/null +++ b/shared/tools/repository-automation/setup-browser.test.mjs @@ -0,0 +1,206 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { + hardenSources, + installBrowser, + installPhase, + prepareUbuntu, +} from "./setup-browser.mjs"; + +test( + "the browser installer requests Chromium's headless shell", + { skip: process.platform !== "linux", timeout: 15_000 }, + async (t) => { + const root = await mkdtemp(join(tmpdir(), "ci-browser-args-")); + t.after(() => rm(root, { recursive: true, force: true })); + const cli = join(root, "fixture.mjs"); + const output = join(root, "args.json"); + await writeFile( + cli, + 'import { writeFileSync } from "node:fs"; writeFileSync(' + + JSON.stringify(output) + + ", JSON.stringify(process.argv.slice(2)));\n", + ); + await installPhase("browser", 10_000, { cli }); + assert.deepEqual(JSON.parse(await readFile(output, "utf8")), [ + "install", + "--only-shell", + "chromium", + ]); + }, +); + +test("Ubuntu preparation supports the hosted runner mirror list and preserves repository signing", async () => { + const path = "/etc/apt/sources.list.d/ubuntu.sources"; + const source = + "Types: deb\nURIs: mirror+file:/etc/apt/apt-mirrors.txt\nSuites: noble noble-updates noble-security\nComponents: main restricted universe multiverse\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n"; + const writes = new Map(); + await prepareUbuntu({ + read: async (file) => { + if (file === path) return source; + throw Object.assign(new Error("missing fixture"), { code: "ENOENT" }); + }, + execute: (command, args, options) => { + assert.equal(command, "sudo"); + assert.deepEqual(args.slice(0, 2), ["-n", "tee"]); + writes.set(args[2], options.input); + }, + }); + assert.equal( + writes.get(path), + source.replace( + "mirror+file:/etc/apt/apt-mirrors.txt", + "https://archive.ubuntu.com/ubuntu", + ), + ); + assert.match( + writes.get("/etc/apt/apt.conf.d/99tutorial-ci"), + /Acquire::https::Timeout "30"/, + ); + assert.equal(writes.size, 2); +}); + +test("Ubuntu preparation also handles legacy direct sources", async () => { + const path = "/etc/apt/sources.list"; + const source = + "deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] http://azure.archive.ubuntu.com/ubuntu/ noble main\n"; + const writes = new Map(); + await prepareUbuntu({ + read: async (file) => { + if (file === path) return source; + throw Object.assign(new Error("missing fixture"), { code: "ENOENT" }); + }, + execute: (_command, args, options) => writes.set(args[2], options.input), + }); + assert.equal( + writes.get(path), + source.replace("http://azure.archive", "https://archive"), + ); + assert.equal(writes.size, 2); +}); + +test("Ubuntu preparation rejects unknown mirror files without changing configuration", async () => { + await assert.rejects( + prepareUbuntu({ + read: async () => "URIs: mirror+file:/tmp/untrusted-mirrors.txt\n", + execute: () => assert.fail("unsupported sources must not be written"), + }), + /Expected official Ubuntu package sources/, + ); +}); + +test("Ubuntu sources use official HTTPS endpoints and retain signing configuration", () => { + const source = + "URIs: http://azure.archive.ubuntu.com/ubuntu/\nSuites: noble noble-updates\nComponents: main universe\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n"; + assert.equal( + hardenSources(source), + source.replace( + "http://azure.archive.ubuntu.com", + "https://archive.ubuntu.com", + ), + ); + assert.equal( + hardenSources("deb http://security.ubuntu.com/ubuntu noble-security main"), + "deb https://security.ubuntu.com/ubuntu noble-security main", + ); + assert.equal( + hardenSources("URIs: https://packages.microsoft.com/repos/code"), + "URIs: https://packages.microsoft.com/repos/code", + ); +}); + +test( + "the installer enforces real process deadlines and cancellation", + { skip: process.platform !== "linux", timeout: 15_000 }, + async (t) => { + const root = await mkdtemp(join(tmpdir(), "ci-installer-")); + t.after(() => rm(root, { recursive: true, force: true })); + const cli = join(root, "fixture.mjs"); + await writeFile(cli, "setInterval(() => {}, 1000);\n"); + await assert.rejects(installPhase("browser", 1000, { cli }), /timed out/); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 100); + try { + await assert.rejects( + installPhase("browser", 10_000, { cli, signal: controller.signal }), + /cancelled/, + ); + } finally { + clearTimeout(timer); + } + }, +); + +test("browser setup retries a transport failure once and keeps successful dependency setup", async () => { + const calls = []; + let downloads = 0; + await installBrowser({ + install: async (phase) => { + calls.push(phase); + if (phase === "browser" && ++downloads === 1) + throw new Error("ECONNRESET"); + }, + }); + assert.deepEqual(calls, ["dependencies", "browser", "browser"]); +}); + +test("browser setup fails after its single network retry", async () => { + let calls = 0; + await assert.rejects( + installBrowser({ + install: async () => { + calls++; + throw new Error("Temporary failure resolving archive.ubuntu.com"); + }, + }), + /Temporary failure/, + ); + assert.equal(calls, 2); +}); + +test("browser setup does not retry verification, package, timeout, or cancellation failures", async () => { + for (const message of [ + "NO_PUBKEY; Failed to fetch", + "Hash Sum mismatch", + "dpkg was interrupted", + "Unable to locate package", + "certificate verification failed", + "setup timed out", + "setup cancelled", + "unrecognized failure", + ]) { + let calls = 0; + await assert.rejects( + installBrowser({ + install: async () => { + calls++; + throw new Error(message); + }, + }), + new RegExp(message), + ); + assert.equal(calls, 1, message); + } +}); + +test("setup reports each phase and a finite remaining attempt budget", async () => { + const reports = []; + await installBrowser({ + install: async (_phase, budget) => + assert.ok(budget > 0 && budget <= 180_000), + report: (row) => reports.push(row), + }); + assert.deepEqual( + reports.map((row) => row.phase), + ["dependencies", "browser"], + ); + assert.ok( + reports.every( + (row) => + row.attempt === 1 && row.result === "success" && row.seconds >= 0, + ), + ); +}); diff --git a/shared/tools/step/README.md b/shared/tools/step/README.md index beaf6cc..c57e168 100644 --- a/shared/tools/step/README.md +++ b/shared/tools/step/README.md @@ -39,5 +39,5 @@ Replay the steps in a separate checkout of that baseline before publishing. Run the helper tests from the repository root: ```sh -node --test shared/tools/step/run.test.mjs +node --test shared/tools/step/run.test.mjs shared/tools/step/source.test.mjs ``` diff --git a/shared/tools/step/run.mjs b/shared/tools/step/run.mjs index 243ca1d..86f2db3 100644 --- a/shared/tools/step/run.mjs +++ b/shared/tools/step/run.mjs @@ -5,14 +5,15 @@ import { lstat, mkdir, open, - readFile, readdir, + readFile, realpath, rename, unlink, } from "node:fs/promises"; import { join, relative, sep } from "node:path"; import { createInterface } from "node:readline/promises"; +import { readSources } from "./source.mjs"; const toolPath = "shared/tools/step"; const statePath = ".local/step"; @@ -196,24 +197,6 @@ function requireCommit(root, sha) { } } -function sourceFile(root, commit, path) { - const entry = git(root, ["ls-tree", "-z", commit, "--", path]).toString( - "utf8", - ); - if (!entry) return null; - const match = /^(100644|100755) blob ([0-9a-f]{40})\t([^\0]+)\0$/.exec(entry); - requireThat( - match && match[3] === path, - `Source must be a regular file: ${path}`, - ); - const size = Number(git(root, ["cat-file", "-s", match[2]]).toString()); - requireThat(size <= maxFileBytes, `Source file exceeds 2 MiB: ${path}`); - return { - bytes: git(root, ["cat-file", "blob", match[2]]), - mode: match[1] === "100755" ? 0o755 : 0o644, - }; -} - // Check every existing component, including ignored state directories. async function safePath(root, path, directory = false) { const parts = path.split("/"); @@ -271,7 +254,7 @@ async function writeAtomic(root, path, bytes, mode = 0o600) { } } -async function verifyTools(root, source) { +async function verifyTools(root, source, sourceFile) { const entries = git(root, [ "ls-tree", "-r", @@ -293,7 +276,7 @@ async function verifyTools(root, source) { requireThat( equal( await currentFile(root, path), - sourceFile(root, source, path)?.bytes ?? null, + sourceFile(source, path)?.bytes ?? null, ), `Helper differs from its pinned source: ${path}. Use a fresh checkout and the matching helper.`, ); @@ -306,15 +289,15 @@ function packageScripts(bytes, label) { return value; } -async function prepare(root, manifest, source, step) { +async function prepare(root, manifest, source, step, sourceFile) { const changes = []; for (const [paths, removing] of [ [step.files, false], [step.remove, true], ]) { for (const path of paths) { - const baseline = sourceFile(root, manifest.baseline, path); - const target = sourceFile(root, source, path); + const baseline = sourceFile(manifest.baseline, path); + const target = sourceFile(source, path); requireThat( removing ? baseline && !target : target, `Missing or inconsistent source operation: ${path}`, @@ -337,10 +320,10 @@ async function prepare(root, manifest, source, step) { requireThat(before !== null, `Missing ${path}`); const current = packageScripts(before, path); const baseline = packageScripts( - sourceFile(root, manifest.baseline, path)?.bytes, + sourceFile(manifest.baseline, path)?.bytes, path, ); - const target = packageScripts(sourceFile(root, source, path)?.bytes, path); + const target = packageScripts(sourceFile(source, path)?.bytes, path); requireThat( current.name === baseline.name && target.name === baseline.name, `Unexpected package name: ${path}`, @@ -517,9 +500,19 @@ export async function runTutorial( ); requireCommit(root, source); requireCommit(root, manifest.baseline); - await verifyTools(root, source); + const sourceFile = readSources( + root, + [source, manifest.baseline], + [ + toolPath, + path, + `${manifest.directory}/package.json`, + ...manifest.steps.flatMap((step) => [...step.files, ...step.remove]), + ], + ); + await verifyTools(root, source, sourceFile); requireThat( - equal(bytes, sourceFile(root, source, path)?.bytes ?? null), + equal(bytes, sourceFile(source, path)?.bytes ?? null), "Project manifest is not registered in the pinned source", ); @@ -561,12 +554,18 @@ export async function runTutorial( } // Check the declared starting files, including the files the integration will delete. for (const step of manifest.steps) { - for (const change of await prepare(root, manifest, source, step)) { + for (const change of await prepare( + root, + manifest, + source, + step, + sourceFile, + )) { if (!change.scripts) requireThat( equal( change.before, - sourceFile(root, manifest.baseline, change.path)?.bytes ?? null, + sourceFile(manifest.baseline, change.path)?.bytes ?? null, ), `Start from the declared baseline; ${change.path} is already changed`, ); @@ -595,7 +594,13 @@ export async function runTutorial( ); let changes; try { - changes = await prepare(root, manifest, source, manifest.steps[index]); + changes = await prepare( + root, + manifest, + source, + manifest.steps[index], + sourceFile, + ); } catch (error) { if (!(error instanceof LocalEditsError)) throw error; const next = manifest.steps[state.completed.length]?.id; diff --git a/shared/tools/step/source.mjs b/shared/tools/step/source.mjs new file mode 100644 index 0000000..263e88a --- /dev/null +++ b/shared/tools/step/source.mjs @@ -0,0 +1,115 @@ +// Reads only the declared immutable Git files for one helper invocation. +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; + +const maxFileBytes = 2 * 1024 * 1024; +const batchBytes = 8 * 1024 * 1024; + +export function readSources( + root, + commits, + paths, + { execute = execFileSync } = {}, +) { + const trees = new Map(); + const objects = new Map(); + const requested = [...new Set(paths)]; + if (!requested.length) throw new Error("No source paths requested"); + const git = (args, input) => + execute("git", ["--literal-pathspecs", "-C", root, ...args], { + input, + maxBuffer: 16 * 1024 * 1024, + stdio: [input ? "pipe" : "ignore", "pipe", "pipe"], + windowsHide: true, + }); + for (const commit of new Set(commits)) { + const entries = new Map(); + // Keep argument lists within Windows' command-line limit too. + const chunks = []; + let chunk = []; + let argumentBytes = 0; + for (const path of requested) { + if (chunk.length && argumentBytes + path.length + 1 > 8192) { + chunks.push(chunk); + chunk = []; + argumentBytes = 0; + } + chunk.push(path); + argumentBytes += path.length + 1; + } + chunks.push(chunk); + for (const paths of chunks) { + const listing = git([ + "ls-tree", + "-r", + "-l", + "-z", + commit, + "--", + ...paths, + ]).toString("utf8"); + for (const line of listing.split("\0").filter(Boolean)) { + const match = + /^(100644|100755) blob ([0-9a-f]{40})\s+(\d+)\t([^\0]+)$/.exec(line); + if (!match) throw new Error("Source must be a regular file"); + const [, mode, oid, length, path] = match; + const size = Number(length); + if (size > maxFileBytes) + throw new Error(`Source file exceeds 2 MiB: ${path}`); + entries.set(path, { oid, mode: mode === "100755" ? 0o755 : 0o644 }); + objects.set(oid, size); + } + } + trees.set(commit, entries); + } + const blobs = new Map(); + let cachedBytes = 0; + function load(batch) { + if (!batch.length) return; + const output = git( + ["cat-file", "--batch"], + `${batch.map(([oid]) => oid).join("\n")}\n`, + ); + let offset = 0; + for (const [oid, size] of batch) { + const end = output.indexOf(10, offset); + const header = output.subarray(offset, end).toString("ascii"); + if ( + end < offset || + header !== `${oid} blob ${size}` || + output.length < end + size + 2 + ) + throw new Error("Invalid or truncated source blob response"); + const bytes = output.subarray(end + 1, end + 1 + size); + const actual = createHash("sha1") + .update(`blob ${size}\0`) + .update(bytes) + .digest("hex"); + offset = end + size + 2; + if (output[offset - 1] !== 10 || actual !== oid) + throw new Error("Corrupt source blob response"); + while (cachedBytes + size > batchBytes && blobs.size) { + const oldest = blobs.keys().next().value; + cachedBytes -= blobs.get(oldest).length; + blobs.delete(oldest); + } + blobs.set(oid, bytes); + cachedBytes += size; + } + if (offset !== output.length) + throw new Error("Unexpected source blob response"); + } + // Small source sets fit in one batch. Larger sets are read on demand so a + // future project's manifest cannot turn the snapshot into an unbounded cache. + if ( + objects.size <= 256 && + [...objects.values()].reduce((sum, size) => sum + size, 0) <= batchBytes + ) + load([...objects]); + return (commit, path) => { + const entry = trees.get(commit)?.get(path); + if (!entry) return null; + if (!blobs.has(entry.oid)) load([[entry.oid, objects.get(entry.oid)]]); + return { bytes: blobs.get(entry.oid), mode: entry.mode }; + }; +} diff --git a/shared/tools/step/source.test.mjs b/shared/tools/step/source.test.mjs new file mode 100644 index 0000000..9cbe491 --- /dev/null +++ b/shared/tools/step/source.test.mjs @@ -0,0 +1,141 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import { readSources } from "./source.mjs"; + +const commit = "a".repeat(40); +const blob = Buffer.from([0, 10, 255, 13, 0]); +const oid = createHash("sha1") + .update(`blob ${blob.length}\0`) + .update(blob) + .digest("hex"); +const tree = (mode = "100755", size = blob.length) => + Buffer.from(`${mode} blob ${oid} ${size}\tapp/[id].js\0`); +const batch = () => + Buffer.concat([ + Buffer.from(`${oid} blob ${blob.length}\n`), + blob, + Buffer.from("\n"), + ]); +const execute = (listing, content) => (_command, args) => + args.includes("ls-tree") ? listing : content; + +test("source snapshots preserve binary bytes, modes, literal paths, and missing entries", () => { + const read = readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(tree(), batch()), + }); + assert.deepEqual(read(commit, "app/[id].js"), { bytes: blob, mode: 0o755 }); + assert.equal(read(commit, "missing.js"), null); +}); + +test("source snapshots reject non-regular and oversized files before reading blobs", () => { + for (const listing of [tree("120000"), tree("100644", 2 * 1024 * 1024 + 1)]) { + assert.throws( + () => + readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(listing, batch()), + }), + /regular file|exceeds 2 MiB/, + ); + } +}); + +test("source snapshots reject truncated, missing, corrupt, and unsolicited blob responses", () => { + for (const content of [ + batch().subarray(0, -2), + Buffer.from(`${oid} missing\n`), + Buffer.concat([ + Buffer.from(`${oid} blob ${blob.length}\n`), + Buffer.alloc(blob.length), + Buffer.from("\n"), + ]), + Buffer.concat([batch(), Buffer.from("extra")]), + ]) { + assert.throws( + () => + readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(tree(), content), + }), + /source blob/i, + ); + } +}); + +test("each invocation reads its own source snapshot", () => { + const first = readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(tree(), batch()), + }); + const second = readSources("/fixture", [commit], ["app/[id].js"], { + execute: execute(Buffer.alloc(0), Buffer.alloc(0)), + }); + assert.deepEqual(first(commit, "app/[id].js").bytes, blob); + assert.equal(second(commit, "app/[id].js"), null); +}); + +test("large source sets remain available after cache eviction", () => { + const files = Array.from({ length: 5 }, (_, index) => { + const bytes = Buffer.alloc(2 * 1024 * 1024, index); + const oid = createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + return { path: `app/file${index}.bin`, oid, bytes }; + }); + const objects = new Map(files.map((file) => [file.oid, file])); + const read = readSources( + "/fixture", + [commit], + files.map((file) => file.path), + { + execute: (_command, args, options) => + args.includes("ls-tree") + ? Buffer.from( + files + .map( + (file) => + `100644 blob ${file.oid} ${file.bytes.length}\t${file.path}\0`, + ) + .join(""), + ) + : Buffer.concat( + options.input + .trim() + .split("\n") + .map((oid) => { + const file = objects.get(oid); + return Buffer.concat([ + Buffer.from(`${oid} blob ${file.bytes.length}\n`), + file.bytes, + Buffer.from("\n"), + ]); + }), + ), + }, + ); + for (const file of [...files, files[0]]) + assert.deepEqual(read(commit, file.path).bytes, file.bytes); +}); + +test("source sets exceeding Windows' argument limit retain every literal path", () => { + const paths = Array.from( + { length: 256 }, + (_, index) => `app/${"x".repeat(180)}-${index}.js`, + ); + const read = readSources("/fixture", [commit], paths, { + execute: (_command, args) => { + assert.ok( + args.join(" ").length < 32767, + "Windows command-line limit exceeded", + ); + if (!args.includes("ls-tree")) return batch(); + return Buffer.from( + args + .slice(args.indexOf("--") + 1) + .map((path) => `100644 blob ${oid} ${blob.length}\t${path}\0`) + .join(""), + ); + }, + }); + for (const path of paths) assert.deepEqual(read(commit, path).bytes, blob); + assert.throws(() => readSources("/fixture", [commit], []), /No source paths/); +});