diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95c1c6fc..52c19aec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,18 +10,22 @@ jobs: build-and-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '20' cache: 'pnpm' - - run: pnpm install --frozen-lockfile - - run: pnpm run supply-chain:check - - run: pnpm run build - - name: Verify context routing benchmark - run: pnpm run benchmark:context - - name: Measure brief token budget + - run: pnpm install --frozen-lockfile + - run: pnpm run supply-chain:check + - run: pnpm run build + # Same check release.yml runs before publishing; exercised here so a + # change to the script can't first fail at release time. + - name: Verify packed packages are publishable + run: pnpm run publish:check + - name: Verify context routing benchmark + run: pnpm run benchmark:context + - name: Measure brief token budget run: node scripts/measure-brief-size.mjs - name: Eval brief orientation coverage run: node scripts/eval-brief-coverage.mjs diff --git a/scripts/assert-packages-publishable.mjs b/scripts/assert-packages-publishable.mjs index 0f4a4b20..794961e4 100644 --- a/scripts/assert-packages-publishable.mjs +++ b/scripts/assert-packages-publishable.mjs @@ -1,8 +1,9 @@ #!/usr/bin/env node import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { spawnSync } from "node:child_process"; +import { gunzipSync } from "node:zlib"; import { fileURLToPath } from "node:url"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -38,6 +39,33 @@ const forbiddenTarballPathPatterns = [ const tempDir = mkdtempSync(join(tmpdir(), "charter-publish-check-")); const failures = []; +// Under `pnpm run`, npm_execpath points at pnpm's JS entry point, which node can +// run directly on every platform. Otherwise use `pnpm` from PATH; on Windows +// that is `pnpm.cmd`, which Node can only start through a shell. +function pnpmCommand(args) { + const execPath = process.env.npm_execpath; + // Match the file name only: npm's npm-cli.js can sit under a pnpm-managed + // Node install, and npm pack would leave workspace: specifiers in place. + if (execPath && /^pnpm\.[cm]?js$/i.test(basename(execPath))) { + return { command: process.execPath, args: [execPath, ...args], shell: false }; + } + if (process.platform === "win32") { + // cmd.exe expands %VAR% even inside double quotes, so refuse rather than + // pack into (and fail to clean up) a different directory. + const unsafe = args.find((arg) => arg.includes("%")); + if (unsafe) { + throw new Error(`Cannot pass ${unsafe} to pnpm through cmd.exe; run via \`pnpm run publish:check\` instead.`); + } + // Pass one pre-quoted command line: Node deprecates args arrays with shell. + return { command: ["pnpm", ...args].map(quoteForCmd).join(" "), args: [], shell: true }; + } + return { command: "pnpm", args, shell: false }; +} + +function quoteForCmd(arg) { + return /^[\w\-.:/\\=]+$/.test(arg) ? arg : `"${arg.replace(/"/g, '""')}"`; +} + function run(command, args, options) { const result = spawnSync(command, args, { ...options, @@ -45,30 +73,100 @@ function run(command, args, options) { stdio: ["ignore", "pipe", "pipe"], }); - if (result.status !== 0) { + if (result.error || result.status !== 0) { throw new Error( [ `${command} ${args.join(" ")} failed in ${options.cwd}`, - result.stdout.trim(), - result.stderr.trim(), + result.error ? `spawn error: ${result.error.message}` : "", + result.status === null && result.signal ? `terminated by signal ${result.signal}` : "", + (result.stdout ?? "").trim(), + (result.stderr ?? "").trim(), ] .filter(Boolean) .join("\n"), ); } - return result.stdout; + return result.stdout ?? ""; } -function packedPackageJson(tarball) { - return JSON.parse(run("tar", ["-xOf", tarball, "package/package.json"], { cwd: root })); +function runPnpm(args, options) { + const invocation = pnpmCommand(args); + return run(invocation.command, invocation.args, { ...options, shell: invocation.shell }); +} + +// Minimal reader for the gzipped ustar archives `pnpm pack` produces. Reading +// the tarball in-process avoids depending on a system `tar`, whose flavour and +// path handling vary by platform (GNU tar under Git Bash reads `C:/...` as a +// remote host). +function readTarball(tarball) { + const data = gunzipSync(readFileSync(tarball)); + const files = new Map(); + const names = []; + let offset = 0; + let pendingPath = null; + let sawEnd = false; + + const field = (block, start, length) => { + const raw = block.subarray(start, start + length); + const nul = raw.indexOf(0); + return raw.subarray(0, nul === -1 ? raw.length : nul).toString("utf8"); + }; + + while (offset + 512 <= data.length) { + const header = data.subarray(offset, offset + 512); + if (header.every((byte) => byte === 0)) { + sawEnd = true; + break; + } + + const size = parseInt(field(header, 124, 12).trim() || "0", 8); + if (!Number.isFinite(size) || offset + 512 + size > data.length) { + throw new Error(`Malformed tar header at offset ${offset} in ${tarball}.`); + } + const type = header[156] === 0 ? "0" : String.fromCharCode(header[156]); + const body = data.subarray(offset + 512, offset + 512 + size); + offset += 512 + Math.ceil(size / 512) * 512; + + if (type === "x") { + const match = /(?:^|\n)\d+ path=([^\n]*)\n/.exec(body.toString("utf8")); + if (match) pendingPath = match[1]; + continue; + } + if (type === "L") { + pendingPath = body.toString("utf8").replace(/\0+$/, ""); + continue; + } + if (type === "g" || type === "K") continue; + + const name = field(header, 0, 100); + // POSIX ustar ("ustar\0" + "00") only; old GNU headers keep times at 345. + const isUstar = header.subarray(257, 265).toString("latin1") === "ustar\u000000"; + const prefix = isUstar ? field(header, 345, 155) : ""; + const path = pendingPath ?? (prefix ? `${prefix}/${name}` : name); + pendingPath = null; + + // Every entry (links and directories too) is checked against the + // forbidden-path patterns; only regular files keep their contents. + names.push(path); + if (type === "0" || type === "7") { + files.set(path, body); + } + } + + if (!sawEnd) { + throw new Error(`Truncated tarball (no end-of-archive marker): ${tarball}.`); + } + + return { files, names }; } -function packedFileList(tarball) { - return run("tar", ["-tf", tarball], { cwd: root }) - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); +function packedPackageJson(archive, tarball) { + const manifest = archive.files.get("package/package.json"); + if (!manifest) { + throw new Error(`${tarball} has no package/package.json.`); + } + return JSON.parse(manifest.toString("utf8")); } function workspaceDependencyEntries(manifest) { @@ -115,11 +213,12 @@ try { const cwd = join(root, packageDir); readFileSync(join(cwd, "package.json"), "utf8"); const beforePack = tarballsIn(tempDir); - const output = run("pnpm", ["pack", "--json", "--pack-destination", tempDir], { cwd }); + const output = runPnpm(["pack", "--json", "--pack-destination", tempDir], { cwd }); const filename = packedFilename(packageDir, output, beforePack); - const packedManifest = packedPackageJson(filename); + const archive = readTarball(filename); + const packedManifest = packedPackageJson(archive, filename); const workspaceEntries = workspaceDependencyEntries(packedManifest); - const forbiddenFiles = packedFileList(filename).filter((file) => + const forbiddenFiles = archive.names.filter((file) => forbiddenTarballPathPatterns.some((pattern) => pattern.test(file)), );