From 79466311061a8dce24212db368b7a928523b0c6f Mon Sep 17 00:00:00 2001 From: 0libote <79215282+0libote@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:09:18 +0000 Subject: [PATCH 1/3] chore(setup): share release bounds, atomic writes, capped reads, chatter JSON Extract single homes for the limits and predicates both release stacks enforce, one atomic tmp+rename writer for the setup store and instruction editors, one capped-read module for skill, instruction and settings reads, and one chatter-tolerant JSON parse per image layer. Document the single-writer contract on the native-settings queues instead of adding a cross-process lock no external writer would honor. --- docker/harness/chatter-json.mjs | 23 +++++ docker/harness/probe.mjs | 9 +- docker/runtime/distribution.mjs | 12 ++- docker/runtime/manager.mjs | 4 +- docker/runtime/release-bounds.mjs | 25 ++++++ docker/runtime/releases.mjs | 15 ++-- docker/setup/atomic-write.mjs | 33 +++++++ docker/setup/bounded-read.mjs | 58 +++++++++++++ docker/setup/cache.mjs | 6 +- docker/setup/chatter-json.mjs | 23 +++++ docker/setup/connect.mjs | 10 +-- docker/setup/context.mjs | 54 ++++-------- docker/setup/mcp/store.mjs | 21 ++--- docker/setup/skills.mjs | 25 ++---- docker/setup/skills/state.mjs | 25 +++--- tests/shared-helpers.test.mjs | 140 ++++++++++++++++++++++++++++++ 16 files changed, 375 insertions(+), 108 deletions(-) create mode 100644 docker/harness/chatter-json.mjs create mode 100644 docker/runtime/release-bounds.mjs create mode 100644 docker/setup/atomic-write.mjs create mode 100644 docker/setup/bounded-read.mjs create mode 100644 docker/setup/chatter-json.mjs create mode 100644 tests/shared-helpers.test.mjs diff --git a/docker/harness/chatter-json.mjs b/docker/harness/chatter-json.mjs new file mode 100644 index 00000000..bbecdd32 --- /dev/null +++ b/docker/harness/chatter-json.mjs @@ -0,0 +1,23 @@ +// Chatter-tolerant JSON parse for harness probes. +// +// Executable probes print log lines around the JSON they are asked for. The +// whole contract is: the document starts at the first "{"; anything before +// it is chatter. Returns null when there is no document or it does not +// parse, never throws. Setup and runtime keep their own identical copies: +// each image layer ships separately, so a cross-layer import would resolve +// in a checkout and break in the image. +export function chatterSlice(text) { + const output = String(text ?? ""); + const start = output.indexOf("{"); + return start < 0 ? null : output.slice(start); +} + +export function jsonFromChatter(text) { + const slice = chatterSlice(text); + if (slice === null) return null; + try { + return JSON.parse(slice); + } catch { + return null; + } +} diff --git a/docker/harness/probe.mjs b/docker/harness/probe.mjs index c0106059..275e3598 100644 --- a/docker/harness/probe.mjs +++ b/docker/harness/probe.mjs @@ -6,6 +6,7 @@ // running version and an authenticated/not/unknown verdict. import path from "node:path"; import {readEnv} from './mise.mjs'; +import { jsonFromChatter } from './chatter-json.mjs'; /** Run the harness's version flag and extract a concrete version. */ export async function probeVersion(ctx, entry, executable) { @@ -143,13 +144,7 @@ async function probeCommand(ctx, executable, args, env = null) { } function parseJson(text) { - const start = text.indexOf("{"); - if (start < 0) return null; - try { - return JSON.parse(text.slice(start)); - } catch { - return null; - } + return jsonFromChatter(text); } function firstLine(text) { diff --git a/docker/runtime/distribution.mjs b/docker/runtime/distribution.mjs index 047a0b14..546e9aa8 100644 --- a/docker/runtime/distribution.mjs +++ b/docker/runtime/distribution.mjs @@ -1,19 +1,17 @@ -import { createHash } from "node:crypto"; import { archiveStem, releaseText } from "./releases.mjs"; import { CHANNELS, channelOf, exactVersion } from "./state.mjs"; import { PATCH_REVISION, ARTIFACT_TYPE, ARCHIVE_TYPE, distributionTag, channelTag, manifestUrl, distributionBlob } from "./distribution-spec.mjs"; +import { ARCHIVE_LIMIT, DOCUMENT_LIMIT, digestOfText as digestOf, isSha256Digest as digest } from "./release-bounds.mjs"; import { registryRequest } from "./registry.mjs"; const INDEX = "application/vnd.oci.image.index.v1+json", MANIFEST = "application/vnd.oci.image.manifest.v1+json"; -const digestOf = (text) => "sha256:" + createHash("sha256").update(text).digest("hex"); -const digest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); export function validateDistributionIndex(index) { if (index?.schemaVersion !== 2 || index.mediaType !== INDEX || index.artifactType !== ARTIFACT_TYPE || index.annotations?.["io.t3code.runtime.patch"] !== PATCH_REVISION || !exactVersion(index.annotations?.["io.t3code.runtime.version"]) || !Array.isArray(index.manifests)) throw new Error("The native runtime index does not match this image's integration patch."); const architectures = new Set(); for (const entry of index.manifests) { const architecture = entry?.platform?.architecture; - if (entry?.platform?.os !== "linux" || !["amd64", "arm64"].includes(architecture) || architectures.has(architecture) || !digest(entry.digest) || entry.mediaType !== MANIFEST || !Number.isSafeInteger(entry.size) || entry.size <= 0 || entry.size > 4 * 1024 * 1024) throw new Error("The native runtime index must contain exactly one tested build for each supported architecture."); + if (entry?.platform?.os !== "linux" || !["amd64", "arm64"].includes(architecture) || architectures.has(architecture) || !digest(entry.digest) || entry.mediaType !== MANIFEST || !Number.isSafeInteger(entry.size) || entry.size <= 0 || entry.size > DOCUMENT_LIMIT) throw new Error("The native runtime index must contain exactly one tested build for each supported architecture."); architectures.add(architecture); } if (architectures.size !== 2) throw new Error("The native runtime index is missing a tested architecture."); @@ -23,7 +21,7 @@ export function validateDistributionIndex(index) { export async function distributionIndex(reference, { fetcher = fetch } = {}) { const response = await registryRequest(manifestUrl(reference), { fetcher, headers: { accept: INDEX } }); if (!response.ok) { await response.body?.cancel(); throw new Error(`Could not fetch the container-maintained T3 runtime (HTTP ${response.status}). Check again later or update the Docker image; the running version was kept.`); } - const text = await releaseText(response, 4 * 1024 * 1024), index = validateDistributionIndex(JSON.parse(text)); + const text = await releaseText(response, DOCUMENT_LIMIT), index = validateDistributionIndex(JSON.parse(text)); const supplied = response.headers.get("docker-content-digest"); if (supplied && supplied !== digestOf(text)) throw new Error("The native runtime index failed its integrity check."); return { index, digest: digestOf(text), version: index.annotations["io.t3code.runtime.version"] }; @@ -35,12 +33,12 @@ async function fromIndex(root, { fetcher, arch }) { if (platforms.length !== 1 || !digest(platforms[0].digest) || platforms[0].mediaType !== MANIFEST) throw new Error("The native runtime index must contain exactly one tested build for this architecture."); const response = await registryRequest(manifestUrl(platforms[0].digest), { fetcher, headers: { accept: MANIFEST } }); if (!response.ok) { await response.body?.cancel(); throw new Error(`The native runtime manifest is unavailable (HTTP ${response.status}).`); } - const text = await releaseText(response, 4 * 1024 * 1024); + const text = await releaseText(response, DOCUMENT_LIMIT); if (digestOf(text) !== platforms[0].digest || Buffer.byteLength(text) !== platforms[0].size) throw new Error("The native runtime manifest failed its integrity check."); const manifest = JSON.parse(text); if (manifest.schemaVersion !== 2 || manifest.mediaType !== MANIFEST || manifest.artifactType !== ARTIFACT_TYPE || manifest.annotations?.["io.t3code.runtime.patch"] !== PATCH_REVISION || manifest.annotations?.["io.t3code.runtime.version"] !== version || manifest.layers?.length !== 1) throw new Error("The native runtime manifest has invalid integration metadata."); const layer = manifest.layers[0]; - if (layer.mediaType !== ARCHIVE_TYPE || !digest(layer.digest) || !Number.isSafeInteger(layer.size) || layer.size <= 0 || layer.size > 300 * 1024 * 1024) throw new Error("The native runtime archive descriptor is invalid."); + if (layer.mediaType !== ARCHIVE_TYPE || !digest(layer.digest) || !Number.isSafeInteger(layer.size) || layer.size <= 0 || layer.size > ARCHIVE_LIMIT) throw new Error("The native runtime archive descriptor is invalid."); return { version, name: "t3-runtime", root: stem, url: distributionBlob(layer.digest), integrity: layer.digest.replace("sha256:", "sha256-"), size: layer.size, patchRevision: PATCH_REVISION, distributionDigest: root.digest, manifestDigest: platforms[0].digest }; } diff --git a/docker/runtime/manager.mjs b/docker/runtime/manager.mjs index d327aed4..1d6ddfc5 100644 --- a/docker/runtime/manager.mjs +++ b/docker/runtime/manager.mjs @@ -3,7 +3,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { unpackRelease, executableVersion } from "./releases.mjs"; +import { unpackRelease, executableVersion, chatterSlice } from "./releases.mjs"; import { packageRelease, releaseForVersion } from "./distribution.mjs"; import { PATCH_REVISION } from "./distribution-spec.mjs"; import { injectSetupBridge } from "./client-shell.mjs"; @@ -28,7 +28,7 @@ export async function discoverRuntime(binary, paths, { run = execute, env = proc const probe = await fs.mkdtemp(path.join(paths.root, ".database-probe-")); try { const issued = await run(binary, ["auth", "session", "issue", "--ttl", "1m", "--subject", "t3-container-probe", "--json"], { env: { ...env, T3CODE_HOME: probe }, timeout: 30_000 }); - const session = JSON.parse(issued.stdout.slice(issued.stdout.indexOf("{"))); + const session = JSON.parse(chatterSlice(issued.stdout) ?? ""); const scopes = session.permissions ?? session.scopes; if (!validScopes(scopes) || !scopes.length) throw new Error("Could not identify this release's supported device permissions safely."); const names = (await fs.readdir(path.join(probe, "userdata"))).filter(databaseName); diff --git a/docker/runtime/release-bounds.mjs b/docker/runtime/release-bounds.mjs new file mode 100644 index 00000000..61a9dc78 --- /dev/null +++ b/docker/runtime/release-bounds.mjs @@ -0,0 +1,25 @@ +// One home for every release size and integrity bound. +// +// The GitHub release stack (releases.mjs) and the container-registry stack +// (distribution.mjs) resolve different backends but stage the same archives +// through the same verifier (unpackRelease). An integrity hardening that +// lands in only one file leaves the other on the old bound, so both stacks +// import these limits and predicates instead of keeping their own copies. +import { createHash } from "node:crypto"; + +// Largest single archive the updater will stage or trust from a manifest. +export const ARCHIVE_LIMIT = 300 * 1024 * 1024; +// Largest extracted tree the updater will accept. +export const UNPACKED_LIMIT = 1024 * 1024 * 1024; +// Free space kept in reserve on the staging volume while downloading. +export const RELEASE_HEADROOM = 64 * 1024 * 1024; +// Largest checksum list accepted from a GitHub release. +export const SUMS_LIMIT = 1024 * 1024; +// Largest registry document (release API page, image index or manifest). +export const DOCUMENT_LIMIT = 4 * 1024 * 1024; + +// Registry blob digest ("sha256:…") used in manifests and blob URLs. +export const isSha256Digest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); +// Download integrity ("sha256-…") carried on a resolved release. +export const isSha256Integrity = (value) => typeof value === "string" && /^sha256-[a-f0-9]{64}$/.test(value); +export const digestOfText = (text) => "sha256:" + createHash("sha256").update(text).digest("hex"); diff --git a/docker/runtime/releases.mjs b/docker/runtime/releases.mjs index b6514cea..5be14027 100644 --- a/docker/runtime/releases.mjs +++ b/docker/runtime/releases.mjs @@ -6,14 +6,12 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { CHANNELS, channelOf, exactVersion, readJson, writeJson } from "./state.mjs"; import { PATCH_REVISION, distributionBlob } from "./distribution-spec.mjs"; +import { ARCHIVE_LIMIT as LIMIT, UNPACKED_LIMIT, RELEASE_HEADROOM as HEADROOM, SUMS_LIMIT, DOCUMENT_LIMIT } from "./release-bounds.mjs"; import { registryRequest } from "./registry.mjs"; const execute = promisify(execFile); const API = "https://api.github.com/repos/pingdotgg/t3code/releases"; const DOWNLOAD = "https://github.com/pingdotgg/t3code/releases/download"; -const LIMIT = 300 * 1024 * 1024; -const UNPACKED_LIMIT = 1024 * 1024 * 1024; -const HEADROOM = 64 * 1024 * 1024; const SPACE_CHECK_BYTES = 8 * 1024 * 1024; const PAGE_SIZE = 20; const downloadHosts = new Set(["github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com", "ghcr.io", "pkg-containers.githubusercontent.com"]); @@ -51,6 +49,13 @@ async function get(url, { fetcher, api = false, token, timeout = 15_000 }) { } export { get as fetchRelease }; +/** Start of the first JSON document in CLI chatter. Null when no document opens. */ +export function chatterSlice(text) { + const output = String(text ?? ""); + const start = output.indexOf("{"); + return start < 0 ? null : output.slice(start); +} + async function boundedText(response, limit) { const chunks = []; let bytes = 0; @@ -73,7 +78,7 @@ export async function releaseForVersion(version, { fetcher = fetch, arch = proce const root = archiveStem(version, arch); const name = `${root}.tar.gz`; const base = `${DOWNLOAD}/v${version}`; - const sums = await boundedText(await get(`${base}/SHA256SUMS`, { fetcher }), 1024 * 1024); + const sums = await boundedText(await get(`${base}/SHA256SUMS`, { fetcher }), SUMS_LIMIT); return { version, name: "t3-runtime", root, url: `${base}/${name}`, integrity: `sha256-${archiveChecksum(sums, name)}` }; } @@ -83,7 +88,7 @@ export async function packageRelease(channel, { fetcher = fetch, arch = process. archiveStem("0.0.0", arch); for (let page = 1; page <= 5; page += 1) { const url = channel === "stable" ? `${API}/latest` : `${API}?per_page=${PAGE_SIZE}&page=${page}`; - const data = JSON.parse(await boundedText(await get(url, { fetcher, api: true, token }), 4 * 1024 * 1024)); + const data = JSON.parse(await boundedText(await get(url, { fetcher, api: true, token }), DOCUMENT_LIMIT)); const entries = channel === "stable" ? [data] : data; if (!Array.isArray(entries)) throw new Error("GitHub did not return a release list."); const release = entries.find((entry) => { diff --git a/docker/setup/atomic-write.mjs b/docker/setup/atomic-write.mjs new file mode 100644 index 00000000..adbdeefc --- /dev/null +++ b/docker/setup/atomic-write.mjs @@ -0,0 +1,33 @@ +// One atomic tmp+rename file writer for the setup console. +// +// The MCP store and the instruction editors each kept their own copy of this +// dance: private temp file, exact mode, rename into place, best-effort temp +// cleanup. A crash between write and rename must leave either the old file or +// the new one, never a half-written one, so both callers share this helper. +// The runtime selection (docker/runtime/state.mjs) and the harness manager +// record (docker/harness/state.mjs) keep their own variants on purpose: the +// former fsyncs for boot-critical durability and the latter writes through an +// injected filesystem seam, and neither image layer can import this one. +import path from "node:path"; +import { randomBytes } from "node:crypto"; + +/** Replace durably: exclusive temp file, exact mode, atomic rename. */ +export async function writeFileAtomic(fs, file, text, mode = 0o600) { + await fs.mkdir(path.dirname(file), { recursive: true }); + const tmp = `${file}.tmp.${process.pid}.${randomBytes(6).toString("hex")}`; + let created = false; + try { + const handle = await fs.open(tmp, "wx", mode); + created = true; + try { await handle.writeFile(text, "utf8"); await handle.chmod(mode); } + finally { await handle.close(); } + await fs.rename(tmp, file); + } finally { + if (created) await fs.rm(tmp, { force: true }); + } +} + +/** Atomic JSON replace with the canonical trailing newline. */ +export async function writeJsonAtomic(fs, file, value, mode = 0o600) { + await writeFileAtomic(fs, file, JSON.stringify(value, null, 2) + "\n", mode); +} diff --git a/docker/setup/bounded-read.mjs b/docker/setup/bounded-read.mjs new file mode 100644 index 00000000..a3e6247d --- /dev/null +++ b/docker/setup/bounded-read.mjs @@ -0,0 +1,58 @@ +// One capped file reader for the setup console. +// +// Skill files, shared instructions and native settings snapshots each read a +// bounded prefix of a file that may be a symlink, may grow between stat and +// read, or may be a raced-in FIFO. The shape is always the same: reject +// non-regular or oversized targets before opening, open without following +// links and without blocking, re-check the opened descriptor, then read at +// most the limit plus one byte (including across short reads) so polling can +// never load a file whole. Callers keep their own messages; this module owns +// the flags and the read loop. +import { constants } from "node:fs"; + +// Never follow a raced-in symlink, and never block on a raced-in FIFO. +export const READ_FLAGS = constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW; + +/** Read at most maxBytes + 1 from an open handle, tolerating short reads. */ +export async function readHandleCapped(handle, maxBytes) { + const buffer = Buffer.alloc(maxBytes + 1); + let bytes = 0; + while (bytes < buffer.length) { + const result = await handle.read(buffer, bytes, buffer.length - bytes, bytes); + if (!result.bytesRead) break; + bytes += result.bytesRead; + } + // `raw` is the exact bytes read: revision hashes must not depend on how + // invalid UTF-8 decodes, so hash this, not the decoded content. + return { content: buffer.toString("utf8", 0, bytes), bytes, raw: buffer.subarray(0, bytes) }; +} + +/** + * Bounded read of a regular file. Throws the caller's `invalid` error for a + * non-regular target and the caller's `tooLarge` error (given the byte count) + * for an oversized one, so messages stay identical at every call site. + * `flags` defaults to non-following and non-blocking; the shared instruction + * file keeps its plain "r" open so a manually symlinked AGENTS.md still reads. + */ +export async function readCappedFile(io, file, maxBytes, { tooLarge, invalid, flags = READ_FLAGS } = {}) { + const failLarge = tooLarge ?? ((bytes) => Object.assign(new Error(`File exceeds the ${maxBytes}-byte read limit.`), { code: "too_large", bytes })); + const failInvalid = invalid ?? (() => Object.assign(new Error("Expected a regular file."), { code: "invalid" })); + const check = (stat) => { + if (!stat.isFile()) throw failInvalid(); + if (stat.size > maxBytes) throw failLarge(stat.size); + }; + // Reject oversized targets before opening them, including through symlinks. + check(await io.stat(file)); + const handle = await io.open(file, flags); + try { + const stat = await handle.stat(); + check(stat); + // A file may grow after stat. Read at most the limit plus one byte, + // including across short reads, so polling can never load it all. + const read = await readHandleCapped(handle, maxBytes); + if (read.bytes > maxBytes) throw failLarge(read.bytes); + return { ...read, mode: stat.mode & 0o777, stat }; + } finally { + await handle.close(); + } +} diff --git a/docker/setup/cache.mjs b/docker/setup/cache.mjs index d2f51ae0..f67a22be 100644 --- a/docker/setup/cache.mjs +++ b/docker/setup/cache.mjs @@ -41,7 +41,8 @@ export async function withTimeout(work, ms) { * Deduplicate concurrent refresh work. While a promise is in flight every * caller shares it; once it settles the next call starts fresh. A rejection * clears the slot without poisoning later calls, and an abandoned refresh - * can never crash the process. + * can never crash the process. Read-path only: this coalesces cache fills, + * it is not a writer mutex (cache entries are replaced, never mutated). */ export function createCoalescer() { let inflight = null; @@ -383,7 +384,8 @@ export function createHarnessCache({ } /** Share pending reads and optionally retain successful answers for a TTL. - * Clearing detaches old reads: their completion cannot replace a newer answer. */ + * Clearing detaches old reads: their completion cannot replace a newer answer. + * Read-path only, like the coalescer above: never used to serialize writes. */ export function createTtlCache({ ttlMs, limit = 200, now = Date.now }) { const entries = new Map(); const cached = (key, load) => { diff --git a/docker/setup/chatter-json.mjs b/docker/setup/chatter-json.mjs new file mode 100644 index 00000000..0ee03b6f --- /dev/null +++ b/docker/setup/chatter-json.mjs @@ -0,0 +1,23 @@ +// Chatter-tolerant JSON parse for setup CLI output. +// +// Native CLIs print log lines around the JSON they are asked for. The whole +// contract is: the document starts at the first "{"; anything before it is +// chatter. Returns null when there is no document or it does not parse, +// never throws. The harness probes and the runtime supervisor keep their own +// identical copies: each image layer ships separately, so a cross-layer +// import would resolve in a checkout and break in the image. +export function chatterSlice(text) { + const output = String(text ?? ""); + const start = output.indexOf("{"); + return start < 0 ? null : output.slice(start); +} + +export function jsonFromChatter(text) { + const slice = chatterSlice(text); + if (slice === null) return null; + try { + return JSON.parse(slice); + } catch { + return null; + } +} diff --git a/docker/setup/connect.mjs b/docker/setup/connect.mjs index 0d94a0c7..d223fcc2 100644 --- a/docker/setup/connect.mjs +++ b/docker/setup/connect.mjs @@ -10,6 +10,7 @@ // t3 connect status --json what is saved // t3 connect unlink off again, keeping the sign-in // +import { jsonFromChatter } from "./chatter-json.mjs"; // The setup service runs those processes. This module parses their output // and reads or saves live preferences through T3 Code's native server API. @@ -89,14 +90,7 @@ export function connectState(json) { /** The JSON object in a CLI's stdout, which may follow lines of chatter. */ export function jsonFrom(stdout) { - const text = String(stdout ?? ""); - const start = text.indexOf("{"); - if (start < 0) return null; - try { - return JSON.parse(text.slice(start)); - } catch { - return null; - } + return jsonFromChatter(stdout); } /** Read and update native preferences without overwriting the other switch. */ diff --git a/docker/setup/context.mjs b/docker/setup/context.mjs index 4ec52d29..a376f498 100644 --- a/docker/setup/context.mjs +++ b/docker/setup/context.mjs @@ -4,6 +4,8 @@ import path from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { agentResourcePaths } from "./agent-resources.mjs"; +import { writeFileAtomic } from "./atomic-write.mjs"; +import { readCappedFile } from "./bounded-read.mjs"; export function contextDirFor(baseDir) { return path.join(baseDir, "setup", "context"); @@ -77,55 +79,31 @@ async function readInstructionFile(fs, file) { const key = path.resolve(file); if (cache?.has(key)) return cache.get(key); const read = (async () => { - const check = (stat) => { - if (!stat.isFile()) throw Object.assign(new Error("Instructions must be a regular file"), { code: "invalid" }); - if (stat.size > MAX_INSTRUCTION_BYTES) throw tooLarge(stat.size); - }; - // Reject oversized targets before opening them, including through symlinks. - check(await fs.stat(file)); - const handle = await fs.open(file, "r"); - try { - const stat = await handle.stat(); - check(stat); - // A file may grow after stat. Read at most the limit plus one byte, - // including across short reads, so status polling can never load it all. - const buffer = Buffer.alloc(MAX_INSTRUCTION_BYTES + 1); - let bytes = 0; - while (bytes < buffer.length) { - const result = await handle.read(buffer, bytes, buffer.length - bytes, bytes); - if (!result.bytesRead) break; - bytes += result.bytesRead; - } - if (bytes > MAX_INSTRUCTION_BYTES) throw tooLarge(bytes); - return { content: buffer.toString("utf8", 0, bytes), bytes, mode: stat.mode & 0o777, modifiedAt: stat.mtimeMs }; - } finally { - await handle.close(); - } + const invalid = () => Object.assign(new Error("Instructions must be a regular file"), { code: "invalid" }); + // The shared file keeps its plain "r" open so a manually symlinked + // AGENTS.md still reads; providers resolve their links before reading. + const { content, bytes, mode, stat } = await readCappedFile(fs, file, MAX_INSTRUCTION_BYTES, { tooLarge, invalid, flags: "r" }); + return { content, bytes, mode, modifiedAt: stat.mtimeMs }; })(); cache?.set(key, read); return read; } -async function writeFileAtomic(fs, file, text, mode = 0o600) { - await fs.mkdir(path.dirname(file), { recursive: true }); - const tmp = `${file}.tmp.${process.pid}.${randomBytes(6).toString("hex")}`; - let created = false; - try { - const handle = await fs.open(tmp, 'wx', mode); - created = true; - try { await handle.writeFile(text, 'utf8'); await handle.chmod(mode); } finally { await handle.close(); } - await fs.rename(tmp, file); - } finally { - if (created) await fs.rm(tmp, { force: true }); - } -} - const revisionFor = (file, content) => content === null ? null : createHash("sha256").update(path.resolve(file)).update("\0").update(content).digest("hex"); const changedInstructions = () => ({ ok: false, code: "conflict", error: "Instructions changed since you opened them. Download your draft, then reopen the editor to load the current file." }); // Shared and provider editors can point at the same file. Keep their revision // checks and atomic writes in one queue per resolved target. +// +// Single-writer contract, same as the MCP store: the setup server is the only +// in-repo writer of these files, so this queue serializes its concurrent +// requests. Hand edits and provider CLIs write outside it, which is why every +// save re-checks its revision inside the queue, re-checks immediately before +// committing, and verifies afterwards instead of assuming the write won. No +// cross-process lock is taken: the harness lock guards mise state, not these +// files, and an advisory lock no external writer honors would only add +// blocking without adding safety. const instructionWrites = new Map(); const instructionModeWrites = new Map(); async function withInstructionWrite(file, work, queues = instructionWrites) { diff --git a/docker/setup/mcp/store.mjs b/docker/setup/mcp/store.mjs index 282af571..5d32fc08 100644 --- a/docker/setup/mcp/store.mjs +++ b/docker/setup/mcp/store.mjs @@ -2,7 +2,8 @@ // files are only touched by explicit link/unlink saves. import path from "node:path"; import { AGENT_NAMES } from "../agent-resources.mjs"; -import { createHash, randomBytes } from "node:crypto"; +import { createHash } from "node:crypto"; +import { writeFileAtomic as atomicWriteFile } from "../atomic-write.mjs"; export const MCP_PROVIDERS = AGENT_NAMES; export const SUPPORTED_MCP_PROVIDERS = Object.freeze(Object.keys(MCP_PROVIDERS)); @@ -98,19 +99,19 @@ async function readStoreFile(fs, file) { return { present: true, servers, bytes: Buffer.byteLength(raw, "utf8") }; } +// Size-guarded atomic replace through the shared tmp+rename writer. async function writeFileAtomic(fs, file, text) { if (Buffer.byteLength(text, "utf8") > MAX_STORE_BYTES) throw problem("too_large", "The shared MCP store would exceed the 256 KiB limit. Reduce the server definition."); - await fs.mkdir(path.dirname(file), { recursive: true }); - const tmp = `${file}.tmp.${process.pid}.${randomBytes(6).toString("hex")}`; - try { - await fs.writeFile(tmp, text, { encoding: "utf8", mode: 0o600 }); - await fs.chmod(tmp, 0o600); - await fs.rename(tmp, file); - } finally { - await fs.rm(tmp, { force: true }); - } + await atomicWriteFile(fs, file, text); } +// Single-writer contract: the setup server is the only in-repo writer of the +// shared store; this queue serializes its concurrent requests. Writers outside +// this process (hand edits on disk) cannot take the queue, so every mutation +// still re-checks the revision inside it and commits atomically; a mismatch +// fails instead of silently winning. No cross-process lock is taken on purpose: +// the harness lock guards mise state, not this file, and an advisory lock no +// external writer honors would only add blocking without adding safety. const storeQueues = new Map(); export async function withStore(file, work) { const key = path.resolve(file); diff --git a/docker/setup/skills.mjs b/docker/setup/skills.mjs index 07e896fa..03294a10 100644 --- a/docker/setup/skills.mjs +++ b/docker/setup/skills.mjs @@ -1,8 +1,8 @@ // Global skill inventory and explicit local file management. No skill execution. import path from "node:path"; import * as fs from "node:fs/promises"; -import { constants } from "node:fs"; import { createHash } from "node:crypto"; +import { readCappedFile } from "./bounded-read.mjs"; import { parseDocument, isMap, isScalar } from "./vendor/yaml.mjs"; import { listCodexSkills, writeCodexSkillConfig } from "./skills/codex.mjs"; import { codexConfigPath, configSnapshot, stateRevision, serializeState } from "./skills/state.mjs"; @@ -51,24 +51,11 @@ export function skillRoots(provider, { home, env = {} }) { // Stat before opening, stat the opened descriptor, then read at most limit + 1. // Nonblocking open prevents a raced-in FIFO from hanging the setup service. export async function readSkillFile(file, maxBytes = DEFAULT_LIMITS.fileBytes, io = fs) { - const check = (stat) => { - if (!stat.isFile()) throw problem("invalid", "The skill must be a regular Markdown file."); - if (stat.size > maxBytes) throw problem("too_large", "This skill exceeds the 64 KiB viewer limit."); - }; - check(await io.stat(file)); - const handle = await io.open(file, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW); - try { - const stat = await handle.stat(); check(stat); - const buffer = Buffer.alloc(maxBytes + 1); - let bytes = 0; - while (bytes < buffer.length) { - const read = await handle.read(buffer, bytes, buffer.length - bytes, bytes); - if (!read.bytesRead) break; - bytes += read.bytesRead; - } - if (bytes > maxBytes) throw problem("too_large", "This skill exceeds the 64 KiB viewer limit."); - return { content: buffer.toString("utf8", 0, bytes), bytes, mode: stat.mode & 0o777, stamp: [stat.dev, stat.ino, stat.mtimeMs, stat.size].join(":") }; - } finally { await handle.close(); } + // Messages stay skill-specific; the flags and the short-read loop are shared. + const invalid = () => problem("invalid", "The skill must be a regular Markdown file."); + const tooLarge = () => problem("too_large", "This skill exceeds the 64 KiB viewer limit."); + const { content, bytes, mode, stat } = await readCappedFile(io, file, maxBytes, { invalid, tooLarge }); + return { content, bytes, mode, stamp: [stat.dev, stat.ino, stat.mtimeMs, stat.size].join(":") }; } export function skillMetadata(content, fallback) { diff --git a/docker/setup/skills/state.mjs b/docker/setup/skills/state.mjs index c4c721cb..caf9f276 100644 --- a/docker/setup/skills/state.mjs +++ b/docker/setup/skills/state.mjs @@ -1,7 +1,7 @@ // Conflict tokens for native settings; settings contents never leave this module. import path from "node:path"; -import { constants } from "node:fs"; import { createHash } from "node:crypto"; +import { READ_FLAGS, readHandleCapped } from "../bounded-read.mjs"; const digest = (value) => createHash("sha256").update(value).digest("hex"); const stamp = (stat) => [stat.dev, stat.ino, stat.mode, stat.size, stat.mtimeMs, stat.ctimeMs].join(":"); @@ -16,19 +16,14 @@ export async function configSnapshot(file, io, includeContent = false) { try { check(await io.stat(file)); const resolved = await io.realpath(file); - handle = await io.open(resolved, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW); + handle = await io.open(resolved, READ_FLAGS); const before = await handle.stat(); check(before); - const buffer = Buffer.alloc(limit + 1); let bytes = 0; - while (bytes < buffer.length) { - const read = await handle.read(buffer, bytes, buffer.length - bytes, bytes); - if (!read.bytesRead) break; - bytes += read.bytesRead; - } + const { content, bytes, raw } = await readHandleCapped(handle, limit); if (bytes > limit || stamp(before) !== stamp(await handle.stat()) || stamp(before) !== stamp(await io.stat(file)) || resolved !== await io.realpath(file)) { throw Object.assign(new Error("Native settings changed while being read. Refresh the list."), { code: "conflict" }); } - const revision = digest(JSON.stringify([file, resolved, stamp(before), digest(buffer.subarray(0, bytes))])); - return includeContent ? { revision, content: buffer.toString("utf8", 0, bytes), resolvedPath: resolved, mode: before.mode & 0o777, exists: true } : revision; + const revision = digest(JSON.stringify([file, resolved, stamp(before), digest(raw)])); + return includeContent ? { revision, content, resolvedPath: resolved, mode: before.mode & 0o777, exists: true } : revision; } catch (error) { if (error.code === "conflict") throw error; if (error.code === "ENOENT" && !handle) { @@ -45,6 +40,16 @@ export async function configSnapshot(file, io, includeContent = false) { export const stateRevision = (config, row) => digest(JSON.stringify([config, row.path, row.resolvedPath, row.scope, row.source, row.name, row.state, row.policyKey, row.nativePolicyVersion])); // All skills share one provider settings file, including across service instances. +// +// Single-writer contract, same as the MCP store: the setup server is the only +// in-repo writer taking this queue, so it serializes its concurrent requests +// per settings file. The native writer itself (the provider CLI) and hand +// edits run outside it, which is why every toggle re-checks its revision +// inside the queue, re-checks immediately before the native write, and +// verifies the result afterwards instead of assuming the write won. No +// cross-process lock is taken: the harness lock guards mise state, not these +// files, and an advisory lock no external writer honors would only add +// blocking without adding safety. const queues = new Map(); export async function serializeState(key, work) { const previous = queues.get(key) || Promise.resolve(); diff --git a/tests/shared-helpers.test.mjs b/tests/shared-helpers.test.mjs new file mode 100644 index 00000000..fe3a5ac8 --- /dev/null +++ b/tests/shared-helpers.test.mjs @@ -0,0 +1,140 @@ +// Parity for the shared helper extraction: release bounds, capped reads, +// atomic writes and chatter-tolerant JSON parsing. Each case pins the exact +// behavior the duplicated copies had, so the shared modules must agree with +// the legacy inline logic and with each other. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import * as fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + ARCHIVE_LIMIT, UNPACKED_LIMIT, RELEASE_HEADROOM, SUMS_LIMIT, DOCUMENT_LIMIT, + isSha256Digest, isSha256Integrity, digestOfText, +} from "../docker/runtime/release-bounds.mjs"; +import { chatterSlice } from "../docker/runtime/releases.mjs"; +import { chatterSlice as harnessSlice, jsonFromChatter as harnessJson } from "../docker/harness/chatter-json.mjs"; +import { chatterSlice as setupSlice, jsonFromChatter as setupJson } from "../docker/setup/chatter-json.mjs"; +import { jsonFrom } from "../docker/setup/connect.mjs"; +import { READ_FLAGS, readCappedFile, readHandleCapped } from "../docker/setup/bounded-read.mjs"; +import { writeFileAtomic, writeJsonAtomic } from "../docker/setup/atomic-write.mjs"; +import { readSkillFile } from "../docker/setup/skills.mjs"; + +test("release bounds match the legacy per-file limits", () => { + assert.equal(ARCHIVE_LIMIT, 300 * 1024 * 1024); + assert.equal(UNPACKED_LIMIT, 1024 * 1024 * 1024); + assert.equal(RELEASE_HEADROOM, 64 * 1024 * 1024); + assert.equal(SUMS_LIMIT, 1024 * 1024); + assert.equal(DOCUMENT_LIMIT, 4 * 1024 * 1024); +}); + +test("shared digest predicates match the legacy inline checks", () => { + const legacyDigest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); + const legacyIntegrity = (value) => typeof value === "string" && /^sha256-[a-f0-9]{64}$/.test(value); + const cases = ["sha256:" + "a".repeat(64), "sha256-" + "b".repeat(64), "sha256:" + "A".repeat(64), + "sha256-" + "c".repeat(63), "sha256:xyz", "", null, undefined, 42, "sha256-" + "d".repeat(65)]; + for (const value of cases) { + assert.equal(isSha256Digest(value), legacyDigest(value), String(value)); + assert.equal(isSha256Integrity(value), legacyIntegrity(value), String(value)); + } + for (const text of ["", "index", "manifest-bytes"]) { + assert.equal(digestOfText(text), "sha256:" + createHash("sha256").update(text).digest("hex")); + } +}); + +test("every chatter parser agrees, including the runtime slice", () => { + const inputs = ["{\"a\":1}", "log line\n{\"a\":1}\ntrailing", "no json here", "{ broken", + "", null, undefined, 42, "{\"nested\":{\"b\":[1,2]}} chatter", "{{}}"]; + for (const input of inputs) { + const slices = [chatterSlice(input), harnessSlice(input), setupSlice(input)]; + assert.deepEqual(slices.slice(1), [slices[0], slices[0]], JSON.stringify(input)); + const parsed = [harnessJson(input), setupJson(input), jsonFrom(input)]; + assert.deepEqual(parsed.slice(1), [parsed[0], parsed[0]], JSON.stringify(input)); + if (slices[0] === null) assert.equal(parsed[0], null); + else { + let expected = null; + try { expected = JSON.parse(slices[0]); } catch { /* stays null */ } + assert.deepEqual(parsed[0], expected); + } + } +}); + +test("capped reads enforce limits before open and bound short reads", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-read-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const exact = path.join(root, "exact.md"); + await fs.writeFile(exact, "x".repeat(64)); + assert.equal((await readCappedFile(fs, exact, 64)).bytes, 64); + const over = path.join(root, "over.md"); + await fs.writeFile(over, "x".repeat(65)); + let opened = 0; + const guarded = { ...fs, open: async () => { opened += 1; throw new Error("oversized file opened"); } }; + await assert.rejects(readCappedFile(guarded, over, 64), /limit/); + assert.equal(opened, 0); + await assert.rejects(readCappedFile(fs, root, 64), /regular/); + // Symlinks are not followed. + const link = path.join(root, "link.md"); + await fs.symlink(exact, link); + await assert.rejects(readCappedFile(fs, link, 64), { code: "ELOOP" }); + // Short reads accumulate until the limit plus one. + const growing = path.join(root, "growing.md"); + await fs.writeFile(growing, "y".repeat(10)); + let calls = 0; + const short = { ...fs, open: async (...args) => { + const handle = await fs.open(...args); + return { ...handle, stat: () => handle.stat(), close: () => handle.close(), + read: async (buffer, offset, length, position) => { + calls += 1; + if (calls === 2) await fs.writeFile(growing, "y".repeat(100)); + return handle.read(buffer, offset, Math.min(length, 3), position); + } }; + } }; + await assert.rejects(readCappedFile(short, growing, 64, { tooLarge: (bytes) => Object.assign(new Error("big:" + bytes), { code: "too_large", bytes }) }), { code: "too_large" }); + assert.ok(calls > 3); + // The plain-"r" open used for the shared instruction file still follows links. + const via = await readCappedFile(fs, link, 64, { flags: "r" }); + assert.equal(via.bytes, 64); +}); + +test("capped reads agree with the skill reader", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-skill-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const file = path.join(root, "SKILL.md"); + await fs.writeFile(file, "---\nname: x\n---\nbody\n"); + const shared = await readCappedFile(fs, file, 64 * 1024, { + invalid: () => Object.assign(new Error("The skill must be a regular Markdown file."), { code: "invalid" }), + tooLarge: () => Object.assign(new Error("This skill exceeds the 64 KiB viewer limit."), { code: "too_large" }), + }); + const direct = await readSkillFile(file); + assert.equal(shared.content, direct.content); + assert.equal(shared.mode, direct.mode); +}); + +test("handle-level reads match the settings snapshot loop", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-handle-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const file = path.join(root, "settings.toml"); + await fs.writeFile(file, "a".repeat(5000)); + const handle = await fs.open(file, READ_FLAGS); + try { + const read = await readHandleCapped(handle, 1024 * 1024); + assert.equal(read.bytes, 5000); + assert.equal(read.raw.length, 5000); + assert.equal(read.content, "a".repeat(5000)); + } finally { await handle.close(); } +}); + +test("atomic writes replace durably with exact modes", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-atomic-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const file = path.join(root, "nested", "store.json"); + await writeJsonAtomic(fs, file, { version: 1, servers: [{ name: "a" }] }); + assert.equal(await fs.readFile(file, "utf8"), '{\n "version": 1,\n "servers": [\n {\n "name": "a"\n }\n ]\n}\n'); + assert.equal((await fs.stat(file)).mode & 0o777, 0o600); + await writeFileAtomic(fs, file, "replaced\n", 0o640); + assert.equal(await fs.readFile(file, "utf8"), "replaced\n"); + assert.equal((await fs.stat(file)).mode & 0o777, 0o640); + const leftovers = (await fs.readdir(path.dirname(file))).filter((name) => name.includes(".tmp.")); + assert.deepEqual(leftovers, []); +}); From d75b14407aca771956b1d48c79539b7fb88157e3 Mon Sep 17 00:00:00 2001 From: Ziyang Zeng Date: Sat, 14 Nov 2026 15:39:56 -0800 Subject: [PATCH 2/3] test: retain Files transfer smoke diagnostics Preserve the exact failing transfer stage, HTTP status and assertion output in the existing smoke diagnostics. Keep all upload, checksum and Trash assertions intact. --- scripts/smoke-test.sh | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 0ae04e7d..b961109d 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -549,20 +549,41 @@ NODE check "Files browses with t3 permissions, preserves managed link guards, and sandboxes downloads" files_browse_works files_changes_work() { - docker exec -i -u t3 -e FILES_SMOKE_KEY="$SETUP_KEY" "$NAME" node --input-type=module <<'NODE' + # check() discards command output. Retain this transfer's assertion and HTTP + # diagnostics so a failure identifies the stage without rerunning it blind. + printf '\nFiles transfer smoke (20 MiB upload, download, Trash, restore)\n' >> "$UI_AUDIT_LOG" + docker exec -i -u t3 -e FILES_SMOKE_KEY="$SETUP_KEY" "$NAME" node --input-type=module <<'NODE' >> "$UI_AUDIT_LOG" 2>&1 import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; import { randomUUID,createHash } from 'node:crypto'; import path from 'node:path'; const root=path.join(process.env.T3_WORKSPACE||'/workspace','.t3-files-smoke-'+randomUUID()); await fs.mkdir(root); -const request=async(route,body)=>{const response=await fetch('http://127.0.0.1:3774'+route,{method:body===undefined?'GET':'POST',headers:{'x-t3-setup-key':process.env.FILES_SMOKE_KEY,...(body===undefined?{}:{'content-type':Buffer.isBuffer(body)?'application/octet-stream':'application/json'})},body:body===undefined?undefined:Buffer.isBuffer(body)?body:JSON.stringify(body)});if(!response.ok)throw new Error(route+': '+await response.text());return response;}; +const request=async(route,body)=>{const response=await fetch('http://127.0.0.1:3774'+route,{method:body===undefined?'GET':'POST',headers:{'x-t3-setup-key':process.env.FILES_SMOKE_KEY,...(body===undefined?{}:{'content-type':Buffer.isBuffer(body)?'application/octet-stream':'application/json'})},body:body===undefined?undefined:Buffer.isBuffer(body)?body:JSON.stringify(body)});if(!response.ok)throw new Error(route+': HTTP '+response.status+' '+await response.text());return response;}; +let stage='starting upload'; try { const bytes=Buffer.alloc(20*1024*1024,0x61),start=await (await request('/files/upload/start',{dest:root,items:[{path:'fixture.bin',size:bytes.length,lastModified:1700000000000}]})).json(),id=start.uploads[0].id; + stage='sending chunks'; for(let offset=0;offsettrue,()=>false),false); - const finished=await (await request('/files/upload/finish',{id})).json(),download=Buffer.from(await (await request('/files/raw?'+new URLSearchParams({path:finished.path}))).arrayBuffer());assert.equal(createHash('sha256').update(download).digest('hex'),createHash('sha256').update(bytes).digest('hex')); - const trashed=await (await request('/files/to-trash',{paths:[finished.path]})).json();assert.equal(await fs.stat(finished.path).then(()=>true,()=>false),false);await request('/files/trash/restore',{ids:trashed.ids});assert.equal((await fs.stat(finished.path)).size,bytes.length); + stage='checking atomic visibility'; + assert.equal(await fs.stat(path.join(root,'fixture.bin')).then(()=>true,()=>false),false,'The upload must remain unpublished until finish.'); + stage='finishing upload'; + const finished=await (await request('/files/upload/finish',{id})).json(); + stage='downloading published bytes'; + const download=Buffer.from(await (await request('/files/raw?'+new URLSearchParams({path:finished.path}))).arrayBuffer()); + stage='verifying download checksum'; + assert.equal(createHash('sha256').update(download).digest('hex'),createHash('sha256').update(bytes).digest('hex'),'Downloaded bytes must match the uploaded fixture.'); + stage='moving to Trash'; + const trashed=await (await request('/files/to-trash',{paths:[finished.path]})).json(); + stage='checking Trash move'; + assert.equal(await fs.stat(finished.path).then(()=>true,()=>false),false,'The trashed fixture must leave its original path.'); + stage='restoring from Trash'; + await request('/files/trash/restore',{ids:trashed.ids}); + stage='checking restored size'; + assert.equal((await fs.stat(finished.path)).size,bytes.length,'The restored fixture must retain its size.'); +} catch(error) { + console.error('Files transfer failed while '+stage+'.'); + throw error; } finally { await fs.rm(root,{recursive:true,force:true}); } NODE } From 465e164965c1f094f7e6183ced8f3eac5d364d98 Mon Sep 17 00:00:00 2001 From: Ziyang Zeng Date: Sat, 14 Nov 2026 15:58:46 -0800 Subject: [PATCH 3/3] fix(ci): match native compiler to each upstream release Read an exact Node engine version from each source tag and pass that official builder image to both architecture jobs. Stable and Nightly can advance independently without breaking ARM64 native SEA packaging. Reject malformed metadata and preserve the bundled runtime Node configuration. --- .github/workflows/native-runtime.yml | 4 +- docs/native-runtime-patches.md | 10 +++- scripts/native-runtime-plan.mjs | 55 ++++++++++++------ tests/native-runtime-plan.test.mjs | 84 ++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 tests/native-runtime-plan.test.mjs diff --git a/.github/workflows/native-runtime.yml b/.github/workflows/native-runtime.yml index 9fc7ecf0..5e254c3f 100644 --- a/.github/workflows/native-runtime.yml +++ b/.github/workflows/native-runtime.yml @@ -81,7 +81,9 @@ jobs: target: t3-runtime-artifact platforms: linux/${{ matrix.arch }} outputs: type=local,dest=native-output - build-args: T3_VERSION=${{ matrix.version }} + build-args: | + T3_VERSION=${{ matrix.version }} + T3_BUILD_NODE_IMAGE=${{ matrix.buildNodeImage }} cache-from: type=gha,scope=native-${{ matrix.version }}-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=native-${{ matrix.version }}-${{ matrix.arch }} - name: Verify the native project API diff --git a/docs/native-runtime-patches.md b/docs/native-runtime-patches.md index 03f3c2c2..53329fa9 100644 --- a/docs/native-runtime-patches.md +++ b/docs/native-runtime-patches.md @@ -67,7 +67,15 @@ Linux ARM64 runners use upstream's native `build-exe` command without a cross-build target, retaining its SEA injection workaround for 16/64 KiB page kernels. The builder reads that path's `dist-exe/t3` output; explicit targets use the platform-named output. Keep the build Node version aligned with the -upstream release's declared version for native ARM64 compilation. +upstream release's declared version for native ARM64 compilation. CI reads +`engines.node` from each exact upstream tag's `package.json` and selects that +version's official Node builder image for both architectures. Stable and +Nightly can therefore use different build Node versions. Missing or non-exact +declarations stop the plan before builds start. A local Docker build of another +T3 release must set `T3_BUILD_NODE_IMAGE` to its declared Node version as well +as setting `T3_VERSION`; the default builder remains aligned with the bundled +Stable release. This changes the compiler image only. The container's runtime +Node version and user-managed toolchains keep their existing configuration. Compilation loads the exact tag's checked-in `.env.example`, allowing only the four public Clerk and relay settings that enable T3 Connect. It excludes local diff --git a/scripts/native-runtime-plan.mjs b/scripts/native-runtime-plan.mjs index 1ed8a355..6a32d045 100644 --- a/scripts/native-runtime-plan.mjs +++ b/scripts/native-runtime-plan.mjs @@ -1,20 +1,43 @@ import fs from "node:fs"; -import { packageRelease } from "../docker/runtime/releases.mjs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { packageRelease, fetchRelease, releaseText } from "../docker/runtime/releases.mjs"; import { exactVersion } from "../docker/runtime/state.mjs"; import { PATCH_REVISION } from "../docker/runtime/distribution-spec.mjs"; +import { DOCUMENT_LIMIT } from "../docker/runtime/release-bounds.mjs"; -const requested = process.env.NATIVE_VERSION?.trim(); -if (requested && !exactVersion(requested)) throw new Error("Choose an exact upstream T3 release."); -const pinned = /ARG T3_VERSION=(\S+)/.exec(fs.readFileSync(new URL("../Dockerfile", import.meta.url), "utf8"))?.[1]; -const versions = requested ? [requested] : process.env.NATIVE_BUNDLED === "true" - ? [pinned, (await packageRelease("nightly")).version] - : (await Promise.all([packageRelease("stable"), packageRelease("nightly")])).map((release) => release.version); -if (!versions.every(exactVersion)) throw new Error("Native build plan contains an invalid version."); -const unique = [...new Set(versions)]; -const matrix = { include: unique.flatMap((version) => [ - { version, arch: "amd64", nodeArch: "x64", runner: "ubuntu-latest" }, - { version, arch: "arm64", nodeArch: "arm64", runner: "ubuntu-24.04-arm" }, -]) }; -const output = { matrix: JSON.stringify(matrix), versions: JSON.stringify(unique), revision: PATCH_REVISION, bundled: pinned, nightly: unique.find((version) => version.includes("-nightly.")) || "" }; -if (process.env.GITHUB_OUTPUT) for (const [name, value] of Object.entries(output)) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); -console.log(JSON.stringify(output)); +// Native ARM64 SEA packaging injects the host Node executable. Its version +// must match the exact source release, which can differ between channels. +export async function buildNodeImage(version, { fetcher = fetch } = {}) { + if (!exactVersion(version)) throw new Error("Choose an exact upstream T3 release."); + const url = `https://raw.githubusercontent.com/pingdotgg/t3code/v${version}/package.json`; + const response = await fetchRelease(url, { fetcher }); + const manifest = JSON.parse(await releaseText(response, DOCUMENT_LIMIT)); + const node = manifest?.engines?.node; + if (typeof node !== "string" || !/^[1-9]\d*\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)$/.test(node) || Number(node.split(".")[0]) < 26) { + throw new Error(`Upstream v${version} must declare an exact Node 26 or newer build version.`); + } + return `node:${node}-trixie-slim`; +} + +export async function nativeRuntimePlan({ requested, bundled = false, pinned, fetcher = fetch, resolveRelease = packageRelease } = {}) { + if (requested && !exactVersion(requested)) throw new Error("Choose an exact upstream T3 release."); + pinned ??= /ARG T3_VERSION=(\S+)/.exec(fs.readFileSync(new URL("../Dockerfile", import.meta.url), "utf8"))?.[1]; + const versions = requested ? [requested] : bundled + ? [pinned, (await resolveRelease("nightly", { fetcher })).version] + : (await Promise.all([resolveRelease("stable", { fetcher }), resolveRelease("nightly", { fetcher })])).map((release) => release.version); + if (!versions.every(exactVersion)) throw new Error("Native build plan contains an invalid version."); + const unique = [...new Set(versions)]; + const releases = await Promise.all(unique.map(async version => ({ version, buildNodeImage: await buildNodeImage(version, { fetcher }) }))); + const matrix = { include: releases.flatMap((release) => [ + { ...release, arch: "amd64", nodeArch: "x64", runner: "ubuntu-latest" }, + { ...release, arch: "arm64", nodeArch: "arm64", runner: "ubuntu-24.04-arm" }, + ]) }; + return { matrix: JSON.stringify(matrix), versions: JSON.stringify(unique), revision: PATCH_REVISION, bundled: pinned, nightly: unique.find((version) => version.includes("-nightly.")) || "" }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + const output = await nativeRuntimePlan({ requested: process.env.NATIVE_VERSION?.trim(), bundled: process.env.NATIVE_BUNDLED === "true" }); + if (process.env.GITHUB_OUTPUT) for (const [name, value] of Object.entries(output)) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); + console.log(JSON.stringify(output)); +} diff --git a/tests/native-runtime-plan.test.mjs b/tests/native-runtime-plan.test.mjs new file mode 100644 index 00000000..530fec9b --- /dev/null +++ b/tests/native-runtime-plan.test.mjs @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { buildNodeImage, nativeRuntimePlan } from "../scripts/native-runtime-plan.mjs"; +import { DOCUMENT_LIMIT } from "../docker/runtime/release-bounds.mjs"; + +const stable = "0.1.9", nightly = "0.1.10-nightly.20261114.4709"; +function upstream(packages = { [stable]: "26.12.1", [nightly]: "26.13.1" }) { + const calls = []; + return { + calls, + fetcher: async (url, options) => { + calls.push({ url, options }); + const version = Object.keys(packages).find(version => url === `https://raw.githubusercontent.com/pingdotgg/t3code/v${version}/package.json`); + assert(version, "Read only the exact release's root package manifest."); + assert.equal(options.redirect, "manual"); + assert(options.signal instanceof AbortSignal); + assert.equal(options.headers.authorization, undefined, "Public source reads must not forward credentials."); + return Response.json({ engines: { node: packages[version] } }); + }, + resolveRelease: async channel => ({ version: channel === "stable" ? stable : nightly }), + }; +} + +test("native CI uses each exact release's Node version on both architectures when channels diverge", async () => { + const world = upstream(); + const output = await nativeRuntimePlan({ bundled: true, pinned: stable, ...world }); + const jobs = JSON.parse(output.matrix).include; + assert.equal(jobs.length, 4); + assert.deepEqual(JSON.parse(output.versions), [stable, nightly]); + assert.equal(output.bundled, stable); + assert.equal(output.nightly, nightly); + for (const version of [stable, nightly]) { + const selected = jobs.filter(job => job.version === version); + assert.deepEqual(selected.map(job => job.arch), ["amd64", "arm64"]); + assert.deepEqual(selected.map(job => job.nodeArch), ["x64", "arm64"]); + assert(selected.every(job => job.buildNodeImage === (version === stable ? "node:26.12.1-trixie-slim" : "node:26.13.1-trixie-slim"))); + } + assert.equal(world.calls.length, 2, "Resolve toolchains once per release, not once per architecture."); +}); + +test("an exact requested release does not follow either moving channel", async () => { + const world = upstream(); + const output = await nativeRuntimePlan({ requested: stable, ...world, resolveRelease: () => assert.fail("No channel lookup for an exact request.") }); + assert.deepEqual(JSON.parse(output.versions), [stable]); + assert.equal(JSON.parse(output.matrix).include.length, 2); + assert.equal(world.calls.length, 1); + assert.equal(output.nightly, ""); +}); + +test("scheduled native plans resolve both channels and deduplicate repeated releases", async () => { + const world = upstream(); + const output = await nativeRuntimePlan({ ...world }); + assert.deepEqual(JSON.parse(output.versions), [stable, nightly]); + const repeated = upstream(); + const one = await nativeRuntimePlan({ ...repeated, resolveRelease: async () => ({ version: stable }) }); + assert.equal(JSON.parse(one.matrix).include.length, 2); + assert.equal(repeated.calls.length, 1); +}); + +test("unsupported Node declarations cannot select arbitrary or floating builder images", async () => { + for (const node of [undefined, null, 26, "", "26", "^26.13.1", ">=26", "v26.13.1", "026.13.1", "26.013.1", "26.13.01", "26.13.1-alpine", "26.13.1\nIMAGE=other", "25.9.0"]) { + await assert.rejects(buildNodeImage(stable, upstream({ [stable]: node })), /exact Node 26 or newer/); + } +}); + +test("invalid release identities fail before a source request", async () => { + const fetcher = () => assert.fail("No network request for an invalid release."); + for (const version of ["latest", "nightly", "../main", "0.1.9\nNODE_IMAGE=other"]) { + await assert.rejects(buildNodeImage(version, { fetcher }), /exact upstream/); + await assert.rejects(nativeRuntimePlan({ requested: version, fetcher }), /exact upstream/); + } +}); + +test("missing, redirected, malformed or oversized source metadata cannot fall back to the bundled toolchain", async () => { + for (const response of [ + new Response("not found", { status: 404 }), + new Response(null, { status: 302, headers: { location: "https://example.test/package.json" } }), + new Response("{bad json"), + new Response(" ".repeat(DOCUMENT_LIMIT + 1)), + Response.json({}), + ]) { + await assert.rejects(nativeRuntimePlan({ requested: nightly, fetcher: async () => response })); + } +});