diff --git a/.github/workflows/native-runtime.yml b/.github/workflows/native-runtime.yml index 9fc7ecf..5e254c3 100644 --- a/.github/workflows/native-runtime.yml +++ b/.github/workflows/native-runtime.yml @@ -81,7 +81,9 @@ jobs: target: t3-runtime-artifact platforms: linux/${{ matrix.arch }} outputs: type=local,dest=native-output - build-args: T3_VERSION=${{ matrix.version }} + build-args: | + T3_VERSION=${{ matrix.version }} + T3_BUILD_NODE_IMAGE=${{ matrix.buildNodeImage }} cache-from: type=gha,scope=native-${{ matrix.version }}-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=native-${{ matrix.version }}-${{ matrix.arch }} - name: Verify the native project API diff --git a/docker/harness/chatter-json.mjs b/docker/harness/chatter-json.mjs new file mode 100644 index 0000000..bbecdd3 --- /dev/null +++ b/docker/harness/chatter-json.mjs @@ -0,0 +1,23 @@ +// Chatter-tolerant JSON parse for harness probes. +// +// Executable probes print log lines around the JSON they are asked for. The +// whole contract is: the document starts at the first "{"; anything before +// it is chatter. Returns null when there is no document or it does not +// parse, never throws. Setup and runtime keep their own identical copies: +// each image layer ships separately, so a cross-layer import would resolve +// in a checkout and break in the image. +export function chatterSlice(text) { + const output = String(text ?? ""); + const start = output.indexOf("{"); + return start < 0 ? null : output.slice(start); +} + +export function jsonFromChatter(text) { + const slice = chatterSlice(text); + if (slice === null) return null; + try { + return JSON.parse(slice); + } catch { + return null; + } +} diff --git a/docker/harness/probe.mjs b/docker/harness/probe.mjs index c010605..275e359 100644 --- a/docker/harness/probe.mjs +++ b/docker/harness/probe.mjs @@ -6,6 +6,7 @@ // running version and an authenticated/not/unknown verdict. import path from "node:path"; import {readEnv} from './mise.mjs'; +import { jsonFromChatter } from './chatter-json.mjs'; /** Run the harness's version flag and extract a concrete version. */ export async function probeVersion(ctx, entry, executable) { @@ -143,13 +144,7 @@ async function probeCommand(ctx, executable, args, env = null) { } function parseJson(text) { - const start = text.indexOf("{"); - if (start < 0) return null; - try { - return JSON.parse(text.slice(start)); - } catch { - return null; - } + return jsonFromChatter(text); } function firstLine(text) { diff --git a/docker/runtime/distribution.mjs b/docker/runtime/distribution.mjs index 047a0b1..546e9aa 100644 --- a/docker/runtime/distribution.mjs +++ b/docker/runtime/distribution.mjs @@ -1,19 +1,17 @@ -import { createHash } from "node:crypto"; import { archiveStem, releaseText } from "./releases.mjs"; import { CHANNELS, channelOf, exactVersion } from "./state.mjs"; import { PATCH_REVISION, ARTIFACT_TYPE, ARCHIVE_TYPE, distributionTag, channelTag, manifestUrl, distributionBlob } from "./distribution-spec.mjs"; +import { ARCHIVE_LIMIT, DOCUMENT_LIMIT, digestOfText as digestOf, isSha256Digest as digest } from "./release-bounds.mjs"; import { registryRequest } from "./registry.mjs"; const INDEX = "application/vnd.oci.image.index.v1+json", MANIFEST = "application/vnd.oci.image.manifest.v1+json"; -const digestOf = (text) => "sha256:" + createHash("sha256").update(text).digest("hex"); -const digest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); export function validateDistributionIndex(index) { if (index?.schemaVersion !== 2 || index.mediaType !== INDEX || index.artifactType !== ARTIFACT_TYPE || index.annotations?.["io.t3code.runtime.patch"] !== PATCH_REVISION || !exactVersion(index.annotations?.["io.t3code.runtime.version"]) || !Array.isArray(index.manifests)) throw new Error("The native runtime index does not match this image's integration patch."); const architectures = new Set(); for (const entry of index.manifests) { const architecture = entry?.platform?.architecture; - if (entry?.platform?.os !== "linux" || !["amd64", "arm64"].includes(architecture) || architectures.has(architecture) || !digest(entry.digest) || entry.mediaType !== MANIFEST || !Number.isSafeInteger(entry.size) || entry.size <= 0 || entry.size > 4 * 1024 * 1024) throw new Error("The native runtime index must contain exactly one tested build for each supported architecture."); + if (entry?.platform?.os !== "linux" || !["amd64", "arm64"].includes(architecture) || architectures.has(architecture) || !digest(entry.digest) || entry.mediaType !== MANIFEST || !Number.isSafeInteger(entry.size) || entry.size <= 0 || entry.size > DOCUMENT_LIMIT) throw new Error("The native runtime index must contain exactly one tested build for each supported architecture."); architectures.add(architecture); } if (architectures.size !== 2) throw new Error("The native runtime index is missing a tested architecture."); @@ -23,7 +21,7 @@ export function validateDistributionIndex(index) { export async function distributionIndex(reference, { fetcher = fetch } = {}) { const response = await registryRequest(manifestUrl(reference), { fetcher, headers: { accept: INDEX } }); if (!response.ok) { await response.body?.cancel(); throw new Error(`Could not fetch the container-maintained T3 runtime (HTTP ${response.status}). Check again later or update the Docker image; the running version was kept.`); } - const text = await releaseText(response, 4 * 1024 * 1024), index = validateDistributionIndex(JSON.parse(text)); + const text = await releaseText(response, DOCUMENT_LIMIT), index = validateDistributionIndex(JSON.parse(text)); const supplied = response.headers.get("docker-content-digest"); if (supplied && supplied !== digestOf(text)) throw new Error("The native runtime index failed its integrity check."); return { index, digest: digestOf(text), version: index.annotations["io.t3code.runtime.version"] }; @@ -35,12 +33,12 @@ async function fromIndex(root, { fetcher, arch }) { if (platforms.length !== 1 || !digest(platforms[0].digest) || platforms[0].mediaType !== MANIFEST) throw new Error("The native runtime index must contain exactly one tested build for this architecture."); const response = await registryRequest(manifestUrl(platforms[0].digest), { fetcher, headers: { accept: MANIFEST } }); if (!response.ok) { await response.body?.cancel(); throw new Error(`The native runtime manifest is unavailable (HTTP ${response.status}).`); } - const text = await releaseText(response, 4 * 1024 * 1024); + const text = await releaseText(response, DOCUMENT_LIMIT); if (digestOf(text) !== platforms[0].digest || Buffer.byteLength(text) !== platforms[0].size) throw new Error("The native runtime manifest failed its integrity check."); const manifest = JSON.parse(text); if (manifest.schemaVersion !== 2 || manifest.mediaType !== MANIFEST || manifest.artifactType !== ARTIFACT_TYPE || manifest.annotations?.["io.t3code.runtime.patch"] !== PATCH_REVISION || manifest.annotations?.["io.t3code.runtime.version"] !== version || manifest.layers?.length !== 1) throw new Error("The native runtime manifest has invalid integration metadata."); const layer = manifest.layers[0]; - if (layer.mediaType !== ARCHIVE_TYPE || !digest(layer.digest) || !Number.isSafeInteger(layer.size) || layer.size <= 0 || layer.size > 300 * 1024 * 1024) throw new Error("The native runtime archive descriptor is invalid."); + if (layer.mediaType !== ARCHIVE_TYPE || !digest(layer.digest) || !Number.isSafeInteger(layer.size) || layer.size <= 0 || layer.size > ARCHIVE_LIMIT) throw new Error("The native runtime archive descriptor is invalid."); return { version, name: "t3-runtime", root: stem, url: distributionBlob(layer.digest), integrity: layer.digest.replace("sha256:", "sha256-"), size: layer.size, patchRevision: PATCH_REVISION, distributionDigest: root.digest, manifestDigest: platforms[0].digest }; } diff --git a/docker/runtime/manager.mjs b/docker/runtime/manager.mjs index d327aed..1d6ddfc 100644 --- a/docker/runtime/manager.mjs +++ b/docker/runtime/manager.mjs @@ -3,7 +3,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { unpackRelease, executableVersion } from "./releases.mjs"; +import { unpackRelease, executableVersion, chatterSlice } from "./releases.mjs"; import { packageRelease, releaseForVersion } from "./distribution.mjs"; import { PATCH_REVISION } from "./distribution-spec.mjs"; import { injectSetupBridge } from "./client-shell.mjs"; @@ -28,7 +28,7 @@ export async function discoverRuntime(binary, paths, { run = execute, env = proc const probe = await fs.mkdtemp(path.join(paths.root, ".database-probe-")); try { const issued = await run(binary, ["auth", "session", "issue", "--ttl", "1m", "--subject", "t3-container-probe", "--json"], { env: { ...env, T3CODE_HOME: probe }, timeout: 30_000 }); - const session = JSON.parse(issued.stdout.slice(issued.stdout.indexOf("{"))); + const session = JSON.parse(chatterSlice(issued.stdout) ?? ""); const scopes = session.permissions ?? session.scopes; if (!validScopes(scopes) || !scopes.length) throw new Error("Could not identify this release's supported device permissions safely."); const names = (await fs.readdir(path.join(probe, "userdata"))).filter(databaseName); diff --git a/docker/runtime/release-bounds.mjs b/docker/runtime/release-bounds.mjs new file mode 100644 index 0000000..61a9dc7 --- /dev/null +++ b/docker/runtime/release-bounds.mjs @@ -0,0 +1,25 @@ +// One home for every release size and integrity bound. +// +// The GitHub release stack (releases.mjs) and the container-registry stack +// (distribution.mjs) resolve different backends but stage the same archives +// through the same verifier (unpackRelease). An integrity hardening that +// lands in only one file leaves the other on the old bound, so both stacks +// import these limits and predicates instead of keeping their own copies. +import { createHash } from "node:crypto"; + +// Largest single archive the updater will stage or trust from a manifest. +export const ARCHIVE_LIMIT = 300 * 1024 * 1024; +// Largest extracted tree the updater will accept. +export const UNPACKED_LIMIT = 1024 * 1024 * 1024; +// Free space kept in reserve on the staging volume while downloading. +export const RELEASE_HEADROOM = 64 * 1024 * 1024; +// Largest checksum list accepted from a GitHub release. +export const SUMS_LIMIT = 1024 * 1024; +// Largest registry document (release API page, image index or manifest). +export const DOCUMENT_LIMIT = 4 * 1024 * 1024; + +// Registry blob digest ("sha256:…") used in manifests and blob URLs. +export const isSha256Digest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); +// Download integrity ("sha256-…") carried on a resolved release. +export const isSha256Integrity = (value) => typeof value === "string" && /^sha256-[a-f0-9]{64}$/.test(value); +export const digestOfText = (text) => "sha256:" + createHash("sha256").update(text).digest("hex"); diff --git a/docker/runtime/releases.mjs b/docker/runtime/releases.mjs index b6514ce..5be1402 100644 --- a/docker/runtime/releases.mjs +++ b/docker/runtime/releases.mjs @@ -6,14 +6,12 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { CHANNELS, channelOf, exactVersion, readJson, writeJson } from "./state.mjs"; import { PATCH_REVISION, distributionBlob } from "./distribution-spec.mjs"; +import { ARCHIVE_LIMIT as LIMIT, UNPACKED_LIMIT, RELEASE_HEADROOM as HEADROOM, SUMS_LIMIT, DOCUMENT_LIMIT } from "./release-bounds.mjs"; import { registryRequest } from "./registry.mjs"; const execute = promisify(execFile); const API = "https://api.github.com/repos/pingdotgg/t3code/releases"; const DOWNLOAD = "https://github.com/pingdotgg/t3code/releases/download"; -const LIMIT = 300 * 1024 * 1024; -const UNPACKED_LIMIT = 1024 * 1024 * 1024; -const HEADROOM = 64 * 1024 * 1024; const SPACE_CHECK_BYTES = 8 * 1024 * 1024; const PAGE_SIZE = 20; const downloadHosts = new Set(["github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com", "ghcr.io", "pkg-containers.githubusercontent.com"]); @@ -51,6 +49,13 @@ async function get(url, { fetcher, api = false, token, timeout = 15_000 }) { } export { get as fetchRelease }; +/** Start of the first JSON document in CLI chatter. Null when no document opens. */ +export function chatterSlice(text) { + const output = String(text ?? ""); + const start = output.indexOf("{"); + return start < 0 ? null : output.slice(start); +} + async function boundedText(response, limit) { const chunks = []; let bytes = 0; @@ -73,7 +78,7 @@ export async function releaseForVersion(version, { fetcher = fetch, arch = proce const root = archiveStem(version, arch); const name = `${root}.tar.gz`; const base = `${DOWNLOAD}/v${version}`; - const sums = await boundedText(await get(`${base}/SHA256SUMS`, { fetcher }), 1024 * 1024); + const sums = await boundedText(await get(`${base}/SHA256SUMS`, { fetcher }), SUMS_LIMIT); return { version, name: "t3-runtime", root, url: `${base}/${name}`, integrity: `sha256-${archiveChecksum(sums, name)}` }; } @@ -83,7 +88,7 @@ export async function packageRelease(channel, { fetcher = fetch, arch = process. archiveStem("0.0.0", arch); for (let page = 1; page <= 5; page += 1) { const url = channel === "stable" ? `${API}/latest` : `${API}?per_page=${PAGE_SIZE}&page=${page}`; - const data = JSON.parse(await boundedText(await get(url, { fetcher, api: true, token }), 4 * 1024 * 1024)); + const data = JSON.parse(await boundedText(await get(url, { fetcher, api: true, token }), DOCUMENT_LIMIT)); const entries = channel === "stable" ? [data] : data; if (!Array.isArray(entries)) throw new Error("GitHub did not return a release list."); const release = entries.find((entry) => { diff --git a/docker/setup/atomic-write.mjs b/docker/setup/atomic-write.mjs new file mode 100644 index 0000000..adbdeef --- /dev/null +++ b/docker/setup/atomic-write.mjs @@ -0,0 +1,33 @@ +// One atomic tmp+rename file writer for the setup console. +// +// The MCP store and the instruction editors each kept their own copy of this +// dance: private temp file, exact mode, rename into place, best-effort temp +// cleanup. A crash between write and rename must leave either the old file or +// the new one, never a half-written one, so both callers share this helper. +// The runtime selection (docker/runtime/state.mjs) and the harness manager +// record (docker/harness/state.mjs) keep their own variants on purpose: the +// former fsyncs for boot-critical durability and the latter writes through an +// injected filesystem seam, and neither image layer can import this one. +import path from "node:path"; +import { randomBytes } from "node:crypto"; + +/** Replace durably: exclusive temp file, exact mode, atomic rename. */ +export async function writeFileAtomic(fs, file, text, mode = 0o600) { + await fs.mkdir(path.dirname(file), { recursive: true }); + const tmp = `${file}.tmp.${process.pid}.${randomBytes(6).toString("hex")}`; + let created = false; + try { + const handle = await fs.open(tmp, "wx", mode); + created = true; + try { await handle.writeFile(text, "utf8"); await handle.chmod(mode); } + finally { await handle.close(); } + await fs.rename(tmp, file); + } finally { + if (created) await fs.rm(tmp, { force: true }); + } +} + +/** Atomic JSON replace with the canonical trailing newline. */ +export async function writeJsonAtomic(fs, file, value, mode = 0o600) { + await writeFileAtomic(fs, file, JSON.stringify(value, null, 2) + "\n", mode); +} diff --git a/docker/setup/bounded-read.mjs b/docker/setup/bounded-read.mjs new file mode 100644 index 0000000..a3e6247 --- /dev/null +++ b/docker/setup/bounded-read.mjs @@ -0,0 +1,58 @@ +// One capped file reader for the setup console. +// +// Skill files, shared instructions and native settings snapshots each read a +// bounded prefix of a file that may be a symlink, may grow between stat and +// read, or may be a raced-in FIFO. The shape is always the same: reject +// non-regular or oversized targets before opening, open without following +// links and without blocking, re-check the opened descriptor, then read at +// most the limit plus one byte (including across short reads) so polling can +// never load a file whole. Callers keep their own messages; this module owns +// the flags and the read loop. +import { constants } from "node:fs"; + +// Never follow a raced-in symlink, and never block on a raced-in FIFO. +export const READ_FLAGS = constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW; + +/** Read at most maxBytes + 1 from an open handle, tolerating short reads. */ +export async function readHandleCapped(handle, maxBytes) { + const buffer = Buffer.alloc(maxBytes + 1); + let bytes = 0; + while (bytes < buffer.length) { + const result = await handle.read(buffer, bytes, buffer.length - bytes, bytes); + if (!result.bytesRead) break; + bytes += result.bytesRead; + } + // `raw` is the exact bytes read: revision hashes must not depend on how + // invalid UTF-8 decodes, so hash this, not the decoded content. + return { content: buffer.toString("utf8", 0, bytes), bytes, raw: buffer.subarray(0, bytes) }; +} + +/** + * Bounded read of a regular file. Throws the caller's `invalid` error for a + * non-regular target and the caller's `tooLarge` error (given the byte count) + * for an oversized one, so messages stay identical at every call site. + * `flags` defaults to non-following and non-blocking; the shared instruction + * file keeps its plain "r" open so a manually symlinked AGENTS.md still reads. + */ +export async function readCappedFile(io, file, maxBytes, { tooLarge, invalid, flags = READ_FLAGS } = {}) { + const failLarge = tooLarge ?? ((bytes) => Object.assign(new Error(`File exceeds the ${maxBytes}-byte read limit.`), { code: "too_large", bytes })); + const failInvalid = invalid ?? (() => Object.assign(new Error("Expected a regular file."), { code: "invalid" })); + const check = (stat) => { + if (!stat.isFile()) throw failInvalid(); + if (stat.size > maxBytes) throw failLarge(stat.size); + }; + // Reject oversized targets before opening them, including through symlinks. + check(await io.stat(file)); + const handle = await io.open(file, flags); + try { + const stat = await handle.stat(); + check(stat); + // A file may grow after stat. Read at most the limit plus one byte, + // including across short reads, so polling can never load it all. + const read = await readHandleCapped(handle, maxBytes); + if (read.bytes > maxBytes) throw failLarge(read.bytes); + return { ...read, mode: stat.mode & 0o777, stat }; + } finally { + await handle.close(); + } +} diff --git a/docker/setup/cache.mjs b/docker/setup/cache.mjs index d2f51ae..f67a22b 100644 --- a/docker/setup/cache.mjs +++ b/docker/setup/cache.mjs @@ -41,7 +41,8 @@ export async function withTimeout(work, ms) { * Deduplicate concurrent refresh work. While a promise is in flight every * caller shares it; once it settles the next call starts fresh. A rejection * clears the slot without poisoning later calls, and an abandoned refresh - * can never crash the process. + * can never crash the process. Read-path only: this coalesces cache fills, + * it is not a writer mutex (cache entries are replaced, never mutated). */ export function createCoalescer() { let inflight = null; @@ -383,7 +384,8 @@ export function createHarnessCache({ } /** Share pending reads and optionally retain successful answers for a TTL. - * Clearing detaches old reads: their completion cannot replace a newer answer. */ + * Clearing detaches old reads: their completion cannot replace a newer answer. + * Read-path only, like the coalescer above: never used to serialize writes. */ export function createTtlCache({ ttlMs, limit = 200, now = Date.now }) { const entries = new Map(); const cached = (key, load) => { diff --git a/docker/setup/chatter-json.mjs b/docker/setup/chatter-json.mjs new file mode 100644 index 0000000..0ee03b6 --- /dev/null +++ b/docker/setup/chatter-json.mjs @@ -0,0 +1,23 @@ +// Chatter-tolerant JSON parse for setup CLI output. +// +// Native CLIs print log lines around the JSON they are asked for. The whole +// contract is: the document starts at the first "{"; anything before it is +// chatter. Returns null when there is no document or it does not parse, +// never throws. The harness probes and the runtime supervisor keep their own +// identical copies: each image layer ships separately, so a cross-layer +// import would resolve in a checkout and break in the image. +export function chatterSlice(text) { + const output = String(text ?? ""); + const start = output.indexOf("{"); + return start < 0 ? null : output.slice(start); +} + +export function jsonFromChatter(text) { + const slice = chatterSlice(text); + if (slice === null) return null; + try { + return JSON.parse(slice); + } catch { + return null; + } +} diff --git a/docker/setup/connect.mjs b/docker/setup/connect.mjs index 0d94a0c..d223fcc 100644 --- a/docker/setup/connect.mjs +++ b/docker/setup/connect.mjs @@ -10,6 +10,7 @@ // t3 connect status --json what is saved // t3 connect unlink off again, keeping the sign-in // +import { jsonFromChatter } from "./chatter-json.mjs"; // The setup service runs those processes. This module parses their output // and reads or saves live preferences through T3 Code's native server API. @@ -89,14 +90,7 @@ export function connectState(json) { /** The JSON object in a CLI's stdout, which may follow lines of chatter. */ export function jsonFrom(stdout) { - const text = String(stdout ?? ""); - const start = text.indexOf("{"); - if (start < 0) return null; - try { - return JSON.parse(text.slice(start)); - } catch { - return null; - } + return jsonFromChatter(stdout); } /** Read and update native preferences without overwriting the other switch. */ diff --git a/docker/setup/context.mjs b/docker/setup/context.mjs index 4ec52d2..a376f49 100644 --- a/docker/setup/context.mjs +++ b/docker/setup/context.mjs @@ -4,6 +4,8 @@ import path from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { agentResourcePaths } from "./agent-resources.mjs"; +import { writeFileAtomic } from "./atomic-write.mjs"; +import { readCappedFile } from "./bounded-read.mjs"; export function contextDirFor(baseDir) { return path.join(baseDir, "setup", "context"); @@ -77,55 +79,31 @@ async function readInstructionFile(fs, file) { const key = path.resolve(file); if (cache?.has(key)) return cache.get(key); const read = (async () => { - const check = (stat) => { - if (!stat.isFile()) throw Object.assign(new Error("Instructions must be a regular file"), { code: "invalid" }); - if (stat.size > MAX_INSTRUCTION_BYTES) throw tooLarge(stat.size); - }; - // Reject oversized targets before opening them, including through symlinks. - check(await fs.stat(file)); - const handle = await fs.open(file, "r"); - try { - const stat = await handle.stat(); - check(stat); - // A file may grow after stat. Read at most the limit plus one byte, - // including across short reads, so status polling can never load it all. - const buffer = Buffer.alloc(MAX_INSTRUCTION_BYTES + 1); - let bytes = 0; - while (bytes < buffer.length) { - const result = await handle.read(buffer, bytes, buffer.length - bytes, bytes); - if (!result.bytesRead) break; - bytes += result.bytesRead; - } - if (bytes > MAX_INSTRUCTION_BYTES) throw tooLarge(bytes); - return { content: buffer.toString("utf8", 0, bytes), bytes, mode: stat.mode & 0o777, modifiedAt: stat.mtimeMs }; - } finally { - await handle.close(); - } + const invalid = () => Object.assign(new Error("Instructions must be a regular file"), { code: "invalid" }); + // The shared file keeps its plain "r" open so a manually symlinked + // AGENTS.md still reads; providers resolve their links before reading. + const { content, bytes, mode, stat } = await readCappedFile(fs, file, MAX_INSTRUCTION_BYTES, { tooLarge, invalid, flags: "r" }); + return { content, bytes, mode, modifiedAt: stat.mtimeMs }; })(); cache?.set(key, read); return read; } -async function writeFileAtomic(fs, file, text, mode = 0o600) { - await fs.mkdir(path.dirname(file), { recursive: true }); - const tmp = `${file}.tmp.${process.pid}.${randomBytes(6).toString("hex")}`; - let created = false; - try { - const handle = await fs.open(tmp, 'wx', mode); - created = true; - try { await handle.writeFile(text, 'utf8'); await handle.chmod(mode); } finally { await handle.close(); } - await fs.rename(tmp, file); - } finally { - if (created) await fs.rm(tmp, { force: true }); - } -} - const revisionFor = (file, content) => content === null ? null : createHash("sha256").update(path.resolve(file)).update("\0").update(content).digest("hex"); const changedInstructions = () => ({ ok: false, code: "conflict", error: "Instructions changed since you opened them. Download your draft, then reopen the editor to load the current file." }); // Shared and provider editors can point at the same file. Keep their revision // checks and atomic writes in one queue per resolved target. +// +// Single-writer contract, same as the MCP store: the setup server is the only +// in-repo writer of these files, so this queue serializes its concurrent +// requests. Hand edits and provider CLIs write outside it, which is why every +// save re-checks its revision inside the queue, re-checks immediately before +// committing, and verifies afterwards instead of assuming the write won. No +// cross-process lock is taken: the harness lock guards mise state, not these +// files, and an advisory lock no external writer honors would only add +// blocking without adding safety. const instructionWrites = new Map(); const instructionModeWrites = new Map(); async function withInstructionWrite(file, work, queues = instructionWrites) { diff --git a/docker/setup/mcp/store.mjs b/docker/setup/mcp/store.mjs index 282af57..5d32fc0 100644 --- a/docker/setup/mcp/store.mjs +++ b/docker/setup/mcp/store.mjs @@ -2,7 +2,8 @@ // files are only touched by explicit link/unlink saves. import path from "node:path"; import { AGENT_NAMES } from "../agent-resources.mjs"; -import { createHash, randomBytes } from "node:crypto"; +import { createHash } from "node:crypto"; +import { writeFileAtomic as atomicWriteFile } from "../atomic-write.mjs"; export const MCP_PROVIDERS = AGENT_NAMES; export const SUPPORTED_MCP_PROVIDERS = Object.freeze(Object.keys(MCP_PROVIDERS)); @@ -98,19 +99,19 @@ async function readStoreFile(fs, file) { return { present: true, servers, bytes: Buffer.byteLength(raw, "utf8") }; } +// Size-guarded atomic replace through the shared tmp+rename writer. async function writeFileAtomic(fs, file, text) { if (Buffer.byteLength(text, "utf8") > MAX_STORE_BYTES) throw problem("too_large", "The shared MCP store would exceed the 256 KiB limit. Reduce the server definition."); - await fs.mkdir(path.dirname(file), { recursive: true }); - const tmp = `${file}.tmp.${process.pid}.${randomBytes(6).toString("hex")}`; - try { - await fs.writeFile(tmp, text, { encoding: "utf8", mode: 0o600 }); - await fs.chmod(tmp, 0o600); - await fs.rename(tmp, file); - } finally { - await fs.rm(tmp, { force: true }); - } + await atomicWriteFile(fs, file, text); } +// Single-writer contract: the setup server is the only in-repo writer of the +// shared store; this queue serializes its concurrent requests. Writers outside +// this process (hand edits on disk) cannot take the queue, so every mutation +// still re-checks the revision inside it and commits atomically; a mismatch +// fails instead of silently winning. No cross-process lock is taken on purpose: +// the harness lock guards mise state, not this file, and an advisory lock no +// external writer honors would only add blocking without adding safety. const storeQueues = new Map(); export async function withStore(file, work) { const key = path.resolve(file); diff --git a/docker/setup/skills.mjs b/docker/setup/skills.mjs index 07e896f..03294a1 100644 --- a/docker/setup/skills.mjs +++ b/docker/setup/skills.mjs @@ -1,8 +1,8 @@ // Global skill inventory and explicit local file management. No skill execution. import path from "node:path"; import * as fs from "node:fs/promises"; -import { constants } from "node:fs"; import { createHash } from "node:crypto"; +import { readCappedFile } from "./bounded-read.mjs"; import { parseDocument, isMap, isScalar } from "./vendor/yaml.mjs"; import { listCodexSkills, writeCodexSkillConfig } from "./skills/codex.mjs"; import { codexConfigPath, configSnapshot, stateRevision, serializeState } from "./skills/state.mjs"; @@ -51,24 +51,11 @@ export function skillRoots(provider, { home, env = {} }) { // Stat before opening, stat the opened descriptor, then read at most limit + 1. // Nonblocking open prevents a raced-in FIFO from hanging the setup service. export async function readSkillFile(file, maxBytes = DEFAULT_LIMITS.fileBytes, io = fs) { - const check = (stat) => { - if (!stat.isFile()) throw problem("invalid", "The skill must be a regular Markdown file."); - if (stat.size > maxBytes) throw problem("too_large", "This skill exceeds the 64 KiB viewer limit."); - }; - check(await io.stat(file)); - const handle = await io.open(file, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW); - try { - const stat = await handle.stat(); check(stat); - const buffer = Buffer.alloc(maxBytes + 1); - let bytes = 0; - while (bytes < buffer.length) { - const read = await handle.read(buffer, bytes, buffer.length - bytes, bytes); - if (!read.bytesRead) break; - bytes += read.bytesRead; - } - if (bytes > maxBytes) throw problem("too_large", "This skill exceeds the 64 KiB viewer limit."); - return { content: buffer.toString("utf8", 0, bytes), bytes, mode: stat.mode & 0o777, stamp: [stat.dev, stat.ino, stat.mtimeMs, stat.size].join(":") }; - } finally { await handle.close(); } + // Messages stay skill-specific; the flags and the short-read loop are shared. + const invalid = () => problem("invalid", "The skill must be a regular Markdown file."); + const tooLarge = () => problem("too_large", "This skill exceeds the 64 KiB viewer limit."); + const { content, bytes, mode, stat } = await readCappedFile(io, file, maxBytes, { invalid, tooLarge }); + return { content, bytes, mode, stamp: [stat.dev, stat.ino, stat.mtimeMs, stat.size].join(":") }; } export function skillMetadata(content, fallback) { diff --git a/docker/setup/skills/state.mjs b/docker/setup/skills/state.mjs index c4c721c..caf9f27 100644 --- a/docker/setup/skills/state.mjs +++ b/docker/setup/skills/state.mjs @@ -1,7 +1,7 @@ // Conflict tokens for native settings; settings contents never leave this module. import path from "node:path"; -import { constants } from "node:fs"; import { createHash } from "node:crypto"; +import { READ_FLAGS, readHandleCapped } from "../bounded-read.mjs"; const digest = (value) => createHash("sha256").update(value).digest("hex"); const stamp = (stat) => [stat.dev, stat.ino, stat.mode, stat.size, stat.mtimeMs, stat.ctimeMs].join(":"); @@ -16,19 +16,14 @@ export async function configSnapshot(file, io, includeContent = false) { try { check(await io.stat(file)); const resolved = await io.realpath(file); - handle = await io.open(resolved, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW); + handle = await io.open(resolved, READ_FLAGS); const before = await handle.stat(); check(before); - const buffer = Buffer.alloc(limit + 1); let bytes = 0; - while (bytes < buffer.length) { - const read = await handle.read(buffer, bytes, buffer.length - bytes, bytes); - if (!read.bytesRead) break; - bytes += read.bytesRead; - } + const { content, bytes, raw } = await readHandleCapped(handle, limit); if (bytes > limit || stamp(before) !== stamp(await handle.stat()) || stamp(before) !== stamp(await io.stat(file)) || resolved !== await io.realpath(file)) { throw Object.assign(new Error("Native settings changed while being read. Refresh the list."), { code: "conflict" }); } - const revision = digest(JSON.stringify([file, resolved, stamp(before), digest(buffer.subarray(0, bytes))])); - return includeContent ? { revision, content: buffer.toString("utf8", 0, bytes), resolvedPath: resolved, mode: before.mode & 0o777, exists: true } : revision; + const revision = digest(JSON.stringify([file, resolved, stamp(before), digest(raw)])); + return includeContent ? { revision, content, resolvedPath: resolved, mode: before.mode & 0o777, exists: true } : revision; } catch (error) { if (error.code === "conflict") throw error; if (error.code === "ENOENT" && !handle) { @@ -45,6 +40,16 @@ export async function configSnapshot(file, io, includeContent = false) { export const stateRevision = (config, row) => digest(JSON.stringify([config, row.path, row.resolvedPath, row.scope, row.source, row.name, row.state, row.policyKey, row.nativePolicyVersion])); // All skills share one provider settings file, including across service instances. +// +// Single-writer contract, same as the MCP store: the setup server is the only +// in-repo writer taking this queue, so it serializes its concurrent requests +// per settings file. The native writer itself (the provider CLI) and hand +// edits run outside it, which is why every toggle re-checks its revision +// inside the queue, re-checks immediately before the native write, and +// verifies the result afterwards instead of assuming the write won. No +// cross-process lock is taken: the harness lock guards mise state, not these +// files, and an advisory lock no external writer honors would only add +// blocking without adding safety. const queues = new Map(); export async function serializeState(key, work) { const previous = queues.get(key) || Promise.resolve(); diff --git a/docs/native-runtime-patches.md b/docs/native-runtime-patches.md index 03f3c2c..53329fa 100644 --- a/docs/native-runtime-patches.md +++ b/docs/native-runtime-patches.md @@ -67,7 +67,15 @@ Linux ARM64 runners use upstream's native `build-exe` command without a cross-build target, retaining its SEA injection workaround for 16/64 KiB page kernels. The builder reads that path's `dist-exe/t3` output; explicit targets use the platform-named output. Keep the build Node version aligned with the -upstream release's declared version for native ARM64 compilation. +upstream release's declared version for native ARM64 compilation. CI reads +`engines.node` from each exact upstream tag's `package.json` and selects that +version's official Node builder image for both architectures. Stable and +Nightly can therefore use different build Node versions. Missing or non-exact +declarations stop the plan before builds start. A local Docker build of another +T3 release must set `T3_BUILD_NODE_IMAGE` to its declared Node version as well +as setting `T3_VERSION`; the default builder remains aligned with the bundled +Stable release. This changes the compiler image only. The container's runtime +Node version and user-managed toolchains keep their existing configuration. Compilation loads the exact tag's checked-in `.env.example`, allowing only the four public Clerk and relay settings that enable T3 Connect. It excludes local diff --git a/scripts/native-runtime-plan.mjs b/scripts/native-runtime-plan.mjs index 1ed8a35..6a32d04 100644 --- a/scripts/native-runtime-plan.mjs +++ b/scripts/native-runtime-plan.mjs @@ -1,20 +1,43 @@ import fs from "node:fs"; -import { packageRelease } from "../docker/runtime/releases.mjs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { packageRelease, fetchRelease, releaseText } from "../docker/runtime/releases.mjs"; import { exactVersion } from "../docker/runtime/state.mjs"; import { PATCH_REVISION } from "../docker/runtime/distribution-spec.mjs"; +import { DOCUMENT_LIMIT } from "../docker/runtime/release-bounds.mjs"; -const requested = process.env.NATIVE_VERSION?.trim(); -if (requested && !exactVersion(requested)) throw new Error("Choose an exact upstream T3 release."); -const pinned = /ARG T3_VERSION=(\S+)/.exec(fs.readFileSync(new URL("../Dockerfile", import.meta.url), "utf8"))?.[1]; -const versions = requested ? [requested] : process.env.NATIVE_BUNDLED === "true" - ? [pinned, (await packageRelease("nightly")).version] - : (await Promise.all([packageRelease("stable"), packageRelease("nightly")])).map((release) => release.version); -if (!versions.every(exactVersion)) throw new Error("Native build plan contains an invalid version."); -const unique = [...new Set(versions)]; -const matrix = { include: unique.flatMap((version) => [ - { version, arch: "amd64", nodeArch: "x64", runner: "ubuntu-latest" }, - { version, arch: "arm64", nodeArch: "arm64", runner: "ubuntu-24.04-arm" }, -]) }; -const output = { matrix: JSON.stringify(matrix), versions: JSON.stringify(unique), revision: PATCH_REVISION, bundled: pinned, nightly: unique.find((version) => version.includes("-nightly.")) || "" }; -if (process.env.GITHUB_OUTPUT) for (const [name, value] of Object.entries(output)) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); -console.log(JSON.stringify(output)); +// Native ARM64 SEA packaging injects the host Node executable. Its version +// must match the exact source release, which can differ between channels. +export async function buildNodeImage(version, { fetcher = fetch } = {}) { + if (!exactVersion(version)) throw new Error("Choose an exact upstream T3 release."); + const url = `https://raw.githubusercontent.com/pingdotgg/t3code/v${version}/package.json`; + const response = await fetchRelease(url, { fetcher }); + const manifest = JSON.parse(await releaseText(response, DOCUMENT_LIMIT)); + const node = manifest?.engines?.node; + if (typeof node !== "string" || !/^[1-9]\d*\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)$/.test(node) || Number(node.split(".")[0]) < 26) { + throw new Error(`Upstream v${version} must declare an exact Node 26 or newer build version.`); + } + return `node:${node}-trixie-slim`; +} + +export async function nativeRuntimePlan({ requested, bundled = false, pinned, fetcher = fetch, resolveRelease = packageRelease } = {}) { + if (requested && !exactVersion(requested)) throw new Error("Choose an exact upstream T3 release."); + pinned ??= /ARG T3_VERSION=(\S+)/.exec(fs.readFileSync(new URL("../Dockerfile", import.meta.url), "utf8"))?.[1]; + const versions = requested ? [requested] : bundled + ? [pinned, (await resolveRelease("nightly", { fetcher })).version] + : (await Promise.all([resolveRelease("stable", { fetcher }), resolveRelease("nightly", { fetcher })])).map((release) => release.version); + if (!versions.every(exactVersion)) throw new Error("Native build plan contains an invalid version."); + const unique = [...new Set(versions)]; + const releases = await Promise.all(unique.map(async version => ({ version, buildNodeImage: await buildNodeImage(version, { fetcher }) }))); + const matrix = { include: releases.flatMap((release) => [ + { ...release, arch: "amd64", nodeArch: "x64", runner: "ubuntu-latest" }, + { ...release, arch: "arm64", nodeArch: "arm64", runner: "ubuntu-24.04-arm" }, + ]) }; + return { matrix: JSON.stringify(matrix), versions: JSON.stringify(unique), revision: PATCH_REVISION, bundled: pinned, nightly: unique.find((version) => version.includes("-nightly.")) || "" }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + const output = await nativeRuntimePlan({ requested: process.env.NATIVE_VERSION?.trim(), bundled: process.env.NATIVE_BUNDLED === "true" }); + if (process.env.GITHUB_OUTPUT) for (const [name, value] of Object.entries(output)) fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); + console.log(JSON.stringify(output)); +} diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 0ae04e7..b961109 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -549,20 +549,41 @@ NODE check "Files browses with t3 permissions, preserves managed link guards, and sandboxes downloads" files_browse_works files_changes_work() { - docker exec -i -u t3 -e FILES_SMOKE_KEY="$SETUP_KEY" "$NAME" node --input-type=module <<'NODE' + # check() discards command output. Retain this transfer's assertion and HTTP + # diagnostics so a failure identifies the stage without rerunning it blind. + printf '\nFiles transfer smoke (20 MiB upload, download, Trash, restore)\n' >> "$UI_AUDIT_LOG" + docker exec -i -u t3 -e FILES_SMOKE_KEY="$SETUP_KEY" "$NAME" node --input-type=module <<'NODE' >> "$UI_AUDIT_LOG" 2>&1 import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; import { randomUUID,createHash } from 'node:crypto'; import path from 'node:path'; const root=path.join(process.env.T3_WORKSPACE||'/workspace','.t3-files-smoke-'+randomUUID()); await fs.mkdir(root); -const request=async(route,body)=>{const response=await fetch('http://127.0.0.1:3774'+route,{method:body===undefined?'GET':'POST',headers:{'x-t3-setup-key':process.env.FILES_SMOKE_KEY,...(body===undefined?{}:{'content-type':Buffer.isBuffer(body)?'application/octet-stream':'application/json'})},body:body===undefined?undefined:Buffer.isBuffer(body)?body:JSON.stringify(body)});if(!response.ok)throw new Error(route+': '+await response.text());return response;}; +const request=async(route,body)=>{const response=await fetch('http://127.0.0.1:3774'+route,{method:body===undefined?'GET':'POST',headers:{'x-t3-setup-key':process.env.FILES_SMOKE_KEY,...(body===undefined?{}:{'content-type':Buffer.isBuffer(body)?'application/octet-stream':'application/json'})},body:body===undefined?undefined:Buffer.isBuffer(body)?body:JSON.stringify(body)});if(!response.ok)throw new Error(route+': HTTP '+response.status+' '+await response.text());return response;}; +let stage='starting upload'; try { const bytes=Buffer.alloc(20*1024*1024,0x61),start=await (await request('/files/upload/start',{dest:root,items:[{path:'fixture.bin',size:bytes.length,lastModified:1700000000000}]})).json(),id=start.uploads[0].id; + stage='sending chunks'; for(let offset=0;offsettrue,()=>false),false); - const finished=await (await request('/files/upload/finish',{id})).json(),download=Buffer.from(await (await request('/files/raw?'+new URLSearchParams({path:finished.path}))).arrayBuffer());assert.equal(createHash('sha256').update(download).digest('hex'),createHash('sha256').update(bytes).digest('hex')); - const trashed=await (await request('/files/to-trash',{paths:[finished.path]})).json();assert.equal(await fs.stat(finished.path).then(()=>true,()=>false),false);await request('/files/trash/restore',{ids:trashed.ids});assert.equal((await fs.stat(finished.path)).size,bytes.length); + stage='checking atomic visibility'; + assert.equal(await fs.stat(path.join(root,'fixture.bin')).then(()=>true,()=>false),false,'The upload must remain unpublished until finish.'); + stage='finishing upload'; + const finished=await (await request('/files/upload/finish',{id})).json(); + stage='downloading published bytes'; + const download=Buffer.from(await (await request('/files/raw?'+new URLSearchParams({path:finished.path}))).arrayBuffer()); + stage='verifying download checksum'; + assert.equal(createHash('sha256').update(download).digest('hex'),createHash('sha256').update(bytes).digest('hex'),'Downloaded bytes must match the uploaded fixture.'); + stage='moving to Trash'; + const trashed=await (await request('/files/to-trash',{paths:[finished.path]})).json(); + stage='checking Trash move'; + assert.equal(await fs.stat(finished.path).then(()=>true,()=>false),false,'The trashed fixture must leave its original path.'); + stage='restoring from Trash'; + await request('/files/trash/restore',{ids:trashed.ids}); + stage='checking restored size'; + assert.equal((await fs.stat(finished.path)).size,bytes.length,'The restored fixture must retain its size.'); +} catch(error) { + console.error('Files transfer failed while '+stage+'.'); + throw error; } finally { await fs.rm(root,{recursive:true,force:true}); } NODE } diff --git a/tests/native-runtime-plan.test.mjs b/tests/native-runtime-plan.test.mjs new file mode 100644 index 0000000..530fec9 --- /dev/null +++ b/tests/native-runtime-plan.test.mjs @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { buildNodeImage, nativeRuntimePlan } from "../scripts/native-runtime-plan.mjs"; +import { DOCUMENT_LIMIT } from "../docker/runtime/release-bounds.mjs"; + +const stable = "0.1.9", nightly = "0.1.10-nightly.20261114.4709"; +function upstream(packages = { [stable]: "26.12.1", [nightly]: "26.13.1" }) { + const calls = []; + return { + calls, + fetcher: async (url, options) => { + calls.push({ url, options }); + const version = Object.keys(packages).find(version => url === `https://raw.githubusercontent.com/pingdotgg/t3code/v${version}/package.json`); + assert(version, "Read only the exact release's root package manifest."); + assert.equal(options.redirect, "manual"); + assert(options.signal instanceof AbortSignal); + assert.equal(options.headers.authorization, undefined, "Public source reads must not forward credentials."); + return Response.json({ engines: { node: packages[version] } }); + }, + resolveRelease: async channel => ({ version: channel === "stable" ? stable : nightly }), + }; +} + +test("native CI uses each exact release's Node version on both architectures when channels diverge", async () => { + const world = upstream(); + const output = await nativeRuntimePlan({ bundled: true, pinned: stable, ...world }); + const jobs = JSON.parse(output.matrix).include; + assert.equal(jobs.length, 4); + assert.deepEqual(JSON.parse(output.versions), [stable, nightly]); + assert.equal(output.bundled, stable); + assert.equal(output.nightly, nightly); + for (const version of [stable, nightly]) { + const selected = jobs.filter(job => job.version === version); + assert.deepEqual(selected.map(job => job.arch), ["amd64", "arm64"]); + assert.deepEqual(selected.map(job => job.nodeArch), ["x64", "arm64"]); + assert(selected.every(job => job.buildNodeImage === (version === stable ? "node:26.12.1-trixie-slim" : "node:26.13.1-trixie-slim"))); + } + assert.equal(world.calls.length, 2, "Resolve toolchains once per release, not once per architecture."); +}); + +test("an exact requested release does not follow either moving channel", async () => { + const world = upstream(); + const output = await nativeRuntimePlan({ requested: stable, ...world, resolveRelease: () => assert.fail("No channel lookup for an exact request.") }); + assert.deepEqual(JSON.parse(output.versions), [stable]); + assert.equal(JSON.parse(output.matrix).include.length, 2); + assert.equal(world.calls.length, 1); + assert.equal(output.nightly, ""); +}); + +test("scheduled native plans resolve both channels and deduplicate repeated releases", async () => { + const world = upstream(); + const output = await nativeRuntimePlan({ ...world }); + assert.deepEqual(JSON.parse(output.versions), [stable, nightly]); + const repeated = upstream(); + const one = await nativeRuntimePlan({ ...repeated, resolveRelease: async () => ({ version: stable }) }); + assert.equal(JSON.parse(one.matrix).include.length, 2); + assert.equal(repeated.calls.length, 1); +}); + +test("unsupported Node declarations cannot select arbitrary or floating builder images", async () => { + for (const node of [undefined, null, 26, "", "26", "^26.13.1", ">=26", "v26.13.1", "026.13.1", "26.013.1", "26.13.01", "26.13.1-alpine", "26.13.1\nIMAGE=other", "25.9.0"]) { + await assert.rejects(buildNodeImage(stable, upstream({ [stable]: node })), /exact Node 26 or newer/); + } +}); + +test("invalid release identities fail before a source request", async () => { + const fetcher = () => assert.fail("No network request for an invalid release."); + for (const version of ["latest", "nightly", "../main", "0.1.9\nNODE_IMAGE=other"]) { + await assert.rejects(buildNodeImage(version, { fetcher }), /exact upstream/); + await assert.rejects(nativeRuntimePlan({ requested: version, fetcher }), /exact upstream/); + } +}); + +test("missing, redirected, malformed or oversized source metadata cannot fall back to the bundled toolchain", async () => { + for (const response of [ + new Response("not found", { status: 404 }), + new Response(null, { status: 302, headers: { location: "https://example.test/package.json" } }), + new Response("{bad json"), + new Response(" ".repeat(DOCUMENT_LIMIT + 1)), + Response.json({}), + ]) { + await assert.rejects(nativeRuntimePlan({ requested: nightly, fetcher: async () => response })); + } +}); diff --git a/tests/shared-helpers.test.mjs b/tests/shared-helpers.test.mjs new file mode 100644 index 0000000..fe3a5ac --- /dev/null +++ b/tests/shared-helpers.test.mjs @@ -0,0 +1,140 @@ +// Parity for the shared helper extraction: release bounds, capped reads, +// atomic writes and chatter-tolerant JSON parsing. Each case pins the exact +// behavior the duplicated copies had, so the shared modules must agree with +// the legacy inline logic and with each other. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import * as fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + ARCHIVE_LIMIT, UNPACKED_LIMIT, RELEASE_HEADROOM, SUMS_LIMIT, DOCUMENT_LIMIT, + isSha256Digest, isSha256Integrity, digestOfText, +} from "../docker/runtime/release-bounds.mjs"; +import { chatterSlice } from "../docker/runtime/releases.mjs"; +import { chatterSlice as harnessSlice, jsonFromChatter as harnessJson } from "../docker/harness/chatter-json.mjs"; +import { chatterSlice as setupSlice, jsonFromChatter as setupJson } from "../docker/setup/chatter-json.mjs"; +import { jsonFrom } from "../docker/setup/connect.mjs"; +import { READ_FLAGS, readCappedFile, readHandleCapped } from "../docker/setup/bounded-read.mjs"; +import { writeFileAtomic, writeJsonAtomic } from "../docker/setup/atomic-write.mjs"; +import { readSkillFile } from "../docker/setup/skills.mjs"; + +test("release bounds match the legacy per-file limits", () => { + assert.equal(ARCHIVE_LIMIT, 300 * 1024 * 1024); + assert.equal(UNPACKED_LIMIT, 1024 * 1024 * 1024); + assert.equal(RELEASE_HEADROOM, 64 * 1024 * 1024); + assert.equal(SUMS_LIMIT, 1024 * 1024); + assert.equal(DOCUMENT_LIMIT, 4 * 1024 * 1024); +}); + +test("shared digest predicates match the legacy inline checks", () => { + const legacyDigest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); + const legacyIntegrity = (value) => typeof value === "string" && /^sha256-[a-f0-9]{64}$/.test(value); + const cases = ["sha256:" + "a".repeat(64), "sha256-" + "b".repeat(64), "sha256:" + "A".repeat(64), + "sha256-" + "c".repeat(63), "sha256:xyz", "", null, undefined, 42, "sha256-" + "d".repeat(65)]; + for (const value of cases) { + assert.equal(isSha256Digest(value), legacyDigest(value), String(value)); + assert.equal(isSha256Integrity(value), legacyIntegrity(value), String(value)); + } + for (const text of ["", "index", "manifest-bytes"]) { + assert.equal(digestOfText(text), "sha256:" + createHash("sha256").update(text).digest("hex")); + } +}); + +test("every chatter parser agrees, including the runtime slice", () => { + const inputs = ["{\"a\":1}", "log line\n{\"a\":1}\ntrailing", "no json here", "{ broken", + "", null, undefined, 42, "{\"nested\":{\"b\":[1,2]}} chatter", "{{}}"]; + for (const input of inputs) { + const slices = [chatterSlice(input), harnessSlice(input), setupSlice(input)]; + assert.deepEqual(slices.slice(1), [slices[0], slices[0]], JSON.stringify(input)); + const parsed = [harnessJson(input), setupJson(input), jsonFrom(input)]; + assert.deepEqual(parsed.slice(1), [parsed[0], parsed[0]], JSON.stringify(input)); + if (slices[0] === null) assert.equal(parsed[0], null); + else { + let expected = null; + try { expected = JSON.parse(slices[0]); } catch { /* stays null */ } + assert.deepEqual(parsed[0], expected); + } + } +}); + +test("capped reads enforce limits before open and bound short reads", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-read-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const exact = path.join(root, "exact.md"); + await fs.writeFile(exact, "x".repeat(64)); + assert.equal((await readCappedFile(fs, exact, 64)).bytes, 64); + const over = path.join(root, "over.md"); + await fs.writeFile(over, "x".repeat(65)); + let opened = 0; + const guarded = { ...fs, open: async () => { opened += 1; throw new Error("oversized file opened"); } }; + await assert.rejects(readCappedFile(guarded, over, 64), /limit/); + assert.equal(opened, 0); + await assert.rejects(readCappedFile(fs, root, 64), /regular/); + // Symlinks are not followed. + const link = path.join(root, "link.md"); + await fs.symlink(exact, link); + await assert.rejects(readCappedFile(fs, link, 64), { code: "ELOOP" }); + // Short reads accumulate until the limit plus one. + const growing = path.join(root, "growing.md"); + await fs.writeFile(growing, "y".repeat(10)); + let calls = 0; + const short = { ...fs, open: async (...args) => { + const handle = await fs.open(...args); + return { ...handle, stat: () => handle.stat(), close: () => handle.close(), + read: async (buffer, offset, length, position) => { + calls += 1; + if (calls === 2) await fs.writeFile(growing, "y".repeat(100)); + return handle.read(buffer, offset, Math.min(length, 3), position); + } }; + } }; + await assert.rejects(readCappedFile(short, growing, 64, { tooLarge: (bytes) => Object.assign(new Error("big:" + bytes), { code: "too_large", bytes }) }), { code: "too_large" }); + assert.ok(calls > 3); + // The plain-"r" open used for the shared instruction file still follows links. + const via = await readCappedFile(fs, link, 64, { flags: "r" }); + assert.equal(via.bytes, 64); +}); + +test("capped reads agree with the skill reader", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-skill-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const file = path.join(root, "SKILL.md"); + await fs.writeFile(file, "---\nname: x\n---\nbody\n"); + const shared = await readCappedFile(fs, file, 64 * 1024, { + invalid: () => Object.assign(new Error("The skill must be a regular Markdown file."), { code: "invalid" }), + tooLarge: () => Object.assign(new Error("This skill exceeds the 64 KiB viewer limit."), { code: "too_large" }), + }); + const direct = await readSkillFile(file); + assert.equal(shared.content, direct.content); + assert.equal(shared.mode, direct.mode); +}); + +test("handle-level reads match the settings snapshot loop", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-handle-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const file = path.join(root, "settings.toml"); + await fs.writeFile(file, "a".repeat(5000)); + const handle = await fs.open(file, READ_FLAGS); + try { + const read = await readHandleCapped(handle, 1024 * 1024); + assert.equal(read.bytes, 5000); + assert.equal(read.raw.length, 5000); + assert.equal(read.content, "a".repeat(5000)); + } finally { await handle.close(); } +}); + +test("atomic writes replace durably with exact modes", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-shared-atomic-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const file = path.join(root, "nested", "store.json"); + await writeJsonAtomic(fs, file, { version: 1, servers: [{ name: "a" }] }); + assert.equal(await fs.readFile(file, "utf8"), '{\n "version": 1,\n "servers": [\n {\n "name": "a"\n }\n ]\n}\n'); + assert.equal((await fs.stat(file)).mode & 0o777, 0o600); + await writeFileAtomic(fs, file, "replaced\n", 0o640); + assert.equal(await fs.readFile(file, "utf8"), "replaced\n"); + assert.equal((await fs.stat(file)).mode & 0o777, 0o640); + const leftovers = (await fs.readdir(path.dirname(file))).filter((name) => name.includes(".tmp.")); + assert.deepEqual(leftovers, []); +});