Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/native-runtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,9 @@ jobs:
target: t3-runtime-artifact
platforms: linux/${{ matrix.arch }}
outputs: type=local,dest=native-output
build-args: T3_VERSION=${{ matrix.version }}
build-args: |
T3_VERSION=${{ matrix.version }}
T3_BUILD_NODE_IMAGE=${{ matrix.buildNodeImage }}
cache-from: type=gha,scope=native-${{ matrix.version }}-${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=native-${{ matrix.version }}-${{ matrix.arch }}
- name: Verify the native project API
Expand Down
23 changes: 23 additions & 0 deletions docker/harness/chatter-json.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Chatter-tolerant JSON parse for harness probes.
//
// Executable probes print log lines around the JSON they are asked for. The
// whole contract is: the document starts at the first "{"; anything before
// it is chatter. Returns null when there is no document or it does not
// parse, never throws. Setup and runtime keep their own identical copies:
// each image layer ships separately, so a cross-layer import would resolve
// in a checkout and break in the image.
export function chatterSlice(text) {
const output = String(text ?? "");
const start = output.indexOf("{");
return start < 0 ? null : output.slice(start);
}

export function jsonFromChatter(text) {
const slice = chatterSlice(text);
if (slice === null) return null;
try {
return JSON.parse(slice);
} catch {
return null;
}
}
9 changes: 2 additions & 7 deletions docker/harness/probe.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
// running version and an authenticated/not/unknown verdict.
import path from "node:path";
import {readEnv} from './mise.mjs';
import { jsonFromChatter } from './chatter-json.mjs';

/** Run the harness's version flag and extract a concrete version. */
export async function probeVersion(ctx, entry, executable) {
Expand Down Expand Up @@ -143,13 +144,7 @@ async function probeCommand(ctx, executable, args, env = null) {
}

function parseJson(text) {
const start = text.indexOf("{");
if (start < 0) return null;
try {
return JSON.parse(text.slice(start));
} catch {
return null;
}
return jsonFromChatter(text);
}

function firstLine(text) {
Expand Down
12 changes: 5 additions & 7 deletions docker/runtime/distribution.mjs
Original file line number Diff line number Diff line change
@@ -1,19 +1,17 @@
import { createHash } from "node:crypto";
import { archiveStem, releaseText } from "./releases.mjs";
import { CHANNELS, channelOf, exactVersion } from "./state.mjs";
import { PATCH_REVISION, ARTIFACT_TYPE, ARCHIVE_TYPE, distributionTag, channelTag, manifestUrl, distributionBlob } from "./distribution-spec.mjs";
import { ARCHIVE_LIMIT, DOCUMENT_LIMIT, digestOfText as digestOf, isSha256Digest as digest } from "./release-bounds.mjs";
import { registryRequest } from "./registry.mjs";

const INDEX = "application/vnd.oci.image.index.v1+json", MANIFEST = "application/vnd.oci.image.manifest.v1+json";
const digestOf = (text) => "sha256:" + createHash("sha256").update(text).digest("hex");
const digest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value);

export function validateDistributionIndex(index) {
if (index?.schemaVersion !== 2 || index.mediaType !== INDEX || index.artifactType !== ARTIFACT_TYPE || index.annotations?.["io.t3code.runtime.patch"] !== PATCH_REVISION || !exactVersion(index.annotations?.["io.t3code.runtime.version"]) || !Array.isArray(index.manifests)) throw new Error("The native runtime index does not match this image's integration patch.");
const architectures = new Set();
for (const entry of index.manifests) {
const architecture = entry?.platform?.architecture;
if (entry?.platform?.os !== "linux" || !["amd64", "arm64"].includes(architecture) || architectures.has(architecture) || !digest(entry.digest) || entry.mediaType !== MANIFEST || !Number.isSafeInteger(entry.size) || entry.size <= 0 || entry.size > 4 * 1024 * 1024) throw new Error("The native runtime index must contain exactly one tested build for each supported architecture.");
if (entry?.platform?.os !== "linux" || !["amd64", "arm64"].includes(architecture) || architectures.has(architecture) || !digest(entry.digest) || entry.mediaType !== MANIFEST || !Number.isSafeInteger(entry.size) || entry.size <= 0 || entry.size > DOCUMENT_LIMIT) throw new Error("The native runtime index must contain exactly one tested build for each supported architecture.");
architectures.add(architecture);
}
if (architectures.size !== 2) throw new Error("The native runtime index is missing a tested architecture.");
Expand All @@ -23,7 +21,7 @@ export function validateDistributionIndex(index) {
export async function distributionIndex(reference, { fetcher = fetch } = {}) {
const response = await registryRequest(manifestUrl(reference), { fetcher, headers: { accept: INDEX } });
if (!response.ok) { await response.body?.cancel(); throw new Error(`Could not fetch the container-maintained T3 runtime (HTTP ${response.status}). Check again later or update the Docker image; the running version was kept.`); }
const text = await releaseText(response, 4 * 1024 * 1024), index = validateDistributionIndex(JSON.parse(text));
const text = await releaseText(response, DOCUMENT_LIMIT), index = validateDistributionIndex(JSON.parse(text));
const supplied = response.headers.get("docker-content-digest");
if (supplied && supplied !== digestOf(text)) throw new Error("The native runtime index failed its integrity check.");
return { index, digest: digestOf(text), version: index.annotations["io.t3code.runtime.version"] };
Expand All @@ -35,12 +33,12 @@ async function fromIndex(root, { fetcher, arch }) {
if (platforms.length !== 1 || !digest(platforms[0].digest) || platforms[0].mediaType !== MANIFEST) throw new Error("The native runtime index must contain exactly one tested build for this architecture.");
const response = await registryRequest(manifestUrl(platforms[0].digest), { fetcher, headers: { accept: MANIFEST } });
if (!response.ok) { await response.body?.cancel(); throw new Error(`The native runtime manifest is unavailable (HTTP ${response.status}).`); }
const text = await releaseText(response, 4 * 1024 * 1024);
const text = await releaseText(response, DOCUMENT_LIMIT);
if (digestOf(text) !== platforms[0].digest || Buffer.byteLength(text) !== platforms[0].size) throw new Error("The native runtime manifest failed its integrity check.");
const manifest = JSON.parse(text);
if (manifest.schemaVersion !== 2 || manifest.mediaType !== MANIFEST || manifest.artifactType !== ARTIFACT_TYPE || manifest.annotations?.["io.t3code.runtime.patch"] !== PATCH_REVISION || manifest.annotations?.["io.t3code.runtime.version"] !== version || manifest.layers?.length !== 1) throw new Error("The native runtime manifest has invalid integration metadata.");
const layer = manifest.layers[0];
if (layer.mediaType !== ARCHIVE_TYPE || !digest(layer.digest) || !Number.isSafeInteger(layer.size) || layer.size <= 0 || layer.size > 300 * 1024 * 1024) throw new Error("The native runtime archive descriptor is invalid.");
if (layer.mediaType !== ARCHIVE_TYPE || !digest(layer.digest) || !Number.isSafeInteger(layer.size) || layer.size <= 0 || layer.size > ARCHIVE_LIMIT) throw new Error("The native runtime archive descriptor is invalid.");
return { version, name: "t3-runtime", root: stem, url: distributionBlob(layer.digest), integrity: layer.digest.replace("sha256:", "sha256-"), size: layer.size, patchRevision: PATCH_REVISION, distributionDigest: root.digest, manifestDigest: platforms[0].digest };
}

Expand Down
4 changes: 2 additions & 2 deletions docker/runtime/manager.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import path from "node:path";
import { randomUUID } from "node:crypto";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { unpackRelease, executableVersion } from "./releases.mjs";
import { unpackRelease, executableVersion, chatterSlice } from "./releases.mjs";
import { packageRelease, releaseForVersion } from "./distribution.mjs";
import { PATCH_REVISION } from "./distribution-spec.mjs";
import { injectSetupBridge } from "./client-shell.mjs";
Expand All @@ -28,7 +28,7 @@ export async function discoverRuntime(binary, paths, { run = execute, env = proc
const probe = await fs.mkdtemp(path.join(paths.root, ".database-probe-"));
try {
const issued = await run(binary, ["auth", "session", "issue", "--ttl", "1m", "--subject", "t3-container-probe", "--json"], { env: { ...env, T3CODE_HOME: probe }, timeout: 30_000 });
const session = JSON.parse(issued.stdout.slice(issued.stdout.indexOf("{")));
const session = JSON.parse(chatterSlice(issued.stdout) ?? "");
const scopes = session.permissions ?? session.scopes;
if (!validScopes(scopes) || !scopes.length) throw new Error("Could not identify this release's supported device permissions safely.");
const names = (await fs.readdir(path.join(probe, "userdata"))).filter(databaseName);
Expand Down
25 changes: 25 additions & 0 deletions docker/runtime/release-bounds.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// One home for every release size and integrity bound.
//
// The GitHub release stack (releases.mjs) and the container-registry stack
// (distribution.mjs) resolve different backends but stage the same archives
// through the same verifier (unpackRelease). An integrity hardening that
// lands in only one file leaves the other on the old bound, so both stacks
// import these limits and predicates instead of keeping their own copies.
import { createHash } from "node:crypto";

// Largest single archive the updater will stage or trust from a manifest.
export const ARCHIVE_LIMIT = 300 * 1024 * 1024;
// Largest extracted tree the updater will accept.
export const UNPACKED_LIMIT = 1024 * 1024 * 1024;
// Free space kept in reserve on the staging volume while downloading.
export const RELEASE_HEADROOM = 64 * 1024 * 1024;
// Largest checksum list accepted from a GitHub release.
export const SUMS_LIMIT = 1024 * 1024;
// Largest registry document (release API page, image index or manifest).
export const DOCUMENT_LIMIT = 4 * 1024 * 1024;

// Registry blob digest ("sha256:…") used in manifests and blob URLs.
export const isSha256Digest = (value) => typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value);
// Download integrity ("sha256-…") carried on a resolved release.
export const isSha256Integrity = (value) => typeof value === "string" && /^sha256-[a-f0-9]{64}$/.test(value);
export const digestOfText = (text) => "sha256:" + createHash("sha256").update(text).digest("hex");
15 changes: 10 additions & 5 deletions docker/runtime/releases.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,12 @@ import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { CHANNELS, channelOf, exactVersion, readJson, writeJson } from "./state.mjs";
import { PATCH_REVISION, distributionBlob } from "./distribution-spec.mjs";
import { ARCHIVE_LIMIT as LIMIT, UNPACKED_LIMIT, RELEASE_HEADROOM as HEADROOM, SUMS_LIMIT, DOCUMENT_LIMIT } from "./release-bounds.mjs";
import { registryRequest } from "./registry.mjs";

const execute = promisify(execFile);
const API = "https://api.github.com/repos/pingdotgg/t3code/releases";
const DOWNLOAD = "https://github.com/pingdotgg/t3code/releases/download";
const LIMIT = 300 * 1024 * 1024;
const UNPACKED_LIMIT = 1024 * 1024 * 1024;
const HEADROOM = 64 * 1024 * 1024;
const SPACE_CHECK_BYTES = 8 * 1024 * 1024;
const PAGE_SIZE = 20;
const downloadHosts = new Set(["github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com", "ghcr.io", "pkg-containers.githubusercontent.com"]);
Expand Down Expand Up @@ -51,6 +49,13 @@ async function get(url, { fetcher, api = false, token, timeout = 15_000 }) {
}
export { get as fetchRelease };

/** Start of the first JSON document in CLI chatter. Null when no document opens. */
export function chatterSlice(text) {
const output = String(text ?? "");
const start = output.indexOf("{");
return start < 0 ? null : output.slice(start);
}

async function boundedText(response, limit) {
const chunks = [];
let bytes = 0;
Expand All @@ -73,7 +78,7 @@ export async function releaseForVersion(version, { fetcher = fetch, arch = proce
const root = archiveStem(version, arch);
const name = `${root}.tar.gz`;
const base = `${DOWNLOAD}/v${version}`;
const sums = await boundedText(await get(`${base}/SHA256SUMS`, { fetcher }), 1024 * 1024);
const sums = await boundedText(await get(`${base}/SHA256SUMS`, { fetcher }), SUMS_LIMIT);
return { version, name: "t3-runtime", root, url: `${base}/${name}`, integrity: `sha256-${archiveChecksum(sums, name)}` };
}

Expand All @@ -83,7 +88,7 @@ export async function packageRelease(channel, { fetcher = fetch, arch = process.
archiveStem("0.0.0", arch);
for (let page = 1; page <= 5; page += 1) {
const url = channel === "stable" ? `${API}/latest` : `${API}?per_page=${PAGE_SIZE}&page=${page}`;
const data = JSON.parse(await boundedText(await get(url, { fetcher, api: true, token }), 4 * 1024 * 1024));
const data = JSON.parse(await boundedText(await get(url, { fetcher, api: true, token }), DOCUMENT_LIMIT));
const entries = channel === "stable" ? [data] : data;
if (!Array.isArray(entries)) throw new Error("GitHub did not return a release list.");
const release = entries.find((entry) => {
Expand Down
33 changes: 33 additions & 0 deletions docker/setup/atomic-write.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// One atomic tmp+rename file writer for the setup console.
//
// The MCP store and the instruction editors each kept their own copy of this
// dance: private temp file, exact mode, rename into place, best-effort temp
// cleanup. A crash between write and rename must leave either the old file or
// the new one, never a half-written one, so both callers share this helper.
// The runtime selection (docker/runtime/state.mjs) and the harness manager
// record (docker/harness/state.mjs) keep their own variants on purpose: the
// former fsyncs for boot-critical durability and the latter writes through an
// injected filesystem seam, and neither image layer can import this one.
import path from "node:path";
import { randomBytes } from "node:crypto";

/** Replace durably: exclusive temp file, exact mode, atomic rename. */
export async function writeFileAtomic(fs, file, text, mode = 0o600) {
await fs.mkdir(path.dirname(file), { recursive: true });
const tmp = `${file}.tmp.${process.pid}.${randomBytes(6).toString("hex")}`;
let created = false;
try {
const handle = await fs.open(tmp, "wx", mode);
created = true;
try { await handle.writeFile(text, "utf8"); await handle.chmod(mode); }
finally { await handle.close(); }
await fs.rename(tmp, file);
} finally {
if (created) await fs.rm(tmp, { force: true });
}
}

/** Atomic JSON replace with the canonical trailing newline. */
export async function writeJsonAtomic(fs, file, value, mode = 0o600) {
await writeFileAtomic(fs, file, JSON.stringify(value, null, 2) + "\n", mode);
}
58 changes: 58 additions & 0 deletions docker/setup/bounded-read.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// One capped file reader for the setup console.
//
// Skill files, shared instructions and native settings snapshots each read a
// bounded prefix of a file that may be a symlink, may grow between stat and
// read, or may be a raced-in FIFO. The shape is always the same: reject
// non-regular or oversized targets before opening, open without following
// links and without blocking, re-check the opened descriptor, then read at
// most the limit plus one byte (including across short reads) so polling can
// never load a file whole. Callers keep their own messages; this module owns
// the flags and the read loop.
import { constants } from "node:fs";

// Never follow a raced-in symlink, and never block on a raced-in FIFO.
export const READ_FLAGS = constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW;

/** Read at most maxBytes + 1 from an open handle, tolerating short reads. */
export async function readHandleCapped(handle, maxBytes) {
const buffer = Buffer.alloc(maxBytes + 1);
let bytes = 0;
while (bytes < buffer.length) {
const result = await handle.read(buffer, bytes, buffer.length - bytes, bytes);
if (!result.bytesRead) break;
bytes += result.bytesRead;
}
// `raw` is the exact bytes read: revision hashes must not depend on how
// invalid UTF-8 decodes, so hash this, not the decoded content.
return { content: buffer.toString("utf8", 0, bytes), bytes, raw: buffer.subarray(0, bytes) };
}

/**
* Bounded read of a regular file. Throws the caller's `invalid` error for a
* non-regular target and the caller's `tooLarge` error (given the byte count)
* for an oversized one, so messages stay identical at every call site.
* `flags` defaults to non-following and non-blocking; the shared instruction
* file keeps its plain "r" open so a manually symlinked AGENTS.md still reads.
*/
export async function readCappedFile(io, file, maxBytes, { tooLarge, invalid, flags = READ_FLAGS } = {}) {
const failLarge = tooLarge ?? ((bytes) => Object.assign(new Error(`File exceeds the ${maxBytes}-byte read limit.`), { code: "too_large", bytes }));
const failInvalid = invalid ?? (() => Object.assign(new Error("Expected a regular file."), { code: "invalid" }));
const check = (stat) => {
if (!stat.isFile()) throw failInvalid();
if (stat.size > maxBytes) throw failLarge(stat.size);
};
// Reject oversized targets before opening them, including through symlinks.
check(await io.stat(file));
const handle = await io.open(file, flags);
try {
const stat = await handle.stat();
check(stat);
// A file may grow after stat. Read at most the limit plus one byte,
// including across short reads, so polling can never load it all.
const read = await readHandleCapped(handle, maxBytes);
if (read.bytes > maxBytes) throw failLarge(read.bytes);
return { ...read, mode: stat.mode & 0o777, stat };
} finally {
await handle.close();
}
}
6 changes: 4 additions & 2 deletions docker/setup/cache.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ export async function withTimeout(work, ms) {
* Deduplicate concurrent refresh work. While a promise is in flight every
* caller shares it; once it settles the next call starts fresh. A rejection
* clears the slot without poisoning later calls, and an abandoned refresh
* can never crash the process.
* can never crash the process. Read-path only: this coalesces cache fills,
* it is not a writer mutex (cache entries are replaced, never mutated).
*/
export function createCoalescer() {
let inflight = null;
Expand Down Expand Up @@ -383,7 +384,8 @@ export function createHarnessCache({
}

/** Share pending reads and optionally retain successful answers for a TTL.
* Clearing detaches old reads: their completion cannot replace a newer answer. */
* Clearing detaches old reads: their completion cannot replace a newer answer.
* Read-path only, like the coalescer above: never used to serialize writes. */
export function createTtlCache({ ttlMs, limit = 200, now = Date.now }) {
const entries = new Map();
const cached = (key, load) => {
Expand Down
23 changes: 23 additions & 0 deletions docker/setup/chatter-json.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Chatter-tolerant JSON parse for setup CLI output.
//
// Native CLIs print log lines around the JSON they are asked for. The whole
// contract is: the document starts at the first "{"; anything before it is
// chatter. Returns null when there is no document or it does not parse,
// never throws. The harness probes and the runtime supervisor keep their own
// identical copies: each image layer ships separately, so a cross-layer
// import would resolve in a checkout and break in the image.
export function chatterSlice(text) {
const output = String(text ?? "");
const start = output.indexOf("{");
return start < 0 ? null : output.slice(start);
}

export function jsonFromChatter(text) {
const slice = chatterSlice(text);
if (slice === null) return null;
try {
return JSON.parse(slice);
} catch {
return null;
}
}
10 changes: 2 additions & 8 deletions docker/setup/connect.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
// t3 connect status --json what is saved
// t3 connect unlink off again, keeping the sign-in
//
import { jsonFromChatter } from "./chatter-json.mjs";
// The setup service runs those processes. This module parses their output
// and reads or saves live preferences through T3 Code's native server API.

Expand Down Expand Up @@ -89,14 +90,7 @@ export function connectState(json) {

/** The JSON object in a CLI's stdout, which may follow lines of chatter. */
export function jsonFrom(stdout) {
const text = String(stdout ?? "");
const start = text.indexOf("{");
if (start < 0) return null;
try {
return JSON.parse(text.slice(start));
} catch {
return null;
}
return jsonFromChatter(stdout);
}

/** Read and update native preferences without overwriting the other switch. */
Expand Down
Loading
Loading