From 601ad01d41000d9f7e728a7cc0724d32d6cdad5a Mon Sep 17 00:00:00 2001 From: 0libote <79215282+0libote@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:06:29 +0000 Subject: [PATCH 1/2] fix(runtime): roll back updates whose release crashes after commit Keep the rollback request and snapshots until the new release survives a health window; an early exit restores the previous release instead of re-booting the crashing one. Boot recovery rolls back an unverified commit, discards stale requests that never matched the selection, and snapshots only the migrating stores rather than all of userdata. --- docker/runtime/supervisor.mjs | 110 +++++++++++++++--- tests/t3-runtime-rollback.test.mjs | 172 +++++++++++++++++++++++++++++ tests/t3-runtime.test.mjs | 4 +- 3 files changed, 267 insertions(+), 19 deletions(-) create mode 100644 tests/t3-runtime-rollback.test.mjs diff --git a/docker/runtime/supervisor.mjs b/docker/runtime/supervisor.mjs index c3e82e3..b7170c2 100644 --- a/docker/runtime/supervisor.mjs +++ b/docker/runtime/supervisor.mjs @@ -61,7 +61,12 @@ async function removeSnapshots(paths) { await syncDirectory(path.dirname(paths.backup)); } -export async function snapshotDatabase(paths, id) { +// A committed release must stay up through this window before its upgrade +// is finalized. An earlier exit restores the previous release instead of +// re-booting the crashing one on every container restart. +const COMMIT_HEALTH_MS = 60_000; + +export async function snapshotDatabase(paths, id, { fromDatabase, targetDatabase } = {}) { await recoverSnapshot(paths); const existing = await readJson(path.join(paths.backup, "complete.json")); if (existing?.id === id) { @@ -77,7 +82,12 @@ export async function snapshotDatabase(paths, id) { // SQLite's backup API includes committed WAL records in a consistent main // database. Sidecars can disappear when a last reader closes, and copying // the main file and WAL separately can capture different checkpoints. - const files = (await databaseFiles(userdata)).filter((file) => /\.(?:sqlite3?|db)$/.test(file)); + // Only the stores a release migration may rewrite are snapshotted. Thread + // histories stay owned by their release; copying all of userdata would make + // every update pay for user data the trial never touches. + const migrating = new Set([fromDatabase, targetDatabase, "state.sqlite", "statev2.sqlite"].filter((name) => typeof name === "string")); + const present = await databaseFiles(userdata); + const files = present.filter((file) => /\.(?:sqlite3?|db)$/.test(file) && migrating.has(file)); const { DatabaseSync, backup } = await import("node:sqlite"); const directories = new Set([stage]); for (const file of files) { @@ -88,7 +98,9 @@ export async function snapshotDatabase(paths, id) { await syncFile(target); for (let dir = path.dirname(target); dir !== stage; dir = path.dirname(dir)) directories.add(dir); } - await writeJson(path.join(stage, "complete.json"), { id, files }); + // Record every database observed, not just the copied ones, so a rollback + // can tell pre-existing histories apart from files the trial introduced. + await writeJson(path.join(stage, "complete.json"), { id, files, present }); for (const dir of [...directories].sort((a, b) => b.length - a.length)) await syncDirectory(dir); // Keep the previous backup until the new one is in place: a crash between // rm and rename used to lose both copies. @@ -114,9 +126,13 @@ export async function snapshotDatabase(paths, id) { export async function restoreDatabase(paths, id) { await recoverSnapshot(paths); const backup = await readJson(path.join(paths.backup, "complete.json")); - if (backup?.id !== id || !Array.isArray(backup.files) || backup.files.some((file) => typeof file !== "string" || path.isAbsolute(file) || file.split(path.sep).includes("..") || !sqlite.test(file))) { + const names = (value) => Array.isArray(value) && value.every((file) => typeof file === "string" && !path.isAbsolute(file) && !file.split(path.sep).includes("..") && sqlite.test(file)); + if (backup?.id !== id || !names(backup.files) || (backup.present !== undefined && !names(backup.present))) { throw new Error("The upgrade's database snapshot is incomplete. Refusing to start against partially restored data."); } + // Pre-existing databases stay owned by their release; only files the trial + // introduced are removed. Snapshots that predate the observed list fall + // back to keeping exactly what they copied. const userdata = path.join(paths.home, "userdata"); await fs.mkdir(userdata, { recursive: true, mode: 0o700 }); const wanted = new Set(backup.files); @@ -163,8 +179,9 @@ export async function restoreDatabase(paths, id) { } // Remove trial-only files after the replacements are durable. Older // snapshots can explicitly contain WALs, so retain every wanted file. + const keep = new Set([...backup.files, ...(backup.present || [])]); for (const file of await databaseFiles(userdata)) { - if (!wanted.has(file)) { + if (!keep.has(file)) { const target = path.join(userdata, file); await fs.rm(target, { force: true }); changed(target); @@ -283,15 +300,54 @@ export async function pruneRuntimes(paths, request) { } } +export async function finalizeCommit(paths, request) { + await removeSnapshots(paths); + await pruneRuntimes(paths, request).catch((error) => console.error(`[t3code] could not trim cached releases: ${error.message}`)); + await writeJson(paths.operation, { ...request, previous: undefined, phase: "succeeded", finishedAt: Date.now() }); + // Keep the request as the cross-process mutex until every cleanup and + // terminal write finishes. A second operator cannot overwrite it early. + await fs.rm(paths.request, { force: true }); + console.error(`[t3code] upgraded to ${request.version} (${request.channel})`); +} + +export async function rollbackCommit(paths, request, error) { + // The committed release exited before proving it can serve. Restore the + // previous selection and database instead of re-booting the bad binary. + await writeJson(paths.selection, request.previous); + await restoreDatabase(paths, request.id); + await fs.rm(paths.request, { force: true }); + await removeSnapshots(paths); + await writeJson(paths.operation, { ...request, previous: undefined, phase: "rolled-back", error: `${error} The previous version and database were restored.`, finishedAt: Date.now() }); + console.error(`[t3code] upgrade rolled back: ${error}`); +} + export async function recoverUpdate(paths) { const pending = await readJson(paths.request); if (!pending) return; const request = validateRequest(pending); + if (request.phase === "committed") { + // The selection was committed but the release never survived its health + // window; the container restarted first. Restore the previous release + // instead of re-booting a version that may crash on every start. + await rollbackCommit(paths, request, "The new T3 Code version did not stay up."); + return; + } const selection = await readSelection(paths); if (selection.updateId === request.id && selection.version === request.version && selection.patchRevision === request.patchRevision) { await writeJson(paths.operation, { ...request, previous: undefined, phase: "succeeded", finishedAt: Date.now() }); } else { - if (selection.version !== request.previous.version || selection.channel !== request.previous.channel || selection.patchRevision !== request.previous.patchRevision) throw new Error("The pending upgrade does not match T3 Code's saved selection."); + if (selection.version !== request.previous.version || selection.channel !== request.previous.channel || selection.patchRevision !== request.previous.patchRevision) { + // A committed selection naming this request must still match it. + if (selection.updateId === request.id) throw new Error("The pending upgrade does not match T3 Code's saved selection."); + // Otherwise the selection was never committed for this request: the + // image was downgraded or the selection was hand-fixed after the + // request was written. Discard the stale request and boot the saved + // selection instead of throwing on every restart. + await recoverSnapshot(paths); + await writeJson(paths.operation, { ...request, previous: undefined, phase: "interrupted", error: "A stale upgrade request did not match T3 Code's saved selection and was discarded.", finishedAt: Date.now() }); + await fs.rm(paths.request, { force: true }); + return; + } if (["starting", "restoring"].includes(request.phase)) { await writeJson(paths.request, { ...request, phase: "restoring" }); await restoreDatabase(paths, request.id); @@ -302,7 +358,7 @@ export async function recoverUpdate(paths) { await removeSnapshots(paths); } -export async function runSupervisor(args, { env = process.env, preparedTimeoutMs = 120_000, authenticationTimeoutMs = 120_000, bundledLauncherProtocol } = {}) { +export async function runSupervisor(args, { env = process.env, preparedTimeoutMs = 120_000, authenticationTimeoutMs = 120_000, bundledLauncherProtocol, commitHealthMs = COMMIT_HEALTH_MS } = {}) { const paths = runtimePaths(env); await fs.mkdir(paths.run, { recursive: true, mode: 0o700 }); const pidFile = path.join(paths.run, "t3.pid"); @@ -394,7 +450,7 @@ export async function runSupervisor(args, { env = process.env, preparedTimeoutMs }; try { await phase("snapshot"); - await snapshotDatabase(paths, request.id); + await snapshotDatabase(paths, request.id, request); snapshot = true; await phase("starting"); await syncAuthentication(paths, request); @@ -417,19 +473,18 @@ export async function runSupervisor(args, { env = process.env, preparedTimeoutMs child = await launch(runtime, { id: request.id, fromVersion: request.fromVersion, targetVersion: request.version, dbPath: path.join(paths.home, "userdata", request.targetDatabase || "statev2.sqlite"), status: "pending" }); await child.prepared; if (stopping) return; + // Persist the committed phase before activation: a crash past this + // point must roll back on the next boot instead of re-booting a + // release that never proved it can serve. + await phase("committed"); await writeJson(paths.selection, { schema: 1, channel: request.channel, version: request.version, ...(request.patchRevision ? { patchRevision: request.patchRevision } : {}), updateId: request.id }); committed = true; await new Promise((resolve, reject) => child.process.send({ type: "committed", updateId: request.id }, (error) => error ? reject(error) : resolve())); - await removeSnapshots(paths); - await pruneRuntimes(paths, request).catch((error) => console.error(`[t3code] could not trim cached releases: ${error.message}`)); - await writeJson(paths.operation, { ...request, previous: undefined, phase: "succeeded", finishedAt: Date.now() }); - // Keep the request as the cross-process mutex until every cleanup and - // terminal write finishes. A second operator cannot overwrite it early. - await fs.rm(paths.request, { force: true }); - console.error(`[t3code] upgraded to ${request.version} (${request.channel})`); + // The request and snapshots stay until the serving loop sees the + // child survive its health window (or rolls back an early exit). } catch (error) { if (stopping) return; - if (committed) throw error; // After activation, ordinary container crash policy applies. + if (committed) throw error; // The committed request rolls back on the next boot. await terminate(child); child = null; if (snapshot) { await phase("restoring"); await restoreDatabase(paths, request.id); } @@ -442,7 +497,28 @@ export async function runSupervisor(args, { env = process.env, preparedTimeoutMs try { while (!stopping) { child ??= await launch(await selectedRuntime(paths)); - const result = await child.done; + let result = null; + const pending = !stopping && await readJson(paths.request); + if (pending?.phase === "committed") { + // The upgrade is committed but unproven. An early exit means the new + // release cannot serve: restore the previous one instead of exiting + // and re-booting the crashing version. Only a full health window + // finalizes the upgrade and its cleanup. + validateRequest(pending); + const settled = await Promise.race([child.done, new Promise((resolve) => setTimeout(resolve, commitHealthMs)).then(() => null)]); + if (settled) { + result = settled; + child = null; + if (!stopping && !await exists(restartFile)) { + await rollbackCommit(paths, pending, "The new T3 Code version exited before it stayed up."); + continue; + } + if (!stopping) await finalizeCommit(paths, pending); + } else if (!stopping) { + await finalizeCommit(paths, pending); + } + } + result ??= await child.done; child = null; if (stopping) return result.code; if (!await exists(restartFile)) return result.code; diff --git a/tests/t3-runtime-rollback.test.mjs b/tests/t3-runtime-rollback.test.mjs new file mode 100644 index 0000000..8e220a3 --- /dev/null +++ b/tests/t3-runtime-rollback.test.mjs @@ -0,0 +1,172 @@ +import assert from "node:assert/strict"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import test from "node:test"; +import { runtimePaths, writeJson, readJson, readSelection, selectedRuntime, releaseDirectory } from "../docker/runtime/state.mjs"; +import { recoverUpdate, restoreDatabase, snapshotDatabase } from "../docker/runtime/supervisor.mjs"; + +const NIGHTLY = "0.0.46-nightly.20261011.2980"; +const cleanups = new WeakMap(); +async function world(t) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "t3-runtime-rollback-test-")); + const stops = []; + cleanups.set(t, stops); + t.after(async () => { + for (const stop of stops.reverse()) await stop(); + await fs.rm(root, { recursive: true, force: true }); + }); + const env = { ...process.env, T3CODE_HOME: path.join(root, "home"), T3_RUN_DIR: path.join(root, "run"), T3_INFRA_BINARY: path.join(root, "bundled/t3") }; + const paths = runtimePaths(env); + await fs.mkdir(path.dirname(paths.binary), { recursive: true }); + await writeJson(path.join(path.dirname(paths.binary), "package.json"), { version: "0.0.45" }); + await fs.mkdir(path.join(paths.home, "userdata"), { recursive: true }); + return { root, env, paths }; +} +const waitFor = async (fn, ms = 5000) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { if (await fn()) return; await new Promise((resolve) => setTimeout(resolve, 20)); } + throw new Error("Timed out waiting for runtime state"); +}; +async function seedDatabase(file, value) { + const { DatabaseSync } = await import("node:sqlite"); + const db = new DatabaseSync(file); + db.exec("CREATE TABLE content (body TEXT)"); + db.prepare("INSERT INTO content VALUES (?)").run(value); + db.close(); +} +async function databaseContent(file) { + const { DatabaseSync } = await import("node:sqlite"); + const db = new DatabaseSync(file, { readOnly: true }); + try { return db.prepare("SELECT body FROM content").get().body; } finally { db.close(); } +} + +async function fakeRuntime(paths, version, mode = "ready", bundled = false) { + const dir = bundled ? path.dirname(paths.binary) : releaseDirectory(paths, { version }); + await fs.mkdir(dir, { recursive: true }); + await writeJson(path.join(dir, "package.json"), { version }); + await writeJson(path.join(dir, ".installed.json"), { version }); + const code = `#!${process.execPath} +const fs = require('node:fs'); +const path = require('node:path'); +const version = ${JSON.stringify(version)}, mode = ${JSON.stringify(mode)}; +const home = process.env.T3CODE_HOME; +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT || '{}'); +if (context.protocol !== 3) process.exit(8); +fs.appendFileSync(path.join(home, 'starts'), version + ' ' + process.pid + '\\n'); +if (context.update) { + const {DatabaseSync} = require('node:sqlite'); + const db = new DatabaseSync(path.join(home, 'userdata/statev2.sqlite')); + db.exec("UPDATE content SET body='trial migration'"); + db.close(); + if (mode === 'crash-after-commit') process.send({type:'prepared',updateId:context.update.id}); + else process.exit(9); +} +process.on('message', msg => { + if (msg.type === 'committed') { + fs.writeFileSync(path.join(home, 'activated'), version); + if (mode === 'crash-after-commit') setTimeout(() => process.exit(9), 300); + } +}); +process.on('SIGTERM', () => process.exit(143)); +setInterval(() => {}, 1000); +`; + await fs.writeFile(path.join(dir, "t3"), code, { mode: 0o755 }); + await fs.mkdir(path.join(dir, "client"), { recursive: true }); + await fs.writeFile(path.join(dir, "client/index.html"), "Fixture native client"); +} + +async function supervisor(t, env, timeout = 2000, commitHealthMs = 100) { + const options = { preparedTimeoutMs: timeout, commitHealthMs }; + const driver = `import {runSupervisor} from ${JSON.stringify(new URL("../docker/runtime/supervisor.mjs", import.meta.url).href)}; process.exitCode = await runSupervisor(['serve'], ${JSON.stringify(options)});`; + const child = spawn(process.execPath, ["--input-type=module", "-e", driver], { env, stdio: "pipe" }); + let log = ""; + child.stderr.on("data", (chunk) => { log += chunk; }); + const done = new Promise((resolve) => child.once("exit", (code) => resolve(code))); + cleanups.get(t).push(async () => { child.kill("SIGTERM"); await done; }); + await waitFor(async () => Boolean(await fs.readFile(path.join(env.T3_RUN_DIR, "t3.pid")).catch(() => null))); + return { child, done, log: () => log }; +} + +async function requestTrial(paths, version, channel = "nightly", database = {}) { + await waitFor(async () => !await readJson(paths.request)); + const request = { id: randomUUID(), version, channel, fromVersion: (await selectedRuntime(paths)).version, phase: "stopping", previous: await readSelection(paths), ...database }; + await writeJson(paths.request, request); + await fs.writeFile(path.join(paths.run, "restart"), "requested"); + return request; +} + +test("a release that crashes after commit is rolled back instead of rebooted", async (t) => { + const { paths, env } = await world(t); + await seedDatabase(path.join(paths.home, "userdata/statev2.sqlite"), "original data"); + await fakeRuntime(paths, "0.0.45", "ready", true); + await fakeRuntime(paths, NIGHTLY, "crash-after-commit"); + const sup = await supervisor(t, env, 2000, 8000); + const starts = () => fs.readFile(path.join(paths.home, "starts"), "utf8").catch(() => ""); + await waitFor(async () => (await starts()).includes("0.0.45")); + await requestTrial(paths, NIGHTLY); + await waitFor(async () => (await readJson(paths.operation))?.phase === "rolled-back", 10_000); + assert.match((await readJson(paths.operation)).error, /exited before it stayed up/); + assert.equal((await selectedRuntime(paths)).version, "0.0.45"); + assert.equal(await databaseContent(path.join(paths.home, "userdata/statev2.sqlite")), "original data"); + assert.equal(await readJson(paths.request), null); + // The supervisor relaunches the previous release instead of exiting, so a + // container restart would boot the good binary rather than crash-loop. + await waitFor(async () => (await starts()).split("\n").filter((line) => line.startsWith("0.0.45 ")).length === 2, 10_000); + assert.doesNotThrow(() => process.kill(sup.child.pid, 0)); +}); + +test("a stale upgrade request boots the saved selection instead of throwing", async (t) => { + const { paths, env } = await world(t); + await fakeRuntime(paths, "0.0.45"); + await writeJson(paths.selection, { schema: 1, channel: "stable", version: "0.0.45" }); + await writeJson(paths.request, { id: randomUUID(), version: NIGHTLY, channel: "nightly", fromVersion: "0.0.45", phase: "stopping", previous: { schema: 1, channel: "stable", version: "0.0.44" } }); + // The image was downgraded (or the selection hand-fixed) after the request + // was written, so it can never match. Booting must discard it and serve. + await supervisor(t, env); + await waitFor(async () => (await fs.readFile(path.join(paths.home, "starts"), "utf8").catch(() => "")).includes("0.0.45")); + assert.equal((await readSelection(paths)).version, "0.0.45"); + assert.equal((await readJson(paths.operation)).phase, "interrupted"); + assert.match((await readJson(paths.operation)).error, /stale upgrade request/); + assert.equal(await readJson(paths.request), null); +}); + +test("a committed selection naming the request still refuses a mismatched patch", async (t) => { + const { paths } = await world(t); + const request = { id: randomUUID(), version: "0.0.45", channel: "stable", fromVersion: "0.0.45", phase: "starting", previous: { schema: 1, channel: "stable", version: "0.0.45" } }; + await writeJson(paths.request, request); + await writeJson(paths.selection, { schema: 1, version: request.version, channel: "stable", updateId: request.id, patchRevision: "a".repeat(64) }); + await assert.rejects(recoverUpdate(paths), /does not match/); + assert.equal((await readJson(paths.request)).id, request.id); +}); + +test("snapshots cover only the migrating stores, not thread histories", async (t) => { + const { paths } = await world(t); + const userdata = path.join(paths.home, "userdata"); + await seedDatabase(path.join(userdata, "statev2.sqlite"), "migrating data"); + await seedDatabase(path.join(userdata, "threads.sqlite"), "thread history"); + await fs.mkdir(path.join(userdata, "cache"), { recursive: true }); + await seedDatabase(path.join(userdata, "cache/history.db"), "nested history"); + await snapshotDatabase(paths, "migrating-only", { fromDatabase: "statev2.sqlite", targetDatabase: "statev2.sqlite" }); + const backup = await readJson(path.join(paths.backup, "complete.json")); + assert.deepEqual(backup.files, ["statev2.sqlite"]); + assert.equal(await fs.readFile(path.join(paths.backup, "files/cache/history.db")).catch(() => null), null); + assert.equal(await databaseContent(path.join(userdata, "threads.sqlite")), "thread history"); + await fs.writeFile(path.join(userdata, "statev2.sqlite"), "trial data"); + await restoreDatabase(paths, "migrating-only"); + assert.equal(await databaseContent(path.join(userdata, "statev2.sqlite")), "migrating data"); + assert.equal(await databaseContent(path.join(userdata, "threads.sqlite")), "thread history"); + assert.equal(await databaseContent(path.join(userdata, "cache/history.db")), "nested history"); +}); + +test("snapshots default to the legacy and current stores without histories", async (t) => { + const { paths } = await world(t); + const userdata = path.join(paths.home, "userdata"); + for (const name of ["state.sqlite", "statev2.sqlite"]) await seedDatabase(path.join(userdata, name), "migrating " + name); + await seedDatabase(path.join(userdata, "threads.sqlite"), "thread history"); + await snapshotDatabase(paths, "defaults"); + const backup = await readJson(path.join(paths.backup, "complete.json")); + assert.deepEqual([...backup.files].sort(), ["state.sqlite", "statev2.sqlite"]); +}); diff --git a/tests/t3-runtime.test.mjs b/tests/t3-runtime.test.mjs index a4a52ae..405f0e9 100644 --- a/tests/t3-runtime.test.mjs +++ b/tests/t3-runtime.test.mjs @@ -319,8 +319,8 @@ test("starting a retained runtime refreshes only its image-owned Setup bridge", assert.deepEqual(await fs.readFile(path.join(directory,'t3')),binaryBefore); }); -async function supervisor(t, env, timeout = 2000, authenticationTimeoutMs) { - const options = { preparedTimeoutMs: timeout, ...(authenticationTimeoutMs === undefined ? {} : { authenticationTimeoutMs }) }; +async function supervisor(t, env, timeout = 2000, authenticationTimeoutMs, commitHealthMs = 100) { + const options = { preparedTimeoutMs: timeout, commitHealthMs, ...(authenticationTimeoutMs === undefined ? {} : { authenticationTimeoutMs }) }; const driver = `import {runSupervisor} from ${JSON.stringify(new URL('../docker/runtime/supervisor.mjs', import.meta.url).href)}; process.exitCode = await runSupervisor(['serve'], ${JSON.stringify(options)});`; const child = spawn(process.execPath, ["--input-type=module", "-e", driver], { env, stdio: "pipe" }); let log = ""; From e2c25d0fcce466655059e928d01fe69547c26a0f Mon Sep 17 00:00:00 2001 From: Ziyang Zeng Date: Sat, 14 Nov 2026 14:40:24 -0800 Subject: [PATCH 2/2] fix: make runtime health-window cleanup restart safe --- docker/runtime/supervisor.mjs | 30 +++++++++++++-- tests/t3-runtime-rollback.test.mjs | 59 +++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/docker/runtime/supervisor.mjs b/docker/runtime/supervisor.mjs index b7170c2..75ce3ff 100644 --- a/docker/runtime/supervisor.mjs +++ b/docker/runtime/supervisor.mjs @@ -301,6 +301,9 @@ export async function pruneRuntimes(paths, request) { } export async function finalizeCommit(paths, request) { + // Persist the healthy decision before deleting its rollback data. Recovery + // must finish cleanup, not try to restore a snapshot already removed. + await writeJson(paths.request, { ...request, phase: "finalizing" }); await removeSnapshots(paths); await pruneRuntimes(paths, request).catch((error) => console.error(`[t3code] could not trim cached releases: ${error.message}`)); await writeJson(paths.operation, { ...request, previous: undefined, phase: "succeeded", finishedAt: Date.now() }); @@ -315,9 +318,17 @@ export async function rollbackCommit(paths, request, error) { // previous selection and database instead of re-booting the bad binary. await writeJson(paths.selection, request.previous); await restoreDatabase(paths, request.id); - await fs.rm(paths.request, { force: true }); - await removeSnapshots(paths); + await finishRollback(paths, { ...request, phase: "rolled-back", rollbackError: error }); +} + +async function finishRollback(paths, request) { + // Restoration is durable before its snapshot is discarded. Retain the + // request mutex until the terminal receipt and cleanup both finish. + await writeJson(paths.request, request); + const error = request.rollbackError; await writeJson(paths.operation, { ...request, previous: undefined, phase: "rolled-back", error: `${error} The previous version and database were restored.`, finishedAt: Date.now() }); + await removeSnapshots(paths); + await fs.rm(paths.request, { force: true }); console.error(`[t3code] upgrade rolled back: ${error}`); } @@ -325,6 +336,17 @@ export async function recoverUpdate(paths) { const pending = await readJson(paths.request); if (!pending) return; const request = validateRequest(pending); + const selection = await readSelection(paths); + if (request.phase === "finalizing") { + if (selection.updateId !== request.id || selection.version !== request.version || selection.channel !== request.channel || selection.patchRevision !== request.patchRevision) throw new Error("The healthy upgrade no longer matches T3 Code's saved selection."); + await finalizeCommit(paths, request); + return; + } + if (request.phase === "rolled-back") { + if (selection.version !== request.previous.version || selection.channel !== request.previous.channel || selection.patchRevision !== request.previous.patchRevision) throw new Error("The restored upgrade no longer matches T3 Code's saved selection."); + await finishRollback(paths, request); + return; + } if (request.phase === "committed") { // The selection was committed but the release never survived its health // window; the container restarted first. Restore the previous release @@ -332,7 +354,6 @@ export async function recoverUpdate(paths) { await rollbackCommit(paths, request, "The new T3 Code version did not stay up."); return; } - const selection = await readSelection(paths); if (selection.updateId === request.id && selection.version === request.version && selection.patchRevision === request.patchRevision) { await writeJson(paths.operation, { ...request, previous: undefined, phase: "succeeded", finishedAt: Date.now() }); } else { @@ -505,7 +526,8 @@ export async function runSupervisor(args, { env = process.env, preparedTimeoutMs // and re-booting the crashing version. Only a full health window // finalizes the upgrade and its cleanup. validateRequest(pending); - const settled = await Promise.race([child.done, new Promise((resolve) => setTimeout(resolve, commitHealthMs)).then(() => null)]); + let healthTimer; + const settled = await Promise.race([child.done, new Promise((resolve) => { healthTimer = setTimeout(() => resolve(null), commitHealthMs); })]).finally(() => clearTimeout(healthTimer)); if (settled) { result = settled; child = null; diff --git a/tests/t3-runtime-rollback.test.mjs b/tests/t3-runtime-rollback.test.mjs index 8e220a3..7836337 100644 --- a/tests/t3-runtime-rollback.test.mjs +++ b/tests/t3-runtime-rollback.test.mjs @@ -6,7 +6,7 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import test from "node:test"; import { runtimePaths, writeJson, readJson, readSelection, selectedRuntime, releaseDirectory } from "../docker/runtime/state.mjs"; -import { recoverUpdate, restoreDatabase, snapshotDatabase } from "../docker/runtime/supervisor.mjs"; +import { finalizeCommit, rollbackCommit, recoverUpdate, restoreDatabase, snapshotDatabase } from "../docker/runtime/supervisor.mjs"; const NIGHTLY = "0.0.46-nightly.20261011.2980"; const cleanups = new WeakMap(); @@ -170,3 +170,60 @@ test("snapshots default to the legacy and current stores without histories", asy const backup = await readJson(path.join(paths.backup, "complete.json")); assert.deepEqual([...backup.files].sort(), ["state.sqlite", "statev2.sqlite"]); }); + +test("interrupted successful cleanup never tries to roll back a removed snapshot", async (t) => { + const { paths } = await world(t); + await seedDatabase(path.join(paths.home, "userdata/statev2.sqlite"), "healthy data"); + const request = { id: randomUUID(), version: NIGHTLY, channel: "nightly", fromVersion: "0.0.45", phase: "committed", previous: { schema: 1, channel: "stable", version: "0.0.45" } }; + await snapshotDatabase(paths, request.id); + await writeJson(paths.request, request); + await writeJson(paths.selection, { schema: 1, channel: request.channel, version: request.version, updateId: request.id }); + // Simulate a crash after snapshot cleanup but before the terminal receipt. + const rename = fs.rename; + fs.rename = async (from, to) => { + if (to === paths.operation) throw new Error("interrupted terminal write"); + return rename(from, to); + }; + try { await assert.rejects(finalizeCommit(paths, request), /interrupted terminal write/); } + finally { fs.rename = rename; } + await recoverUpdate(paths); + assert.equal((await readSelection(paths)).version, NIGHTLY); + assert.equal((await readJson(paths.operation)).phase, "succeeded"); + assert.equal(await databaseContent(path.join(paths.home, "userdata/statev2.sqlite")), "healthy data"); + assert.equal(await readJson(paths.request), null); +}); + +test("stopping during the commit window releases its timer promptly", async (t) => { + const { paths, env } = await world(t); + await seedDatabase(path.join(paths.home, "userdata/statev2.sqlite"), "original data"); + await fakeRuntime(paths, "0.0.45", "ready", true); + await fakeRuntime(paths, NIGHTLY, "crash-after-commit"); + const sup = await supervisor(t, env, 2000, 8000); + await requestTrial(paths, NIGHTLY); + await waitFor(async () => (await fs.readFile(path.join(paths.home, "activated"), "utf8").catch(() => "")) === NIGHTLY); + await new Promise(resolve => setTimeout(resolve, 50)); + sup.child.kill("SIGTERM"); + const exited = await Promise.race([sup.done.then(() => true), new Promise(resolve => { const timer=setTimeout(()=>resolve(false),1500);timer.unref(); })]); + assert.equal(exited, true, "the health timer must not hold shutdown open"); +}); + +test("interrupted rollback cleanup resumes without an already removed snapshot", async (t) => { + const { paths } = await world(t); + await seedDatabase(path.join(paths.home, "userdata/statev2.sqlite"), "original data"); + const request = { id: randomUUID(), version: NIGHTLY, channel: "nightly", fromVersion: "0.0.45", phase: "committed", previous: { schema: 1, channel: "stable", version: "0.0.45" } }; + await snapshotDatabase(paths, request.id); + await writeJson(paths.request, request); + await writeJson(paths.selection, { schema: 1, channel: request.channel, version: request.version, updateId: request.id }); + const rm = fs.rm; + fs.rm = async (file, options) => { + await rm(file, options); + if (file === paths.backup) throw new Error("interrupted rollback cleanup"); + }; + try { await assert.rejects(rollbackCommit(paths, request, "early exit"), /interrupted rollback cleanup/); } + finally { fs.rm = rm; } + await recoverUpdate(paths); + assert.equal((await readSelection(paths)).version, "0.0.45"); + assert.equal((await readJson(paths.operation)).phase, "rolled-back"); + assert.equal(await databaseContent(path.join(paths.home, "userdata/statev2.sqlite")), "original data"); + assert.equal(await readJson(paths.request), null); +});