From 8bc2edaeb355850517b7b35e6ae80c62e75cebc3 Mon Sep 17 00:00:00 2001 From: Marve10s Date: Thu, 15 Oct 2026 02:04:54 +0300 Subject: [PATCH 01/17] fix(server): a remote update points the service unit at the runtime it committed --- apps/server/src/cloud/bootService.ts | 10 +- apps/server/src/cloud/serviceProtocol.ts | 3 + apps/server/src/serviceLauncher.test.ts | 147 ++++++++++++++++++++++- apps/server/src/serviceLauncher.ts | 108 ++++++++++++++++- docs/internals/server-updates.md | 7 +- 5 files changed, 264 insertions(+), 11 deletions(-) diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 88d477974ae0a..9af7c30f27d08 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -28,6 +28,7 @@ import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_RESTART_PENDING_FILE, SERVICE_STATE_FILE, + SERVICE_UNIT_ENV, compareExactServiceVersions, parseServiceState, serviceStateActiveVersion, @@ -41,7 +42,6 @@ export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; // (com.t3tools.t3code), so launchd and TCC records never collide. export const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service"; export const BOOT_SERVICE_PLIST_FILE = `${BOOT_SERVICE_LAUNCHD_LABEL}.plist`; -const BOOT_SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; export const BOOT_SERVICE_LOG_FILE = "boot-service.log"; /** systemd expands `%` specifiers, including in unquoted append-log paths. */ @@ -104,7 +104,7 @@ export function renderBootServiceUnit(plan: BootServicePlan): string { "Type=simple", "WorkingDirectory=%h", `Environment=T3CODE_HOME=${quoteSystemdValue(plan.baseDir)}`, - `Environment=${BOOT_SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`, + `Environment=${SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`, `ExecStart=${plan.program.map(quoteSystemdValue).join(" ")}`, // Let the launcher mark an explicit stop before it signals the server. // systemd still SIGKILLs the whole cgroup if graceful shutdown times out. @@ -162,7 +162,7 @@ export function renderBootServicePlist( ` ${escapeXmlText(options.environmentPath)}`, ` T3CODE_HOME`, ` ${escapeXmlText(plan.baseDir)}`, - ` ${BOOT_SERVICE_UNIT_ENV}`, + ` ${SERVICE_UNIT_ENV}`, ` ${BOOT_SERVICE_PLIST_FILE}`, ` `, ` WorkingDirectory`, @@ -606,7 +606,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const xdgConfigHome = yield* Config.String("XDG_CONFIG_HOME").pipe(Config.withDefault("")); const installerPath = yield* Config.String("PATH").pipe(Config.withDefault("")); const runningInsideService = - (yield* Config.String(BOOT_SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) === + (yield* Config.String(SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) === BOOT_SERVICE_UNIT_FILE; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -890,7 +890,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { // Transient properties reach systemd literally, unlike unit-file specifiers. `--property=StandardOutput=append:${logPath}`, `--property=StandardError=append:${logPath}`, - `--setenv=${BOOT_SERVICE_UNIT_ENV}=`, + `--setenv=${SERVICE_UNIT_ENV}=`, runtimePaths.entryPath, "service", "install", diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 3b8ea548d93da..4452a91005e98 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -5,6 +5,9 @@ export const SERVICE_LAUNCHER_PROTOCOL = 3 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_ENV_FILE = "service.env"; export const SERVICE_STATE_FILE = "service-state.json"; +/** Set by the unit or launch agent to its own file name, so the launcher can + find the definition that started it. */ +export const SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; /** Written before explicit stop to distinguish shutdown from update handoff. */ export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */ diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 274b86573ed3c..5dfcd7d97d871 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -12,6 +12,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import { vi } from "vite-plus/test"; +import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootService.ts"; import { databaseBackupBlocker, Launcher, @@ -19,6 +20,8 @@ import { readServiceEnvironment, type BackupSpaceProbe, readServiceState, + repointServiceUnit, + type ServiceUnit, writeServiceState, } from "./serviceLauncher.ts"; import { @@ -111,6 +114,35 @@ it("rejects contradictory service state", () => { ); }); +const unitPlan = (root: string, version: string) => ({ + program: [`${root}/runtime/versions/${version}/t3`, "__service-launcher"], + baseDir: root, + logPath: `${root}/logs/boot-service.log`, + unitPath: `${root}/unit/t3code.service`, +}); + +it("points a unit or launch agent at another runtime and leaves the rest alone", () => { + for (const root of ["/home/user/.t3", "/home/user name/.t3", "/home/100%/.t3"]) { + assert.equal( + repointServiceUnit(renderBootServiceUnit(unitPlan(root, "1.0.0")), "1.1.0"), + renderBootServiceUnit(unitPlan(root, "1.1.0")), + ); + } + const root = "/Users/user/.t3 & co"; + const plistOptions = { + homeDir: "/Users/user", + environmentPath: `/usr/bin:${root}/runtime/versions/1.0.0`, + }; + assert.equal( + repointServiceUnit(renderBootServicePlist(unitPlan(root, "1.0.0"), plistOptions), "1.1.0"), + renderBootServicePlist(unitPlan(root, "1.1.0"), plistOptions), + ); + for (const version of ["1.1.0", "1.2.0"]) { + const unit = renderBootServiceUnit(unitPlan(root, version)); + assert.equal(repointServiceUnit(unit, "1.1.0"), unit); + } +}); + const writeFakeRuntime = ( fs: FileSystem.FileSystem, path: Path.Path, @@ -129,6 +161,28 @@ const writeFakeRuntime = ( return entryPath; }); +// The unit the launcher repoints after a commit, written where its plan says. +const fakeServiceUnit = ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + version: string, +) => + Effect.gen(function* () { + const plan = unitPlan(root, version); + yield* fs.makeDirectory(path.dirname(plan.unitPath), { recursive: true }); + yield* fs.writeFileString(plan.unitPath, renderBootServiceUnit(plan)); + const unit: ServiceUnit & { reloads: number } = { + path: plan.unitPath, + reloads: 0, + reload: () => { + unit.reloads += 1; + return Promise.resolve(); + }, + }; + return unit; + }); + it.layer(NodeServices.layer)("service state persistence", (it) => { it.effect( "loads optional service.env literally without changing inherited or managed variables", @@ -294,8 +348,85 @@ if (context.update?.status === "pending") { activeVersion: "1.0.0", }), ); + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + serviceUnit: unit, + }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update?.status, "committed"); + assert.equal( + yield* fs.readFileString(unit.path), + renderBootServiceUnit(unitPlan(root, "1.1.0")), + ); + assert.equal(unit.reloads, 1); + }), + ); + + it.effect("commits even when the unit cannot be repointed", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-unit-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + process.send({ type: "prepared", updateId: context.update.id }); + process.on("message", (message) => { + if (message.type === "committed") process.exit(0); + }); +} else if (context.update === undefined) { + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); + setInterval(() => {}, 1_000); +} else { + process.exit(0); +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + let reloads = 0; + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + serviceUnit: { + path: path.join(root, "unit", "missing.service"), + reload: () => { + reloads += 1; + return Promise.resolve(); + }, + }, + }, + ); yield* Effect.promise(() => launcher.run().then( () => Promise.reject(new Error("launcher unexpectedly completed")), @@ -306,6 +437,7 @@ if (context.update?.status === "pending") { const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.1.0"); assert.equal(state.update?.status, "committed"); + assert.equal(reloads, 0); }), ); @@ -356,7 +488,15 @@ if (context.update?.status === "pending") { activeVersion: "1.0.0", }), ); + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + serviceUnit: unit, + }, + ); yield* Effect.promise(() => main().then( () => Promise.reject(new Error("launcher unexpectedly completed")), @@ -367,6 +507,11 @@ if (context.update?.status === "pending") { const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.0.0"); assert.equal(state.update?.status, "rolled-back"); + assert.equal( + yield* fs.readFileString(unit.path), + renderBootServiceUnit(unitPlan(root, "1.0.0")), + ); + assert.equal(unit.reloads, 0); assert.equal( state.update?.status === "rolled-back" ? state.update.reason : undefined, "invalid-prepared", diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index ee05c63297843..7d6a040a71204 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -27,6 +27,7 @@ import { SERVICE_STATE_FILE, SERVICE_RESTART_PENDING_FILE, SERVICE_STOP_MARKER_FILE, + SERVICE_UNIT_ENV, } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; @@ -64,6 +65,76 @@ const runtimeSpawnArguments = (paths: ReturnType) => ({ args: ["serve"], }); +/** + * The unit or launch agent that `t3 service install` wrote. Its program names + * the runtime that ran the install, and a remote update never replaces it, so + * after a commit the launcher points it at the committed runtime. Otherwise the + * next start lands on a runtime that nothing in the service state records. + */ +export interface ServiceUnit { + readonly path: string; + /** systemd keeps a loaded copy of the unit; launchd re-reads the plist at bootstrap. */ + readonly reload: () => Promise; +} + +export function serviceUnitFromEnvironment( + env: NodeJS.ProcessEnv = process.env, +): ServiceUnit | undefined { + const unitName = env[SERVICE_UNIT_ENV]?.trim(); + const home = env.HOME?.trim(); + if (!unitName || !home) return undefined; + if (unitName.endsWith(".plist")) { + return { + path: NodePath.join(home, "Library", "LaunchAgents", unitName), + reload: () => Promise.resolve(), + }; + } + return { + path: NodePath.join(home, ".config", "systemd", "user", unitName), + reload: () => + new Promise((resolve, reject) => { + NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], { + stdio: "ignore", + timeout: 30_000, + }) + .once("error", reject) + .once("exit", (code, signal) => + code === 0 + ? resolve() + : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)), + ); + }), + }; +} + +// The executable the unit or launch agent starts: systemd's ExecStart value, +// or the plist string that precedes the __service-launcher argument. Only the +// version segment changes, so paths the renderer had to quote or escape stay +// as they are. The launch agent's PATH names a versions directory without the +// executable and is left alone. +const SERVICE_UNIT_PROGRAMS = [ + /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m, + /([^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*__service-launcher<\/string>)/, +] as const; + +/** + * The unit contents with their program pointed at `version`'s runtime. A unit + * already at that version or a newer one is returned unchanged: `t3 update` + * may have repointed it to a later version while this commit was in flight. + */ +export function repointServiceUnit(contents: string, version: string): string { + for (const program of SERVICE_UNIT_PROGRAMS) { + const match = program.exec(contents); + if (match === null) continue; + const [, prefix = "", current = "", suffix = ""] = match; + if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) { + return contents; + } + return contents.replace(program, () => `${prefix}${version}${suffix}`); + } + return contents; +} + const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const; const RESTORE_MARKER = ".restore-pending"; @@ -361,8 +432,12 @@ export async function readServiceState(filePath: string): Promise } export async function writeServiceState(filePath: string, state: ServiceState): Promise { + await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 }); + await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`); +} + +async function replaceFileDurably(filePath: string, contents: string): Promise { const directory = NodePath.dirname(filePath); - await NodeFSP.mkdir(directory, { recursive: true, mode: 0o700 }); const tempPath = NodePath.join( directory, `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`, @@ -370,7 +445,7 @@ export async function writeServiceState(filePath: string, state: ServiceState): let handle: NodeFSP.FileHandle | undefined; try { handle = await NodeFSP.open(tempPath, "wx", 0o600); - await handle.writeFile(`${JSON.stringify(state, null, 2)}\n`, "utf8"); + await handle.writeFile(contents, "utf8"); await handle.sync(); await handle.close(); handle = undefined; @@ -458,6 +533,7 @@ export class Launcher { #done = false; readonly #completion = Promise.withResolvers(); readonly #backupSpaceProbe: Partial | undefined; + readonly #serviceUnit: ServiceUnit | undefined; readonly #environment: Readonly; constructor( @@ -465,11 +541,13 @@ export class Launcher { state: ServiceState, options: { readonly backupSpaceProbe?: Partial; + readonly serviceUnit?: ServiceUnit; readonly environment?: Readonly; } = {}, ) { this.#baseDir = baseDir; this.#backupSpaceProbe = options.backupSpaceProbe; + this.#serviceUnit = options.serviceUnit; this.#environment = { ...(options.environment ?? process.env) }; this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); this.#state = state; @@ -739,6 +817,25 @@ export class Launcher { child.role = "active"; await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); await sendMessage(child.process, { type: "committed", updateId: committed.id }); + await this.#repointServiceUnit(committed.targetVersion); + } + + // The commit is durable before this runs: a unit that could not be rewritten + // leaves the service on the new version, pointed at the old runtime, as before. + async #repointServiceUnit(version: string): Promise { + const unit = this.#serviceUnit; + if (unit === undefined) return; + try { + const contents = await NodeFSP.readFile(unit.path, "utf8"); + const repointed = repointServiceUnit(contents, version); + if (repointed === contents) return; + await replaceFileDurably(unit.path, repointed); + await unit.reload(); + } catch (error) { + process.stderr.write( + `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`, + ); + } } async #handlePreparedTimeout(child: ManagedChild): Promise { @@ -837,5 +934,10 @@ export async function main(): Promise { const environment = await readServiceEnvironment(baseDir); const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); const state = await readServiceState(statePath); - await new Launcher(baseDir, state, { environment }).run(); + const serviceUnit = serviceUnitFromEnvironment(); + await new Launcher( + baseDir, + state, + serviceUnit === undefined ? { environment } : { environment, serviceUnit }, + ).run(); } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 1536884578d4b..e48cc6668d8bf 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -10,8 +10,11 @@ processes do not self-update. Exact-version installs keep restarts independent of npm cache eviction or a moving release tag. Installation and preflight happen in staging before publishing an immutable runtime. Preflight checks the launcher protocol because a target that -needs new rollback guarantees cannot safely run under an older launcher. Upgrading -that launcher requires a local service update. +needs new rollback guarantees cannot safely run under an older launcher. After a +commit the launcher points the unit at the committed runtime, so the next service +start also runs that version's launcher; the protocol check is what makes that safe. +launchd reads the plist only at its next bootstrap, so a crash respawn before then +still runs the loaded definition. ## Commit boundary From 2d2fe299f8df895f849064cc9f50450a464d4f6e Mon Sep 17 00:00:00 2001 From: Marve10s Date: Thu, 15 Oct 2026 02:10:43 +0300 Subject: [PATCH 02/17] fix(server): keep the launcher's unit lookup private --- apps/server/src/serviceLauncher.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 7d6a040a71204..63203a0ad40a8 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -77,9 +77,7 @@ export interface ServiceUnit { readonly reload: () => Promise; } -export function serviceUnitFromEnvironment( - env: NodeJS.ProcessEnv = process.env, -): ServiceUnit | undefined { +function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined { const unitName = env[SERVICE_UNIT_ENV]?.trim(); const home = env.HOME?.trim(); if (!unitName || !home) return undefined; From cfa360dc35cac5b236ba598ca189433457194695 Mon Sep 17 00:00:00 2001 From: Marve10s Date: Thu, 15 Oct 2026 02:11:54 +0300 Subject: [PATCH 03/17] fix(server): the unit is repointed even when the committed child is gone --- apps/server/src/serviceLauncher.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 63203a0ad40a8..021d94aa0d1a5 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -814,8 +814,11 @@ export class Launcher { this.#state = next; child.role = "active"; await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); - await sendMessage(child.process, { type: "committed", updateId: committed.id }); - await this.#repointServiceUnit(committed.targetVersion); + try { + await sendMessage(child.process, { type: "committed", updateId: committed.id }); + } finally { + await this.#repointServiceUnit(committed.targetVersion); + } } // The commit is durable before this runs: a unit that could not be rewritten From 5b576e506560de19edd198fe2dd17d658cd430c6 Mon Sep 17 00:00:00 2001 From: Marve10s Date: Fri, 16 Oct 2026 22:45:44 +0300 Subject: [PATCH 04/17] fix(server): a restarted launcher repoints the unit a committed update left behind --- apps/server/src/serviceLauncher.test.ts | 49 +++++++++++++++++++++++++ apps/server/src/serviceLauncher.ts | 3 ++ 2 files changed, 52 insertions(+) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 5dfcd7d97d871..f5f4e97f7266f 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -270,6 +270,55 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); + it.effect("a fresh launcher repoints a unit left behind by an interrupted commit", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + "setInterval(() => {}, 1_000);\n", + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "committed", + }, + }), + ); + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); + const run = () => + Effect.gen(function* () { + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { serviceUnit: unit }, + ); + const running = launcher.run(); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }); + + yield* run(); + assert.equal( + yield* fs.readFileString(unit.path), + renderBootServiceUnit(unitPlan(root, "1.1.0")), + ); + assert.equal(unit.reloads, 1); + + yield* run(); + assert.equal(unit.reloads, 1); + }), + ); + it.effect("stays idle instead of exiting when another server owns the T3 home", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 021d94aa0d1a5..aeac6dd0e918f 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -625,6 +625,9 @@ export class Launcher { if (update !== undefined) { await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); } + if (update?.status === "committed") { + await this.#repointServiceUnit(this.#state.activeVersion); + } await this.#startChild(this.#state.activeVersion, "active", update); return; } From a6ff33cbf5ec97f6c61749c743d3dfe6c4109a82 Mon Sep 17 00:00:00 2001 From: Marve10s Date: Fri, 16 Oct 2026 22:57:32 +0300 Subject: [PATCH 05/17] fix(server): a restarted launcher repoints the unit after any finished update --- apps/server/src/serviceLauncher.test.ts | 94 +++++++++++++------------ apps/server/src/serviceLauncher.ts | 2 - 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index f5f4e97f7266f..055ca8e72adf1 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -270,53 +270,55 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect("a fresh launcher repoints a unit left behind by an interrupted commit", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", "1.1.0"), - "setInterval(() => {}, 1_000);\n", - ); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.1.0", - update: { - id: "update-1", - fromVersion: "1.0.0", - targetVersion: "1.1.0", - status: "committed", - }, - }), - ); - const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); - const run = () => - Effect.gen(function* () { - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - { serviceUnit: unit }, - ); - const running = launcher.run(); - yield* Effect.promise(() => launcher.stop("SIGTERM")); - yield* Effect.promise(() => running); - }); - - yield* run(); - assert.equal( - yield* fs.readFileString(unit.path), - renderBootServiceUnit(unitPlan(root, "1.1.0")), - ); - assert.equal(unit.reloads, 1); + it.effect.each(["committed", "rolled-back"] as const)( + "a fresh launcher repoints a unit left behind by a %s update", + (status) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + "setInterval(() => {}, 1_000);\n", + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-1", + fromVersion: status === "committed" ? "1.0.0" : "1.1.0", + targetVersion: status === "committed" ? "1.1.0" : "1.2.0", + status, + }, + }), + ); + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); + const run = () => + Effect.gen(function* () { + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { serviceUnit: unit }, + ); + const running = launcher.run(); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }); + + yield* run(); + assert.equal( + yield* fs.readFileString(unit.path), + renderBootServiceUnit(unitPlan(root, "1.1.0")), + ); + assert.equal(unit.reloads, 1); - yield* run(); - assert.equal(unit.reloads, 1); - }), + yield* run(); + assert.equal(unit.reloads, 1); + }), ); it.effect("stays idle instead of exiting when another server owns the T3 home", () => diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index aeac6dd0e918f..257f787058134 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -624,8 +624,6 @@ export class Launcher { if (update?.status !== "pending") { if (update !== undefined) { await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); - } - if (update?.status === "committed") { await this.#repointServiceUnit(this.#state.activeVersion); } await this.#startChild(this.#state.activeVersion, "active", update); From 8f436a3c23908662bc6206fc3525911af4782659 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 06:22:03 +0000 Subject: [PATCH 06/17] test(server): cover service unit recovery and launcher upgrade gate --- apps/server/src/serviceLauncher.test.ts | 238 ++++++++++++------------ 1 file changed, 124 insertions(+), 114 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 055ca8e72adf1..268874296e56c 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -2,6 +2,8 @@ import type * as NodeChildProcess from "node:child_process"; // @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback. import * as NodeFS from "node:fs"; +// @effect-diagnostics-next-line nodeBuiltinImport:off - models the service manager reading disk through the launcher's Promise callback. +import * as NodeFSP from "node:fs/promises"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -20,8 +22,6 @@ import { readServiceEnvironment, type BackupSpaceProbe, readServiceState, - repointServiceUnit, - type ServiceUnit, writeServiceState, } from "./serviceLauncher.ts"; import { @@ -121,28 +121,6 @@ const unitPlan = (root: string, version: string) => ({ unitPath: `${root}/unit/t3code.service`, }); -it("points a unit or launch agent at another runtime and leaves the rest alone", () => { - for (const root of ["/home/user/.t3", "/home/user name/.t3", "/home/100%/.t3"]) { - assert.equal( - repointServiceUnit(renderBootServiceUnit(unitPlan(root, "1.0.0")), "1.1.0"), - renderBootServiceUnit(unitPlan(root, "1.1.0")), - ); - } - const root = "/Users/user/.t3 & co"; - const plistOptions = { - homeDir: "/Users/user", - environmentPath: `/usr/bin:${root}/runtime/versions/1.0.0`, - }; - assert.equal( - repointServiceUnit(renderBootServicePlist(unitPlan(root, "1.0.0"), plistOptions), "1.1.0"), - renderBootServicePlist(unitPlan(root, "1.1.0"), plistOptions), - ); - for (const version of ["1.1.0", "1.2.0"]) { - const unit = renderBootServiceUnit(unitPlan(root, version)); - assert.equal(repointServiceUnit(unit, "1.1.0"), unit); - } -}); - const writeFakeRuntime = ( fs: FileSystem.FileSystem, path: Path.Path, @@ -161,23 +139,27 @@ const writeFakeRuntime = ( return entryPath; }); -// The unit the launcher repoints after a commit, written where its plan says. const fakeServiceUnit = ( fs: FileSystem.FileSystem, path: Path.Path, root: string, version: string, + render = renderBootServiceUnit, ) => Effect.gen(function* () { const plan = unitPlan(root, version); yield* fs.makeDirectory(path.dirname(plan.unitPath), { recursive: true }); - yield* fs.writeFileString(plan.unitPath, renderBootServiceUnit(plan)); - const unit: ServiceUnit & { reloads: number } = { + const contents = render(plan); + yield* fs.writeFileString(plan.unitPath, contents); + const unit = { path: plan.unitPath, reloads: 0, - reload: () => { + loadedContents: contents, + failReload: false, + reload: async () => { unit.reloads += 1; - return Promise.resolve(); + if (unit.failReload) throw new Error("daemon-reload failed"); + unit.loadedContents = await NodeFSP.readFile(unit.path, "utf8"); }, }; return unit; @@ -270,9 +252,9 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect.each(["committed", "rolled-back"] as const)( - "a fresh launcher repoints a unit left behind by a %s update", - (status) => + it.effect.each(["1.0.0", "1.2.0"])( + "recovers a rolled-back update without downgrading the installed unit at %s", + (unitVersion) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -290,13 +272,16 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { activeVersion: "1.1.0", update: { id: "update-1", - fromVersion: status === "committed" ? "1.0.0" : "1.1.0", - targetVersion: status === "committed" ? "1.1.0" : "1.2.0", - status, + fromVersion: "1.1.0", + targetVersion: "1.2.0", + status: "rolled-back", }, }), ); - const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); + const unit = yield* fakeServiceUnit(fs, path, root, unitVersion); + const expected = renderBootServiceUnit( + unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion), + ); const run = () => Effect.gen(function* () { const launcher = new Launcher( @@ -310,14 +295,12 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }); yield* run(); - assert.equal( - yield* fs.readFileString(unit.path), - renderBootServiceUnit(unitPlan(root, "1.1.0")), - ); - assert.equal(unit.reloads, 1); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); yield* run(); - assert.equal(unit.reloads, 1); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); }), ); @@ -361,90 +344,71 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect("commits only after the trial reports prepared", () => + it.effect("serializes shutdown with launcher recovery", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-stop-" }); const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - yield* fs.writeFileString(databasePath, "before trial"); - const encodedDatabasePath = JSON.stringify(databasePath); - const childSource = ` -const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); -if (context.update?.status === "pending") { - process.send({ type: "prepared", updateId: context.update.id }); - process.on("message", (message) => { - if (message.type === "committed") process.exit(0); - }); -} else if (context.update === undefined) { - process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); - setInterval(() => {}, 1_000); -} else { - process.exit(0); -} -`; - for (const version of ["1.0.0", "1.1.0"]) { - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", version), - childSource, - ); - } + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + "setInterval(() => {}, 1_000);\n", + ); yield* Effect.promise(() => writeServiceState(statePath, { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); - const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); - - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - { - serviceUnit: unit, - }, - ); - yield* Effect.promise(() => - launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), - ), - ); - const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.1.0"); - assert.equal(state.update?.status, "committed"); - assert.equal( - yield* fs.readFileString(unit.path), - renderBootServiceUnit(unitPlan(root, "1.1.0")), - ); - assert.equal(unit.reloads, 1); + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const running = launcher.run(); + const stopping = launcher.stop("SIGTERM"); + assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); + yield* Effect.promise(() => stopping); + yield* Effect.promise(() => running); }), ); - it.effect("commits even when the unit cannot be repointed", () => + const commitUpdate = (render = renderBootServiceUnit, missingUnit = false) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-unit-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service 100% & -" }); const statePath = path.join(root, "runtime", "service-state.json"); const databasePath = path.join(root, "userdata", "state.sqlite"); + const activatedPath = path.join(root, "activated"); yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); yield* fs.writeFileString(databasePath, "before trial"); - const encodedDatabasePath = JSON.stringify(databasePath); + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0", render); + if (missingUnit) yield* fs.remove(unit.path); const childSource = ` +import assert from "node:assert/strict"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { - process.send({ type: "prepared", updateId: context.update.id }); + const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8")); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update.status, "pending"); + ${ + missingUnit + ? `assert.equal(existsSync(${JSON.stringify(unit.path)}), false);` + : `assert.equal(readFileSync(${JSON.stringify(unit.path)}, "utf8"), ${JSON.stringify(unit.loadedContents)});` + } process.on("message", (message) => { - if (message.type === "committed") process.exit(0); + if (message.type === "committed") { + const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8")); + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update.status, "committed"); + writeFileSync(${JSON.stringify(activatedPath)}, message.updateId); + process.exit(0); + } }); + process.send({ type: "prepared", updateId: context.update.id }); } else if (context.update === undefined) { - process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} }); setInterval(() => {}, 1_000); } else { process.exit(0); @@ -464,34 +428,80 @@ if (context.update?.status === "pending") { activeVersion: "1.0.0", }), ); - let reloads = 0; const launcher = new Launcher( root, yield* Effect.promise(() => readServiceState(statePath)), - { - serviceUnit: { - path: path.join(root, "unit", "missing.service"), - reload: () => { - reloads += 1; - return Promise.resolve(); - }, - }, - }, + { serviceUnit: unit }, ); - yield* Effect.promise(() => + const failure = yield* Effect.promise(() => launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), + () => "launcher unexpectedly completed", + (cause: unknown) => String(cause), ), ); - + assert.include(failure, "Active child exited unexpectedly (0)"); const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.1.0"); assert.equal(state.update?.status, "committed"); - assert.equal(reloads, 0); + assert.equal(yield* fs.readFileString(activatedPath), state.update?.id); + const recover = () => + Effect.gen(function* () { + const fresh = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { serviceUnit: unit }, + ); + const running = fresh.run(); + yield* Effect.promise(() => fresh.stop("SIGTERM")); + yield* Effect.promise(() => running); + }); + return { fs, root, unit, recover }; + }); + + it.effect.each([ + { name: "systemd", render: renderBootServiceUnit }, + { + name: "launchd", + render: (plan: Parameters[0]) => + renderBootServicePlist(plan, { + homeDir: "/Users/test & ", + environmentPath: "/usr/bin:/custom/runtime/versions/1.0.0", + }), + }, + ])("commits and repoints $name only after the trial reports prepared", ({ render }) => + Effect.gen(function* () { + const { fs, root, unit } = yield* commitUpdate(render); + const expected = render(unitPlan(root, "1.1.0")); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + assert.equal(unit.reloads, 1); }), ); + it.effect( + "keeps a durable commit through unit failure and retries a failed reload on restart", + () => + Effect.gen(function* () { + const { fs, root, unit, recover } = yield* commitUpdate(renderBootServiceUnit, true); + assert.equal(unit.reloads, 0); + const original = renderBootServiceUnit(unitPlan(root, "1.0.0")); + const expected = renderBootServiceUnit(unitPlan(root, "1.1.0")); + assert.equal(unit.loadedContents, original); + yield* fs.writeFileString(unit.path, original); + unit.failReload = true; + yield* recover(); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.reloads, 1); + assert.equal(unit.loadedContents, original); + + unit.failReload = false; + yield* recover(); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + assert.equal(unit.reloads, 2); + }), + ); + it.effect("rolls back invalid trials with the startup environment", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; From 9d2e78b54a78b9dc60d3ea169c062291ab1973a2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 06:36:29 +0000 Subject: [PATCH 07/17] fix(server): coordinate service units with remote updates --- apps/server/src/cloud/bootService.test.ts | 182 ++++++++++++++++++ apps/server/src/cloud/bootService.ts | 137 ++++++++----- apps/server/src/cloud/serviceProtocol.ts | 6 +- apps/server/src/cloud/serviceSelectionLock.ts | 71 +++++++ apps/server/src/serviceLauncher.ts | 66 +++++-- docs/internals/server-updates.md | 14 +- 6 files changed, 404 insertions(+), 72 deletions(-) create mode 100644 apps/server/src/cloud/serviceSelectionLock.ts diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 814d470cad410..d227536a80db8 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Cross-process launcher contention uses real filesystem leases and IPC barriers. +import * as NodeChildProcess from "node:child_process"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -5,11 +7,13 @@ import * as ConfigProvider from "effect/ConfigProvider"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import { CliError } from "effect/cli"; import { HttpClient } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import { vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; import * as BootService from "./bootService.ts"; @@ -22,6 +26,24 @@ import { serviceStateHasPendingUpdate, } from "./serviceProtocol.ts"; +const leaseObserver = vi.hoisted(() => ({ contended: (_destination: string) => {} })); +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + renameSync: (...args: Parameters) => { + try { + return actual.renameSync(...args); + } catch (error) { + if (["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) { + leaseObserver.contended(String(args[1])); + } + throw error; + } + }, + }; +}); + it("reads the served T3 home back out of a rendered unit or plist", () => { const plan = (baseDir: string) => ({ program: [`${baseDir}/runtime/versions/1.2.3/t3`, "__service-launcher"], @@ -442,6 +464,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => { current: false, installedVersion: "1.2.4-nightly.1", }); + yield* service.install({ allowDowngrade: true }); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + }); } }), ); @@ -578,6 +605,156 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); + it.effect( + "serializes a local install with another process reconciling an older unit snapshot", + () => + Effect.gen(function* () { + const { service, fs, statePath, makeService, runtime } = yield* makeHarness(); + const path = yield* Path.Path; + const plan = yield* service.install(); + const unit = yield* fs.readFileString(plan.unitPath); + yield* fs.writeFileString( + plan.unitPath, + unit.replace("versions/1.2.3/t3", "versions/1.2.2/t3"), + ); + yield* fs.writeFileString( + statePath, + JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + update: { + id: "committed", + fromVersion: "1.2.2", + targetVersion: "1.2.3", + status: "committed", + }, + }), + ); + const rejection = path.join(plan.baseDir, "rejected-update"); + const databasePath = path.join(plan.baseDir, "state.sqlite"); + yield* fs.writeFileString(databasePath, "database"); + yield* fs.writeFileString( + runtime.entryPath, + `#!${process.execPath} +import { writeFileSync } from "node:fs"; +process.on("message", message => { + if (message === "request-update") process.send({ type: "request-update", targetVersion: "1.2.5", dbPath: ${JSON.stringify(databasePath)} }); + if (message.type === "update-rejected") { + writeFileSync(${JSON.stringify(rejection)}, message.reason); + process.exit(0); + } +}); +`, + ); + yield* fs.chmod(runtime.entryPath, 0o755); + const newer = yield* makeService(undefined, "1.2.4"); + yield* makeService(undefined, "1.2.5"); + const snapshot = Promise.withResolvers(); + const milestone = Promise.withResolvers(); + const script = path.join(plan.baseDir, "launcher-contention.mjs"); + yield* fs.writeFileString( + script, + ` +import fs from "node:fs/promises"; +import cp from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +const server = Promise.withResolvers(); +const originalSpawn = cp.spawn; +cp.spawn = (...args) => { const child = originalSpawn(...args); server.resolve(child); return child; }; +process.on("message", async message => { + if (message === "installed") (await server.promise).send("request-update"); +}); +const originalRead = fs.readFile; +fs.readFile = async (...args) => { + const contents = await originalRead(...args); + if (args[0] === ${JSON.stringify(plan.unitPath)}) { + fs.readFile = originalRead; + syncBuiltinESMExports(); + const resume = new Promise(resolve => process.once("message", resolve)); + process.send("snapshot"); + await resume; + } + return contents; +}; +syncBuiltinESMExports(); +const { Launcher, readServiceState } = await import(${JSON.stringify(import.meta.resolve("../serviceLauncher.ts"))}); +const launcher = new Launcher(${JSON.stringify(plan.baseDir)}, await readServiceState(${JSON.stringify(statePath)}), { + serviceUnit: { path: ${JSON.stringify(plan.unitPath)}, reload: async () => {} }, +}); +try { await launcher.run(); throw Error("unexpected completion"); } +catch (error) { + if (!error.message.includes("Active child exited unexpectedly (0)")) throw error; +} +const { acquireServiceSelectionLock } = await import(${JSON.stringify(import.meta.resolve("./serviceSelectionLock.ts"))}); +// Exit owning the real lease, as a crashed process would. +await acquireServiceSelectionLock(${JSON.stringify(plan.unitPath)}); +process.disconnect(); +`, + ); + const child = yield* Effect.acquireRelease( + Effect.sync(() => + NodeChildProcess.fork(script, [], { + execArgv: [], + stdio: ["ignore", "inherit", "inherit", "ipc"], + }), + ), + (child) => + Effect.sync(() => { + if (child.exitCode === null && child.signalCode === null) child.kill(); + }), + ); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => + code === 0 ? resolve() : reject(new Error(`launcher exited ${code}`)), + ); + }); + child.on("message", (message) => { + if (message === "snapshot") snapshot.resolve(); + }); + yield* Effect.promise(() => + Promise.race([ + snapshot.promise, + exited.then(() => { + throw new Error("launcher exited before snapshot"); + }), + ]), + ); + leaseObserver.contended = (destination) => { + if (destination === plan.unitPath + ".selection-lock") milestone.resolve(); + }; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + leaseObserver.contended = () => {}; + }), + ); + const installing = yield* newer + .install({ start: false }) + .pipe(Effect.ensuring(Effect.sync(() => milestone.resolve())), Effect.forkChild); + // Either observe real OS contention or let an uncoordinated installer finish + // before releasing the stale snapshot. The latter deterministically loses C. + yield* Effect.promise(() => milestone.promise); + child.send("resume"); + yield* Fiber.join(installing); + child.send("installed"); + yield* Effect.promise(() => exited); + expect(yield* fs.readFileString(rejection)).toContain("Service selection changed"); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.4", + }); + expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3"); + expect( + yield* fs.readFileString( + path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE), + ), + ).toBe("1.2.4\n"); + // The next installer reclaims the dead owner and releases its own lease. + yield* newer.install({ start: false }); + expect(yield* fs.exists(plan.unitPath + ".selection-lock")).toBe(false); + }), + ); + it.effect("install with start=false keeps the marker when a later write fails", () => Effect.gen(function* () { const { service, fs, statePath, makeService } = yield* makeHarness(); @@ -617,6 +794,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const error = yield* service.install({ start: false }).pipe(Effect.flip); expect(error._tag).toBe("BootServiceUpdatePendingError"); expect(yield* fs.readFileString(statePath)).toBe(pendingState); + yield* fs.remove((yield* service.status).unitPath); + expect((yield* service.install({ start: false }).pipe(Effect.flip))._tag).toBe( + "BootServiceUpdatePendingError", + ); + expect(yield* fs.readFileString(statePath)).toBe(pendingState); }), ); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 9af7c30f27d08..fd09a7ea34b02 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -16,6 +16,7 @@ import * as Schema from "effect/Schema"; import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease"; import * as ProcessRunner from "../processRunner.ts"; +import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts"; import { ensurePinnedRuntimeInstalled, launcherOwnsVersionsDir, @@ -799,6 +800,16 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { if (remaining[0]) return yield* new BootServicePrerequisiteError({ problem: remaining[0] }); }); + const withSelectionLock = (selectionPath: string, operation: Effect.Effect) => + Effect.acquireUseRelease( + Effect.tryPromise({ + try: () => acquireServiceSelectionLock(selectionPath), + catch: (cause) => new BootServiceInstallError({ cause }), + }), + () => operation, + (release) => Effect.sync(release), + ); + const install = Effect.fn("cloud.boot_service.install")(function* (options?: { readonly allowDowngrade?: boolean; readonly start?: boolean; @@ -876,8 +887,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const installed = yield* fs .exists(unitPath) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - // With start=false, changing startup-only files is safe except during a remote update, - // when the launcher also writes its state file. + // Stop outside the selection lock: the launcher may be waiting to acquire it. const start = options?.start !== false; if (installed && start && runningInsideService && manager.kind === "systemd") { // Stopping the unit from inside it kills this process before the start. @@ -905,58 +915,81 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { yield* runSteps(manager.stop); } - yield* Effect.gen(function* () { - if (installed) { - const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); - if (Option.isSome(previousStateText)) { - if (serviceStateHasPendingUpdate(previousStateText.value)) { - return yield* new BootServiceUpdatePendingError(); + // State first, then unit. The launcher acquires only one at a time, and a + // failure to lock the unit must never prevent its durable update commit. + yield* withSelectionLock( + statePath, + withSelectionLock( + unitPath, + Effect.gen(function* () { + const currentlyInstalled = yield* fs.exists(unitPath); + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if (Option.isSome(previousStateText)) { + if (serviceStateHasPendingUpdate(previousStateText.value)) { + return yield* new BootServiceUpdatePendingError(); + } + // A remote update can finish after the CLI checks status. Read its + // final version after the launcher stops and before changing files. + const installedVersion = serviceStateActiveVersion(previousStateText.value); + if ( + installedVersion !== undefined && + options?.allowDowngrade !== true && + compareExactServiceVersions(input.cliVersion, installedVersion) < 0 + ) { + return yield* new BootServiceDowngradeRefusedError({ + installedVersion, + targetVersion: input.cliVersion, + }); + } } - // Read the final version after stopping the launcher; a remote update may finish after the status check. - const installedVersion = serviceStateActiveVersion(previousStateText.value); - if ( - installedVersion !== undefined && - options?.allowDowngrade !== true && - compareExactServiceVersions(input.cliVersion, installedVersion) < 0 - ) { - return yield* new BootServiceDowngradeRefusedError({ - installedVersion, - targetVersion: input.cliVersion, - }); + yield* fs + .makeDirectory(path.dirname(unitPath), { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + if (!start && (currentlyInstalled || Option.isSome(previousStateText))) { + // Written first: once the files below name the new version, the running + // service is behind them, and a failure between the two writes must not + // leave it looking current. + yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 }); } - } - } - yield* fs - .makeDirectory(path.dirname(unitPath), { recursive: true }) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - // Files can match the new version even when activation fails; keep that install retryable. - yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 }); - yield* writeDurably( - statePath, - // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document. - `${JSON.stringify( - { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: input.cliVersion, - } satisfies ServiceState, - null, - 2, - )}\n`, - ); - if (!start && installed) { - // A concurrent remote update can overwrite launcher state; detect it before repointing the unit. - const written = yield* fs.readFileString(statePath); - if (serviceStateActiveVersion(written) !== input.cliVersion) { - return yield* new BootServiceUpdatePendingError(); - } - } - yield* writeDurably(unitPath, manager.render(plan)); - - if (start) { - yield* runSteps(manager.activate); - yield* fs.remove(restartPendingPath, { force: true }); - } - }).pipe( + yield* writeDurably( + statePath, + // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document. + `${JSON.stringify( + { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: input.cliVersion, + } satisfies ServiceState, + null, + 2, + )}\n`, + ); + if (!start && (currentlyInstalled || Option.isSome(previousStateText))) { + const written = yield* fs.readFileString(statePath); + if (serviceStateActiveVersion(written) !== input.cliVersion) { + return yield* new BootServiceUpdatePendingError(); + } + } + yield* writeDurably(unitPath, manager.render(plan)); + }), + ), + ).pipe( + Effect.andThen( + Effect.gen(function* () { + if (start) { + yield* runSteps(manager.activate); + yield* withSelectionLock( + statePath, + Effect.gen(function* () { + // A concurrent deferred install may now be waiting for a newer restart. + const awaited = yield* fs.readFileString(restartPendingPath).pipe(Effect.option); + if (Option.isSome(awaited) && awaited.value.trim() === input.cliVersion) { + yield* fs.remove(restartPendingPath, { force: true }); + } + }), + ); + } + }), + ), Effect.mapError((cause) => cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause, ), diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 4452a91005e98..035df149002a7 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -1,7 +1,9 @@ import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; -// Bump for incompatible runtime layout changes; launchers survive self-updates. Protocol 3 requires standalone executables. -export const SERVICE_LAUNCHER_PROTOCOL = 3 as const; +// Protocol 4 requires coordinated selection writes and unit reconciliation. An +// installed protocol-3 launcher needs one local update or service install first: +// remote updates replace its child, so cannot add these launcher guarantees. +export const SERVICE_LAUNCHER_PROTOCOL = 4 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_ENV_FILE = "service.env"; export const SERVICE_STATE_FILE = "service-state.json"; diff --git a/apps/server/src/cloud/serviceSelectionLock.ts b/apps/server/src/cloud/serviceSelectionLock.ts new file mode 100644 index 0000000000000..2d085bd77dab1 --- /dev/null +++ b/apps/server/src/cloud/serviceSelectionLock.ts @@ -0,0 +1,71 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Shared with the standalone launcher, which has no Effect runtime. +// @effect-diagnostics globalTimers:off -- Bounded cross-process lease acquisition in the standalone launcher. +// @effect-diagnostics globalDate:off -- The standalone launcher has no Effect clock for its acquisition deadline. +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +/** Serialize durable selection and unit writes, never manager stop/start commands. */ +export async function acquireServiceSelectionLock(selectionPath: string): Promise<() => void> { + const directory = NodePath.dirname(selectionPath); + NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const lock = selectionPath + ".selection-lock"; + const owner = `${process.pid}.${NodeCrypto.randomUUID()}`; + const candidate = `${lock}.${owner}`; + const deadline = Date.now() + 30_000; + const removeEmptyLock = () => { + try { + NodeFS.rmdirSync(lock); + } catch (error) { + if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes((error as NodeJS.ErrnoException).code ?? "")) + throw error; + } + }; + NodeFS.mkdirSync(candidate, { mode: 0o700 }); + try { + NodeFS.writeFileSync(NodePath.join(candidate, owner), "", { mode: 0o600 }); + while (true) { + try { + // Publish the owner atomically; rename cannot replace another populated directory. + NodeFS.renameSync(candidate, lock); + return () => { + NodeFS.unlinkSync(NodePath.join(lock, owner)); + removeEmptyLock(); + }; + } catch (error) { + if (!["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) + throw error; + } + let owners: string[]; + try { + owners = NodeFS.readdirSync(lock); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + if (owners.length === 1) { + const previous = owners[0]!; + const pid = Number(previous.split(".", 1)[0]); + if (!Number.isSafeInteger(pid) || pid <= 0) + throw new Error("Invalid service selection lock owner."); + try { + process.kill(pid, 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + // Remove only this dead owner's unique file, never a replacement owner's. + try { + NodeFS.unlinkSync(NodePath.join(lock, previous)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + } + removeEmptyLock(); + if (Date.now() >= deadline) + throw new Error("Service selection is locked. Retry when the other operation finishes."); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } finally { + NodeFS.rmSync(candidate, { recursive: true, force: true }); + } +} diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 257f787058134..9954a1a5637d8 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -7,6 +7,7 @@ import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; +import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts"; import type { PendingServiceUpdate, @@ -120,7 +121,7 @@ const SERVICE_UNIT_PROGRAMS = [ * already at that version or a newer one is returned unchanged: `t3 update` * may have repointed it to a later version while this commit was in flight. */ -export function repointServiceUnit(contents: string, version: string): string { +function repointServiceUnit(contents: string, version: string): string { for (const program of SERVICE_UNIT_PROGRAMS) { const match = program.exec(contents); if (match === null) continue; @@ -612,13 +613,38 @@ export class Launcher { this.#timer = undefined; } + async #selectState(next: ServiceState): Promise { + const release = await acquireServiceSelectionLock(this.#statePath); + try { + const selected = await readServiceState(this.#statePath); + // Local installs replace the selection and clear its update record. An old + // launcher must not restore its own selection when asked to update again. + if ( + selected.activeVersion !== this.#state.activeVersion || + selected.update?.id !== this.#state.update?.id || + selected.update?.status !== this.#state.update?.status + ) { + throw new Error("Service selection changed. Restart the service before updating."); + } + await writeServiceState(this.#statePath, next); + this.#state = next; + } finally { + release(); + } + } + async #recover(): Promise { // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); const restartPending = restartPendingPath(this.#baseDir); - const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined); - if (awaitedVersion?.trim() === this.#state.activeVersion) { - await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined); + const release = await acquireServiceSelectionLock(this.#statePath); + try { + const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined); + if (awaitedVersion?.trim() === this.#state.activeVersion) { + await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined); + } + } finally { + release(); } const update = this.#state.update; if (update?.status !== "pending") { @@ -773,8 +799,12 @@ export class Launcher { status: "pending", }; const next: ServiceState = { ...this.#state, update: pending }; - await writeServiceState(this.#statePath, next); - this.#state = next; + try { + await this.#selectState(next); + } catch (error) { + await reject(error instanceof Error ? error.message : String(error)); + return; + } await sendMessage(child.process, { type: "update-accepted", updateId: pending.id }); this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS); } @@ -811,8 +841,7 @@ export class Launcher { activeVersion: pending.targetVersion, update: committed, }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#selectState(next); child.role = "active"; await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); try { @@ -828,11 +857,16 @@ export class Launcher { const unit = this.#serviceUnit; if (unit === undefined) return; try { - const contents = await NodeFSP.readFile(unit.path, "utf8"); - const repointed = repointServiceUnit(contents, version); - if (repointed === contents) return; - await replaceFileDurably(unit.path, repointed); - await unit.reload(); + const release = await acquireServiceSelectionLock(unit.path); + try { + const contents = await NodeFSP.readFile(unit.path, "utf8"); + const repointed = repointServiceUnit(contents, version); + if (repointed !== contents) await replaceFileDurably(unit.path, repointed); + // A previous rename may have succeeded before reload failed. + await unit.reload(); + } finally { + release(); + } } catch (error) { process.stderr.write( `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`, @@ -893,8 +927,7 @@ export class Launcher { activeVersion: pending.fromVersion, update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#selectState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); } // Signal listeners alone do not keep Node alive. @@ -921,8 +954,7 @@ export class Launcher { activeVersion: pending.fromVersion, update: outcome, }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#selectState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); await this.#startChild(next.activeVersion, "active", outcome); } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index e48cc6668d8bf..a911693b6ac05 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -4,7 +4,12 @@ A [stable launcher](../../apps/server/src/serviceLauncher.ts) owns the runtime selected by systemd or launchd. It is the only runtime writer of durable service state. Server children request updates over inherited IPC; they never rewrite their service definition or select their own replacement. Local service commands -may replace the launcher and state while the service is stopped. Foreground CLI +may replace the launcher and state, optionally deferring the restart. Both writers +serialize state selection and unit writes across processes. The installer takes +the state lease before the unit lease; the launcher never holds both, so unit +failures cannot prevent durable commit. Manager stop/start commands stay outside +both leases so stopping a launcher cannot deadlock. A running launcher refuses +another update after a local command changes its selection. Foreground CLI processes do not self-update. Exact-version installs keep restarts independent of npm cache eviction or a moving @@ -13,6 +18,13 @@ immutable runtime. Preflight checks the launcher protocol because a target that needs new rollback guarantees cannot safely run under an older launcher. After a commit the launcher points the unit at the committed runtime, so the next service start also runs that version's launcher; the protocol check is what makes that safe. +Protocol 4 adds these guarantees. An installed protocol-3 launcher requires one +local `t3 update` or `t3 service install` before remote updates are accepted; a +child update cannot upgrade its supervising launcher. Terminal-state recovery +retries reconciliation and reload even when the file already names the selected +runtime, since a previous reload may have failed. A newer locally installed unit +is preserved. + launchd reads the plist only at its next bootstrap, so a crash respawn before then still runs the loaded definition. From 55edc19ab847c9819aac991f5e9b0400b5397fad Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 06:49:41 +0000 Subject: [PATCH 08/17] test(server): cover local upgrade from protocol 3 --- apps/server/src/serviceLauncher.test.ts | 49 +++++++++++++++++-------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 268874296e56c..90bfff385ff56 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -372,7 +372,11 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - const commitUpdate = (render = renderBootServiceUnit, missingUnit = false) => + const commitUpdate = ( + render = renderBootServiceUnit, + missingUnit = false, + legacyInstall = false, + ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -388,6 +392,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { import assert from "node:assert/strict"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +assert.equal(context.protocol, ${SERVICE_LAUNCHER_PROTOCOL}); if (context.update?.status === "pending") { const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8")); assert.equal(state.activeVersion, "1.0.0"); @@ -400,6 +405,7 @@ if (context.update?.status === "pending") { process.on("message", (message) => { if (message.type === "committed") { const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8")); + assert.equal(state.protocol, ${SERVICE_LAUNCHER_PROTOCOL}); assert.equal(state.activeVersion, "1.1.0"); assert.equal(state.update.status, "committed"); writeFileSync(${JSON.stringify(activatedPath)}, message.updateId); @@ -422,11 +428,17 @@ if (context.update?.status === "pending") { childSource, ); } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), + // The old CLI installs the new runtime but writes state with its own protocol. + yield* fs.writeFileString( + statePath, + `${JSON.stringify( + { + protocol: legacyInstall ? 3 : SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }, + null, + 2, + )}\n`, ); const launcher = new Launcher( root, @@ -459,23 +471,30 @@ if (context.update?.status === "pending") { }); it.effect.each([ - { name: "systemd", render: renderBootServiceUnit }, + { + name: "systemd after a protocol-3 local upgrade", + render: renderBootServiceUnit, + legacyInstall: true, + }, { name: "launchd", + legacyInstall: false, render: (plan: Parameters[0]) => renderBootServicePlist(plan, { homeDir: "/Users/test & ", environmentPath: "/usr/bin:/custom/runtime/versions/1.0.0", }), }, - ])("commits and repoints $name only after the trial reports prepared", ({ render }) => - Effect.gen(function* () { - const { fs, root, unit } = yield* commitUpdate(render); - const expected = render(unitPlan(root, "1.1.0")); - assert.equal(yield* fs.readFileString(unit.path), expected); - assert.equal(unit.loadedContents, expected); - assert.equal(unit.reloads, 1); - }), + ])( + "commits and repoints $name only after the trial reports prepared", + ({ render, legacyInstall }) => + Effect.gen(function* () { + const { fs, root, unit } = yield* commitUpdate(render, false, legacyInstall); + const expected = render(unitPlan(root, "1.1.0")); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + assert.equal(unit.reloads, 1); + }), ); it.effect( From 0121bd916961df0a53bdef0d5dc79796b3c279d2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 06:55:39 +0000 Subject: [PATCH 09/17] test(server): cover restart marker after newer service install --- apps/server/src/cloud/bootService.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index d227536a80db8..b30b2a4b22e5c 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -602,6 +602,18 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect(yield* newer.restart).toBe("restarted"); expect((yield* newer.status).problems).not.toContain("restart-pending"); expect((yield* newer.status).current).toBe(true); + + const latest = yield* makeService(undefined, "1.2.5"); + yield* latest.install({ start: true }); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.5", + }); + expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.5/t3"); + expect((yield* latest.status).current).toBe(true); + expect( + yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE)), + ).toBe(false); }), ); From bb9bc4690f6442bb39bbe2501af4d8a4b044c492 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:01:05 +0000 Subject: [PATCH 10/17] fix(server): recover service upgrades from legacy launcher state --- apps/server/src/cloud/bootService.ts | 20 +++++++++++++----- apps/server/src/cloud/serviceProtocol.ts | 5 +++-- apps/server/src/cloud/serviceSelectionLock.ts | 21 ++++++++++++------- apps/server/src/serviceLauncher.ts | 11 +++++++++- docs/internals/server-updates.md | 9 +++++--- 5 files changed, 47 insertions(+), 19 deletions(-) diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index fd09a7ea34b02..0f98d82ff0c12 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -497,9 +497,12 @@ export class BootServicePrerequisiteError extends Schema.TaggedError()( "BootServiceUpdatePendingError", - {}, + { recoveryCommand: Schema.optional(Schema.String) }, ) { override get message(): string { + if (this.recoveryCommand !== undefined) { + return `A remote server update is still pending, but its service definition is missing. With the service stopped, run this on the server machine: ${this.recoveryCommand}\nWait for the update to finish or roll back, stop the foreground launcher with Ctrl-C, then retry the install.`; + } return "A remote server update is still pending. Wait for it to finish, then retry."; } } @@ -925,12 +928,19 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const currentlyInstalled = yield* fs.exists(unitPath); const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); if (Option.isSome(previousStateText)) { + const installedVersion = serviceStateActiveVersion(previousStateText.value); if (serviceStateHasPendingUpdate(previousStateText.value)) { - return yield* new BootServiceUpdatePendingError(); + const quote = (word: string) => `'${word.replaceAll("'", "'\\''")}'`; + return yield* new BootServiceUpdatePendingError( + !currentlyInstalled && installedVersion !== undefined + ? { + recoveryCommand: `T3CODE_HOME=${quote(input.baseDir)} ${quote(pinnedRuntimePaths(path, input.baseDir, installedVersion, platform).entryPath)} __service-launcher`, + } + : {}, + ); } // A remote update can finish after the CLI checks status. Read its // final version after the launcher stops and before changing files. - const installedVersion = serviceStateActiveVersion(previousStateText.value); if ( installedVersion !== undefined && options?.allowDowngrade !== true && @@ -981,8 +991,8 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { statePath, Effect.gen(function* () { // A concurrent deferred install may now be waiting for a newer restart. - const awaited = yield* fs.readFileString(restartPendingPath).pipe(Effect.option); - if (Option.isSome(awaited) && awaited.value.trim() === input.cliVersion) { + const selected = parseServiceState(yield* fs.readFileString(statePath)); + if (selected?.activeVersion === input.cliVersion && selected.update === undefined) { yield* fs.remove(restartPendingPath, { force: true }); } }), diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 035df149002a7..c465ce9a9c138 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -1,7 +1,7 @@ import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; // Protocol 4 requires coordinated selection writes and unit reconciliation. An -// installed protocol-3 launcher needs one local update or service install first: +// installed protocol-3 launcher needs one local update or service install with a restart: // remote updates replace its child, so cannot add these launcher guarantees. export const SERVICE_LAUNCHER_PROTOCOL = 4 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; @@ -148,7 +148,8 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { if (!isRecord(value)) return undefined; const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update); if ( - value.protocol !== SERVICE_LAUNCHER_PROTOCOL || + // Protocol 3 has the same durable format; its installer can select a protocol-4 runtime. + (value.protocol !== 3 && value.protocol !== SERVICE_LAUNCHER_PROTOCOL) || typeof value.activeVersion !== "string" || !isExactServiceVersion(value.activeVersion) || (value.update !== undefined && update === undefined) || diff --git a/apps/server/src/cloud/serviceSelectionLock.ts b/apps/server/src/cloud/serviceSelectionLock.ts index 2d085bd77dab1..9280f1bde3ab4 100644 --- a/apps/server/src/cloud/serviceSelectionLock.ts +++ b/apps/server/src/cloud/serviceSelectionLock.ts @@ -47,22 +47,27 @@ export async function acquireServiceSelectionLock(selectionPath: string): Promis const previous = owners[0]!; const pid = Number(previous.split(".", 1)[0]); if (!Number.isSafeInteger(pid) || pid <= 0) - throw new Error("Invalid service selection lock owner."); + throw new Error(`Invalid service selection lock owner at ${lock}.`); try { process.kill(pid, 0); } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - // Remove only this dead owner's unique file, never a replacement owner's. - try { - NodeFS.unlinkSync(NodePath.join(lock, previous)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ESRCH" && code !== "EPERM") throw error; + if (code === "ESRCH") { + // Remove only this dead owner's unique file, never a replacement owner's. + try { + NodeFS.unlinkSync(NodePath.join(lock, previous)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } } } } removeEmptyLock(); if (Date.now() >= deadline) - throw new Error("Service selection is locked. Retry when the other operation finishes."); + throw new Error( + `Service selection is locked at ${lock}. Retry when the other operation finishes.`, + ); await new Promise((resolve) => setTimeout(resolve, 50)); } } finally { diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 9954a1a5637d8..3569edf6c7898 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -639,8 +639,17 @@ export class Launcher { const restartPending = restartPendingPath(this.#baseDir); const release = await acquireServiceSelectionLock(this.#statePath); try { + const selected = await readServiceState(this.#statePath); const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined); - if (awaitedVersion?.trim() === this.#state.activeVersion) { + const awaited = awaitedVersion?.trim(); + if ( + selected.activeVersion === this.#state.activeVersion && + selected.update?.id === this.#state.update?.id && + selected.update?.status === this.#state.update?.status && + awaited !== undefined && + isExactServiceVersion(awaited) && + compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0 + ) { await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined); } } finally { diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index a911693b6ac05..97ddd548d7bd9 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -18,15 +18,18 @@ immutable runtime. Preflight checks the launcher protocol because a target that needs new rollback guarantees cannot safely run under an older launcher. After a commit the launcher points the unit at the committed runtime, so the next service start also runs that version's launcher; the protocol check is what makes that safe. -Protocol 4 adds these guarantees. An installed protocol-3 launcher requires one -local `t3 update` or `t3 service install` before remote updates are accepted; a +Protocol 4 adds these guarantees and accepts valid protocol-3 durable state, whose +format is unchanged. An installed protocol-3 launcher requires one local +`t3 update --yes` or `t3 service install` that restarts the service before remote +updates are accepted. Deferring the restart leaves the old launcher running; a child update cannot upgrade its supervising launcher. Terminal-state recovery retries reconciliation and reload even when the file already names the selected runtime, since a previous reload may have failed. A newer locally installed unit is preserved. launchd reads the plist only at its next bootstrap, so a crash respawn before then -still runs the loaded definition. +still runs the loaded definition. After a deferred protocol-3 upgrade, that loaded +launcher cannot read protocol-4 state; the local restart is required to recover. ## Commit boundary From 894ef00b86ecd33b1962c2a09af7014b57f2a644 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:11:04 +0000 Subject: [PATCH 11/17] test(server): protect service selection from stale launchers --- apps/server/src/serviceLauncher.test.ts | 69 +++++++++++++++++++++---- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 90bfff385ff56..8e74c9773c3af 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -252,9 +252,12 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect.each(["1.0.0", "1.2.0"])( - "recovers a rolled-back update without downgrading the installed unit at %s", - (unitVersion) => + it.effect.each([ + { unitVersion: "1.0.0", replacement: "an intentional downgrade" }, + { unitVersion: "1.2.0", replacement: "another T3 home" }, + ])( + "recovers a rolled-back update with unit $unitVersion and preserves $replacement", + ({ unitVersion, replacement }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -282,25 +285,69 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const expected = renderBootServiceUnit( unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion), ); - const run = () => + const createLauncher = Effect.gen(function* () { + return new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)), { + serviceUnit: unit, + }); + }); + const run = (launcher: Launcher) => Effect.gen(function* () { - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - { serviceUnit: unit }, - ); const running = launcher.run(); yield* Effect.promise(() => launcher.stop("SIGTERM")); yield* Effect.promise(() => running); }); - yield* run(); + yield* run(yield* createLauncher); assert.equal(yield* fs.readFileString(unit.path), expected); assert.equal(unit.loadedContents, expected); - yield* run(); + yield* run(yield* createLauncher); assert.equal(yield* fs.readFileString(unit.path), expected); assert.equal(unit.loadedContents, expected); + + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-2", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "committed", + }, + }), + ); + const staleLauncher = yield* createLauncher; + // A local install can replace the selection or service owner without stopping this launcher. + const selectedRoot = + replacement === "another T3 home" ? path.join(root, "other-home") : root; + yield* writeFakeRuntime( + fs, + path, + path.join(selectedRoot, "runtime", "versions", "1.0.0"), + "setInterval(() => {}, 1_000);\n", + ); + const selectedStatePath = path.join(selectedRoot, "runtime", "service-state.json"); + yield* Effect.promise(() => + writeServiceState(selectedStatePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const selectedUnit = renderBootServiceUnit({ + ...unitPlan(selectedRoot, "1.0.0"), + unitPath: unit.path, + }); + yield* fs.writeFileString(unit.path, selectedUnit); + yield* Effect.promise(() => unit.reload()); + const oldHomeState = yield* fs.readFileString(statePath); + const selectedState = yield* fs.readFileString(selectedStatePath); + + yield* run(staleLauncher); + assert.equal(yield* fs.readFileString(statePath), oldHomeState); + assert.equal(yield* fs.readFileString(selectedStatePath), selectedState); + assert.equal(yield* fs.readFileString(unit.path), selectedUnit, replacement); + assert.equal(unit.loadedContents, selectedUnit, replacement); }), ); From 99c99f2427c6389be8cb224418df5f9e6eb527ed Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:13:57 +0000 Subject: [PATCH 12/17] test(server): require unit reconciliation after live rollback --- apps/server/src/serviceLauncher.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 8e74c9773c3af..f0e92b6f0ebcb 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -615,7 +615,7 @@ if (context.update?.status === "pending") { activeVersion: "1.0.0", }), ); - const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0"); + const unit = yield* fakeServiceUnit(fs, path, root, "0.9.0"); const launcher = new Launcher( root, @@ -638,7 +638,7 @@ if (context.update?.status === "pending") { yield* fs.readFileString(unit.path), renderBootServiceUnit(unitPlan(root, "1.0.0")), ); - assert.equal(unit.reloads, 0); + assert.equal(unit.reloads, 1); assert.equal( state.update?.status === "rolled-back" ? state.update.reason : undefined, "invalid-prepared", From 57d12538d7eaa2482b1d590d1394f5e83a678d17 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:18:39 +0000 Subject: [PATCH 13/17] fix(server): preserve service selection during unit reconciliation --- apps/server/src/cloud/bootService.ts | 29 ++--------- apps/server/src/cloud/serviceProtocol.ts | 17 ++++++ apps/server/src/serviceLauncher.ts | 66 +++++++++++++++++------- docs/internals/server-updates.md | 7 +-- docs/user/updating.md | 3 ++ 5 files changed, 75 insertions(+), 47 deletions(-) diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 0f98d82ff0c12..94eb8f1f6df53 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -31,12 +31,15 @@ import { SERVICE_STATE_FILE, SERVICE_UNIT_ENV, compareExactServiceVersions, + serviceUnitBaseDirOf as bootServiceBaseDirOf, parseServiceState, serviceStateActiveVersion, serviceStateHasPendingUpdate, type ServiceState, } from "./serviceProtocol.ts"; +export { serviceUnitBaseDirOf as bootServiceBaseDirOf } from "./serviceProtocol.ts"; + const BOOT_SERVICE_NAME = "t3code"; export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; // `.service` suffix keeps the label distinct from the desktop app's bundle id @@ -57,32 +60,6 @@ function quoteSystemdValue(value: string): string { : escaped; } -const unquoteSystemdValue = (raw: string) => - (raw.startsWith('"') && raw.endsWith('"') - ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\") - : raw - ).replaceAll("%%", "%"); - -const unescapeXmlText = (value: string) => - value.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); - -export function bootServiceBaseDirOf(contents: string): string | undefined { - const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1]; - if (systemd !== undefined) return unquoteSystemdValue(systemd.trim()); - const plist = /T3CODE_HOME<\/key>\s*([^<]*)<\/string>/.exec(contents)?.[1]; - return plist === undefined ? undefined : unescapeXmlText(plist); -} - -/** The executable an installed unit or plist starts: the launcher's runtime. */ -export function bootServiceProgramOf(contents: string): string | undefined { - const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1]; - if (systemd !== undefined) return unquoteSystemdValue(systemd); - const plist = /ProgramArguments<\/key>\s*\s*([^<]*)<\/string>/.exec( - contents, - )?.[1]; - return plist === undefined ? undefined : unescapeXmlText(plist); -} - export interface BootServicePlan { readonly program: ReadonlyArray; readonly baseDir: string; diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index c465ce9a9c138..07c5289035263 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -15,6 +15,23 @@ export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */ export const SERVICE_RESTART_PENDING_FILE = ".restart-pending"; +export function serviceUnitBaseDirOf(contents: string): string | undefined { + const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1]; + if (systemd !== undefined) { + const raw = systemd.trim(); + const unquoted = + raw.startsWith('"') && raw.endsWith('"') + ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\") + : raw; + return unquoted.replaceAll("%%", "%"); + } + const plist = /T3CODE_HOME<\/key>\s*([^<]*)<\/string>/.exec(contents)?.[1]; + if (plist !== undefined) { + return plist.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); + } + return undefined; +} + export interface PendingServiceUpdate { readonly id: string; readonly fromVersion: string; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 3569edf6c7898..e04367f0c8e92 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -22,6 +22,7 @@ import { decodeServiceLauncherChildMessage, isExactServiceVersion, parseServiceState, + serviceUnitBaseDirOf, SERVICE_ENV_FILE, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, @@ -637,23 +638,31 @@ export class Launcher { // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); const restartPending = restartPendingPath(this.#baseDir); - const release = await acquireServiceSelectionLock(this.#statePath); try { - const selected = await readServiceState(this.#statePath); - const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined); - const awaited = awaitedVersion?.trim(); - if ( - selected.activeVersion === this.#state.activeVersion && - selected.update?.id === this.#state.update?.id && - selected.update?.status === this.#state.update?.status && - awaited !== undefined && - isExactServiceVersion(awaited) && - compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0 - ) { - await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined); + const release = await acquireServiceSelectionLock(this.#statePath); + try { + const selected = await readServiceState(this.#statePath); + const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch( + () => undefined, + ); + const awaited = awaitedVersion?.trim(); + if ( + selected.activeVersion === this.#state.activeVersion && + selected.update?.id === this.#state.update?.id && + selected.update?.status === this.#state.update?.status && + awaited !== undefined && + isExactServiceVersion(awaited) && + compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0 + ) { + await NodeFSP.rm(restartPending, { force: true }); + } + } finally { + release(); } - } finally { - release(); + } catch (error) { + process.stderr.write( + `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`, + ); } const update = this.#state.update; if (update?.status !== "pending") { @@ -860,15 +869,31 @@ export class Launcher { } } - // The commit is durable before this runs: a unit that could not be rewritten - // leaves the service on the new version, pointed at the old runtime, as before. + // Selection is durable before this best-effort unit reconciliation. async #repointServiceUnit(version: string): Promise { const unit = this.#serviceUnit; if (unit === undefined) return; try { const release = await acquireServiceSelectionLock(unit.path); try { + // The installer writes state before the unit under this lease. Do not + // acquire its state lease here: the installer takes that lease first. + const selected = await readServiceState(this.#statePath); + if ( + selected.activeVersion !== this.#state.activeVersion || + selected.update?.id !== this.#state.update?.id || + selected.update?.status !== this.#state.update?.status + ) { + return; + } const contents = await NodeFSP.readFile(unit.path, "utf8"); + const installedBaseDir = serviceUnitBaseDirOf(contents); + if ( + installedBaseDir === undefined || + NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir) + ) { + return; + } const repointed = repointServiceUnit(contents, version); if (repointed !== contents) await replaceFileDurably(unit.path, repointed); // A previous rename may have succeeded before reload failed. @@ -938,6 +963,7 @@ export class Launcher { }; await this.#selectState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); + await this.#repointServiceUnit(next.activeVersion); } // Signal listeners alone do not keep Node alive. this.#timer = setInterval(() => {}, 2_147_483_647); @@ -965,7 +991,11 @@ export class Launcher { }; await this.#selectState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); - await this.#startChild(next.activeVersion, "active", outcome); + try { + await this.#startChild(next.activeVersion, "active", outcome); + } finally { + await this.#repointServiceUnit(next.activeVersion); + } } } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 97ddd548d7bd9..0a26fce41eb1f 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -27,9 +27,10 @@ retries reconciliation and reload even when the file already names the selected runtime, since a previous reload may have failed. A newer locally installed unit is preserved. -launchd reads the plist only at its next bootstrap, so a crash respawn before then -still runs the loaded definition. After a deferred protocol-3 upgrade, that loaded -launcher cannot read protocol-4 state; the local restart is required to recover. +launchd keeps its loaded definition until the next bootstrap; systemd keeps its +loaded definition until daemon-reload. After a deferred protocol-3 upgrade, either +manager can therefore respawn the old launcher, which cannot read protocol-4 +state. One local service restart is required to load the updated launcher. ## Commit boundary diff --git a/docs/user/updating.md b/docs/user/updating.md index 7e62c7bf580fc..1a016db76e173 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -106,6 +106,9 @@ The offered action depends on how the server runs: | **Copy update command** | Run the command on the named host to update the detected global npm install, then restart the server with your usual options. | | **Copy relaunch command** | Stop the command-line server on its host and relaunch with the copied command, keeping your usual subcommand and options. This does not update an installed `t3` command. | +Older background services need one local `t3 update --yes` or `t3 service install` +that restarts the service before they can accept remote updates. + On the host, run: ```sh From d3f1e35bba94c4823322ee22d508d8b7fa7de36e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 07:31:16 +0000 Subject: [PATCH 14/17] fix(server): start recovered service before unit reconciliation --- apps/server/src/cloud/bootService.ts | 16 +++++------- apps/server/src/cloud/serviceProtocol.ts | 33 +++++++++++++++--------- apps/server/src/serviceLauncher.test.ts | 32 ++--------------------- apps/server/src/serviceLauncher.ts | 15 ++++++----- docs/internals/server-updates.md | 5 ++-- docs/user/updating.md | 5 ++-- 6 files changed, 44 insertions(+), 62 deletions(-) diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 94eb8f1f6df53..e7de29ad8953d 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -38,7 +38,10 @@ import { type ServiceState, } from "./serviceProtocol.ts"; -export { serviceUnitBaseDirOf as bootServiceBaseDirOf } from "./serviceProtocol.ts"; +export { + serviceUnitBaseDirOf as bootServiceBaseDirOf, + serviceUnitProgramOf as bootServiceProgramOf, +} from "./serviceProtocol.ts"; const BOOT_SERVICE_NAME = "t3code"; export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; @@ -916,8 +919,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { : {}, ); } - // A remote update can finish after the CLI checks status. Read its - // final version after the launcher stops and before changing files. + // Read the final version after stopping the launcher; a remote update may finish after the status check. if ( installedVersion !== undefined && options?.allowDowngrade !== true && @@ -932,12 +934,8 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { yield* fs .makeDirectory(path.dirname(unitPath), { recursive: true }) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - if (!start && (currentlyInstalled || Option.isSome(previousStateText))) { - // Written first: once the files below name the new version, the running - // service is behind them, and a failure between the two writes must not - // leave it looking current. - yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 }); - } + // Files can match the new version even when activation fails; keep that install retryable. + yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 }); yield* writeDurably( statePath, // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document. diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 07c5289035263..8958c945aab84 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -15,21 +15,30 @@ export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */ export const SERVICE_RESTART_PENDING_FILE = ".restart-pending"; +const unquoteSystemdValue = (raw: string) => + (raw.startsWith('"') && raw.endsWith('"') + ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\") + : raw + ).replaceAll("%%", "%"); + +const unescapeXmlText = (value: string) => + value.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); + export function serviceUnitBaseDirOf(contents: string): string | undefined { const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1]; - if (systemd !== undefined) { - const raw = systemd.trim(); - const unquoted = - raw.startsWith('"') && raw.endsWith('"') - ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\") - : raw; - return unquoted.replaceAll("%%", "%"); - } + if (systemd !== undefined) return unquoteSystemdValue(systemd.trim()); const plist = /T3CODE_HOME<\/key>\s*([^<]*)<\/string>/.exec(contents)?.[1]; - if (plist !== undefined) { - return plist.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); - } - return undefined; + return plist === undefined ? undefined : unescapeXmlText(plist); +} + +/** The executable an installed unit or plist starts: the launcher's runtime. */ +export function serviceUnitProgramOf(contents: string): string | undefined { + const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1]; + if (systemd !== undefined) return unquoteSystemdValue(systemd); + const plist = /ProgramArguments<\/key>\s*\s*([^<]*)<\/string>/.exec( + contents, + )?.[1]; + return plist === undefined ? undefined : unescapeXmlText(plist); } export interface PendingServiceUpdate { diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index f0e92b6f0ebcb..01a63e4a675b9 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -18,7 +18,6 @@ import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootServi import { databaseBackupBlocker, Launcher, - main, readServiceEnvironment, type BackupSpaceProbe, readServiceState, @@ -391,34 +390,6 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect("serializes shutdown with launcher recovery", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-stop-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", "1.0.0"), - "setInterval(() => {}, 1_000);\n", - ); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); - const running = launcher.run(); - const stopping = launcher.stop("SIGTERM"); - assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); - yield* Effect.promise(() => stopping); - yield* Effect.promise(() => running); - }), - ); - const commitUpdate = ( render = renderBootServiceUnit, missingUnit = false, @@ -622,10 +593,11 @@ if (context.update?.status === "pending") { yield* Effect.promise(() => readServiceState(statePath)), { serviceUnit: unit, + environment: yield* Effect.promise(() => readServiceEnvironment(root)), }, ); yield* Effect.promise(() => - main().then( + launcher.run().then( () => Promise.reject(new Error("launcher unexpectedly completed")), () => Promise.resolve(), ), diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index e04367f0c8e92..90200b2571550 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -107,11 +107,9 @@ function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): Servi }; } -// The executable the unit or launch agent starts: systemd's ExecStart value, -// or the plist string that precedes the __service-launcher argument. Only the -// version segment changes, so paths the renderer had to quote or escape stay -// as they are. The launch agent's PATH names a versions directory without the -// executable and is left alone. +// systemd's ExecStart value, or the plist string before the __service-launcher argument. +// Only the version segment changes, so quoted or escaped paths stay as rendered. +// The launch agent's PATH names a versions directory without the executable; leave it. const SERVICE_UNIT_PROGRAMS = [ /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m, /([^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*__service-launcher<\/string>)/, @@ -668,9 +666,12 @@ export class Launcher { if (update?.status !== "pending") { if (update !== undefined) { await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); - await this.#repointServiceUnit(this.#state.activeVersion); } - await this.#startChild(this.#state.activeVersion, "active", update); + try { + await this.#startChild(this.#state.activeVersion, "active", update); + } finally { + if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion); + } return; } if (await databaseRestorePending(this.#baseDir, update)) { diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 0a26fce41eb1f..ee0c1f223559f 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -29,8 +29,9 @@ is preserved. launchd keeps its loaded definition until the next bootstrap; systemd keeps its loaded definition until daemon-reload. After a deferred protocol-3 upgrade, either -manager can therefore respawn the old launcher, which cannot read protocol-4 -state. One local service restart is required to load the updated launcher. +manager can therefore respawn the old launcher, but the new runtime refuses to run +under it. The service cannot recover until a local restart through +`t3 update --yes` or `t3 service install` loads the updated launcher. ## Commit boundary diff --git a/docs/user/updating.md b/docs/user/updating.md index 1a016db76e173..beb51bb86f8c1 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -106,8 +106,9 @@ The offered action depends on how the server runs: | **Copy update command** | Run the command on the named host to update the detected global npm install, then restart the server with your usual options. | | **Copy relaunch command** | Stop the command-line server on its host and relaunch with the copied command, keeping your usual subcommand and options. This does not update an installed `t3` command. | -Older background services need one local `t3 update --yes` or `t3 service install` -that restarts the service before they can accept remote updates. +For older background services, run `t3 update --yes` or `t3 service install` locally +once and restart when prompted; deferring can leave the service stopped after a +crash or restart until you run `t3 service restart`. On the host, run: From c6dc7b8f55803c01da739f2c064cdd6cbb2ffb5c Mon Sep 17 00:00:00 2001 From: Marve10s Date: Fri, 30 Oct 2026 13:42:36 +0300 Subject: [PATCH 15/17] fix(server): a service selection lease from an earlier boot is reclaimed A crash while a lease was held left its owner file behind across a reboot. When that PID belonged to a live process again, every acquisition waited out the deadline and failed, which made update commits fatal until someone deleted the lock directory. Lease owners now record the boot they were taken in, and an owner from another boot counts as dead. --- .../src/cloud/serviceSelectionLock.test.ts | 30 ++++++++++++++ apps/server/src/cloud/serviceSelectionLock.ts | 40 ++++++++++++------- 2 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 apps/server/src/cloud/serviceSelectionLock.test.ts diff --git a/apps/server/src/cloud/serviceSelectionLock.test.ts b/apps/server/src/cloud/serviceSelectionLock.test.ts new file mode 100644 index 0000000000000..d2567de4c0bc5 --- /dev/null +++ b/apps/server/src/cloud/serviceSelectionLock.test.ts @@ -0,0 +1,30 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts"; + +it.layer(NodeServices.layer)("service selection lock", (it) => { + it.effect("reclaims a lease from an earlier boot even when its PID is live again", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-selection-lock-" }); + const selectionPath = path.join(root, "service-state.json"); + const lock = `${selectionPath}.selection-lock`; + const stale = `${process.pid}.1.00000000-0000-4000-8000-000000000000`; + yield* fs.makeDirectory(lock, { recursive: true }); + yield* fs.writeFileString(path.join(lock, stale), ""); + + const release = yield* Effect.promise(() => acquireServiceSelectionLock(selectionPath)); + const owners = yield* fs.readDirectory(lock); + assert.equal(owners.length, 1); + assert.notEqual(owners[0], stale); + assert.isTrue(owners[0]!.startsWith(`${process.pid}.`)); + release(); + assert.isFalse(yield* fs.exists(lock)); + }), + ); +}); diff --git a/apps/server/src/cloud/serviceSelectionLock.ts b/apps/server/src/cloud/serviceSelectionLock.ts index 9280f1bde3ab4..4cbd123c22f7e 100644 --- a/apps/server/src/cloud/serviceSelectionLock.ts +++ b/apps/server/src/cloud/serviceSelectionLock.ts @@ -3,14 +3,18 @@ // @effect-diagnostics globalDate:off -- The standalone launcher has no Effect clock for its acquisition deadline. import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; +// Coarse boot time: a lease from an earlier boot is dead even when its PID is live again. +const bootEpochSeconds = () => Math.round(Date.now() / 1000 - NodeOS.uptime()); + /** Serialize durable selection and unit writes, never manager stop/start commands. */ export async function acquireServiceSelectionLock(selectionPath: string): Promise<() => void> { const directory = NodePath.dirname(selectionPath); NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 }); const lock = selectionPath + ".selection-lock"; - const owner = `${process.pid}.${NodeCrypto.randomUUID()}`; + const owner = `${process.pid}.${bootEpochSeconds()}.${NodeCrypto.randomUUID()}`; const candidate = `${lock}.${owner}`; const deadline = Date.now() + 30_000; const removeEmptyLock = () => { @@ -45,21 +49,29 @@ export async function acquireServiceSelectionLock(selectionPath: string): Promis } if (owners.length === 1) { const previous = owners[0]!; - const pid = Number(previous.split(".", 1)[0]); + const [pidText, bootText] = previous.split("."); + const pid = Number(pidText); if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error(`Invalid service selection lock owner at ${lock}.`); - try { - process.kill(pid, 0); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== "ESRCH" && code !== "EPERM") throw error; - if (code === "ESRCH") { - // Remove only this dead owner's unique file, never a replacement owner's. - try { - NodeFS.unlinkSync(NodePath.join(lock, previous)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } + const ownerBoot = Number(bootText); + const fromEarlierBoot = + Number.isSafeInteger(ownerBoot) && Math.abs(ownerBoot - bootEpochSeconds()) > 60; + let dead = fromEarlierBoot; + if (!dead) { + try { + process.kill(pid, 0); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ESRCH" && code !== "EPERM") throw error; + dead = code === "ESRCH"; + } + } + if (dead) { + // Remove only this dead owner's unique file, never a replacement owner's. + try { + NodeFS.unlinkSync(NodePath.join(lock, previous)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } } } From d43d89395a55d6198b7defb86cb526fc00b0d665 Mon Sep 17 00:00:00 2001 From: Marve10s Date: Mon, 9 Nov 2026 14:30:45 +0300 Subject: [PATCH 16/17] fix(server): the launcher repoints a unit under a relocated systemd config --- apps/server/src/serviceLauncher.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 90200b2571550..0dd9c1d670399 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -89,8 +89,14 @@ function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): Servi reload: () => Promise.resolve(), }; } + // The manager's own environment, which decides where it reads user units. + const xdgConfigHome = env.XDG_CONFIG_HOME?.trim(); + const configHome = + xdgConfigHome && NodePath.isAbsolute(xdgConfigHome) + ? xdgConfigHome + : NodePath.join(home, ".config"); return { - path: NodePath.join(home, ".config", "systemd", "user", unitName), + path: NodePath.join(configHome, "systemd", "user", unitName), reload: () => new Promise((resolve, reject) => { NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], { From 3a20f33b763c53cb623f38a0a7d395f17475728f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 06:21:03 +0000 Subject: [PATCH 17/17] docs(server): tighten the service unit reconciliation notes AI-assisted; verified with the tests and tools listed above. --- apps/server/src/cloud/bootService.test.ts | 4 +-- docs/internals/server-updates.md | 44 +++++++++++------------ docs/user/updating.md | 9 ++--- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index b30b2a4b22e5c..841aa78f6c879 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -743,8 +743,8 @@ process.disconnect(); const installing = yield* newer .install({ start: false }) .pipe(Effect.ensuring(Effect.sync(() => milestone.resolve())), Effect.forkChild); - // Either observe real OS contention or let an uncoordinated installer finish - // before releasing the stale snapshot. The latter deterministically loses C. + // Resume the launcher once the installer contends for the unit lease, or once an + // uncoordinated installer has finished, which makes the stale snapshot win. yield* Effect.promise(() => milestone.promise); child.send("resume"); yield* Fiber.join(installing); diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index ee0c1f223559f..e0f1d286a1608 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -4,34 +4,30 @@ A [stable launcher](../../apps/server/src/serviceLauncher.ts) owns the runtime selected by systemd or launchd. It is the only runtime writer of durable service state. Server children request updates over inherited IPC; they never rewrite their service definition or select their own replacement. Local service commands -may replace the launcher and state, optionally deferring the restart. Both writers -serialize state selection and unit writes across processes. The installer takes -the state lease before the unit lease; the launcher never holds both, so unit -failures cannot prevent durable commit. Manager stop/start commands stay outside -both leases so stopping a launcher cannot deadlock. A running launcher refuses -another update after a local command changes its selection. Foreground CLI -processes do not self-update. +may replace the launcher and state, optionally deferring the restart. Foreground +CLI processes do not self-update. + +Both writers serialize state and unit writes through +[file leases](../../apps/server/src/cloud/serviceSelectionLock.ts). The installer +takes the state lease before the unit lease; the launcher never holds both, so a +unit failure cannot block a durable commit. Manager stop/start commands run outside +both leases, since stopping a launcher that waits on a lease would deadlock. A +running launcher refuses further updates once a local command changes its selection. Exact-version installs keep restarts independent of npm cache eviction or a moving release tag. Installation and preflight happen in staging before publishing an immutable runtime. Preflight checks the launcher protocol because a target that -needs new rollback guarantees cannot safely run under an older launcher. After a -commit the launcher points the unit at the committed runtime, so the next service -start also runs that version's launcher; the protocol check is what makes that safe. -Protocol 4 adds these guarantees and accepts valid protocol-3 durable state, whose -format is unchanged. An installed protocol-3 launcher requires one local -`t3 update --yes` or `t3 service install` that restarts the service before remote -updates are accepted. Deferring the restart leaves the old launcher running; a -child update cannot upgrade its supervising launcher. Terminal-state recovery -retries reconciliation and reload even when the file already names the selected -runtime, since a previous reload may have failed. A newer locally installed unit -is preserved. - -launchd keeps its loaded definition until the next bootstrap; systemd keeps its -loaded definition until daemon-reload. After a deferred protocol-3 upgrade, either -manager can therefore respawn the old launcher, but the new runtime refuses to run -under it. The service cannot recover until a local restart through -`t3 update --yes` or `t3 service install` loads the updated launcher. +needs new rollback guarantees cannot safely run under an older launcher. A remote +update replaces only the child, so upgrading the launcher requires a local service +update with a restart. + +After a commit or rollback the launcher points the unit at the selected runtime and +reloads the manager, so the next service start runs that version's launcher; the +protocol check is what makes that safe. Each start after a finished update retries this, because +a previous reload may have failed, and leaves a unit naming a newer runtime or another +T3 home alone. Until a deferred local upgrade restarts, systemd (before daemon-reload) +and launchd (before bootstrap) can respawn the old launcher, which the new runtime +refuses to run under. ## Commit boundary diff --git a/docs/user/updating.md b/docs/user/updating.md index beb51bb86f8c1..123306077f8f5 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -106,10 +106,6 @@ The offered action depends on how the server runs: | **Copy update command** | Run the command on the named host to update the detected global npm install, then restart the server with your usual options. | | **Copy relaunch command** | Stop the command-line server on its host and relaunch with the copied command, keeping your usual subcommand and options. This does not update an installed `t3` command. | -For older background services, run `t3 update --yes` or `t3 service install` locally -once and restart when prompted; deferring can leave the service stopped after a -crash or restart until you run `t3 service restart`. - On the host, run: ```sh @@ -122,6 +118,11 @@ asks before restarting the background service; if you decline, run stop it and start it again afterwards with your usual options such as `--host` or `--tailscale-serve`. +A background service installed by an older release refuses **Update server** +until you update it once on the host and let that update restart it. If you +decline the restart, the service can stay down after a crash or reboot until you +run `t3 service restart`. + If you run the server with `npx` rather than an installed `t3`, there is nothing to update on the host: stop the server and relaunch it as `npx t3@` with the same subcommand and options.