diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 814d470cad410..841aa78f6c879 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Cross-process launcher contention uses real filesystem leases and IPC barriers. +import * as NodeChildProcess from "node:child_process"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -5,11 +7,13 @@ import * as ConfigProvider from "effect/ConfigProvider"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import { CliError } from "effect/cli"; import { HttpClient } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import { vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; import * as BootService from "./bootService.ts"; @@ -22,6 +26,24 @@ import { serviceStateHasPendingUpdate, } from "./serviceProtocol.ts"; +const leaseObserver = vi.hoisted(() => ({ contended: (_destination: string) => {} })); +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + renameSync: (...args: Parameters) => { + try { + return actual.renameSync(...args); + } catch (error) { + if (["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) { + leaseObserver.contended(String(args[1])); + } + throw error; + } + }, + }; +}); + it("reads the served T3 home back out of a rendered unit or plist", () => { const plan = (baseDir: string) => ({ program: [`${baseDir}/runtime/versions/1.2.3/t3`, "__service-launcher"], @@ -442,6 +464,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => { current: false, installedVersion: "1.2.4-nightly.1", }); + yield* service.install({ allowDowngrade: true }); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + }); } }), ); @@ -575,6 +602,168 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect(yield* newer.restart).toBe("restarted"); expect((yield* newer.status).problems).not.toContain("restart-pending"); expect((yield* newer.status).current).toBe(true); + + const latest = yield* makeService(undefined, "1.2.5"); + yield* latest.install({ start: true }); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.5", + }); + expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.5/t3"); + expect((yield* latest.status).current).toBe(true); + expect( + yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE)), + ).toBe(false); + }), + ); + + it.effect( + "serializes a local install with another process reconciling an older unit snapshot", + () => + Effect.gen(function* () { + const { service, fs, statePath, makeService, runtime } = yield* makeHarness(); + const path = yield* Path.Path; + const plan = yield* service.install(); + const unit = yield* fs.readFileString(plan.unitPath); + yield* fs.writeFileString( + plan.unitPath, + unit.replace("versions/1.2.3/t3", "versions/1.2.2/t3"), + ); + yield* fs.writeFileString( + statePath, + JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + update: { + id: "committed", + fromVersion: "1.2.2", + targetVersion: "1.2.3", + status: "committed", + }, + }), + ); + const rejection = path.join(plan.baseDir, "rejected-update"); + const databasePath = path.join(plan.baseDir, "state.sqlite"); + yield* fs.writeFileString(databasePath, "database"); + yield* fs.writeFileString( + runtime.entryPath, + `#!${process.execPath} +import { writeFileSync } from "node:fs"; +process.on("message", message => { + if (message === "request-update") process.send({ type: "request-update", targetVersion: "1.2.5", dbPath: ${JSON.stringify(databasePath)} }); + if (message.type === "update-rejected") { + writeFileSync(${JSON.stringify(rejection)}, message.reason); + process.exit(0); + } +}); +`, + ); + yield* fs.chmod(runtime.entryPath, 0o755); + const newer = yield* makeService(undefined, "1.2.4"); + yield* makeService(undefined, "1.2.5"); + const snapshot = Promise.withResolvers(); + const milestone = Promise.withResolvers(); + const script = path.join(plan.baseDir, "launcher-contention.mjs"); + yield* fs.writeFileString( + script, + ` +import fs from "node:fs/promises"; +import cp from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +const server = Promise.withResolvers(); +const originalSpawn = cp.spawn; +cp.spawn = (...args) => { const child = originalSpawn(...args); server.resolve(child); return child; }; +process.on("message", async message => { + if (message === "installed") (await server.promise).send("request-update"); +}); +const originalRead = fs.readFile; +fs.readFile = async (...args) => { + const contents = await originalRead(...args); + if (args[0] === ${JSON.stringify(plan.unitPath)}) { + fs.readFile = originalRead; + syncBuiltinESMExports(); + const resume = new Promise(resolve => process.once("message", resolve)); + process.send("snapshot"); + await resume; + } + return contents; +}; +syncBuiltinESMExports(); +const { Launcher, readServiceState } = await import(${JSON.stringify(import.meta.resolve("../serviceLauncher.ts"))}); +const launcher = new Launcher(${JSON.stringify(plan.baseDir)}, await readServiceState(${JSON.stringify(statePath)}), { + serviceUnit: { path: ${JSON.stringify(plan.unitPath)}, reload: async () => {} }, +}); +try { await launcher.run(); throw Error("unexpected completion"); } +catch (error) { + if (!error.message.includes("Active child exited unexpectedly (0)")) throw error; +} +const { acquireServiceSelectionLock } = await import(${JSON.stringify(import.meta.resolve("./serviceSelectionLock.ts"))}); +// Exit owning the real lease, as a crashed process would. +await acquireServiceSelectionLock(${JSON.stringify(plan.unitPath)}); +process.disconnect(); +`, + ); + const child = yield* Effect.acquireRelease( + Effect.sync(() => + NodeChildProcess.fork(script, [], { + execArgv: [], + stdio: ["ignore", "inherit", "inherit", "ipc"], + }), + ), + (child) => + Effect.sync(() => { + if (child.exitCode === null && child.signalCode === null) child.kill(); + }), + ); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => + code === 0 ? resolve() : reject(new Error(`launcher exited ${code}`)), + ); + }); + child.on("message", (message) => { + if (message === "snapshot") snapshot.resolve(); + }); + yield* Effect.promise(() => + Promise.race([ + snapshot.promise, + exited.then(() => { + throw new Error("launcher exited before snapshot"); + }), + ]), + ); + leaseObserver.contended = (destination) => { + if (destination === plan.unitPath + ".selection-lock") milestone.resolve(); + }; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + leaseObserver.contended = () => {}; + }), + ); + const installing = yield* newer + .install({ start: false }) + .pipe(Effect.ensuring(Effect.sync(() => milestone.resolve())), Effect.forkChild); + // Resume the launcher once the installer contends for the unit lease, or once an + // uncoordinated installer has finished, which makes the stale snapshot win. + yield* Effect.promise(() => milestone.promise); + child.send("resume"); + yield* Fiber.join(installing); + child.send("installed"); + yield* Effect.promise(() => exited); + expect(yield* fs.readFileString(rejection)).toContain("Service selection changed"); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.4", + }); + expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3"); + expect( + yield* fs.readFileString( + path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE), + ), + ).toBe("1.2.4\n"); + // The next installer reclaims the dead owner and releases its own lease. + yield* newer.install({ start: false }); + expect(yield* fs.exists(plan.unitPath + ".selection-lock")).toBe(false); }), ); @@ -617,6 +806,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const error = yield* service.install({ start: false }).pipe(Effect.flip); expect(error._tag).toBe("BootServiceUpdatePendingError"); expect(yield* fs.readFileString(statePath)).toBe(pendingState); + yield* fs.remove((yield* service.status).unitPath); + expect((yield* service.install({ start: false }).pipe(Effect.flip))._tag).toBe( + "BootServiceUpdatePendingError", + ); + expect(yield* fs.readFileString(statePath)).toBe(pendingState); }), ); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 88d477974ae0a..e7de29ad8953d 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -16,6 +16,7 @@ import * as Schema from "effect/Schema"; import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease"; import * as ProcessRunner from "../processRunner.ts"; +import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts"; import { ensurePinnedRuntimeInstalled, launcherOwnsVersionsDir, @@ -28,20 +29,26 @@ import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_RESTART_PENDING_FILE, SERVICE_STATE_FILE, + SERVICE_UNIT_ENV, compareExactServiceVersions, + serviceUnitBaseDirOf as bootServiceBaseDirOf, parseServiceState, serviceStateActiveVersion, serviceStateHasPendingUpdate, type ServiceState, } from "./serviceProtocol.ts"; +export { + serviceUnitBaseDirOf as bootServiceBaseDirOf, + serviceUnitProgramOf as bootServiceProgramOf, +} from "./serviceProtocol.ts"; + const BOOT_SERVICE_NAME = "t3code"; export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; // `.service` suffix keeps the label distinct from the desktop app's bundle id // (com.t3tools.t3code), so launchd and TCC records never collide. export const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service"; export const BOOT_SERVICE_PLIST_FILE = `${BOOT_SERVICE_LAUNCHD_LABEL}.plist`; -const BOOT_SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; export const BOOT_SERVICE_LOG_FILE = "boot-service.log"; /** systemd expands `%` specifiers, including in unquoted append-log paths. */ @@ -56,32 +63,6 @@ function quoteSystemdValue(value: string): string { : escaped; } -const unquoteSystemdValue = (raw: string) => - (raw.startsWith('"') && raw.endsWith('"') - ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\") - : raw - ).replaceAll("%%", "%"); - -const unescapeXmlText = (value: string) => - value.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); - -export function bootServiceBaseDirOf(contents: string): string | undefined { - const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1]; - if (systemd !== undefined) return unquoteSystemdValue(systemd.trim()); - const plist = /T3CODE_HOME<\/key>\s*([^<]*)<\/string>/.exec(contents)?.[1]; - return plist === undefined ? undefined : unescapeXmlText(plist); -} - -/** The executable an installed unit or plist starts: the launcher's runtime. */ -export function bootServiceProgramOf(contents: string): string | undefined { - const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1]; - if (systemd !== undefined) return unquoteSystemdValue(systemd); - const plist = /ProgramArguments<\/key>\s*\s*([^<]*)<\/string>/.exec( - contents, - )?.[1]; - return plist === undefined ? undefined : unescapeXmlText(plist); -} - export interface BootServicePlan { readonly program: ReadonlyArray; readonly baseDir: string; @@ -104,7 +85,7 @@ export function renderBootServiceUnit(plan: BootServicePlan): string { "Type=simple", "WorkingDirectory=%h", `Environment=T3CODE_HOME=${quoteSystemdValue(plan.baseDir)}`, - `Environment=${BOOT_SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`, + `Environment=${SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`, `ExecStart=${plan.program.map(quoteSystemdValue).join(" ")}`, // Let the launcher mark an explicit stop before it signals the server. // systemd still SIGKILLs the whole cgroup if graceful shutdown times out. @@ -162,7 +143,7 @@ export function renderBootServicePlist( ` ${escapeXmlText(options.environmentPath)}`, ` T3CODE_HOME`, ` ${escapeXmlText(plan.baseDir)}`, - ` ${BOOT_SERVICE_UNIT_ENV}`, + ` ${SERVICE_UNIT_ENV}`, ` ${BOOT_SERVICE_PLIST_FILE}`, ` `, ` WorkingDirectory`, @@ -496,9 +477,12 @@ export class BootServicePrerequisiteError extends Schema.TaggedError()( "BootServiceUpdatePendingError", - {}, + { recoveryCommand: Schema.optional(Schema.String) }, ) { override get message(): string { + if (this.recoveryCommand !== undefined) { + return `A remote server update is still pending, but its service definition is missing. With the service stopped, run this on the server machine: ${this.recoveryCommand}\nWait for the update to finish or roll back, stop the foreground launcher with Ctrl-C, then retry the install.`; + } return "A remote server update is still pending. Wait for it to finish, then retry."; } } @@ -606,7 +590,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const xdgConfigHome = yield* Config.String("XDG_CONFIG_HOME").pipe(Config.withDefault("")); const installerPath = yield* Config.String("PATH").pipe(Config.withDefault("")); const runningInsideService = - (yield* Config.String(BOOT_SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) === + (yield* Config.String(SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) === BOOT_SERVICE_UNIT_FILE; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -799,6 +783,16 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { if (remaining[0]) return yield* new BootServicePrerequisiteError({ problem: remaining[0] }); }); + const withSelectionLock = (selectionPath: string, operation: Effect.Effect) => + Effect.acquireUseRelease( + Effect.tryPromise({ + try: () => acquireServiceSelectionLock(selectionPath), + catch: (cause) => new BootServiceInstallError({ cause }), + }), + () => operation, + (release) => Effect.sync(release), + ); + const install = Effect.fn("cloud.boot_service.install")(function* (options?: { readonly allowDowngrade?: boolean; readonly start?: boolean; @@ -876,8 +870,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const installed = yield* fs .exists(unitPath) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - // With start=false, changing startup-only files is safe except during a remote update, - // when the launcher also writes its state file. + // Stop outside the selection lock: the launcher may be waiting to acquire it. const start = options?.start !== false; if (installed && start && runningInsideService && manager.kind === "systemd") { // Stopping the unit from inside it kills this process before the start. @@ -890,7 +883,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { // Transient properties reach systemd literally, unlike unit-file specifiers. `--property=StandardOutput=append:${logPath}`, `--property=StandardError=append:${logPath}`, - `--setenv=${BOOT_SERVICE_UNIT_ENV}=`, + `--setenv=${SERVICE_UNIT_ENV}=`, runtimePaths.entryPath, "service", "install", @@ -905,58 +898,83 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { yield* runSteps(manager.stop); } - yield* Effect.gen(function* () { - if (installed) { - const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); - if (Option.isSome(previousStateText)) { - if (serviceStateHasPendingUpdate(previousStateText.value)) { - return yield* new BootServiceUpdatePendingError(); + // State first, then unit. The launcher acquires only one at a time, and a + // failure to lock the unit must never prevent its durable update commit. + yield* withSelectionLock( + statePath, + withSelectionLock( + unitPath, + Effect.gen(function* () { + const currentlyInstalled = yield* fs.exists(unitPath); + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if (Option.isSome(previousStateText)) { + const installedVersion = serviceStateActiveVersion(previousStateText.value); + if (serviceStateHasPendingUpdate(previousStateText.value)) { + const quote = (word: string) => `'${word.replaceAll("'", "'\\''")}'`; + return yield* new BootServiceUpdatePendingError( + !currentlyInstalled && installedVersion !== undefined + ? { + recoveryCommand: `T3CODE_HOME=${quote(input.baseDir)} ${quote(pinnedRuntimePaths(path, input.baseDir, installedVersion, platform).entryPath)} __service-launcher`, + } + : {}, + ); + } + // Read the final version after stopping the launcher; a remote update may finish after the status check. + if ( + installedVersion !== undefined && + options?.allowDowngrade !== true && + compareExactServiceVersions(input.cliVersion, installedVersion) < 0 + ) { + return yield* new BootServiceDowngradeRefusedError({ + installedVersion, + targetVersion: input.cliVersion, + }); + } } - // Read the final version after stopping the launcher; a remote update may finish after the status check. - const installedVersion = serviceStateActiveVersion(previousStateText.value); - if ( - installedVersion !== undefined && - options?.allowDowngrade !== true && - compareExactServiceVersions(input.cliVersion, installedVersion) < 0 - ) { - return yield* new BootServiceDowngradeRefusedError({ - installedVersion, - targetVersion: input.cliVersion, - }); + yield* fs + .makeDirectory(path.dirname(unitPath), { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + // Files can match the new version even when activation fails; keep that install retryable. + yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 }); + yield* writeDurably( + statePath, + // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document. + `${JSON.stringify( + { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: input.cliVersion, + } satisfies ServiceState, + null, + 2, + )}\n`, + ); + if (!start && (currentlyInstalled || Option.isSome(previousStateText))) { + const written = yield* fs.readFileString(statePath); + if (serviceStateActiveVersion(written) !== input.cliVersion) { + return yield* new BootServiceUpdatePendingError(); + } } - } - } - yield* fs - .makeDirectory(path.dirname(unitPath), { recursive: true }) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - // Files can match the new version even when activation fails; keep that install retryable. - yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 }); - yield* writeDurably( - statePath, - // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document. - `${JSON.stringify( - { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: input.cliVersion, - } satisfies ServiceState, - null, - 2, - )}\n`, - ); - if (!start && installed) { - // A concurrent remote update can overwrite launcher state; detect it before repointing the unit. - const written = yield* fs.readFileString(statePath); - if (serviceStateActiveVersion(written) !== input.cliVersion) { - return yield* new BootServiceUpdatePendingError(); - } - } - yield* writeDurably(unitPath, manager.render(plan)); - - if (start) { - yield* runSteps(manager.activate); - yield* fs.remove(restartPendingPath, { force: true }); - } - }).pipe( + yield* writeDurably(unitPath, manager.render(plan)); + }), + ), + ).pipe( + Effect.andThen( + Effect.gen(function* () { + if (start) { + yield* runSteps(manager.activate); + yield* withSelectionLock( + statePath, + Effect.gen(function* () { + // A concurrent deferred install may now be waiting for a newer restart. + const selected = parseServiceState(yield* fs.readFileString(statePath)); + if (selected?.activeVersion === input.cliVersion && selected.update === undefined) { + yield* fs.remove(restartPendingPath, { force: true }); + } + }), + ); + } + }), + ), Effect.mapError((cause) => cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause, ), diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 3b8ea548d93da..8958c945aab84 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -1,15 +1,46 @@ import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; -// Bump for incompatible runtime layout changes; launchers survive self-updates. Protocol 3 requires standalone executables. -export const SERVICE_LAUNCHER_PROTOCOL = 3 as const; +// Protocol 4 requires coordinated selection writes and unit reconciliation. An +// installed protocol-3 launcher needs one local update or service install with a restart: +// remote updates replace its child, so cannot add these launcher guarantees. +export const SERVICE_LAUNCHER_PROTOCOL = 4 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_ENV_FILE = "service.env"; export const SERVICE_STATE_FILE = "service-state.json"; +/** Set by the unit or launch agent to its own file name, so the launcher can + find the definition that started it. */ +export const SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; /** Written before explicit stop to distinguish shutdown from update handoff. */ export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */ export const SERVICE_RESTART_PENDING_FILE = ".restart-pending"; +const unquoteSystemdValue = (raw: string) => + (raw.startsWith('"') && raw.endsWith('"') + ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\") + : raw + ).replaceAll("%%", "%"); + +const unescapeXmlText = (value: string) => + value.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); + +export function serviceUnitBaseDirOf(contents: string): string | undefined { + const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1]; + if (systemd !== undefined) return unquoteSystemdValue(systemd.trim()); + const plist = /T3CODE_HOME<\/key>\s*([^<]*)<\/string>/.exec(contents)?.[1]; + return plist === undefined ? undefined : unescapeXmlText(plist); +} + +/** The executable an installed unit or plist starts: the launcher's runtime. */ +export function serviceUnitProgramOf(contents: string): string | undefined { + const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1]; + if (systemd !== undefined) return unquoteSystemdValue(systemd); + const plist = /ProgramArguments<\/key>\s*\s*([^<]*)<\/string>/.exec( + contents, + )?.[1]; + return plist === undefined ? undefined : unescapeXmlText(plist); +} + export interface PendingServiceUpdate { readonly id: string; readonly fromVersion: string; @@ -143,7 +174,8 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { if (!isRecord(value)) return undefined; const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update); if ( - value.protocol !== SERVICE_LAUNCHER_PROTOCOL || + // Protocol 3 has the same durable format; its installer can select a protocol-4 runtime. + (value.protocol !== 3 && value.protocol !== SERVICE_LAUNCHER_PROTOCOL) || typeof value.activeVersion !== "string" || !isExactServiceVersion(value.activeVersion) || (value.update !== undefined && update === undefined) || diff --git a/apps/server/src/cloud/serviceSelectionLock.test.ts b/apps/server/src/cloud/serviceSelectionLock.test.ts new file mode 100644 index 0000000000000..d2567de4c0bc5 --- /dev/null +++ b/apps/server/src/cloud/serviceSelectionLock.test.ts @@ -0,0 +1,30 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts"; + +it.layer(NodeServices.layer)("service selection lock", (it) => { + it.effect("reclaims a lease from an earlier boot even when its PID is live again", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-selection-lock-" }); + const selectionPath = path.join(root, "service-state.json"); + const lock = `${selectionPath}.selection-lock`; + const stale = `${process.pid}.1.00000000-0000-4000-8000-000000000000`; + yield* fs.makeDirectory(lock, { recursive: true }); + yield* fs.writeFileString(path.join(lock, stale), ""); + + const release = yield* Effect.promise(() => acquireServiceSelectionLock(selectionPath)); + const owners = yield* fs.readDirectory(lock); + assert.equal(owners.length, 1); + assert.notEqual(owners[0], stale); + assert.isTrue(owners[0]!.startsWith(`${process.pid}.`)); + release(); + assert.isFalse(yield* fs.exists(lock)); + }), + ); +}); diff --git a/apps/server/src/cloud/serviceSelectionLock.ts b/apps/server/src/cloud/serviceSelectionLock.ts new file mode 100644 index 0000000000000..4cbd123c22f7e --- /dev/null +++ b/apps/server/src/cloud/serviceSelectionLock.ts @@ -0,0 +1,88 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Shared with the standalone launcher, which has no Effect runtime. +// @effect-diagnostics globalTimers:off -- Bounded cross-process lease acquisition in the standalone launcher. +// @effect-diagnostics globalDate:off -- The standalone launcher has no Effect clock for its acquisition deadline. +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +// Coarse boot time: a lease from an earlier boot is dead even when its PID is live again. +const bootEpochSeconds = () => Math.round(Date.now() / 1000 - NodeOS.uptime()); + +/** Serialize durable selection and unit writes, never manager stop/start commands. */ +export async function acquireServiceSelectionLock(selectionPath: string): Promise<() => void> { + const directory = NodePath.dirname(selectionPath); + NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const lock = selectionPath + ".selection-lock"; + const owner = `${process.pid}.${bootEpochSeconds()}.${NodeCrypto.randomUUID()}`; + const candidate = `${lock}.${owner}`; + const deadline = Date.now() + 30_000; + const removeEmptyLock = () => { + try { + NodeFS.rmdirSync(lock); + } catch (error) { + if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes((error as NodeJS.ErrnoException).code ?? "")) + throw error; + } + }; + NodeFS.mkdirSync(candidate, { mode: 0o700 }); + try { + NodeFS.writeFileSync(NodePath.join(candidate, owner), "", { mode: 0o600 }); + while (true) { + try { + // Publish the owner atomically; rename cannot replace another populated directory. + NodeFS.renameSync(candidate, lock); + return () => { + NodeFS.unlinkSync(NodePath.join(lock, owner)); + removeEmptyLock(); + }; + } catch (error) { + if (!["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) + throw error; + } + let owners: string[]; + try { + owners = NodeFS.readdirSync(lock); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + if (owners.length === 1) { + const previous = owners[0]!; + const [pidText, bootText] = previous.split("."); + const pid = Number(pidText); + if (!Number.isSafeInteger(pid) || pid <= 0) + throw new Error(`Invalid service selection lock owner at ${lock}.`); + const ownerBoot = Number(bootText); + const fromEarlierBoot = + Number.isSafeInteger(ownerBoot) && Math.abs(ownerBoot - bootEpochSeconds()) > 60; + let dead = fromEarlierBoot; + if (!dead) { + try { + process.kill(pid, 0); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ESRCH" && code !== "EPERM") throw error; + dead = code === "ESRCH"; + } + } + if (dead) { + // Remove only this dead owner's unique file, never a replacement owner's. + try { + NodeFS.unlinkSync(NodePath.join(lock, previous)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + } + removeEmptyLock(); + if (Date.now() >= deadline) + throw new Error( + `Service selection is locked at ${lock}. Retry when the other operation finishes.`, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } finally { + NodeFS.rmSync(candidate, { recursive: true, force: true }); + } +} diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 0006d371cf2c9..3304211d9e7c6 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -2,6 +2,8 @@ import type * as NodeChildProcess from "node:child_process"; // @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback. import * as NodeFS from "node:fs"; +// @effect-diagnostics-next-line nodeBuiltinImport:off - models the service manager reading disk through the launcher's Promise callback. +import * as NodeFSP from "node:fs/promises"; import * as NodeSqlite from "node:sqlite"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -13,10 +15,10 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import { vi } from "vite-plus/test"; +import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootService.ts"; import { databaseBackupBlocker, Launcher, - main, readServiceEnvironment, type BackupSpaceProbe, readServiceState, @@ -112,6 +114,13 @@ it("rejects contradictory service state", () => { ); }); +const unitPlan = (root: string, version: string) => ({ + program: [`${root}/runtime/versions/${version}/t3`, "__service-launcher"], + baseDir: root, + logPath: `${root}/logs/boot-service.log`, + unitPath: `${root}/unit/t3code.service`, +}); + const writeFakeRuntime = ( fs: FileSystem.FileSystem, path: Path.Path, @@ -130,6 +139,32 @@ const writeFakeRuntime = ( return entryPath; }); +const fakeServiceUnit = ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + version: string, + render = renderBootServiceUnit, +) => + Effect.gen(function* () { + const plan = unitPlan(root, version); + yield* fs.makeDirectory(path.dirname(plan.unitPath), { recursive: true }); + const contents = render(plan); + yield* fs.writeFileString(plan.unitPath, contents); + const unit = { + path: plan.unitPath, + reloads: 0, + loadedContents: contents, + failReload: false, + reload: async () => { + unit.reloads += 1; + if (unit.failReload) throw new Error("daemon-reload failed"); + unit.loadedContents = await NodeFSP.readFile(unit.path, "utf8"); + }, + }; + return unit; + }); + it.layer(NodeServices.layer)("service state persistence", (it) => { it.effect( "loads optional service.env literally without changing inherited or managed variables", @@ -217,6 +252,105 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); + it.effect.each([ + { unitVersion: "1.0.0", replacement: "an intentional downgrade" }, + { unitVersion: "1.2.0", replacement: "another T3 home" }, + ])( + "recovers a rolled-back update with unit $unitVersion and preserves $replacement", + ({ unitVersion, replacement }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + "setInterval(() => {}, 1_000);\n", + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-1", + fromVersion: "1.1.0", + targetVersion: "1.2.0", + status: "rolled-back", + }, + }), + ); + const unit = yield* fakeServiceUnit(fs, path, root, unitVersion); + const expected = renderBootServiceUnit( + unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion), + ); + const createLauncher = Effect.gen(function* () { + return new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)), { + serviceUnit: unit, + }); + }); + const run = (launcher: Launcher) => + Effect.gen(function* () { + const running = launcher.run(); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }); + + yield* run(yield* createLauncher); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + + yield* run(yield* createLauncher); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-2", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "committed", + }, + }), + ); + const staleLauncher = yield* createLauncher; + // A local install can replace the selection or service owner without stopping this launcher. + const selectedRoot = + replacement === "another T3 home" ? path.join(root, "other-home") : root; + yield* writeFakeRuntime( + fs, + path, + path.join(selectedRoot, "runtime", "versions", "1.0.0"), + "setInterval(() => {}, 1_000);\n", + ); + const selectedStatePath = path.join(selectedRoot, "runtime", "service-state.json"); + yield* Effect.promise(() => + writeServiceState(selectedStatePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const selectedUnit = renderBootServiceUnit({ + ...unitPlan(selectedRoot, "1.0.0"), + unitPath: unit.path, + }); + yield* fs.writeFileString(unit.path, selectedUnit); + yield* Effect.promise(() => unit.reload()); + const oldHomeState = yield* fs.readFileString(statePath); + const selectedState = yield* fs.readFileString(selectedStatePath); + + yield* run(staleLauncher); + assert.equal(yield* fs.readFileString(statePath), oldHomeState); + assert.equal(yield* fs.readFileString(selectedStatePath), selectedState); + assert.equal(yield* fs.readFileString(unit.path), selectedUnit, replacement); + assert.equal(unit.loadedContents, selectedUnit, replacement); + }), + ); + it.effect("stays idle instead of exiting when another server owns the T3 home", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -257,25 +391,49 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect("commits only after the trial reports prepared", () => + const commitUpdate = ( + render = renderBootServiceUnit, + missingUnit = false, + legacyInstall = false, + ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service 100% & -" }); const statePath = path.join(root, "runtime", "service-state.json"); const databasePath = path.join(root, "userdata", "state.sqlite"); + const activatedPath = path.join(root, "activated"); yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); yield* fs.writeFileString(databasePath, "before trial"); - const encodedDatabasePath = JSON.stringify(databasePath); + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0", render); + if (missingUnit) yield* fs.remove(unit.path); const childSource = ` +import assert from "node:assert/strict"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +assert.equal(context.protocol, ${SERVICE_LAUNCHER_PROTOCOL}); if (context.update?.status === "pending") { - process.send({ type: "prepared", updateId: context.update.id }); + const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8")); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update.status, "pending"); + ${ + missingUnit + ? `assert.equal(existsSync(${JSON.stringify(unit.path)}), false);` + : `assert.equal(readFileSync(${JSON.stringify(unit.path)}, "utf8"), ${JSON.stringify(unit.loadedContents)});` + } process.on("message", (message) => { - if (message.type === "committed") process.exit(0); + if (message.type === "committed") { + const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8")); + assert.equal(state.protocol, ${SERVICE_LAUNCHER_PROTOCOL}); + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update.status, "committed"); + writeFileSync(${JSON.stringify(activatedPath)}, message.updateId); + process.exit(0); + } }); + process.send({ type: "prepared", updateId: context.update.id }); } else if (context.update === undefined) { - process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} }); setInterval(() => {}, 1_000); } else { process.exit(0); @@ -289,25 +447,97 @@ if (context.update?.status === "pending") { childSource, ); } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), + // The old CLI installs the new runtime but writes state with its own protocol. + yield* fs.writeFileString( + statePath, + `${JSON.stringify( + { + protocol: legacyInstall ? 3 : SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }, + null, + 2, + )}\n`, ); - - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); - yield* Effect.promise(() => + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { serviceUnit: unit }, + ); + const failure = yield* Effect.promise(() => launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), + () => "launcher unexpectedly completed", + (cause: unknown) => String(cause), ), ); - + assert.include(failure, "Active child exited unexpectedly (0)"); const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.1.0"); assert.equal(state.update?.status, "committed"); - }), + assert.equal(yield* fs.readFileString(activatedPath), state.update?.id); + const recover = () => + Effect.gen(function* () { + const fresh = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { serviceUnit: unit }, + ); + const running = fresh.run(); + yield* Effect.promise(() => fresh.stop("SIGTERM")); + yield* Effect.promise(() => running); + }); + return { fs, root, unit, recover }; + }); + + it.effect.each([ + { + name: "systemd after a protocol-3 local upgrade", + render: renderBootServiceUnit, + legacyInstall: true, + }, + { + name: "launchd", + legacyInstall: false, + render: (plan: Parameters[0]) => + renderBootServicePlist(plan, { + homeDir: "/Users/test & ", + environmentPath: "/usr/bin:/custom/runtime/versions/1.0.0", + }), + }, + ])( + "commits and repoints $name only after the trial reports prepared", + ({ render, legacyInstall }) => + Effect.gen(function* () { + const { fs, root, unit } = yield* commitUpdate(render, false, legacyInstall); + const expected = render(unitPlan(root, "1.1.0")); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + assert.equal(unit.reloads, 1); + }), + ); + + it.effect( + "keeps a durable commit through unit failure and retries a failed reload on restart", + () => + Effect.gen(function* () { + const { fs, root, unit, recover } = yield* commitUpdate(renderBootServiceUnit, true); + assert.equal(unit.reloads, 0); + const original = renderBootServiceUnit(unitPlan(root, "1.0.0")); + const expected = renderBootServiceUnit(unitPlan(root, "1.1.0")); + assert.equal(unit.loadedContents, original); + yield* fs.writeFileString(unit.path, original); + unit.failReload = true; + yield* recover(); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.reloads, 1); + assert.equal(unit.loadedContents, original); + + unit.failReload = false; + yield* recover(); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + assert.equal(unit.reloads, 2); + }), ); it.effect("rolls back invalid trials with the startup environment", () => @@ -357,9 +587,18 @@ if (context.update?.status === "pending") { activeVersion: "1.0.0", }), ); + const unit = yield* fakeServiceUnit(fs, path, root, "0.9.0"); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + serviceUnit: unit, + environment: yield* Effect.promise(() => readServiceEnvironment(root)), + }, + ); yield* Effect.promise(() => - main().then( + launcher.run().then( () => Promise.reject(new Error("launcher unexpectedly completed")), () => Promise.resolve(), ), @@ -368,6 +607,11 @@ if (context.update?.status === "pending") { const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.0.0"); assert.equal(state.update?.status, "rolled-back"); + assert.equal( + yield* fs.readFileString(unit.path), + renderBootServiceUnit(unitPlan(root, "1.0.0")), + ); + assert.equal(unit.reloads, 1); assert.equal( state.update?.status === "rolled-back" ? state.update.reason : undefined, "invalid-prepared", diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index b46083cccbcd8..e6d3431f39b3b 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -7,6 +7,7 @@ import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; +import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts"; import type { PendingServiceUpdate, @@ -21,12 +22,14 @@ import { decodeServiceLauncherChildMessage, isExactServiceVersion, parseServiceState, + serviceUnitBaseDirOf, SERVICE_ENV_FILE, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, SERVICE_RESTART_PENDING_FILE, SERVICE_STOP_MARKER_FILE, + SERVICE_UNIT_ENV, } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; @@ -64,6 +67,78 @@ const runtimeSpawnArguments = (paths: ReturnType) => ({ args: ["serve"], }); +/** + * The unit or launch agent that `t3 service install` wrote. Its program names + * the runtime that ran the install, and a remote update never replaces it, so + * after a commit the launcher points it at the committed runtime. Otherwise the + * next start lands on a runtime that nothing in the service state records. + */ +export interface ServiceUnit { + readonly path: string; + /** systemd keeps a loaded copy of the unit; launchd re-reads the plist at bootstrap. */ + readonly reload: () => Promise; +} + +function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined { + const unitName = env[SERVICE_UNIT_ENV]?.trim(); + const home = env.HOME?.trim(); + if (!unitName || !home) return undefined; + if (unitName.endsWith(".plist")) { + return { + path: NodePath.join(home, "Library", "LaunchAgents", unitName), + reload: () => Promise.resolve(), + }; + } + // The manager's own environment, which decides where it reads user units. + const xdgConfigHome = env.XDG_CONFIG_HOME?.trim(); + const configHome = + xdgConfigHome && NodePath.isAbsolute(xdgConfigHome) + ? xdgConfigHome + : NodePath.join(home, ".config"); + return { + path: NodePath.join(configHome, "systemd", "user", unitName), + reload: () => + new Promise((resolve, reject) => { + NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], { + stdio: "ignore", + timeout: 30_000, + }) + .once("error", reject) + .once("exit", (code, signal) => + code === 0 + ? resolve() + : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)), + ); + }), + }; +} + +// systemd's ExecStart value, or the plist string before the __service-launcher argument. +// Only the version segment changes, so quoted or escaped paths stay as rendered. +// The launch agent's PATH names a versions directory without the executable; leave it. +const SERVICE_UNIT_PROGRAMS = [ + /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m, + /([^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*__service-launcher<\/string>)/, +] as const; + +/** + * The unit contents with their program pointed at `version`'s runtime. A unit + * already at that version or a newer one is returned unchanged: `t3 update` + * may have repointed it to a later version while this commit was in flight. + */ +function repointServiceUnit(contents: string, version: string): string { + for (const program of SERVICE_UNIT_PROGRAMS) { + const match = program.exec(contents); + if (match === null) continue; + const [, prefix = "", current = "", suffix = ""] = match; + if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) { + return contents; + } + return contents.replace(program, () => `${prefix}${version}${suffix}`); + } + return contents; +} + const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const; const RESTORE_MARKER = ".restore-pending"; @@ -445,8 +520,12 @@ export async function readServiceState(filePath: string): Promise } export async function writeServiceState(filePath: string, state: ServiceState): Promise { + await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 }); + await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`); +} + +async function replaceFileDurably(filePath: string, contents: string): Promise { const directory = NodePath.dirname(filePath); - await NodeFSP.mkdir(directory, { recursive: true, mode: 0o700 }); const tempPath = NodePath.join( directory, `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`, @@ -454,7 +533,7 @@ export async function writeServiceState(filePath: string, state: ServiceState): let handle: NodeFSP.FileHandle | undefined; try { handle = await NodeFSP.open(tempPath, "wx", 0o600); - await handle.writeFile(`${JSON.stringify(state, null, 2)}\n`, "utf8"); + await handle.writeFile(contents, "utf8"); await handle.sync(); await handle.close(); handle = undefined; @@ -542,6 +621,7 @@ export class Launcher { #done = false; readonly #completion = Promise.withResolvers(); readonly #backupSpaceProbe: Partial | undefined; + readonly #serviceUnit: ServiceUnit | undefined; readonly #environment: Readonly; constructor( @@ -549,11 +629,13 @@ export class Launcher { state: ServiceState, options: { readonly backupSpaceProbe?: Partial; + readonly serviceUnit?: ServiceUnit; readonly environment?: Readonly; } = {}, ) { this.#baseDir = baseDir; this.#backupSpaceProbe = options.backupSpaceProbe; + this.#serviceUnit = options.serviceUnit; this.#environment = { ...(options.environment ?? process.env) }; this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); this.#state = state; @@ -620,20 +702,66 @@ export class Launcher { this.#timer = undefined; } + async #selectState(next: ServiceState): Promise { + const release = await acquireServiceSelectionLock(this.#statePath); + try { + const selected = await readServiceState(this.#statePath); + // Local installs replace the selection and clear its update record. An old + // launcher must not restore its own selection when asked to update again. + if ( + selected.activeVersion !== this.#state.activeVersion || + selected.update?.id !== this.#state.update?.id || + selected.update?.status !== this.#state.update?.status + ) { + throw new Error("Service selection changed. Restart the service before updating."); + } + await writeServiceState(this.#statePath, next); + this.#state = next; + } finally { + release(); + } + } + async #recover(): Promise { // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); const restartPending = restartPendingPath(this.#baseDir); - const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined); - if (awaitedVersion?.trim() === this.#state.activeVersion) { - await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined); + try { + const release = await acquireServiceSelectionLock(this.#statePath); + try { + const selected = await readServiceState(this.#statePath); + const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch( + () => undefined, + ); + const awaited = awaitedVersion?.trim(); + if ( + selected.activeVersion === this.#state.activeVersion && + selected.update?.id === this.#state.update?.id && + selected.update?.status === this.#state.update?.status && + awaited !== undefined && + isExactServiceVersion(awaited) && + compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0 + ) { + await NodeFSP.rm(restartPending, { force: true }); + } + } finally { + release(); + } + } catch (error) { + process.stderr.write( + `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`, + ); } const update = this.#state.update; if (update?.status !== "pending") { if (update !== undefined) { await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); } - await this.#startChild(this.#state.activeVersion, "active", update); + try { + await this.#startChild(this.#state.activeVersion, "active", update); + } finally { + if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion); + } return; } if (await databaseRestorePending(this.#baseDir, update)) { @@ -780,8 +908,12 @@ export class Launcher { status: "pending", }; const next: ServiceState = { ...this.#state, update: pending }; - await writeServiceState(this.#statePath, next); - this.#state = next; + try { + await this.#selectState(next); + } catch (error) { + await reject(error instanceof Error ? error.message : String(error)); + return; + } await sendMessage(child.process, { type: "update-accepted", updateId: pending.id }); this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS); } @@ -827,11 +959,53 @@ export class Launcher { activeVersion: pending.targetVersion, update: committed, }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#selectState(next); child.role = "active"; await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); - await sendMessage(child.process, { type: "committed", updateId: committed.id }); + try { + await sendMessage(child.process, { type: "committed", updateId: committed.id }); + } finally { + await this.#repointServiceUnit(committed.targetVersion); + } + } + + // Selection is durable before this best-effort unit reconciliation. + async #repointServiceUnit(version: string): Promise { + const unit = this.#serviceUnit; + if (unit === undefined) return; + try { + const release = await acquireServiceSelectionLock(unit.path); + try { + // The installer writes state before the unit under this lease. Do not + // acquire its state lease here: the installer takes that lease first. + const selected = await readServiceState(this.#statePath); + if ( + selected.activeVersion !== this.#state.activeVersion || + selected.update?.id !== this.#state.update?.id || + selected.update?.status !== this.#state.update?.status + ) { + return; + } + const contents = await NodeFSP.readFile(unit.path, "utf8"); + const installedBaseDir = serviceUnitBaseDirOf(contents); + if ( + installedBaseDir === undefined || + NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir) + ) { + return; + } + const repointed = repointServiceUnit(contents, version); + if (repointed !== contents) await replaceFileDurably(unit.path, repointed); + // A previous rename may have succeeded before reload failed. + await unit.reload(); + } finally { + release(); + } + } catch (error) { + process.stderr.write( + `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`, + ); + } } async #handlePreparedTimeout(child: ManagedChild): Promise { @@ -887,9 +1061,9 @@ export class Launcher { activeVersion: pending.fromVersion, update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#selectState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); + await this.#repointServiceUnit(next.activeVersion); } // Signal listeners alone do not keep Node alive. this.#timer = setInterval(() => {}, 2_147_483_647); @@ -915,10 +1089,13 @@ export class Launcher { activeVersion: pending.fromVersion, update: outcome, }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#selectState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); - await this.#startChild(next.activeVersion, "active", outcome); + try { + await this.#startChild(next.activeVersion, "active", outcome); + } finally { + await this.#repointServiceUnit(next.activeVersion); + } } } @@ -930,5 +1107,10 @@ export async function main(): Promise { const environment = await readServiceEnvironment(baseDir); const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); const state = await readServiceState(statePath); - await new Launcher(baseDir, state, { environment }).run(); + const serviceUnit = serviceUnitFromEnvironment(); + await new Launcher( + baseDir, + state, + serviceUnit === undefined ? { environment } : { environment, serviceUnit }, + ).run(); } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 1536884578d4b..e0f1d286a1608 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -4,14 +4,30 @@ A [stable launcher](../../apps/server/src/serviceLauncher.ts) owns the runtime selected by systemd or launchd. It is the only runtime writer of durable service state. Server children request updates over inherited IPC; they never rewrite their service definition or select their own replacement. Local service commands -may replace the launcher and state while the service is stopped. Foreground CLI -processes do not self-update. +may replace the launcher and state, optionally deferring the restart. Foreground +CLI processes do not self-update. + +Both writers serialize state and unit writes through +[file leases](../../apps/server/src/cloud/serviceSelectionLock.ts). The installer +takes the state lease before the unit lease; the launcher never holds both, so a +unit failure cannot block a durable commit. Manager stop/start commands run outside +both leases, since stopping a launcher that waits on a lease would deadlock. A +running launcher refuses further updates once a local command changes its selection. Exact-version installs keep restarts independent of npm cache eviction or a moving release tag. Installation and preflight happen in staging before publishing an immutable runtime. Preflight checks the launcher protocol because a target that -needs new rollback guarantees cannot safely run under an older launcher. Upgrading -that launcher requires a local service update. +needs new rollback guarantees cannot safely run under an older launcher. A remote +update replaces only the child, so upgrading the launcher requires a local service +update with a restart. + +After a commit or rollback the launcher points the unit at the selected runtime and +reloads the manager, so the next service start runs that version's launcher; the +protocol check is what makes that safe. Each start after a finished update retries this, because +a previous reload may have failed, and leaves a unit naming a newer runtime or another +T3 home alone. Until a deferred local upgrade restarts, systemd (before daemon-reload) +and launchd (before bootstrap) can respawn the old launcher, which the new runtime +refuses to run under. ## Commit boundary diff --git a/docs/user/updating.md b/docs/user/updating.md index 7e62c7bf580fc..123306077f8f5 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -118,6 +118,11 @@ asks before restarting the background service; if you decline, run stop it and start it again afterwards with your usual options such as `--host` or `--tailscale-serve`. +A background service installed by an older release refuses **Update server** +until you update it once on the host and let that update restart it. If you +decline the restart, the service can stay down after a crash or reboot until you +run `t3 service restart`. + If you run the server with `npx` rather than an installed `t3`, there is nothing to update on the host: stop the server and relaunch it as `npx t3@` with the same subcommand and options.