diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index 4358165958a1b..42c2b636e274a 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -16,6 +16,11 @@ import { serviceCommand } from "./service.ts"; afterEach(() => vi.restoreAllMocks()); +vi.mock("../cloud/selfUpdate.ts", async () => { + const Effect = await import("effect/Effect"); + return { updateStoppedRuntime: () => Effect.void }; +}); + const status = { supported: true, installed: true, @@ -55,6 +60,42 @@ function makeTestService( } it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands", (it) => { + it.effect.each([ + { label: "missing", installed: false }, + { label: "foreign", installed: true, installedBaseDir: "/another/t3-home" }, + ])("update-stopped refuses a $label unit before touching runtime state", (input) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-cli-test-" }); + const { service, installOptions, restarts } = makeTestService({ ...status, ...input }); + vi.spyOn(BootService, "layer").mockReturnValue( + Layer.succeed(BootService.BootService, service), + ); + const result = yield* Command.runWith(serviceCommand, { version: packageJson.version })([ + "update-stopped", + "1.2.3", + "--base-dir", + baseDir, + "--unit", + "crew.service", + ]).pipe( + Effect.provideService(HostProcess.Environment, {}), + Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} }))), + Effect.result, + ); + expect(result).toMatchObject({ + _tag: "Failure", + failure: { + _tag: "BootServiceCommandError", + step: "checking the selected service unit and T3 home", + }, + }); + expect(yield* fs.exists(`${baseDir}/runtime`)).toBe(false); + expect(installOptions).toEqual([]); + expect(restarts).toEqual([]); + }), + ); + it.effect.each(["install", "update"] as const)( "%s refuses a downgrade before changing the service", (command) => @@ -62,14 +103,16 @@ it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands const fs = yield* FileSystem.FileSystem; const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-cli-test-" }); const { service, installOptions } = makeTestService(newerServiceStatus); - vi.spyOn(BootService, "layer").mockReturnValue( - Layer.succeed(BootService.BootService, service), - ); + const serviceLayer = vi + .spyOn(BootService, "layer") + .mockReturnValue(Layer.succeed(BootService.BootService, service)); const error = yield* Command.runWith(serviceCommand, { version: packageJson.version })([ command, "--base-dir", baseDir, + "--unit", + "crew.service", ]).pipe( Effect.provideService(HostProcess.Environment, {}), Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} }))), @@ -82,6 +125,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands targetVersion: packageJson.version, }); expect(installOptions).toEqual([]); + expect(serviceLayer).toHaveBeenCalledWith( + expect.objectContaining({ baseDir, unit: "crew.service" }), + ); }), ); diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index b78ee31c9fa4d..9da8438502cbe 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -5,8 +5,9 @@ import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; -import { Command, Flag, GlobalFlag, Prompt } from "effect/cli"; +import { Argument, Command, Flag, GlobalFlag, Prompt } from "effect/cli"; import { FetchHttpClient } from "effect/http"; import packageJson from "../../package.json" with { type: "json" }; @@ -14,15 +15,17 @@ import * as BootService from "../cloud/bootService.ts"; import * as ServiceProtection from "../cloud/serviceProtection.ts"; import { compareExactServiceVersions, SERVICE_ENV_FILE } from "../cloud/serviceProtocol.ts"; import * as ServerConfig from "../config.ts"; +import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; import * as ProcessRunner from "../processRunner.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; import { resolveEntryPath } from "./invocation.ts"; -export const layer = (config: ServerConfig.ServerConfig["Service"]) => +export const layer = (config: ServerConfig.ServerConfig["Service"], unit?: string) => BootService.layer({ baseDir: config.baseDir, logsDir: config.logsDir, cliVersion: packageJson.version, + ...(unit === undefined ? {} : { unit }), }).pipe(Layer.provide(ProcessRunner.layer), Layer.provide(FetchHttpClient.layer)); export type ServiceReconcileResult = @@ -154,7 +157,8 @@ function formatServiceStatus( } const installedVersion = status.installedVersion ?? cliVersion; const problems = (status.problems ?? []).map( - (problem) => ` [${problem}] ${BootService.formatBootServiceProblem(problem)}`, + (problem) => + ` [${problem}] ${BootService.formatBootServiceProblem(problem, status.unitPath.split("/").at(-1))}`, ); if ( !status.current && @@ -183,16 +187,31 @@ function formatServiceStatus( } const runServiceCommand = Effect.fn("cli.service.run")(function* ( - flags: { readonly baseDir: Parameters[0]["baseDir"] }, + flags: { + readonly baseDir: Parameters[0]["baseDir"]; + readonly unit?: Option.Option; + }, run: Effect.Effect, ) { const logLevel = yield* GlobalFlag.LogLevel; const config = yield* resolveCliAuthConfig(flags, logLevel); - return yield* run.pipe(Effect.provide(Layer.merge(layer(config), ServerConfig.layer(config)))); + const unit = flags.unit === undefined ? undefined : Option.getOrUndefined(flags.unit); + return yield* run.pipe( + Effect.provide(Layer.merge(layer(config, unit), ServerConfig.layer(config))), + ); }); -const serviceReconcileFlags = { +const serviceFlags = { ...projectLocationFlags, + unit: Flag.String("unit").pipe( + Flag.withDescription("Select the systemd user service unit."), + Flag.withSchema(BootService.BootServiceUnitName), + Flag.optional, + ), +}; + +const serviceReconcileFlags = { + ...serviceFlags, allowDowngrade: Flag.Boolean("allow-downgrade").pipe( Flag.withDescription("Allow replacing a newer installed service with this older CLI version."), Flag.withDefault(false), @@ -246,7 +265,43 @@ const serviceUpdateCommand = Command.make("update", serviceReconcileFlags).pipe( ), ); -const serviceRestartCommand = Command.make("restart", projectLocationFlags).pipe( +const serviceUpdateStoppedCommand = Command.make("update-stopped", { + ...serviceFlags, + targetVersion: Argument.String("target-version"), +}).pipe( + Command.withDescription("Update an already stopped service to an exact release version."), + Command.withHandler((flags) => + runServiceCommand( + flags, + Effect.gen(function* () { + const status = yield* (yield* BootService.BootService).status; + const { baseDir } = yield* ServerConfig.ServerConfig; + const path = yield* Path.Path; + if ( + !status.installed || + status.installedBaseDir === undefined || + path.resolve(status.installedBaseDir) !== path.resolve(baseDir) + ) { + return yield* new BootService.BootServiceCommandError({ + step: "checking the selected service unit and T3 home", + cause: new Error("The selected service unit is missing or serves another T3 home."), + }); + } + return yield* Effect.promise(() => import("../cloud/selfUpdate.ts")); + }).pipe( + Effect.flatMap(({ updateStoppedRuntime }) => + updateStoppedRuntime({ targetVersion: flags.targetVersion }), + ), + Effect.provide( + Layer.mergeAll(ProcessRunner.layer, FetchHttpClient.layer, ManagedPolicy.layer), + ), + Effect.andThen(Console.log(`Service active version: t3@${flags.targetVersion}`)), + ), + ), + ), +); + +const serviceRestartCommand = Command.make("restart", serviceFlags).pipe( Command.withDescription( "Restart the background service. Picks up a version installed by `t3 update` that was not restarted at the time.", ), @@ -271,7 +326,7 @@ const serviceRestartCommand = Command.make("restart", projectLocationFlags).pipe ), ); -const serviceUninstallCommand = Command.make("uninstall", projectLocationFlags).pipe( +const serviceUninstallCommand = Command.make("uninstall", serviceFlags).pipe( Command.withDescription("Stop and remove the T3 Code background service."), Command.withHandler((flags) => runServiceCommand( @@ -287,7 +342,7 @@ const serviceUninstallCommand = Command.make("uninstall", projectLocationFlags). ), ); -const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe( +const serviceStatusCommand = Command.make("status", serviceFlags).pipe( Command.withDescription("Show whether the T3 Code background service is installed."), Command.withHandler((flags) => runServiceCommand( @@ -295,9 +350,14 @@ const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe( Effect.gen(function* () { const service = yield* BootService.BootService; const path = yield* Path.Path; + const { baseDir } = yield* ServerConfig.ServerConfig; const status = yield* service.status; const protection = - status.installed && (yield* HostProcess.Platform) === "linux" + status.installed && + status.installedBaseDir !== undefined && + path.resolve(status.installedBaseDir) === path.resolve(baseDir) && + path.basename(status.unitPath) === BootService.BOOT_SERVICE_UNIT_FILE && + (yield* HostProcess.Platform) === "linux" ? yield* ServiceProtection.readProtection(path.dirname(status.unitPath)).pipe( Effect.provide(ProcessRunner.layer), ) @@ -426,5 +486,6 @@ export const serviceCommand = Command.make("service").pipe( serviceStatusCommand, serviceProtectCommand, serviceUpdateCommand, + serviceUpdateStoppedCommand, ]), ); diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 7d80aee9af6b0..0ea4f69d2dda6 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -127,10 +127,12 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( const failed = command === control.failCommand; if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; - if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; - if (!failed && command === "systemctl --user restart t3code.service") control.active = true; + if (!failed && input.command === "systemctl" && input.args[1] === "enable") + control.enabled = true; + if (!failed && input.command === "systemctl" && input.args[1] === "restart") + control.active = true; // A timed-out stop may still stop the service. - if (command === "systemctl --user stop t3code.service") control.active = false; + if (input.command === "systemctl" && input.args[1] === "stop") control.active = false; if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || @@ -169,8 +171,9 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( environmentPath: string | undefined = installerPath, cliVersion = "1.2.3", serviceBaseDir = baseDir, - insideService = false, + insideService: boolean | string = false, hostExecPath = "/usr/bin/t3", + unit?: string, ) => Effect.gen(function* () { const paths = pinnedRuntimePaths(path, serviceBaseDir, cliVersion, platform); @@ -181,6 +184,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( baseDir: serviceBaseDir, logsDir: path.join(serviceBaseDir, "userdata", "logs"), cliVersion, + ...(unit === undefined ? {} : { unit }), host: { execPath: hostExecPath }, }); }).pipe( @@ -201,7 +205,12 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( ...(environmentPath === undefined || environmentPath === "" ? {} : { PATH: environmentPath }), - ...(insideService ? { T3_BOOT_SERVICE_UNIT: "t3code.service" } : {}), + ...(insideService + ? { + T3_BOOT_SERVICE_UNIT: + typeof insideService === "string" ? insideService : "t3code.service", + } + : {}), }, }), ), @@ -318,20 +327,52 @@ it.layer(NodeServices.layer)("boot service install", (it) => { it.effect("installs, reports current state, and uninstalls", () => Effect.gen(function* () { const { service, fs, statePath, timeouts, runtime } = yield* makeHarness(); + const path = yield* Path.Path; const plan = yield* service.install(); + const launcherPath = path.join(plan.baseDir, "runtime", "launcher"); expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.2.3", }); - expect(plan.program).toEqual([runtime.entryPath, "__service-launcher"]); + expect(plan.program).toEqual([launcherPath, "__service-launcher"]); const unit = yield* fs.readFileString(plan.unitPath); - expect(BootService.bootServiceProgramOf(unit)).toBe(runtime.entryPath); + expect(BootService.bootServiceProgramOf(unit)).toBe(launcherPath); + expect(yield* fs.realPath(launcherPath)).toBe(runtime.entryPath); expect(unit).toMatch(/^ExecStart=(?:"(?:\\.|[^"\\])*"|\S+) __service-launcher$/m); expect(yield* service.status).toMatchObject({ current: true, installedVersion: "1.2.3", }); + yield* fs.writeFileString(plan.unitPath, unit.replace(launcherPath, runtime.entryPath)); + expect(yield* service.status).toMatchObject({ installed: true, current: false }); + const repairedState = { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + update: { + id: "previous", + fromVersion: "1.2.2", + targetVersion: "1.2.3", + status: "committed", + }, + preparation: { + fromVersion: "1.2.2", + targetVersion: "1.2.3", + installationId: "a".repeat(64), + sha256: "b".repeat(64), + }, + }; + yield* fs.writeFileString(statePath, JSON.stringify(repairedState)); + yield* service.install(); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual(repairedState); + expect(yield* fs.readFileString(plan.unitPath)).toBe(unit); + yield* fs.remove(launcherPath); + expect((yield* service.status).current).toBe(false); + yield* fs.symlink(runtime.sentinelPath, launcherPath); + expect((yield* service.status).current).toBe(false); + yield* fs.remove(launcherPath); + yield* fs.symlink(runtime.entryPath, launcherPath); + expect((yield* service.status).current).toBe(true); const pendingState = JSON.stringify({ protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.2.3", @@ -489,8 +530,15 @@ it.layer(NodeServices.layer)("boot service install", (it) => { activeVersion: "1.2.4", }); expect(BootService.bootServiceProgramOf(yield* fs.readFileString(plan.unitPath))).toBe( - pinnedRuntimePaths(path, baseDir, "1.2.4", platform).entryPath, + platform === "linux" + ? path.join(baseDir, "runtime", "launcher") + : pinnedRuntimePaths(path, baseDir, "1.2.4", platform).entryPath, ); + if (platform === "linux") { + expect(yield* fs.realPath(plan.program[0]!)).toBe( + pinnedRuntimePaths(path, baseDir, "1.2.4", platform).entryPath, + ); + } expect( commands.filter( (command) => @@ -514,10 +562,9 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const path = yield* Path.Path; yield* service.install(); const newer = yield* makeService(undefined, "1.2.4"); - const unitPath = (yield* service.status).unitPath; - yield* fs.remove(unitPath); - yield* fs.makeDirectory(unitPath); - yield* fs.writeFileString(path.join(unitPath, "occupied"), ""); + const launcherPath = path.join(path.dirname(statePath), "launcher"); + yield* fs.remove(launcherPath); + yield* fs.writeFileString(launcherPath, "unowned launcher"); const error = yield* newer.install({ start: false }).pipe(Effect.flip); expect(error._tag).toBe("BootServiceInstallError"); @@ -612,7 +659,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); - it.effect("restart leaves a service that serves another T3 home alone", () => + it.effect("leaves a service that serves another T3 home alone", () => Effect.gen(function* () { const { service, fs, commands, makeService } = yield* makeHarness(); yield* service.install(); @@ -622,22 +669,121 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const other = yield* makeService(undefined, "1.2.3", path.join(otherHome, ".t3")); expect(yield* other.restart).toBe("skipped"); + expect(yield* other.uninstall).toBe(false); + expect((yield* other.install().pipe(Effect.flip))._tag).toBe("BootServiceInstallError"); + expect(yield* other.status).toMatchObject({ installed: true, current: false }); expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([]); }), ); + it.effect("manages only the selected custom unit and rejects invalid unit names", () => + Effect.gen(function* () { + const { service, fs, commands, invocations, makeService, baseDir, statePath } = + yield* makeHarness(); + const original = yield* service.install(); + const originalUnit = yield* fs.readFileString(original.unitPath); + const custom = yield* makeService( + undefined, + undefined, + undefined, + false, + undefined, + "crew.service", + ); + commands.length = 0; + const plan = yield* custom.install(); + expect(plan.unitPath).toBe(original.unitPath.replace("t3code.service", "crew.service")); + expect(yield* fs.readFileString(plan.unitPath)).toContain( + "Environment=T3_BOOT_SERVICE_UNIT=crew.service", + ); + expect((yield* custom.status).current).toBe(true); + expect(commands).toContain("systemctl --user is-active crew.service"); + expect(commands).toContain("systemctl --user is-enabled crew.service"); + const inherited = yield* makeService(undefined, undefined, undefined, "crew.service"); + expect((yield* inherited.status).unitPath).toBe(plan.unitPath); + expect(yield* inherited.restart).toBe("queued"); + const explicit = yield* makeService( + undefined, + undefined, + undefined, + "t3code.service", + undefined, + "crew.service", + ); + expect((yield* explicit.status).unitPath).toBe(plan.unitPath); + invocations.length = 0; + expect(yield* explicit.restart).toBe("restarted"); + expect(invocations.some(({ args }) => args.includes("--no-block"))).toBe(false); + commands.length = 0; + yield* custom.withStopped(Effect.void); + expect(commands).toEqual([ + "systemctl --user is-enabled crew.service", + "systemctl --user is-active crew.service", + "systemctl --user stop crew.service", + "systemctl --user restart crew.service", + ]); + const otherDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-custom-other-" }); + const other = yield* makeService( + undefined, + undefined, + otherDir, + false, + undefined, + "crew.service", + ); + commands.length = 0; + expect(yield* other.restart).toBe("skipped"); + yield* other.withStopped(Effect.void); + expect(yield* other.uninstall).toBe(false); + const state = yield* fs.readFileString(statePath); + expect((yield* other.install().pipe(Effect.flip))._tag).toBe("BootServiceInstallError"); + expect((yield* other.status).current).toBe(false); + expect(commands).toEqual([]); + expect(yield* fs.readFileString(statePath)).toBe(state); + expect(yield* custom.uninstall).toBe(true); + expect(yield* fs.readFileString(original.unitPath)).toBe(originalUnit); + expect((yield* service.status).current).toBe(true); + for (const unit of [ + "../crew.service", + "/tmp/crew.service", + "--crew.service", + "crew.timer", + "crew\n.service", + ]) { + commands.length = 0; + const error = yield* makeService( + undefined, + undefined, + baseDir, + false, + undefined, + unit, + ).pipe(Effect.flip); + expect(error._tag).toBe("BootServiceInstallError"); + expect(commands).toEqual([]); + } + }), + ); + it.effect("hands the whole install to systemd without changing the running service", () => Effect.gen(function* () { const { fs, commands, invocations, control, makeService } = yield* makeHarness(); const path = yield* Path.Path; const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-handoff-" }); const baseDir = path.join(home, "T3 Data 100%", "${UNSET}", "$literal"); - const outside = yield* makeService(undefined, "1.2.3", baseDir); + const outside = yield* makeService( + undefined, + "1.2.3", + baseDir, + false, + undefined, + "crew.service", + ); const plan = yield* outside.install(); const statePath = path.join(baseDir, "runtime", "service-state.json"); const state = yield* fs.readFileString(statePath); const unit = yield* fs.readFileString(plan.unitPath); - const inside = yield* makeService(undefined, "1.2.4", baseDir, true); + const inside = yield* makeService(undefined, "1.2.4", baseDir, "crew.service"); commands.length = 0; invocations.length = 0; @@ -645,7 +791,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect(yield* fs.readFileString(statePath)).toBe(state); expect(yield* fs.readFileString(plan.unitPath)).toBe(unit); - expect(commands).not.toContain("systemctl --user stop t3code.service"); + expect(commands).not.toContain("systemctl --user stop crew.service"); expect(result).toMatchObject({ handedOff: true, logPath: plan.logPath }); const handoffs = invocations.filter(({ command }) => command === "systemd-run"); expect(handoffs).toHaveLength(1); @@ -661,6 +807,8 @@ it.layer(NodeServices.layer)("boot service install", (it) => { "install", "--base-dir", path.join(home, "T3 Data 100%", "$${UNSET}", "$$literal"), + "--unit", + "crew.service", "--allow-downgrade", ]); @@ -671,7 +819,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect(yield* fs.readFileString(statePath)).toBe(state); expect(yield* fs.readFileString(plan.unitPath)).toBe(unit); expect(control.active).toBe(true); - expect(commands).not.toContain("systemctl --user stop t3code.service"); + expect(commands).not.toContain("systemctl --user stop crew.service"); }), ); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index efa18413d979c..005cda4d19cdc 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -15,6 +15,7 @@ import * as Schema from "effect/Schema"; import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease"; import * as ProcessRunner from "../processRunner.ts"; +import { syncServiceLauncherEntry } from "../serviceLauncher.ts"; import { ensurePinnedRuntimeInstalled, launcherOwnsVersionsDir, @@ -36,6 +37,11 @@ import { const BOOT_SERVICE_NAME = "t3code"; export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; +export const BootServiceUnitName = Schema.String.check( + Schema.isMaxLength(255), + Schema.isPattern(/^[A-Za-z0-9][A-Za-z0-9_.@:-]*\.service$/), +); +const decodeBootServiceUnitName = Schema.decodeUnknownEffect(BootServiceUnitName); // `.service` suffix keeps the label distinct from the desktop app's bundle id // (com.t3tools.t3code), so launchd and TCC records never collide. export const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service"; @@ -86,6 +92,7 @@ export interface BootServicePlan { readonly baseDir: string; readonly logPath: string; readonly unitPath: string; + readonly unit?: string; /** The manager accepted a detached install; its outcome is written to `logPath`. */ readonly handedOff?: boolean; } @@ -103,7 +110,7 @@ export function renderBootServiceUnit(plan: BootServicePlan): string { "Type=simple", "WorkingDirectory=%h", `Environment=T3CODE_HOME=${quoteSystemdValue(plan.baseDir)}`, - `Environment=${BOOT_SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`, + `Environment=${BOOT_SERVICE_UNIT_ENV}=${plan.unit ?? BOOT_SERVICE_UNIT_FILE}`, `ExecStart=${plan.program.map(quoteSystemdValue).join(" ")}`, // Let the launcher mark an explicit stop before it signals the server. // systemd still SIGKILLs the whole cgroup if graceful shutdown times out. @@ -214,14 +221,9 @@ export interface BootServiceManager { function systemdManager(input: { readonly path: Path.Path; readonly homeDir: string; + readonly unit: string; }): BootServiceManager { - const unitPath = input.path.join( - input.homeDir, - ".config", - "systemd", - "user", - BOOT_SERVICE_UNIT_FILE, - ); + const unitPath = input.path.join(input.homeDir, ".config", "systemd", "user", input.unit); const prepareActivation = [ { step: "reloading systemd user units", @@ -231,7 +233,7 @@ function systemdManager(input: { { step: "enabling the service", command: "systemctl", - args: ["--user", "enable", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "enable", input.unit], }, ]; return { @@ -242,7 +244,7 @@ function systemdManager(input: { { step: "stopping the installed service", command: "systemctl", - args: ["--user", "stop", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "stop", input.unit], timeout: STOP_STEP_TIMEOUT, }, ], @@ -251,14 +253,14 @@ function systemdManager(input: { { step: "starting the service", command: "systemctl", - args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "restart", input.unit], }, ], restart: [ { step: "restarting the service after a failed update", command: "systemctl", - args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "restart", input.unit], }, ], handoff: [ @@ -266,14 +268,14 @@ function systemdManager(input: { { step: "restarting the service", command: "systemctl", - args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "restart", "--no-block", input.unit], }, ], deactivate: [ { step: "stopping the service", command: "systemctl", - args: ["--user", "disable", "--now", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "disable", "--now", input.unit], timeout: STOP_STEP_TIMEOUT, }, ], @@ -361,12 +363,13 @@ function selectBootServiceManager(input: { readonly uid: number | undefined; readonly path: Path.Path; readonly environmentPath: string; + readonly unit: string; }): BootServiceManager | undefined { if (input.homeDir === "") { return undefined; } if (input.platform === "linux") { - return systemdManager({ path: input.path, homeDir: input.homeDir }); + return systemdManager({ path: input.path, homeDir: input.homeDir, unit: input.unit }); } if (input.platform === "darwin" && input.uid !== undefined) { return launchdManager({ @@ -424,7 +427,10 @@ const BootServiceProblem = Schema.Literals([ ]); type BootServiceProblem = typeof BootServiceProblem.Type; -export function formatBootServiceProblem(problem: BootServiceProblem): string { +export function formatBootServiceProblem( + problem: BootServiceProblem, + unit = BOOT_SERVICE_UNIT_FILE, +): string { switch (problem) { case "user-manager-unavailable": return "Cannot reach the systemd user manager. Run `systemctl --user status` in a login session for the service user. Install your distribution's systemd user-session support if it is missing; do not run T3 with sudo."; @@ -435,7 +441,7 @@ export function formatBootServiceProblem(problem: BootServiceProblem): string { case "service-disabled": return "The service is not enabled to start automatically. Run `t3 service install` to repair it."; case "service-stopped": - return "The service is not running. Check the service log and `systemctl --user status t3code.service`, then run `t3 service install`."; + return `The service is not running. Check the service log and \`systemctl --user status ${unit}\`, then run \`t3 service install\`.`; case "restart-pending": return "A newer version is installed but the service is still running the previous one. Run `t3 service restart` to switch."; } @@ -526,6 +532,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { readonly logsDir: string; readonly cliVersion: string; readonly host?: BootServiceHost; + readonly unit?: string; }) { const hostExecPath = yield* HostProcess.ExecutablePath; const platform = yield* HostProcess.Platform; @@ -537,9 +544,17 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ); const homeDir = yield* Config.String("HOME").pipe(Config.withDefault("")); const installerPath = yield* Config.String("PATH").pipe(Config.withDefault("")); - const runningInsideService = - (yield* Config.String(BOOT_SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) === - BOOT_SERVICE_UNIT_FILE; + const inheritedUnit = yield* Config.String(BOOT_SERVICE_UNIT_ENV).pipe(Config.withDefault("")); + const selectedUnit = + platform === "linux" + ? (input.unit ?? (inheritedUnit || BOOT_SERVICE_UNIT_FILE)) + : BOOT_SERVICE_UNIT_FILE; + if (platform === "linux") { + yield* decodeBootServiceUnitName(selectedUnit).pipe( + Effect.mapError((cause) => new BootServiceInstallError({ cause })), + ); + } + const runningInsideService = inheritedUnit === selectedUnit; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const runner = yield* ProcessRunner.ProcessRunner; @@ -579,6 +594,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { uid, path, environmentPath, + unit: selectedUnit, }); const unitPath = detectedManager?.unitPath ?? ""; const logPath = path.join(input.logsDir, BOOT_SERVICE_LOG_FILE); @@ -604,10 +620,23 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { }), ).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); const plan: BootServicePlan = { - program: [runtimePaths.entryPath, "__service-launcher"], + program: [ + platform === "linux" + ? path.join(input.baseDir, "runtime", "launcher") + : runtimePaths.entryPath, + "__service-launcher", + ], baseDir: input.baseDir, logPath, unitPath, + ...(platform === "linux" ? { unit: selectedUnit } : {}), + }; + const servesBaseDir = (contents: string) => { + const installedBaseDir = bootServiceBaseDirOf(contents); + return ( + installedBaseDir !== undefined && + path.resolve(installedBaseDir) === path.resolve(input.baseDir) + ); }; const requireManager = Effect.suspend(() => @@ -687,8 +716,8 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { if (includeService && succeeded(manager)) { const [enabled, active] = yield* Effect.all( [ - probe("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]), - probe("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]), + probe("systemctl", ["--user", "is-enabled", selectedUnit]), + probe("systemctl", ["--user", "is-active", selectedUnit]), ], { concurrency: "unbounded" }, ); @@ -726,6 +755,19 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { readonly start?: boolean; }) { const manager = yield* requireManager; + const installed = yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + if (installed) { + const unit = yield* fs + .readFileString(unitPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + if (!servesBaseDir(unit)) { + return yield* new BootServiceInstallError({ + cause: new Error("The selected unit serves another T3 home."), + }); + } + } yield* fs .makeDirectory(input.logsDir, { recursive: true }) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); @@ -734,7 +776,13 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { yield* requireSystemdPrerequisites.pipe(Effect.tapError(logFailure)); } - yield* ensurePinnedRuntimeInstalled({ + const updatePending = + installed && + (yield* fs.readFileString(statePath).pipe( + Effect.map(serviceStateHasPendingUpdate), + Effect.orElseSucceed(() => false), + )); + const installRuntime = ensurePinnedRuntimeInstalled({ baseDir: input.baseDir, version: input.cliVersion, fs, @@ -786,13 +834,13 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { : new BootServiceInstallError({ cause: error }), ), ); - const installed = yield* fs - .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + // Pending updates must reach the stopped-state guard before retention admission. + if (!updatePending) yield* installRuntime; // With start=false, changing startup-only files is safe except during a remote update, // when the launcher also writes its state file. const start = options?.start !== false; if (installed && start && runningInsideService && manager.kind === "systemd") { + if (updatePending) return yield* new BootServiceUpdatePendingError(); // Stopping the unit from inside it kills this process before the start. // A transient unit runs the same install from outside, stop first, so it // still serialises with a remote update the launcher may be starting. @@ -810,6 +858,8 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { "--base-dir", // systemd expands $VAR in arguments (not in the executable path). input.baseDir.replaceAll("$", () => "$$"), + "--unit", + selectedUnit, ...(options?.allowDowngrade === true ? ["--allow-downgrade"] : []), ]); return { ...plan, handedOff: true }; @@ -819,12 +869,14 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { } yield* Effect.gen(function* () { + let previousState: ServiceState | undefined; if (installed) { const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); if (Option.isSome(previousStateText)) { if (serviceStateHasPendingUpdate(previousStateText.value)) { return yield* new BootServiceUpdatePendingError(); } + previousState = parseServiceState(previousStateText.value); // Read the final version after stopping the launcher; a remote update may finish after the status check. const installedVersion = serviceStateActiveVersion(previousStateText.value); if ( @@ -839,6 +891,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { } } } + if (updatePending) yield* installRuntime; yield* fs .makeDirectory(path.dirname(unitPath), { recursive: true }) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); @@ -848,10 +901,12 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { statePath, // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned document. `${JSON.stringify( - { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: input.cliVersion, - } satisfies ServiceState, + previousState?.activeVersion === input.cliVersion + ? previousState + : ({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: input.cliVersion, + } satisfies ServiceState), null, 2, )}\n`, @@ -863,6 +918,12 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { return yield* new BootServiceUpdatePendingError(); } } + if (manager.kind === "systemd") { + yield* Effect.tryPromise({ + try: () => syncServiceLauncherEntry(input.baseDir, input.cliVersion), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + } yield* writeDurably(unitPath, manager.render(plan)); if (start) { @@ -884,11 +945,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const manager = yield* requireManager; const unit = yield* fs.readFileString(unitPath).pipe(Effect.option); if (Option.isNone(unit)) return "skipped" as const; - const installedBaseDir = bootServiceBaseDirOf(unit.value); - if ( - installedBaseDir === undefined || - path.resolve(installedBaseDir) !== path.resolve(input.baseDir) - ) { + if (!servesBaseDir(unit.value)) { return "skipped" as const; } // Writes no state, and the launcher lets an update transition in flight @@ -917,11 +974,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const manager = detectedManager; const unit = yield* fs.readFileString(unitPath).pipe(Effect.option); if (Option.isNone(unit)) return yield* operation; - const installedBaseDir = bootServiceBaseDirOf(unit.value); - if ( - installedBaseDir === undefined || - path.resolve(installedBaseDir) !== path.resolve(input.baseDir) - ) { + if (!servesBaseDir(unit.value)) { return yield* operation; } // `is-active` misses crash-looping systemd units; launchd can be loaded without a pid. @@ -932,8 +985,8 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ? yield* Effect.map( Effect.all( [ - probe("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]), - probe("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]), + probe("systemctl", ["--user", "is-enabled", selectedUnit]), + probe("systemctl", ["--user", "is-active", selectedUnit]), ], { concurrency: "unbounded" }, ), @@ -961,12 +1014,14 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () { const manager = yield* requireManager; - if ( - !(yield* fs - .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })))) - ) - return false; + const unit = yield* fs.readFileString(unitPath).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.void, + ), + Effect.mapError((cause) => new BootServiceInstallError({ cause })), + ); + if (unit === undefined || !servesBaseDir(unit)) return false; yield* runSteps(manager.deactivate); yield* fs .remove(unitPath) @@ -982,11 +1037,12 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { if (!(yield* fs.exists(unitPath))) { return { supported: true, installed: false, current: false, unitPath, logPath }; } - const [unit, runtimeEntryExists, runtimeSentinel, stateText] = yield* Effect.all([ + const [unit, runtimeEntry, runtimeSentinel, stateText, launcherEntry] = yield* Effect.all([ fs.readFileString(unitPath), - fs.exists(runtimePaths.entryPath), + fs.realPath(runtimePaths.entryPath).pipe(Effect.option), fs.readFileString(runtimePaths.sentinelPath).pipe(Effect.option), fs.readFileString(statePath).pipe(Effect.option), + fs.realPath(plan.program[0]!).pipe(Effect.option), ]); const state = Option.isSome(stateText) ? parseServiceState(stateText.value) : undefined; const installedVersion = Option.isSome(stateText) @@ -998,7 +1054,9 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ? contents.replace(/(PATH<\/key>\n\s*)[^<]*(<\/string>)/, "$1$2") : contents; const problems: BootServiceProblem[] = - detectedManager.kind === "systemd" ? [...(yield* readSystemdProblems(true))] : []; + detectedManager.kind === "systemd" && servesBaseDir(unit) + ? [...(yield* readSystemdProblems(true))] + : []; if (yield* fs.exists(restartPendingPath)) problems.push("restart-pending"); return { supported: true, @@ -1009,7 +1067,9 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { current: problems.length === 0 && normalizeUnit(unit) === normalizeUnit(detectedManager.render(plan)) && - runtimeEntryExists && + Option.isSome(runtimeEntry) && + (detectedManager.kind !== "systemd" || + (Option.isSome(launcherEntry) && launcherEntry.value === runtimeEntry.value)) && Option.isSome(runtimeSentinel) && runtimeSentinel.value.trim() === input.cliVersion && state?.activeVersion === input.cliVersion && @@ -1030,4 +1090,5 @@ export const layer = (input: { readonly logsDir: string; readonly cliVersion: string; readonly host?: BootServiceHost; + readonly unit?: string; }) => Layer.effect(BootService, make(input)); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 91fc2c0b42bfa..29a9ad17d3596 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -126,6 +126,10 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( }), }); const launcher = ServiceLauncherClient.ServiceLauncherClient.of({ + prepareUpdate: () => + Effect.sync(() => { + order.push("prepare"); + }), managed: options.managed ?? true, requestUpdate: options.requestUpdate ?? @@ -181,7 +185,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { method: "boot-service", updateId: "launcher-id", }); - expect(order).toEqual(["download", "extract", "preflight", "accept"]); + expect(order).toEqual(["prepare", "download", "extract", "preflight", "accept"]); const history = (yield* selfUpdate.history!.list).entries; expect(history[0]?.status).toBe("awaiting-restart"); expect(history[0]?.receiptId).toBe("launcher-id"); @@ -211,7 +215,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { method: "boot-service", updateId: "launcher-id", }); - expect(order).toEqual(["download", "extract", "preflight", "accept"]); + expect(order).toEqual(["prepare", "download", "extract", "preflight", "accept"]); }), ); @@ -358,7 +362,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { yield* Deferred.succeed(releaseDownload, undefined); yield* Deferred.await(accepted); - expect(order).toEqual(["download", "extract", "preflight", "accept"]); + expect(order).toEqual(["prepare", "download", "extract", "preflight", "accept"]); expect((yield* selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip)).reason).toBe( "A server update is already in progress.", ); diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index f9d8f83f77e84..f618db5c70d35 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -32,6 +32,7 @@ import { } from "@t3tools/shared/cliRelease"; import packageJson from "../../package.json" with { type: "json" }; +import { readServiceState, updateStoppedServer } from "../serviceLauncher.ts"; import * as ServerConfig from "../config.ts"; import * as DesktopAppUpdate from "../desktopUpdate/DesktopAppUpdate.ts"; import { databaseMigrationHead } from "../persistence/databaseMigrationHead.ts"; @@ -135,6 +136,146 @@ export function trackSwitchBlockReason(input: { : null; } +const validateServiceRuntime = Effect.fn("cloud.server_self_update.validate_runtime")(function* ( + runtime: ReturnType, + dbPath: string, + targetVersion: string, +) { + const runner = yield* ProcessRunner.ProcessRunner; + return yield* runner + .run({ + command: pinnedRuntimeCommand(runtime).command, + args: [ + ...pinnedRuntimeCommand(runtime).args, + "__service-preflight", + "--database-path", + dbPath, + "--launcher-protocol", + String(SERVICE_LAUNCHER_PROTOCOL), + ], + timeout: PREFLIGHT_TIMEOUT, + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "running the staged service preflight", + cause, + }), + ), + Effect.flatMap( + ( + result, + ): Effect.Effect => { + if (result.code !== 0) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "running the staged service preflight", + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ); + } + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout.trim()); + } catch (cause) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "decoding the staged service preflight", + cause, + }), + ); + } + const preflight = decodeServicePreflightResult(parsed); + if (preflight === undefined || preflight.version !== targetVersion) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "verifying the staged service preflight", + }), + ); + } + return preflight.status === "ready" + ? Effect.void + : Effect.fail( + new PinnedRuntimePreflightBlockedError({ + version: targetVersion, + reason: preflight.reason, + }), + ); + }, + ), + ); +}); + +export const updateStoppedRuntime = Effect.fn("cloud.server_self_update.update_stopped")( + function* (input: { readonly targetVersion: string }) { + const config = yield* ServerConfig.ServerConfig; + const policy = yield* ManagedPolicy.ManagedPolicy; + const runner = yield* ProcessRunner.ProcessRunner; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; + const arch = yield* HostProcess.Architecture; + const httpClient = yield* HttpClient.HttpClient; + const targetVersion = input.targetVersion.trim(); + const refuse = (reason: string, cause?: unknown) => + new ServerSelfUpdateError({ reason, ...(cause === undefined ? {} : { cause }) }); + if (!isExactServiceVersion(targetVersion)) + return yield* refuse("An exact target runtime version is required."); + const policyRefusal = managedUpdateRefusal(policy.document, cliReleaseChannelOf(targetVersion)); + if (policyRefusal !== undefined) return yield* refuse(policyRefusal); + const state = yield* Effect.tryPromise({ + try: () => readServiceState(path.join(config.baseDir, "runtime", "service-state.json")), + catch: (cause) => refuse("Could not read the managed runtime state.", cause), + }); + const trackRefusal = trackSwitchBlockReason({ + serverVersion: state.activeVersion, + targetVersion, + targetDatabaseMigration: undefined, + databaseMigration: () => databaseMigrationHead(config.dbPath), + }); + if (trackRefusal !== null) return yield* refuse(trackRefusal); + const releaseBaseUrl = Option.getOrUndefined( + yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), + ); + const install = (dbPath: string) => + ensurePinnedRuntimeInstalled({ + baseDir: config.baseDir, + version: targetVersion, + fs, + path, + runner, + httpClient, + platform, + arch, + releaseBaseUrl, + validate: (runtime) => + validateServiceRuntime(runtime, dbPath, targetVersion).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + ), + }).pipe(Effect.asVoid); + const runInstall = Effect.runPromiseWith( + yield* Effect.context>>(), + ); + return yield* Effect.tryPromise({ + try: (signal) => + updateStoppedServer( + config.baseDir, + targetVersion, + (dbPath, installSignal) => runInstall(install(dbPath), { signal: installSignal }), + signal, + ), + catch: (cause) => + refuse( + "The stopped server update could not complete; its rollback and recovery state remain protected.", + cause, + ), + }); + }, +); + export const make = Effect.fn("cloud.server_self_update.make")(function* () { const serverConfig = yield* ServerConfig.ServerConfig; const desktopAppUpdate = yield* DesktopAppUpdate.DesktopAppUpdate; @@ -235,6 +376,9 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { let handoffAccepted = false; return yield* Effect.gen(function* () { yield* reportProgress("downloading"); + yield* launcher + .prepareUpdate({ targetVersion, dbPath: serverConfig.dbPath }) + .pipe(Effect.mapError((cause) => failWith(cause.message, cause))); const paths = yield* ensurePinnedRuntimeInstalled({ baseDir: serverConfig.baseDir, version: targetVersion, @@ -246,74 +390,9 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { arch, releaseBaseUrl, validate: (runtime) => - runner - .run({ - command: pinnedRuntimeCommand(runtime).command, - args: [ - ...pinnedRuntimeCommand(runtime).args, - "__service-preflight", - "--database-path", - serverConfig.dbPath, - "--launcher-protocol", - String(SERVICE_LAUNCHER_PROTOCOL), - ], - timeout: PREFLIGHT_TIMEOUT, - }) - .pipe( - Effect.mapError( - (cause) => - new PinnedRuntimeInstallError({ - step: "running the staged service preflight", - cause, - }), - ), - Effect.flatMap( - ( - result, - ): Effect.Effect< - void, - PinnedRuntimeInstallError | PinnedRuntimePreflightBlockedError - > => { - if (result.code !== 0) { - return Effect.fail( - new PinnedRuntimeInstallError({ - step: "running the staged service preflight", - exitCode: Number(result.code), - stdoutLength: result.stdout.length, - stderrLength: result.stderr.length, - }), - ); - } - let parsed: unknown; - try { - parsed = JSON.parse(result.stdout.trim()); - } catch (cause) { - return Effect.fail( - new PinnedRuntimeInstallError({ - step: "decoding the staged service preflight", - cause, - }), - ); - } - const preflight = decodeServicePreflightResult(parsed); - if (preflight === undefined || preflight.version !== targetVersion) { - return Effect.fail( - new PinnedRuntimeInstallError({ - step: "verifying the staged service preflight", - }), - ); - } - return preflight.status === "ready" - ? Effect.void - : Effect.fail( - new PinnedRuntimePreflightBlockedError({ - version: targetVersion, - reason: preflight.reason, - }), - ); - }, - ), - ), + validateServiceRuntime(runtime, serverConfig.dbPath, targetVersion).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + ), }).pipe( Effect.mapError((error) => error._tag === "PinnedRuntimePreflightBlockedError" diff --git a/apps/server/src/cloud/serverRestore.test.ts b/apps/server/src/cloud/serverRestore.test.ts index 09fe787dee50a..216d3b83d9b10 100644 --- a/apps/server/src/cloud/serverRestore.test.ts +++ b/apps/server/src/cloud/serverRestore.test.ts @@ -12,7 +12,12 @@ import * as FileSystem from "effect/FileSystem"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { vi } from "vite-plus/test"; -import { Launcher, readServiceState, writeServiceState } from "../serviceLauncher.ts"; +import { + Launcher, + readServiceState, + writeServiceState, + updateStoppedServer, +} from "../serviceLauncher.ts"; import { acquireServiceLauncherLock } from "./serverRestoreControl.ts"; import { SERVER_RESTORE_MANIFEST, @@ -75,7 +80,7 @@ vi.mock("node:child_process", async (importOriginal) => { return { ...actual, spawn, execFile }; }); -async function fixture(baseDir: string, behavior: "ready" | "crash" = "ready") { +async function fixture(baseDir: string, behavior: "ready" | "crash" = "ready", payloadBytes = 0) { const userdata = NodePath.join(baseDir, "userdata"); await NodeFSP.mkdir(userdata, { recursive: true }); await NodeFSP.writeFile(NodePath.join(userdata, "environment-id"), "original-environment\n"); @@ -115,6 +120,8 @@ process.on("message", (message) => { `; await NodeFSP.writeFile(executable, `#!${process.execPath}\n${source}`, { mode: 0o755 }); if (process.platform === "win32") await NodeFSP.writeFile(`${executable}.cjs`, source); + if (payloadBytes > 0) + await NodeFSP.writeFile(NodePath.join(directory, "payload"), Buffer.alloc(payloadBytes)); await NodeFSP.writeFile( NodePath.join(directory, SERVER_RESTORE_MANIFEST), JSON.stringify({ @@ -425,6 +432,89 @@ it.layer(NodeServices.layer, { excludeTestServices: true })("offline server rest }), ); + it.effect( + "prepares successive updates when the active and target runtimes fit but three do not", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-restore-successive-" }); + const setup = yield* Effect.promise(() => fixture(baseDir, "ready", 32_768)); + yield* Effect.promise(async () => { + const versions = NodePath.join(baseDir, "runtime", "versions"); + const initial = await inspectServerRestore(baseDir); + assert.equal(initial.candidates[0]?.blockedReason, null); + const launcher = NodePath.join(baseDir, "t3"); + if (process.platform === "linux") { + await NodeFSP.symlink(NodePath.join(versions, "1.0.0", "t3"), launcher); + await NodeFSP.writeFile( + NodePath.join(baseDir, "runtime", "launchers"), + `${launcher}\n`, + ); + } + const runtimeBytes = initial.totalBytes / 2; + const maxBytes = initial.totalBytes + 8192; + assert.isAbove(runtimeBytes * 3, maxBytes); + await NodeFSP.writeFile( + NodePath.join(baseDir, "runtime", "restore-limits.json"), + JSON.stringify({ maxVersions: 2, maxBytes }), + ); + await writeServiceState(setup.statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "2.0.0", + update: { + id: "update-2", + fromVersion: "1.0.0", + targetVersion: "2.0.0", + status: "committed", + }, + }); + await expect( + updateStoppedServer(baseDir, "3.0.0", async () => { + throw new Error("download failed"); + }), + ).rejects.toThrow("download failed"); + const recovered = await readServiceState(setup.statePath); + assert.equal(recovered.activeVersion, "2.0.0"); + assert.equal(recovered.update?.id, "update-2"); + assert.equal(recovered.preparation?.fromVersion, "2.0.0"); + assert.deepEqual(await NodeFSP.readdir(versions), ["2.0.0"]); + for (const target of [3, 4]) { + const version = `${target}.0.0`; + await updateStoppedServer(baseDir, version, async () => { + const retention = await acquireServerRuntimeRetention(baseDir); + try { + await retention.admit(version); + await retention.check(runtimeBytes + retention.metadataBytes(version)); + const staging = await NodeFSP.mkdtemp(NodePath.join(versions, ".staging-")); + await retention.reserve(version, staging); + await expect( + retention.prepare(await readServiceState(setup.statePath), `${target + 1}.0.0`), + ).rejects.toThrow(/concurrent/); + await setup.installRuntime(version, staging); + await retention.check(0); + await NodeFSP.rename(staging, NodePath.join(versions, version)); + } finally { + retention.release(); + } + }); + const committed = await readServiceState(setup.statePath); + assert.equal(committed.activeVersion, version); + assert.equal(committed.update?.status, "committed"); + assert.equal(committed.update?.fromVersion, `${target - 1}.0.0`); + const inventory = await inspectServerRestore(baseDir); + assert.isAtMost(inventory.totalBytes, maxBytes); + assert.equal(inventory.candidates[0]?.blockedReason, null); + assert.deepEqual(await NodeFSP.readdir(versions), [`${target - 1}.0.0`, version]); + if (process.platform === "linux") + assert.equal( + await NodeFSP.realpath(launcher), + NodePath.join(versions, version, "t3"), + ); + } + }); + }), + ); + it.effect("retains a persistent original launcher through repeated updates", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/cloud/serverRuntimeRetention.ts b/apps/server/src/cloud/serverRuntimeRetention.ts index 97302618c566c..b96ca7850ec64 100644 --- a/apps/server/src/cloud/serverRuntimeRetention.ts +++ b/apps/server/src/cloud/serverRuntimeRetention.ts @@ -13,9 +13,15 @@ import { serverRestoreLimits, serverRuntimeStorage, } from "./serverRestoreStore.ts"; -import { verifyServerRestoreRuntime } from "./serverRestoreManifest.ts"; +import { + serverRestoreDatabaseBlocker, + serverRestoreIdentity, + serverRestoreQualificationBlocker, + verifyServerRestoreRuntime, +} from "./serverRestoreManifest.ts"; import { isExactServiceVersion, + compareExactServiceVersions, parseServiceState, SERVICE_STATE_FILE, type ServiceState, @@ -291,6 +297,7 @@ async function runtimeReferences(baseDir: string) { ].filter(NodePath.isAbsolute); for (const launcher of new Set([ ...recorded, + ...(process.platform === "linux" ? [NodePath.join(baseDir, "runtime", "launcher")] : []), ...directories.map((directory) => NodePath.join(directory, launcherName)), ])) { try { @@ -331,10 +338,22 @@ async function retentionSnapshot(baseDir: string, nextState?: ServiceState) { // Legacy homes cannot qualify a predecessor until their profile has initialized. if (inventory === undefined) for (const version of (await serverRuntimeStorage(baseDir)).entries) keep.add(version); - const predecessor = inventory?.candidates.find((candidate) => candidate.blockedReason === null); - if (predecessor !== undefined) keep.add(predecessor.version); - else if (state.update?.status === "committed" && state.update.restore === undefined) - keep.add(state.update.fromVersion); + const preparation = state.preparation; + if (preparation?.fromVersion === state.activeVersion) { + const baseline = await qualifiedActiveRuntime(baseDir, state.activeVersion); + if ( + baseline.installationId !== preparation.installationId || + baseline.sha256 !== preparation.sha256 + ) + throw new Error("The prepared rollback baseline changed; no runtime was removed."); + } else { + const predecessor = inventory?.candidates.find( + (candidate) => candidate.blockedReason === null, + ); + if (predecessor !== undefined) keep.add(predecessor.version); + else if (state.update?.status === "committed" && state.update.restore === undefined) + keep.add(state.update.fromVersion); + } } const references = await runtimeReferences(baseDir); const fold = (value: string) => (process.platform === "win32" ? value.toLowerCase() : value); @@ -350,6 +369,19 @@ async function retentionSnapshot(baseDir: string, nextState?: ServiceState) { return { versionsDir, state, keep, referenced }; } +async function qualifiedActiveRuntime(baseDir: string, version: string) { + const identity = await serverRestoreIdentity(baseDir); + const directory = NodePath.join(identity.baseDir, "runtime", "versions", version); + if ((await readOptional(NodePath.join(directory, ".install-complete")))?.trim() !== version) + throw new Error("The active runtime is incomplete; its predecessor remains protected."); + const verified = await verifyServerRestoreRuntime(directory, version); + const blocker = + (await serverRestoreQualificationBlocker(identity, version, verified.sha256)) ?? + serverRestoreDatabaseBlocker(identity.dbPath, verified.manifest); + if (blocker !== undefined) throw new Error(blocker); + return { installationId: identity.installationId, sha256: verified.sha256 }; +} + async function cleanUnused(baseDir: string, reuseVersion?: string) { const snapshot = await retentionSnapshot(baseDir); if (reuseVersion !== undefined) snapshot.keep.add(reuseVersion); @@ -493,6 +525,28 @@ export async function acquireServerRuntimeRetention( return { release: () => db.close(), metadataBytes, + prepare: async (state: ServiceState, targetVersion: string): Promise => { + const snapshot = await cleanUnused(baseDir); + if ( + snapshot.candidate !== undefined || + snapshot.state?.update?.status === "pending" || + JSON.stringify(snapshot.state) !== JSON.stringify(state) + ) + throw new Error("A concurrent runtime transaction needs recovery before preparation."); + if ( + !isExactServiceVersion(targetVersion) || + compareExactServiceVersions(targetVersion, state.activeVersion) <= 0 + ) + throw new Error("Update preparation requires a newer exact runtime version."); + return { + ...state, + preparation: { + fromVersion: state.activeVersion, + targetVersion, + ...(await qualifiedActiveRuntime(baseDir, state.activeVersion)), + }, + }; + }, clean: async () => { await cleanUnused(baseDir); await check(); @@ -505,6 +559,10 @@ export async function acquireServerRuntimeRetention( ); if ( state?.update?.status === "pending" || + (state?.preparation?.fromVersion === state?.activeVersion && + state?.preparation !== undefined && + state.preparation.targetVersion !== version && + version !== state.activeVersion) || (candidate !== undefined && (!cached || candidate.version !== version)) ) throw new Error( diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.ts index f60b8bb9cf886..3e225c019488f 100644 --- a/apps/server/src/cloud/serviceLauncherClient.test.ts +++ b/apps/server/src/cloud/serviceLauncherClient.test.ts @@ -86,11 +86,19 @@ it.effect("returns the launcher-generated ID only after update acceptance", () = childVersion: "1.0.0", }); const client = yield* makeClient(host, "1.0.0"); + const preparation = yield* Effect.forkChild( + client.prepareUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }), + { startImmediately: true }, + ); + expect(host.sent).toEqual([ + { type: "prepare-update", targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }, + ]); + host.emit({ type: "update-prepared", targetVersion: "1.1.0" }); + yield* Fiber.join(preparation); const requested = yield* Effect.forkChild( client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }), { startImmediately: true }, ); - yield* Effect.yieldNow; host.emit({ type: "update-accepted", updateId: "launcher-id", diff --git a/apps/server/src/cloud/serviceLauncherClient.ts b/apps/server/src/cloud/serviceLauncherClient.ts index 3685416e1094c..e312866c8b15f 100644 --- a/apps/server/src/cloud/serviceLauncherClient.ts +++ b/apps/server/src/cloud/serviceLauncherClient.ts @@ -98,6 +98,10 @@ export class ServiceLauncherClient extends Context.Service< ServiceLauncherClient, { readonly managed: boolean; + readonly prepareUpdate: (input: { + readonly targetVersion: string; + readonly dbPath: string; + }) => Effect.Effect; readonly requestUpdate: (input: { readonly targetVersion: string; readonly dbPath: string; @@ -214,6 +218,27 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o ), ); + const prepareUpdate: ServiceLauncherClient["Service"]["prepareUpdate"] = (input) => + exchange( + { type: "prepare-update", ...input }, + (reply) => + (reply.type === "update-prepared" && reply.targetVersion === input.targetVersion) || + reply.type === "update-rejected", + ).pipe( + Effect.flatMap((reply) => + reply.type === "update-prepared" + ? Effect.void + : reply.type === "update-rejected" + ? Effect.fail( + new ServiceLauncherRejectedError({ + targetVersion: input.targetVersion, + reason: reply.reason, + }), + ) + : Effect.die("service launcher returned an impossible preparation response"), + ), + ); + const pending = context?.update?.status === "pending" ? context.update : undefined; const outcome = context?.update === undefined || context.update.status === "pending" @@ -260,6 +285,7 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o return ServiceLauncherClient.of({ managed, requestUpdate, + prepareUpdate, prepareTrial, }); }); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 4c1caacabf26a..478c25af29c78 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -36,6 +36,12 @@ export interface ServiceState { readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL; readonly activeVersion: string; readonly update?: ServiceUpdateRecord; + readonly preparation?: { + readonly fromVersion: string; + readonly targetVersion: string; + readonly installationId: string; + readonly sha256: string; + }; } export interface ServiceLauncherContext { @@ -45,6 +51,11 @@ export interface ServiceLauncherContext { } export type ServiceLauncherChildMessage = + | { + readonly type: "prepare-update"; + readonly targetVersion: string; + readonly dbPath: string; + } | { readonly type: "request-update"; readonly targetVersion: string; @@ -57,6 +68,7 @@ export type ServiceLauncherChildMessage = }; export type ServiceLauncherParentMessage = + | { readonly type: "update-prepared"; readonly targetVersion: string } | { readonly type: "update-accepted"; readonly updateId: string; @@ -183,11 +195,23 @@ export function compareExactServiceVersions(left: string, right: string): number export function decodeServiceState(value: unknown): ServiceState | undefined { if (!isRecord(value)) return undefined; const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update); + const preparation = value.preparation; if ( value.protocol !== SERVICE_LAUNCHER_PROTOCOL || typeof value.activeVersion !== "string" || !isExactServiceVersion(value.activeVersion) || (value.update !== undefined && update === undefined) || + (preparation !== undefined && + (!isRecord(preparation) || + typeof preparation.fromVersion !== "string" || + !isExactServiceVersion(preparation.fromVersion) || + typeof preparation.targetVersion !== "string" || + !isExactServiceVersion(preparation.targetVersion) || + compareExactServiceVersions(preparation.targetVersion, preparation.fromVersion) <= 0 || + typeof preparation.installationId !== "string" || + !/^[a-f0-9]{64}$/.test(preparation.installationId) || + typeof preparation.sha256 !== "string" || + !/^[a-f0-9]{64}$/.test(preparation.sha256))) || (update !== undefined && (update.restore === undefined ? compareExactServiceVersions(update.targetVersion, update.fromVersion) <= 0 @@ -203,6 +227,16 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: value.activeVersion, ...(update === undefined ? {} : { update }), + ...(isRecord(preparation) + ? { + preparation: { + fromVersion: preparation.fromVersion as string, + targetVersion: preparation.targetVersion as string, + installationId: preparation.installationId as string, + sha256: preparation.sha256 as string, + }, + } + : {}), }; } @@ -274,7 +308,7 @@ export function decodeServiceLauncherChildMessage( ): ServiceLauncherChildMessage | undefined { if (!isRecord(value)) return undefined; if ( - value.type === "request-update" && + (value.type === "request-update" || value.type === "prepare-update") && typeof value.targetVersion === "string" && typeof value.dbPath === "string" ) { @@ -299,6 +333,8 @@ export function decodeServiceLauncherParentMessage( value: unknown, ): ServiceLauncherParentMessage | undefined { if (!isRecord(value)) return undefined; + if (value.type === "update-prepared" && typeof value.targetVersion === "string") + return { type: value.type, targetVersion: value.targetVersion }; if (value.type === "update-rejected" && typeof value.reason === "string") { return { type: value.type, reason: value.reason }; } diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index e5943e0731fbf..73d0b418c169e 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -13,7 +13,6 @@ import { vi } from "vite-plus/test"; import { databaseBackupBlocker, Launcher, - main, readServiceEnvironment, type BackupSpaceProbe, readServiceState, @@ -536,7 +535,14 @@ if (context.update?.status === "pending") { Effect.sync(() => vi.spyOn(console, "error")), (spy) => Effect.sync(() => spy.mockRestore()), ); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const handoff = vi.fn<(version: string) => void>(); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + handoff, + }, + ); yield* Effect.promise(() => launcher.run().then( () => Promise.reject(new Error("launcher unexpectedly completed")), @@ -547,6 +553,7 @@ if (context.update?.status === "pending") { const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.1.0"); assert.equal(state.update?.status, "committed"); + assert.deepEqual(handoff.mock.calls, [["1.1.0"]]); assert.isEmpty(cleanupErrors.mock.calls); }), ); @@ -599,8 +606,17 @@ if (context.update?.status === "pending") { }), ); + const handoff = vi.fn<(version: string) => void>(); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + environment: yield* Effect.promise(() => readServiceEnvironment(root)), + handoff, + }, + ); yield* Effect.promise(() => - main().then( + launcher.run().then( () => Promise.reject(new Error("launcher unexpectedly completed")), () => Promise.resolve(), ), @@ -609,6 +625,7 @@ if (context.update?.status === "pending") { const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.0.0"); assert.equal(state.update?.status, "rolled-back"); + assert.deepEqual(handoff.mock.calls, [["1.1.0"], ["1.0.0"]]); assert.equal( state.update?.status === "rolled-back" ? state.update.reason : undefined, "invalid-prepared", diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 33ae062835359..60f0616fa321c 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -24,6 +24,7 @@ import { } from "./cloud/serverRestoreManifest.ts"; import { preflightServerRestore } from "./cloud/serverRestoreStore.ts"; import { + acquireServerRuntimeRetention, assertServerRuntimeCommitFits, cleanupServerRuntimeRetention, readServerRuntimeCandidate, @@ -393,6 +394,70 @@ export async function writeServiceState(filePath: string, state: ServiceState): } } +export async function syncServiceLauncherEntry(baseDir: string, version: string): Promise { + const entry = runtimePaths(baseDir, version).entryPath; + // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. + if (process.platform !== "linux") return entry; + if (!isExactServiceVersion(version) || !(await runtimeExists(baseDir, version))) + throw new Error("The selected service launcher runtime is incomplete."); + const link = NodePath.join(baseDir, "runtime", "launcher"); + const stat = await NodeFSP.lstat(link).catch((cause: NodeJS.ErrnoException) => { + if (cause.code === "ENOENT") return undefined; + throw cause; + }); + if (stat !== undefined) { + if (!stat.isSymbolicLink()) throw new Error("The service launcher entry is not an owned link."); + const target = NodePath.resolve(NodePath.dirname(link), await NodeFSP.readlink(link)); + if ( + NodePath.dirname(NodePath.dirname(target)) !== + NodePath.resolve(baseDir, "runtime", "versions") || + !isExactServiceVersion(NodePath.basename(NodePath.dirname(target))) || + NodePath.basename(target) !== "t3" + ) + throw new Error("The service launcher entry belongs to a different installation."); + } + const temporary = `${link}.${process.pid}.${NodeCrypto.randomUUID()}`; + try { + await NodeFSP.symlink(entry, temporary); + await NodeFSP.rename(temporary, link); + await syncDirectory(NodePath.dirname(link)); + } finally { + await NodeFSP.rm(temporary, { force: true }); + } + const recorded = await NodeFSP.readFile( + NodePath.join(baseDir, "runtime", "launchers"), + "utf8", + ).catch((cause: NodeJS.ErrnoException) => { + if (cause.code === "ENOENT") return ""; + throw cause; + }); + for (const launcher of new Set(recorded.split(/\r?\n/).filter(NodePath.isAbsolute))) { + if (launcher === link) continue; + const target = await NodeFSP.readlink(launcher).catch((cause: NodeJS.ErrnoException) => { + if (cause.code === "ENOENT" || cause.code === "EINVAL") return undefined; + throw cause; + }); + if (target === undefined) continue; + const resolved = NodePath.resolve(NodePath.dirname(launcher), target); + if ( + NodePath.dirname(NodePath.dirname(resolved)) !== + NodePath.resolve(baseDir, "runtime", "versions") || + !isExactServiceVersion(NodePath.basename(NodePath.dirname(resolved))) || + NodePath.basename(resolved) !== "t3" + ) + continue; + const pending = `${launcher}.${process.pid}.${NodeCrypto.randomUUID()}`; + try { + await NodeFSP.symlink(link, pending); + await NodeFSP.rename(pending, launcher); + await syncDirectory(NodePath.dirname(launcher)); + } finally { + await NodeFSP.rm(pending, { force: true }); + } + } + return link; +} + async function runtimeExists(baseDir: string, version: string): Promise { const paths = runtimePaths(baseDir, version); try { @@ -473,6 +538,13 @@ export class Launcher { readonly #backupSpaceProbe: Partial | undefined; readonly #environment: Readonly; readonly #initialRestore: AppRestoreInput | undefined; + readonly #handoff: ((version: string) => void) | undefined; + readonly #initialUpdate: + | { + readonly targetVersion: string; + readonly install: (dbPath: string, signal: AbortSignal) => Promise; + } + | undefined; constructor( baseDir: string, @@ -481,6 +553,11 @@ export class Launcher { readonly backupSpaceProbe?: Partial; readonly environment?: Readonly; readonly initialRestore?: AppRestoreInput; + readonly handoff?: (version: string) => void; + readonly initialUpdate?: { + readonly targetVersion: string; + readonly install: (dbPath: string, signal: AbortSignal) => Promise; + }; } = {}, ) { this.#baseDir = baseDir; @@ -489,20 +566,68 @@ export class Launcher { this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); this.#state = state; this.#initialRestore = options.initialRestore; + this.#handoff = options.handoff; + this.#initialUpdate = options.initialUpdate; } - async run(): Promise { + async run(signal?: AbortSignal): Promise { let releaseLock: (() => void) | undefined; let closeControl: (() => Promise) | undefined; const onSigterm = () => void this.stop("SIGTERM"); const onSigint = () => void this.stop("SIGINT"); + const onAbort = () => void this.stop("SIGTERM"); + signal?.throwIfAborted(); + signal?.addEventListener("abort", onAbort, { once: true }); process.once("SIGTERM", onSigterm); process.once("SIGINT", onSigint); try { this.#enqueue(async () => { releaseLock = await acquireServiceLauncherLock(this.#baseDir); this.#state = await readServiceState(this.#statePath); + if (this.#stopRequested) return; let freshRestoreId: string | undefined; + if (this.#initialUpdate !== undefined) { + const releaseProfile = await acquireStoppedServerProfile(this.#baseDir); + try { + const targetVersion = this.#initialUpdate.targetVersion; + if (await serverUpdateHold(this.#baseDir)) + throw new Error("Server updates are on hold after restore."); + if (this.#state.update?.status === "pending") { + if ( + this.#state.update.restore !== undefined || + this.#state.update.targetVersion !== targetVersion + ) + throw new Error("A different managed transaction needs recovery first."); + } else { + await this.#prepareUpdate(targetVersion); + const { dbPath } = await serverRestoreIdentity(this.#baseDir); + await this.#initialUpdate.install(dbPath, this.#retentionController.signal); + if (this.#stopRequested) throw new Error("Update was cancelled before handoff."); + if (!(await runtimeExists(this.#baseDir, targetVersion))) + throw new Error("The target runtime is incomplete."); + const blocker = await databaseBackupBlocker( + this.#baseDir, + dbPath, + this.#backupSpaceProbe, + ); + if (blocker !== undefined) throw new Error(blocker); + const next: ServiceState = { + ...this.#state, + update: { + id: NodeCrypto.randomUUID(), + fromVersion: this.#state.activeVersion, + targetVersion, + dbPath, + status: "pending", + }, + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + } + } finally { + releaseProfile(); + } + } if (this.#stopRequested && this.#initialRestore !== undefined) return; if (this.#initialRestore !== undefined) { await assertServerRestoreStopped(this.#baseDir); @@ -520,46 +645,55 @@ export class Launcher { freshRestoreId = this.#state.update?.id; } } - closeControl = await serveServerRestoreControl( - this.#baseDir, - (request) => - new Promise((resolve) => { - this.#enqueue(async () => { - try { - if (request.action === "release-hold") { - if (this.#state.update?.status === "pending") - throw new Error("A version transaction is still pending."); - await writeServerUpdateHold(this.#baseDir, false); - resolve({ - status: "restored", - version: this.#state.activeVersion, - reason: null, + closeControl = + this.#initialUpdate === undefined + ? await serveServerRestoreControl( + this.#baseDir, + (request) => + new Promise((resolve) => { + this.#enqueue(async () => { + try { + if (request.action === "release-hold") { + if (this.#state.update?.status === "pending") + throw new Error("A version transaction is still pending."); + await writeServerUpdateHold(this.#baseDir, false); + resolve({ + status: "restored", + version: this.#state.activeVersion, + reason: null, + }); + return; + } + if (this.#child?.role !== "active") + throw new Error("The managed server is not ready for restore."); + const child = this.#child; + await this.#prepareRestore(request.input); + this.#timer = setTimeout( + () => this.#enqueue(() => this.#beginTrial(child)), + HANDOFF_DELAY_MS, + ); + resolve({ + status: "scheduled", + version: request.input.version, + reason: null, + }); + } catch (cause) { + resolve({ + status: "failed", + version: this.#state.activeVersion, + reason: + cause instanceof Error ? cause.message : "Restore preflight failed.", + }); + } }); - return; - } - if (this.#child?.role !== "active") - throw new Error("The managed server is not ready for restore."); - const child = this.#child; - await this.#prepareRestore(request.input); - this.#timer = setTimeout( - () => this.#enqueue(() => this.#beginTrial(child)), - HANDOFF_DELAY_MS, - ); - resolve({ status: "scheduled", version: request.input.version, reason: null }); - } catch (cause) { - resolve({ - status: "failed", - version: this.#state.activeVersion, - reason: cause instanceof Error ? cause.message : "Restore preflight failed.", - }); - } - }); - }), - ); + }), + ) + : undefined; await this.#recover(freshRestoreId); }); await this.#completion.promise; } finally { + signal?.removeEventListener("abort", onAbort); process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); await closeControl?.(); @@ -697,6 +831,7 @@ export class Launcher { if (update !== undefined) { await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); } + await syncServiceLauncherEntry(this.#baseDir, this.#state.activeVersion); await cleanupServerRuntimeRetention(this.#baseDir, { signal: this.#retentionController.signal, }).catch((cause: unknown) => { @@ -817,16 +952,41 @@ export class Launcher { async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise { if (this.#child !== child || this.#stopping) return; - if (message.type === "request-update") { + if (message.type === "request-update" || message.type === "prepare-update") { await this.#handleUpdateRequest(child, message); return; } await this.#handlePrepared(child, message); } + async #prepareUpdate(targetVersion: string, child?: ManagedChild): Promise { + const retention = await acquireServerRuntimeRetention(this.#baseDir, { + signal: this.#retentionController.signal, + }); + try { + if (child !== undefined) + await qualifyServerRestoreRuntime( + await serverRestoreIdentity(this.#baseDir), + child.version, + ); + const next = await retention.prepare(this.#state, targetVersion); + if (this.#stopRequested || (child !== undefined && !child.process.connected)) + throw new Error("Update preparation was cancelled before it was committed."); + await syncServiceLauncherEntry(this.#baseDir, this.#state.activeVersion); + await writeServiceState(this.#statePath, next); + this.#state = next; + await retention.clean(); + } finally { + retention.release(); + } + } + async #handleUpdateRequest( child: ManagedChild, - message: Extract, + message: Extract< + ServiceLauncherChildMessage, + { readonly type: "request-update" | "prepare-update" } + >, ): Promise { const reject = (reason: string) => sendMessage(child.process, { type: "update-rejected", reason }); @@ -858,6 +1018,25 @@ export class Launcher { await reject("The requested database path is not absolute."); return; } + if (message.type === "prepare-update") { + try { + await this.#prepareUpdate(message.targetVersion, child); + await sendMessage(child.process, { + type: "update-prepared", + targetVersion: message.targetVersion, + }); + } catch (cause) { + await reject(cause instanceof Error ? cause.message : "Update preparation failed."); + } + return; + } + if ( + this.#state.preparation?.fromVersion === child.version && + this.#state.preparation.targetVersion !== message.targetVersion + ) { + await reject("A different runtime update was prepared."); + return; + } if (!(await runtimeExists(this.#baseDir, message.targetVersion))) { await reject("The requested target runtime is missing or incomplete."); return; @@ -909,6 +1088,8 @@ export class Launcher { this.#timer = undefined; this.#child = null; await terminateChild(child.process); + if (pending.restore === undefined && !this.#stopRequested) + this.#handoff?.(pending.targetVersion); await this.#startTrial(pending); } @@ -982,6 +1163,10 @@ export class Launcher { await writeServiceState(this.#statePath, next); this.#state = next; child.role = "active"; + await sendMessage(child.process, { type: "committed", updateId: committed.id }); + await syncServiceLauncherEntry(this.#baseDir, next.activeVersion).catch((cause: unknown) => + NodeFS.writeSync(2, `${NodeUtil.format("Service launcher entry needs recovery.", cause)}\n`), + ); await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); await serverRestoreIdentity(this.#baseDir) .then((identity) => qualifyServerRestoreRuntime(identity, child.version)) @@ -995,8 +1180,8 @@ export class Launcher { `${NodeUtil.format("Committed runtime retention cleanup could not complete.", cause)}\n`, ); }); - await sendMessage(child.process, { type: "committed", updateId: committed.id }); - if (this.#initialRestore !== undefined) void this.stop("SIGTERM"); + if (this.#initialRestore !== undefined || this.#initialUpdate !== undefined) + void this.stop("SIGTERM"); } async #handlePreparedTimeout(child: ManagedChild): Promise { @@ -1099,8 +1284,10 @@ export class Launcher { await writeServiceState(this.#statePath, next); this.#state = next; await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); + if (pending.restore === undefined && !this.#stopRequested) this.#handoff?.(next.activeVersion); await this.#startChild(next.activeVersion, "active", outcome); - if (this.#initialRestore !== undefined) void this.stop("SIGTERM"); + if (this.#initialRestore !== undefined || this.#initialUpdate !== undefined) + void this.stop("SIGTERM"); } } @@ -1125,6 +1312,27 @@ export async function restoreStoppedServer( }; } +export async function updateStoppedServer( + baseDir: string, + targetVersion: string, + install: (dbPath: string, signal: AbortSignal) => Promise, + signal?: AbortSignal, +): Promise { + const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); + const state = await readServiceState(statePath); + const environment = await readServiceEnvironment(baseDir); + await new Launcher(baseDir, state, { + environment, + initialUpdate: { targetVersion, install }, + }).run(signal); + const outcome = await readServiceState(statePath); + if (outcome.activeVersion !== targetVersion || outcome.update?.status !== "committed") + throw new Error( + "The stopped server update did not commit; inspect the managed recovery state.", + ); + return outcome.activeVersion; +} + export async function main(): Promise { const baseDir = process.env.T3CODE_HOME?.trim(); if (baseDir === undefined || baseDir === "") { @@ -1133,5 +1341,34 @@ export async function main(): Promise { const environment = await readServiceEnvironment(baseDir); const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); const state = await readServiceState(statePath); - await new Launcher(baseDir, state, { environment }).run(); + const realVersionsDir = await NodeFSP.realpath(NodePath.join(baseDir, "runtime", "versions")); + const nativeLauncher = + // oxlint-disable-next-line t3code/no-global-process-runtime -- Native exec handoff runs before Effect startup. + process.platform === "linux" && + NodePath.dirname(NodePath.dirname(process.execPath)) === realVersionsDir; + const handoff = nativeLauncher + ? (version: string) => { + if (process.execve === undefined) + throw new Error("This launcher runtime cannot hand off its executable safely."); + const entry = runtimePaths(baseDir, version).entryPath; + process.execve(entry, [entry, "__service-launcher"], environment); + } + : undefined; + const selected = + state.update?.status === "pending" && + state.update.restore === undefined && + (await runtimeExists(baseDir, state.update.targetVersion)) + ? state.update.targetVersion + : state.activeVersion; + if ( + handoff !== undefined && + (await NodeFSP.realpath(runtimePaths(baseDir, selected).entryPath)) !== process.execPath + ) { + await verifyServerRestoreRuntime(runtimePaths(baseDir, selected).versionDir, selected); + handoff(selected); + } + await new Launcher(baseDir, state, { + environment, + ...(handoff === undefined ? {} : { handoff }), + }).run(); } diff --git a/docs/user/background-service.md b/docs/user/background-service.md index c36527b2c425c..ee5ccb4952a9a 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -22,6 +22,10 @@ Uninstalling the service leaves your projects, threads, and settings intact. Running `t3 service install` again repairs a service that `t3 service status` reports as broken. +For a custom Linux unit, pass its exact name to service commands, for example +`t3 service status --unit crew.service`. Keep `--base-dir` consistent with that +unit's T3 home. Service commands leave units for another T3 home alone. + `t3 update` downloads the newest release on your channel and switches `t3` and the service to it. Restarting interrupts running agent turns, terminals, and remote clients, so it asks first; answer no and the service keeps running @@ -31,6 +35,17 @@ to pick up the new version. Wait for any remote update already in progress before updating; to match a remote client's version, follow [Updating T3 Code](./updating.md). +If an older service cannot prepare its next runtime within the retention limit, +use an up-to-date CLI on that machine. Stop its exact systemd user unit, then run +`t3 service update-stopped VERSION --base-dir /path/to/t3-home --unit UNIT.service` and start +that unit again. This command requires a stopped server and launcher, verifies +the download against the same profile, and commits through the normal trial and +database-backup path. A failed preparation keeps the current runtime available; +retry while stopped or start the unit to run its recorded recovery. Linux units +installed by the current CLI use `runtime/launcher` under T3 home as their stable executable. For an older custom unit, +point `ExecStart` there with `__service-launcher` before starting it again, keeping +its existing environment and other settings. + Pass an exact version (`t3 update 0.0.42`) to pin one, `--channel nightly` to switch trains, or `--allow-downgrade` to move backwards. `preview` is a maintainers' test train: its builds can be broken and are never offered as