From 19e73d7b6677252fe33c01029754ebce62b5278a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 21 Oct 2026 22:56:47 +0000 Subject: [PATCH 1/2] test(server): reproduce #17490 service state losing the launcher runtime The service unit keeps running the launcher from the runtime that installed it, but service-state.json never records that runtime, so state-based cleanup deletes it and the next restart fails with 203/EXEC. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/bootService.test.ts | 3 +++ apps/server/src/cloud/pinnedRuntime.test.ts | 29 +++++++++++++++++++++ apps/server/src/serviceLauncher.test.ts | 8 ++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index d3a8af2320c4..a6b1c674c712 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -300,9 +300,12 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const { service, fs, statePath, timeouts, runtime } = yield* makeHarness(); const plan = yield* service.install(); + // The unit runs the launcher from this runtime even after remote updates move + // activeVersion on, so state has to record it for anything that cleans up runtimes. expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.2.3", + launcherVersion: "1.2.3", }); expect(plan.program).toEqual([runtime.entryPath, "__service-launcher"]); expect(yield* fs.readFileString(plan.unitPath)).toContain( diff --git a/apps/server/src/cloud/pinnedRuntime.test.ts b/apps/server/src/cloud/pinnedRuntime.test.ts index 9d536f471e76..34f29b75e924 100644 --- a/apps/server/src/cloud/pinnedRuntime.test.ts +++ b/apps/server/src/cloud/pinnedRuntime.test.ts @@ -428,6 +428,35 @@ it.layer(NodeServices.layer)("prunePinnedRuntimes", (it) => { }), ); + it.effect("keeps the runtime the service state records as the launcher", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { baseDir } = yield* setup(["0.1.1", "0.1.2", "0.1.3"]); + // Installed from 0.1.1, then updated remotely twice. The service is stopped, so nothing + // runs 0.1.1, but its unit will start the launcher from it. + yield* fs.writeFileString( + path.join(baseDir, "runtime", "service-state.json"), + JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "0.1.3", + launcherVersion: "0.1.1", + update: { id: "u", fromVersion: "0.1.2", targetVersion: "0.1.3", status: "committed" }, + }), + ); + + const removed = yield* prunePinnedRuntimes({ baseDir, keep: ["0.1.3"] }).pipe( + Effect.provideService( + ProcessRunner.ProcessRunner, + lsofRunner("p1\nftxt\nn/sbin/launchd\n"), + ), + Effect.provideService(HostProcess.Platform, "darwin"), + Effect.provideService(HostProcess.Environment, { T3CODE_INSTALL_BIN_DIR: baseDir }), + ); + assert.deepEqual(removed, []); + }), + ); + it.effect("matches a Windows shim whatever its casing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 70550f237927..ee747cc2ebe7 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -285,10 +285,12 @@ if (context.update?.status === "pending") { childSource, ); } - yield* Effect.promise(() => - writeServiceState(statePath, { + yield* fs.writeFileString( + statePath, + JSON.stringify({ protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", + launcherVersion: "1.0.0", }), ); @@ -303,6 +305,8 @@ if (context.update?.status === "pending") { const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.1.0"); assert.equal(state.update?.status, "committed"); + // The service unit still starts the launcher from 1.0.0. + assert.propertyVal(state, "launcherVersion", "1.0.0"); }), ); From 00024c7f87dd842a0184500d39f5a42856178bd6 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 00:05:31 +0000 Subject: [PATCH 2/2] fix(server): service state records the runtime its launcher runs from Remote updates move activeVersion on but leave the unit starting the launcher from the runtime that installed it, so state-based cleanup could delete that runtime and the next restart failed with 203/EXEC. Installs now record launcherVersion, pruning keeps it, and the launcher carries it through update transitions, taking the on-disk value so a `t3 update` that repointed the unit without a restart is not overwritten. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/bootService.test.ts | 2 + apps/server/src/cloud/bootService.ts | 1 + apps/server/src/cloud/pinnedRuntime.ts | 7 +- apps/server/src/cloud/serviceProtocol.ts | 10 +++ apps/server/src/serviceLauncher.test.ts | 92 +++++++++++++---------- apps/server/src/serviceLauncher.ts | 30 ++++++-- 6 files changed, 92 insertions(+), 50 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index a6b1c674c712..da1d0384bef2 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -471,6 +471,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.2.4", + launcherVersion: "1.2.4", }); expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3"); expect( @@ -766,6 +767,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.2.3", + launcherVersion: "1.2.3", }); expect(yield* fs.readFileString(plan.unitPath)).toContain( ` ${runtime.entryPath}\n __service-launcher`, diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index c1be1fc2c5ef..747637d7e90d 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -869,6 +869,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: input.cliVersion, + launcherVersion: input.cliVersion, } satisfies ServiceState, null, 2, diff --git a/apps/server/src/cloud/pinnedRuntime.ts b/apps/server/src/cloud/pinnedRuntime.ts index 0093e7a6220e..9b153a403dee 100644 --- a/apps/server/src/cloud/pinnedRuntime.ts +++ b/apps/server/src/cloud/pinnedRuntime.ts @@ -551,6 +551,7 @@ const selectUnusedRuntimes = Effect.fn("cloud.pinned_runtime.select_unused")(fun const state = parseServiceState(stateText.value); if (state === undefined) return pruned; keep.add(state.activeVersion); + if (state.launcherVersion !== undefined) keep.add(state.launcherVersion); if (state.update !== undefined) { keep.add(state.update.fromVersion); keep.add(state.update.targetVersion); @@ -600,9 +601,9 @@ const selectUnusedRuntimes = Effect.fn("cloud.pinned_runtime.select_unused")(fun /** * Every update publishes a new immutable runtime, so without pruning they pile up. Removes * completed runtimes nothing can start again and keeps: the caller's versions, everything the - * service state references (active, rollback, pending), and runtimes that a running process - * executes, a `t3` launcher points at, or `references` (such as the service unit) name, plus - * anything published within the last hour. Staging directories, symlinks, and unexpected entries + * service state references (active, launcher, rollback, pending), and runtimes that a running + * process executes, a `t3` launcher points at, or `references` (such as the service unit) name, + * plus anything published within the last hour. Staging directories, symlinks, and unexpected entries * are never touched. Returns the versions removed. */ export const prunePinnedRuntimes = Effect.fn("cloud.pinned_runtime.prune")(function* (input: { diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 5704c38b8af5..62f0fa10e4ad 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -29,6 +29,10 @@ export type ServiceUpdateRecord = PendingServiceUpdate | ServerSelfUpdateOutcome export interface ServiceState { readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL; readonly activeVersion: string; + /** The runtime the installed unit starts the launcher from. Remote updates move + `activeVersion` on without touching the unit, so cleanup has to keep this one. + Absent in state written by older installers, and launchers before it drop it. */ + readonly launcherVersion?: string; readonly update?: ServiceUpdateRecord; } @@ -152,6 +156,9 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { value.protocol !== SERVICE_LAUNCHER_PROTOCOL || typeof value.activeVersion !== "string" || !isExactServiceVersion(value.activeVersion) || + (value.launcherVersion !== undefined && + (typeof value.launcherVersion !== "string" || + !isExactServiceVersion(value.launcherVersion))) || (value.update !== undefined && update === undefined) || (update !== undefined && compareExactServiceVersions(update.targetVersion, update.fromVersion) <= 0) || @@ -165,6 +172,9 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { return { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: value.activeVersion, + ...(typeof value.launcherVersion === "string" + ? { launcherVersion: value.launcherVersion } + : {}), ...(update === undefined ? {} : { update }), }; } diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index ee747cc2ebe7..b166afec74ab 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -253,17 +253,20 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect("commits only after the trial reports prepared", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - yield* fs.writeFileString(databasePath, "before trial"); - const encodedDatabasePath = JSON.stringify(databasePath); - const childSource = ` + // `t3 update` without a restart repoints the unit while the launcher keeps running. + it.effect.each([undefined, "1.1.0"])( + "commits only after the trial reports prepared (unit repointed to: %s)", + (repointedLauncher) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { process.send({ type: "prepared", updateId: context.update.id }); @@ -277,37 +280,48 @@ if (context.update?.status === "pending") { process.exit(0); } `; - for (const version of ["1.0.0", "1.1.0"]) { - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", version), - childSource, + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + launcherVersion: "1.0.0", + }), ); - } - yield* fs.writeFileString( - statePath, - JSON.stringify({ - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - launcherVersion: "1.0.0", - }), - ); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); - yield* Effect.promise(() => - launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), - ), - ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + if (repointedLauncher !== undefined) { + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + launcherVersion: repointedLauncher, + }), + ); + } + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); - const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.1.0"); - assert.equal(state.update?.status, "committed"); - // The service unit still starts the launcher from 1.0.0. - assert.propertyVal(state, "launcherVersion", "1.0.0"); - }), + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update?.status, "committed"); + // The service unit still starts the launcher from 1.0.0 unless it was repointed. + assert.equal(state.launcherVersion, repointedLauncher ?? "1.0.0"); + }), ); it.effect("rolls back invalid trials with the startup environment", () => diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 9926159be285..42770ad34566 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -557,6 +557,24 @@ export class Launcher { this.#timer = undefined; } + async #writeState(next: ServiceState): Promise { + // `t3 update` without a restart repoints the unit and records its launcher + // runtime while this launcher keeps running, so what the file says wins. + const current = await NodeFSP.readFile(this.#statePath, "utf8").then( + parseServiceState, + () => undefined, + ); + const { launcherVersion } = current ?? next; + const state: ServiceState = { + protocol: next.protocol, + activeVersion: next.activeVersion, + ...(launcherVersion === undefined ? {} : { launcherVersion }), + ...(next.update === undefined ? {} : { update: next.update }), + }; + await writeServiceState(this.#statePath, state); + this.#state = state; + } + async #recover(): Promise { // A fresh launcher means servers are running again: any stop marker from // a previous explicit stop is stale and must not make a future update @@ -724,8 +742,7 @@ export class Launcher { status: "pending", }; const next: ServiceState = { ...this.#state, update: pending }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#writeState(next); await sendMessage(child.process, { type: "update-accepted", updateId: pending.id }); this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS); } @@ -762,8 +779,7 @@ export class Launcher { activeVersion: pending.targetVersion, update: committed, }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#writeState(next); child.role = "active"; await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); await sendMessage(child.process, { type: "committed", updateId: committed.id }); @@ -826,8 +842,7 @@ export class Launcher { activeVersion: pending.fromVersion, update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#writeState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); } // Signal listeners alone do not keep Node alive. @@ -854,8 +869,7 @@ export class Launcher { activeVersion: pending.fromVersion, update: outcome, }; - await writeServiceState(this.#statePath, next); - this.#state = next; + await this.#writeState(next); await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); await this.#startChild(next.activeVersion, "active", outcome); }