diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts
index d3a8af2320c4..da1d0384bef2 100644
--- a/apps/server/src/cloud/bootService.test.ts
+++ b/apps/server/src/cloud/bootService.test.ts
@@ -300,9 +300,12 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
const { service, fs, statePath, timeouts, runtime } = yield* makeHarness();
const plan = yield* service.install();
+ // The unit runs the launcher from this runtime even after remote updates move
+ // activeVersion on, so state has to record it for anything that cleans up runtimes.
expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
protocol: SERVICE_LAUNCHER_PROTOCOL,
activeVersion: "1.2.3",
+ launcherVersion: "1.2.3",
});
expect(plan.program).toEqual([runtime.entryPath, "__service-launcher"]);
expect(yield* fs.readFileString(plan.unitPath)).toContain(
@@ -468,6 +471,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
protocol: SERVICE_LAUNCHER_PROTOCOL,
activeVersion: "1.2.4",
+ launcherVersion: "1.2.4",
});
expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3");
expect(
@@ -763,6 +767,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
protocol: SERVICE_LAUNCHER_PROTOCOL,
activeVersion: "1.2.3",
+ launcherVersion: "1.2.3",
});
expect(yield* fs.readFileString(plan.unitPath)).toContain(
` ${runtime.entryPath}\n __service-launcher`,
diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts
index c1be1fc2c5ef..747637d7e90d 100644
--- a/apps/server/src/cloud/bootService.ts
+++ b/apps/server/src/cloud/bootService.ts
@@ -869,6 +869,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
{
protocol: SERVICE_LAUNCHER_PROTOCOL,
activeVersion: input.cliVersion,
+ launcherVersion: input.cliVersion,
} satisfies ServiceState,
null,
2,
diff --git a/apps/server/src/cloud/pinnedRuntime.test.ts b/apps/server/src/cloud/pinnedRuntime.test.ts
index 9d536f471e76..34f29b75e924 100644
--- a/apps/server/src/cloud/pinnedRuntime.test.ts
+++ b/apps/server/src/cloud/pinnedRuntime.test.ts
@@ -428,6 +428,35 @@ it.layer(NodeServices.layer)("prunePinnedRuntimes", (it) => {
}),
);
+ it.effect("keeps the runtime the service state records as the launcher", () =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const { baseDir } = yield* setup(["0.1.1", "0.1.2", "0.1.3"]);
+ // Installed from 0.1.1, then updated remotely twice. The service is stopped, so nothing
+ // runs 0.1.1, but its unit will start the launcher from it.
+ yield* fs.writeFileString(
+ path.join(baseDir, "runtime", "service-state.json"),
+ JSON.stringify({
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
+ activeVersion: "0.1.3",
+ launcherVersion: "0.1.1",
+ update: { id: "u", fromVersion: "0.1.2", targetVersion: "0.1.3", status: "committed" },
+ }),
+ );
+
+ const removed = yield* prunePinnedRuntimes({ baseDir, keep: ["0.1.3"] }).pipe(
+ Effect.provideService(
+ ProcessRunner.ProcessRunner,
+ lsofRunner("p1\nftxt\nn/sbin/launchd\n"),
+ ),
+ Effect.provideService(HostProcess.Platform, "darwin"),
+ Effect.provideService(HostProcess.Environment, { T3CODE_INSTALL_BIN_DIR: baseDir }),
+ );
+ assert.deepEqual(removed, []);
+ }),
+ );
+
it.effect("matches a Windows shim whatever its casing", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
diff --git a/apps/server/src/cloud/pinnedRuntime.ts b/apps/server/src/cloud/pinnedRuntime.ts
index 0093e7a6220e..9b153a403dee 100644
--- a/apps/server/src/cloud/pinnedRuntime.ts
+++ b/apps/server/src/cloud/pinnedRuntime.ts
@@ -551,6 +551,7 @@ const selectUnusedRuntimes = Effect.fn("cloud.pinned_runtime.select_unused")(fun
const state = parseServiceState(stateText.value);
if (state === undefined) return pruned;
keep.add(state.activeVersion);
+ if (state.launcherVersion !== undefined) keep.add(state.launcherVersion);
if (state.update !== undefined) {
keep.add(state.update.fromVersion);
keep.add(state.update.targetVersion);
@@ -600,9 +601,9 @@ const selectUnusedRuntimes = Effect.fn("cloud.pinned_runtime.select_unused")(fun
/**
* Every update publishes a new immutable runtime, so without pruning they pile up. Removes
* completed runtimes nothing can start again and keeps: the caller's versions, everything the
- * service state references (active, rollback, pending), and runtimes that a running process
- * executes, a `t3` launcher points at, or `references` (such as the service unit) name, plus
- * anything published within the last hour. Staging directories, symlinks, and unexpected entries
+ * service state references (active, launcher, rollback, pending), and runtimes that a running
+ * process executes, a `t3` launcher points at, or `references` (such as the service unit) name,
+ * plus anything published within the last hour. Staging directories, symlinks, and unexpected entries
* are never touched. Returns the versions removed.
*/
export const prunePinnedRuntimes = Effect.fn("cloud.pinned_runtime.prune")(function* (input: {
diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts
index 5704c38b8af5..62f0fa10e4ad 100644
--- a/apps/server/src/cloud/serviceProtocol.ts
+++ b/apps/server/src/cloud/serviceProtocol.ts
@@ -29,6 +29,10 @@ export type ServiceUpdateRecord = PendingServiceUpdate | ServerSelfUpdateOutcome
export interface ServiceState {
readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL;
readonly activeVersion: string;
+ /** The runtime the installed unit starts the launcher from. Remote updates move
+ `activeVersion` on without touching the unit, so cleanup has to keep this one.
+ Absent in state written by older installers, and launchers before it drop it. */
+ readonly launcherVersion?: string;
readonly update?: ServiceUpdateRecord;
}
@@ -152,6 +156,9 @@ export function decodeServiceState(value: unknown): ServiceState | undefined {
value.protocol !== SERVICE_LAUNCHER_PROTOCOL ||
typeof value.activeVersion !== "string" ||
!isExactServiceVersion(value.activeVersion) ||
+ (value.launcherVersion !== undefined &&
+ (typeof value.launcherVersion !== "string" ||
+ !isExactServiceVersion(value.launcherVersion))) ||
(value.update !== undefined && update === undefined) ||
(update !== undefined &&
compareExactServiceVersions(update.targetVersion, update.fromVersion) <= 0) ||
@@ -165,6 +172,9 @@ export function decodeServiceState(value: unknown): ServiceState | undefined {
return {
protocol: SERVICE_LAUNCHER_PROTOCOL,
activeVersion: value.activeVersion,
+ ...(typeof value.launcherVersion === "string"
+ ? { launcherVersion: value.launcherVersion }
+ : {}),
...(update === undefined ? {} : { update }),
};
}
diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts
index 70550f237927..b166afec74ab 100644
--- a/apps/server/src/serviceLauncher.test.ts
+++ b/apps/server/src/serviceLauncher.test.ts
@@ -253,17 +253,20 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
}),
);
- it.effect("commits only after the trial reports prepared", () =>
- Effect.gen(function* () {
- const fs = yield* FileSystem.FileSystem;
- const path = yield* Path.Path;
- const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });
- const statePath = path.join(root, "runtime", "service-state.json");
- const databasePath = path.join(root, "userdata", "state.sqlite");
- yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
- yield* fs.writeFileString(databasePath, "before trial");
- const encodedDatabasePath = JSON.stringify(databasePath);
- const childSource = `
+ // `t3 update` without a restart repoints the unit while the launcher keeps running.
+ it.effect.each([undefined, "1.1.0"])(
+ "commits only after the trial reports prepared (unit repointed to: %s)",
+ (repointedLauncher) =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });
+ const statePath = path.join(root, "runtime", "service-state.json");
+ const databasePath = path.join(root, "userdata", "state.sqlite");
+ yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
+ yield* fs.writeFileString(databasePath, "before trial");
+ const encodedDatabasePath = JSON.stringify(databasePath);
+ const childSource = `
const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
if (context.update?.status === "pending") {
process.send({ type: "prepared", updateId: context.update.id });
@@ -277,33 +280,48 @@ if (context.update?.status === "pending") {
process.exit(0);
}
`;
- for (const version of ["1.0.0", "1.1.0"]) {
- yield* writeFakeRuntime(
- fs,
- path,
- path.join(root, "runtime", "versions", version),
- childSource,
+ for (const version of ["1.0.0", "1.1.0"]) {
+ yield* writeFakeRuntime(
+ fs,
+ path,
+ path.join(root, "runtime", "versions", version),
+ childSource,
+ );
+ }
+ yield* Effect.promise(() =>
+ writeServiceState(statePath, {
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
+ activeVersion: "1.0.0",
+ launcherVersion: "1.0.0",
+ }),
);
- }
- yield* Effect.promise(() =>
- writeServiceState(statePath, {
- protocol: SERVICE_LAUNCHER_PROTOCOL,
- activeVersion: "1.0.0",
- }),
- );
- const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
- yield* Effect.promise(() =>
- launcher.run().then(
- () => Promise.reject(new Error("launcher unexpectedly completed")),
- () => Promise.resolve(),
- ),
- );
+ const launcher = new Launcher(
+ root,
+ yield* Effect.promise(() => readServiceState(statePath)),
+ );
+ if (repointedLauncher !== undefined) {
+ yield* Effect.promise(() =>
+ writeServiceState(statePath, {
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
+ activeVersion: "1.0.0",
+ launcherVersion: repointedLauncher,
+ }),
+ );
+ }
+ yield* Effect.promise(() =>
+ launcher.run().then(
+ () => Promise.reject(new Error("launcher unexpectedly completed")),
+ () => Promise.resolve(),
+ ),
+ );
- const state = yield* Effect.promise(() => readServiceState(statePath));
- assert.equal(state.activeVersion, "1.1.0");
- assert.equal(state.update?.status, "committed");
- }),
+ const state = yield* Effect.promise(() => readServiceState(statePath));
+ assert.equal(state.activeVersion, "1.1.0");
+ assert.equal(state.update?.status, "committed");
+ // The service unit still starts the launcher from 1.0.0 unless it was repointed.
+ assert.equal(state.launcherVersion, repointedLauncher ?? "1.0.0");
+ }),
);
it.effect("rolls back invalid trials with the startup environment", () =>
diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts
index 9926159be285..42770ad34566 100644
--- a/apps/server/src/serviceLauncher.ts
+++ b/apps/server/src/serviceLauncher.ts
@@ -557,6 +557,24 @@ export class Launcher {
this.#timer = undefined;
}
+ async #writeState(next: ServiceState): Promise {
+ // `t3 update` without a restart repoints the unit and records its launcher
+ // runtime while this launcher keeps running, so what the file says wins.
+ const current = await NodeFSP.readFile(this.#statePath, "utf8").then(
+ parseServiceState,
+ () => undefined,
+ );
+ const { launcherVersion } = current ?? next;
+ const state: ServiceState = {
+ protocol: next.protocol,
+ activeVersion: next.activeVersion,
+ ...(launcherVersion === undefined ? {} : { launcherVersion }),
+ ...(next.update === undefined ? {} : { update: next.update }),
+ };
+ await writeServiceState(this.#statePath, state);
+ this.#state = state;
+ }
+
async #recover(): Promise {
// A fresh launcher means servers are running again: any stop marker from
// a previous explicit stop is stale and must not make a future update
@@ -724,8 +742,7 @@ export class Launcher {
status: "pending",
};
const next: ServiceState = { ...this.#state, update: pending };
- await writeServiceState(this.#statePath, next);
- this.#state = next;
+ await this.#writeState(next);
await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
}
@@ -762,8 +779,7 @@ export class Launcher {
activeVersion: pending.targetVersion,
update: committed,
};
- await writeServiceState(this.#statePath, next);
- this.#state = next;
+ await this.#writeState(next);
child.role = "active";
await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
await sendMessage(child.process, { type: "committed", updateId: committed.id });
@@ -826,8 +842,7 @@ export class Launcher {
activeVersion: pending.fromVersion,
update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
};
- await writeServiceState(this.#statePath, next);
- this.#state = next;
+ await this.#writeState(next);
await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
}
// Signal listeners alone do not keep Node alive.
@@ -854,8 +869,7 @@ export class Launcher {
activeVersion: pending.fromVersion,
update: outcome,
};
- await writeServiceState(this.#statePath, next);
- this.#state = next;
+ await this.#writeState(next);
await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
await this.#startChild(next.activeVersion, "active", outcome);
}