diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index fca51dcac1e09..c0cedd588fdc4 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -4,11 +4,14 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as Queue from "effect/Queue"; import { HttpClient } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as TestClock from "effect/testing/TestClock"; import * as ProcessRunner from "../processRunner.ts"; import * as BootService from "./bootService.ts"; @@ -87,15 +90,22 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( const timeouts = new Map(); const control: { failCommand: string | undefined; + failOnce?: boolean; stateAfterStop?: string; linger: string; enabled: boolean; active: boolean; + missingExitCode: number; + drainPolls: number; + onDrainPoll: Effect.Effect; } = { failCommand: undefined, linger: "yes", enabled: true, active: true, + missingExitCode: 113, + drainPolls: 0, + onDrainPoll: Effect.void, }; const runner = ProcessRunner.ProcessRunner.of({ run: Effect.fn("test.run_boot_service_command")(function* ( @@ -106,16 +116,30 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( invocations.push(input); timeouts.set(command, input.timeout); const failed = command === control.failCommand; + if (failed && control.failOnce) { + control.failCommand = undefined; + control.failOnce = false; + } if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; if (!failed && command === "systemctl --user restart t3code.service") control.active = true; // A timed-out stop may still stop the service. if (command === "systemctl --user stop t3code.service") control.active = false; + // On macOS `active` means loaded. A booted-out job still shows in + // `print` for `drainPolls` more calls while it drains. + if (!failed && command.startsWith("launchctl bootstrap ")) control.active = true; + if (!failed && command.startsWith("launchctl bootout ")) control.active = false; + const launchdPrint = !failed && command.startsWith("launchctl print "); + const draining = launchdPrint && !control.active && control.drainPolls > 0; + if (draining) { + control.drainPolls -= 1; + yield* control.onDrainPoll; + } if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || - command.startsWith("launchctl bootout --wait ")) + command.startsWith("launchctl bootout ")) ) { yield* fs.writeFileString(statePath, control.stateAfterStop).pipe(Effect.orDie); } @@ -132,11 +156,15 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( : "", stderr: "", code: ChildProcessSpawner.ExitCode( - failed || - (input.args[1] === "is-active" && !control.active) || - (input.args[1] === "is-enabled" && !control.enabled) - ? 1 - : 0, + launchdPrint + ? control.active || draining + ? 0 + : control.missingExitCode + : failed || + (input.args[1] === "is-active" && !control.active) || + (input.args[1] === "is-enabled" && !control.enabled) + ? 1 + : 0, ), timedOut: false, stdoutTruncated: false, @@ -403,7 +431,8 @@ it.layer(NodeServices.layer)("boot service install", (it) => { platform === "linux" ? ["systemctl --user stop t3code.service", "systemctl --user restart t3code.service"] : [ - "launchctl bootout --wait gui/501/com.t3tools.t3code.service", + "launchctl bootout gui/501/com.t3tools.t3code.service", + "launchctl print gui/501/com.t3tools.t3code.service", `launchctl bootstrap gui/501 ${plan.unitPath}`, ], ); @@ -769,7 +798,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect((yield* service.status).installed).toBe(false); expect(commands.some((command) => command.startsWith("systemctl "))).toBe(false); // launchd can take 90s to bootout; the runner's 60s default could let bootstrap race it. - expect(timeouts.get("launchctl bootout --wait gui/501/com.t3tools.t3code.service")).toEqual( + expect(timeouts.get("launchctl bootout gui/501/com.t3tools.t3code.service")).toEqual( Duration.seconds(120), ); }), @@ -827,17 +856,152 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); - it.effect("ignores a bootout for an agent that is not loaded", () => + it.effect.each([112, 113])("ignores a bootout for an agent missing with exit %s", (code) => Effect.gen(function* () { const { service, control } = yield* makeHarness("darwin"); yield* service.install(); - control.failCommand = "launchctl bootout --wait gui/501/com.t3tools.t3code.service"; + control.active = false; + control.missingExitCode = code; + control.failCommand = "launchctl bootout gui/501/com.t3tools.t3code.service"; yield* service.install(); expect((yield* service.status).current).toBe(true); }), ); + it.effect( + "requires launchd shutdown confirmation for maintenance after a failed status probe", + () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness("darwin"); + yield* service.install(); + control.failCommand = "launchctl print gui/501/com.t3tools.t3code.service"; + let ran = false; + + const error = yield* service + .withStopped( + Effect.sync(() => { + ran = true; + }), + ) + .pipe(Effect.flip); + + expect(error).toMatchObject({ + _tag: "BootServiceCommandError", + step: "waiting for the launch agent to stop", + exitCode: 1, + }); + expect(ran).toBe(false); + }), + ); + + it.effect( + "restores launchd after a failed drain probe only when a final probe confirms removal", + () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + const plan = yield* service.install(); + const polls = yield* Queue.unbounded(); + control.drainPolls = 1; + control.onDrainPoll = Effect.sync(() => { + control.failCommand = "launchctl print gui/501/com.t3tools.t3code.service"; + control.failOnce = true; + }).pipe(Effect.andThen(Queue.offer(polls, undefined)), Effect.asVoid); + commands.length = 0; + + const maintenance = yield* service + .withStopped(Effect.sync(() => commands.push("repair"))) + .pipe(Effect.flip, Effect.forkChild); + yield* Queue.take(polls); + yield* TestClock.adjust("1 second"); + + expect(yield* Fiber.join(maintenance)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "waiting for the launch agent to stop", + }); + expect(commands).not.toContain("repair"); + expect(commands.at(-2)).toBe("launchctl print gui/501/com.t3tools.t3code.service"); + expect(commands.at(-1)).toBe(`launchctl bootstrap gui/501 ${plan.unitPath}`); + }), + ); + + // regression: #11995 macOS 15 rejects `bootout --wait`, so the stop waits on `print` + it.effect("starts a booted-out launch agent again only once launchd has removed it", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + const plan = yield* service.install(); + const polls = yield* Queue.unbounded(); + control.drainPolls = 2; + control.onDrainPoll = Queue.offer(polls, undefined).pipe(Effect.asVoid); + commands.length = 0; + + const reinstall = yield* service.install().pipe(Effect.forkChild); + yield* Queue.take(polls); + yield* TestClock.adjust("1 second"); + yield* Queue.take(polls); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(reinstall); + + const print = "launchctl print gui/501/com.t3tools.t3code.service"; + expect(commands.filter((command) => command.startsWith("launchctl "))).toEqual([ + "launchctl bootout gui/501/com.t3tools.t3code.service", + print, + print, + print, + "launchctl enable gui/501/com.t3tools.t3code.service", + `launchctl bootstrap gui/501 ${plan.unitPath}`, + ]); + }), + ); + + it.effect.each(["install", "maintenance"] as const)( + "fails %s while launchd is still draining after the stop timeout", + (operation) => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + const plan = yield* service.install(); + const polls = yield* Queue.unbounded(); + control.drainPolls = Number.POSITIVE_INFINITY; + control.onDrainPoll = Queue.offer(polls, undefined).pipe(Effect.asVoid); + commands.length = 0; + + const run = + operation === "install" + ? service.install().pipe(Effect.asVoid) + : service.withStopped( + Effect.sync(() => { + commands.push("repair"); + }), + ); + const reinstall = yield* run.pipe(Effect.flip, Effect.forkChild); + yield* Queue.take(polls); + yield* TestClock.adjust("120 seconds"); + + expect(yield* Fiber.join(reinstall)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "waiting for the launch agent to stop", + }); + expect(commands).not.toContain(`launchctl bootstrap gui/501 ${plan.unitPath}`); + expect(commands).not.toContain("repair"); + }), + ); + + it.effect("fails an install when launchd cannot say whether the agent stopped", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + const plan = yield* service.install(); + control.failCommand = "launchctl print gui/501/com.t3tools.t3code.service"; + commands.length = 0; + + expect(yield* service.install().pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "waiting for the launch agent to stop", + exitCode: 1, + }); + expect(commands).not.toContain(`launchctl bootstrap gui/501 ${plan.unitPath}`); + }), + ); + it.effect("keeps failed launch agent installs pending until activation succeeds", () => Effect.gen(function* () { const { service, fs, statePath, commands, control } = yield* makeHarness("darwin"); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 15b3b16a7bc0f..1bb74f42fdcab 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -10,6 +10,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import { HttpClient } from "effect/http"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease"; @@ -185,6 +186,8 @@ export interface BootServiceStep { /** Non-zero exit is logged and ignored. */ readonly optional?: boolean; readonly timeout?: Duration.Input; + /** Poll successful probes until one of these exits, bounded by STOP_STEP_TIMEOUT. */ + readonly untilExit?: ReadonlyArray; } /** Must exceed systemd TimeoutStopSec and launchd ExitTimeOut (both 90s) to avoid shutdown races. */ @@ -293,6 +296,14 @@ function launchdManager(input: { ); const domainTarget = `gui/${input.uid}`; const serviceTarget = `${domainTarget}/${BOOT_SERVICE_LAUNCHD_LABEL}`; + // bootout returns before draining finishes; bootstrap races fail EIO until print reports + // no such service (113) or domain (112). Other failures must not permit bootstrap. + const waitForStop: BootServiceStep = { + step: "waiting for the launch agent to stop", + command: "launchctl", + args: ["print", serviceTarget], + untilExit: [112, 113], + }; // Unloaded jobs may reject bootout/enable. Bootstrap starts RunAtLoad/KeepAlive jobs; kickstart would restart them. return { kind: "launchd", @@ -302,15 +313,15 @@ function launchdManager(input: { homeDir: input.homeDir, environmentPath: input.environmentPath, }), - // bootout --wait is undocumented but waits through ExitTimeOut, preventing bootstrap EIO races. stop: [ { step: "stopping the installed launch agent", command: "launchctl", - args: ["bootout", "--wait", serviceTarget], + args: ["bootout", serviceTarget], optional: true, timeout: STOP_STEP_TIMEOUT, }, + waitForStop, ], activate: [ // A persisted `launchctl disable` override refuses bootstrap; clear it. @@ -338,10 +349,11 @@ function launchdManager(input: { { step: "stopping the service", command: "launchctl", - args: ["bootout", "--wait", serviceTarget], + args: ["bootout", serviceTarget], optional: true, timeout: STOP_STEP_TIMEOUT, }, + waitForStop, ], finalize: [], }; @@ -498,7 +510,7 @@ export class BootService extends Context.Service< readonly restart: Effect.Effect<"restarted" | "queued" | "skipped", BootServiceError>; /** * Runs the operation with this home's service stopped; other homes are left alone. - * Restores a previously enabled or loaded service even if stopping failed. + * Restores a previously enabled or loaded service; launchd requires confirmed removal. * The operation's failure takes precedence over a restart failure. */ readonly withStopped: ( @@ -613,12 +625,26 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { step: string, command: string, args: ReadonlyArray, - options?: { readonly timeout?: Duration.Input }, + options?: Pick, ) { - return yield* runner.run({ command, args, timeout: options?.timeout }).pipe( + const untilExit = options?.untilExit; + const commandRun = runner.run({ command, args, timeout: options?.timeout }); + const run = + untilExit === undefined + ? commandRun + : commandRun.pipe( + Effect.repeat({ + schedule: Schedule.spaced(Duration.seconds(1)), + while: (result) => result.code === 0, + }), + Effect.timeout(STOP_STEP_TIMEOUT), + Effect.withSpan("cloud.boot_service.wait_for_step"), + ); + return yield* run.pipe( Effect.mapError((cause) => new BootServiceCommandError({ step, cause })), Effect.filterOrFail( - (result) => result.code === 0, + (result) => + untilExit === undefined ? result.code === 0 : untilExit.includes(Number(result.code)), (result) => new BootServiceCommandError({ step, @@ -635,12 +661,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { Effect.forEach( steps, (entry) => { - const run = runStep( - entry.step, - entry.command, - entry.args, - entry.timeout === undefined ? undefined : { timeout: entry.timeout }, - ); + const run = runStep(entry.step, entry.command, entry.args, entry); return entry.optional === true ? run.pipe(Effect.ignore) : run.pipe(Effect.asVoid); }, { discard: true }, @@ -929,18 +950,30 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ); return yield* Effect.acquireUseRelease( // Require every stop before repair when restoring, so the server releases the database. - // Otherwise stops are best-effort; launchd rejects bootout of an unloaded job. - // A failed stop may still stop the service, so restoration runs either way. - runSteps(manager.stop.map((step) => ({ ...step, optional: !restore }))).pipe(Effect.exit), + // Launchd drain confirmation stays strict even when the initial status probe failed. + runSteps( + manager.stop.map((step) => ({ + ...step, + optional: !restore && step.untilExit === undefined, + })), + ).pipe(Effect.exit), (stopped): Effect.Effect => Exit.isSuccess(stopped) ? operation : Effect.failCause(stopped.cause), - (_, exit) => { - if (!restore) return Effect.void; - // Start only: `activate` would also enable a unit the user disabled. - // When the operation itself failed, its error is the one worth reporting. - const start = runSteps(manager.restart); - return Exit.isFailure(exit) ? Effect.ignore(start) : start; - }, + (stopped, exit) => + Effect.gen(function* () { + if (!restore) return; + if (manager.kind === "launchd" && Exit.isFailure(stopped)) { + const result = yield* probe("launchctl", [ + "print", + `gui/${uid}/${BOOT_SERVICE_LAUNCHD_LABEL}`, + ]); + if (Option.isNone(result) || ![112, 113].includes(Number(result.value.code))) return; + } + // Start only: `activate` would also enable a unit the user disabled. + // When the operation itself failed, its error is the one worth reporting. + const start = runSteps(manager.restart); + return yield* Exit.isFailure(exit) ? Effect.ignore(start) : start; + }), ); });