diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index fca51dcac1e09..2c5de1dcfe56b 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -87,12 +87,15 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( const timeouts = new Map(); const control: { failCommand: string | undefined; + /** A command that fails with `code` for its next `remaining` invocations. */ + transient: { command: string; code: number; remaining: number } | undefined; stateAfterStop?: string; linger: string; enabled: boolean; active: boolean; } = { failCommand: undefined, + transient: undefined, linger: "yes", enabled: true, active: true, @@ -105,7 +108,12 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( commands.push(command); invocations.push(input); timeouts.set(command, input.timeout); - const failed = command === control.failCommand; + const transientHit = + control.transient !== undefined && + control.transient.command === command && + control.transient.remaining > 0; + if (transientHit) control.transient!.remaining -= 1; + const failed = command === control.failCommand || transientHit; if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; @@ -115,7 +123,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || - command.startsWith("launchctl bootout --wait ")) + command.startsWith("launchctl bootout ")) ) { yield* fs.writeFileString(statePath, control.stateAfterStop).pipe(Effect.orDie); } @@ -132,11 +140,13 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( : "", stderr: "", code: ChildProcessSpawner.ExitCode( - failed || - (input.args[1] === "is-active" && !control.active) || - (input.args[1] === "is-enabled" && !control.enabled) - ? 1 - : 0, + transientHit + ? control.transient!.code + : failed || + (input.args[1] === "is-active" && !control.active) || + (input.args[1] === "is-enabled" && !control.enabled) + ? 1 + : 0, ), timedOut: false, stdoutTruncated: false, @@ -775,6 +785,44 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); + // regression: macOS 15 parses `bootout --wait` as a target specifier and + // exits 64. The step is optional, so the old job stayed loaded and the + // following bootstrap failed EIO, leaving the previous release serving the + // T3 home after `t3 service install` had already written the new version. + it.effect("still stops the launch agent when the OS rejects bootout --wait", () => + Effect.gen(function* () { + const { service, control, commands } = yield* makeHarness("darwin"); + yield* service.install(); + + const rejectedFlag = "launchctl bootout --wait gui/501/com.t3tools.t3code.service"; + control.transient = { command: rejectedFlag, code: 64, remaining: 1 }; + commands.length = 0; + yield* service.install(); + + expect(commands).toContain("launchctl bootout gui/501/com.t3tools.t3code.service"); + expect(commands.some((command) => command.startsWith("launchctl bootstrap "))).toBe(true); + expect((yield* service.status).current).toBe(true); + }), + ); + + // withStopped makes every stop step required when the agent is loaded, so + // the rejected `--wait` must not abort the sequence before the plain + // bootout runs; `t3 doctor` repairs go through this path. + it.effect.each(["darwin"] as const)("repairs through the bootout fallback on %s", (platform) => + Effect.gen(function* () { + const { service, control, commands } = yield* makeHarness(platform); + yield* service.install(); + + const rejectedFlag = "launchctl bootout --wait gui/501/com.t3tools.t3code.service"; + control.transient = { command: rejectedFlag, code: 64, remaining: 1 }; + commands.length = 0; + yield* service.withStopped(Effect.sync(() => commands.push("repair"))); + + expect(commands).toContain("launchctl bootout gui/501/com.t3tools.t3code.service"); + expect(commands).toContain("repair"); + }), + ); + // regression: #8173 keep provider CLIs available in the macOS service it.effect("reconstructs a launch agent search path when the installer has no PATH", () => Effect.gen(function* () { diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 15b3b16a7bc0f..bdfca2a069224 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -9,6 +9,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; import { HttpClient } from "effect/http"; import * as Schema from "effect/Schema"; @@ -184,12 +185,45 @@ export interface BootServiceStep { readonly args: ReadonlyArray; /** Non-zero exit is logged and ignored. */ readonly optional?: boolean; + /** + * Exit codes that still count as success. launchd reports a plain "nothing + * to stop" (3) and, on releases without the flag, an unrecognized `--wait` + * target specifier (64); neither means the job is still loaded. + */ + readonly tolerateExitCodes?: ReadonlyArray; + /** + * Arguments tried once if this step's own arguments fail. The step fails only + * when both attempts fail, so it can stay required while offering a fallback. + */ + readonly fallbackArgs?: ReadonlyArray; + /** + * Exit codes worth retrying within the step's timeout. launchd reports EIO + * while it is still tearing down the previous job, so a start that races a + * drain succeeds on a later attempt without any other change. + */ + readonly retryOnExitCodes?: ReadonlyArray; readonly timeout?: Duration.Input; } /** Must exceed systemd TimeoutStopSec and launchd ExitTimeOut (both 90s) to avoid shutdown races. */ const STOP_STEP_TIMEOUT = Duration.seconds(120); +/** + * launchd drops a job from the domain asynchronously: a `bootstrap` that lands + * while the previous job drains fails EIO. Retrying a few times rides out that + * window, which is otherwise the only thing standing between a version update + * and a service that keeps running the release it was meant to replace. + */ +const LAUNCHD_RETRY_BASE_MS = 250; +const LAUNCHD_RETRY_CAP_MS = 2_000; +// 250ms doubling to a 2s cap. A short drain resolves on the first or second +// attempt; the 17 attempts add up to about 30s, which covers a plist's default +// ExitTimeOut without making a broken install wait the full 90s. +const LAUNCHD_RETRY_ATTEMPTS = 17; + +const launchdRetryDelay = (attempt: number) => + Duration.millis(Math.min(LAUNCHD_RETRY_BASE_MS * 2 ** attempt, LAUNCHD_RETRY_CAP_MS)); + export interface BootServiceManager { readonly kind: "systemd" | "launchd"; readonly unitPath: string; @@ -294,6 +328,9 @@ function launchdManager(input: { const domainTarget = `gui/${input.uid}`; const serviceTarget = `${domainTarget}/${BOOT_SERVICE_LAUNCHD_LABEL}`; // Unloaded jobs may reject bootout/enable. Bootstrap starts RunAtLoad/KeepAlive jobs; kickstart would restart them. + // macOS 15 rejects `bootout --wait` as an unknown target specifier (64), so the plain form is the fallback. + const LAUNCHD_EIO = 5; + const LAUNCHD_NOTHING_TO_STOP = 3; return { kind: "launchd", unitPath, @@ -308,7 +345,10 @@ function launchdManager(input: { step: "stopping the installed launch agent", command: "launchctl", args: ["bootout", "--wait", serviceTarget], + // Fallback inside the step: it fails only if both forms fail. + fallbackArgs: ["bootout", serviceTarget], optional: true, + tolerateExitCodes: [LAUNCHD_NOTHING_TO_STOP], timeout: STOP_STEP_TIMEOUT, }, ], @@ -324,6 +364,7 @@ function launchdManager(input: { step: "starting the service", command: "launchctl", args: ["bootstrap", domainTarget, unitPath], + retryOnExitCodes: [LAUNCHD_EIO], }, ], restart: [ @@ -331,6 +372,7 @@ function launchdManager(input: { step: "restarting the service after a failed update", command: "launchctl", args: ["bootstrap", domainTarget, unitPath], + retryOnExitCodes: [LAUNCHD_EIO], }, ], // Persisted launchctl disable overrides would block reinstall. @@ -339,7 +381,9 @@ function launchdManager(input: { step: "stopping the service", command: "launchctl", args: ["bootout", "--wait", serviceTarget], + fallbackArgs: ["bootout", serviceTarget], optional: true, + tolerateExitCodes: [LAUNCHD_NOTHING_TO_STOP], timeout: STOP_STEP_TIMEOUT, }, ], @@ -613,24 +657,63 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { step: string, command: string, args: ReadonlyArray, - options?: { readonly timeout?: Duration.Input }, + options?: { + readonly timeout?: Duration.Input; + readonly retryOnExitCodes?: ReadonlyArray; + readonly tolerateExitCodes?: ReadonlyArray; + readonly fallbackArgs?: ReadonlyArray; + }, ) { - return yield* runner.run({ command, args, timeout: options?.timeout }).pipe( - Effect.mapError((cause) => new BootServiceCommandError({ step, cause })), - Effect.filterOrFail( - (result) => result.code === 0, - (result) => - new BootServiceCommandError({ - step, - exitCode: Number(result.code), - stdoutLength: result.stdout.length, - stderrLength: result.stderr.length, - }), - ), - Effect.tapError(logFailure), - ); + const tolerated = options?.tolerateExitCodes ?? []; + // `suspend` matters: retry re-runs this effect, and each attempt must call + // the runner again instead of replaying one already-built command result. + const invoke = (argsToRun: ReadonlyArray) => + Effect.suspend(() => + runner.run({ command, args: argsToRun, timeout: options?.timeout }), + ).pipe( + Effect.mapError((cause) => new BootServiceCommandError({ step, cause })), + Effect.filterOrFail( + (result) => result.code === 0 || tolerated.includes(Number(result.code)), + (result) => + new BootServiceCommandError({ + step, + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ), + ); + const attempt = invoke(args); + const retryable = options?.retryOnExitCodes; + const primary = retryable === undefined ? attempt : retryableStep(attempt, retryable); + const fallback = options?.fallbackArgs; + return yield* ( + fallback === undefined ? primary : primary.pipe(Effect.catch(() => invoke(fallback))) + ).pipe(Effect.tapError(logFailure)); }); + const retryableStep = ( + // The cap keeps a drain that never ends from stalling the step past its + // timeout instead of reporting the failure. + step: Effect.Effect, + codes: ReadonlyArray, + ): Effect.Effect => { + const attempt = ( + remaining: number, + ): Effect.Effect => + step.pipe( + Effect.catchIf( + (error) => + error.exitCode !== undefined && codes.includes(error.exitCode) && remaining > 0, + () => + Effect.sleep(launchdRetryDelay(LAUNCHD_RETRY_ATTEMPTS - remaining)).pipe( + Effect.andThen(attempt(remaining - 1)), + ), + ), + ); + return attempt(LAUNCHD_RETRY_ATTEMPTS); + }; + const runSteps = (steps: ReadonlyArray) => Effect.forEach( steps, @@ -639,7 +722,21 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { entry.step, entry.command, entry.args, - entry.timeout === undefined ? undefined : { timeout: entry.timeout }, + entry.timeout === undefined && + entry.retryOnExitCodes === undefined && + entry.tolerateExitCodes === undefined && + entry.fallbackArgs === undefined + ? undefined + : { + ...(entry.timeout === undefined ? {} : { timeout: entry.timeout }), + ...(entry.retryOnExitCodes === undefined + ? {} + : { retryOnExitCodes: entry.retryOnExitCodes }), + ...(entry.tolerateExitCodes === undefined + ? {} + : { tolerateExitCodes: entry.tolerateExitCodes }), + ...(entry.fallbackArgs === undefined ? {} : { fallbackArgs: entry.fallbackArgs }), + }, ); return entry.optional === true ? run.pipe(Effect.ignore) : run.pipe(Effect.asVoid); },